Fix instance exit notification

Fix improper notification on Factorio server exit causing plugins and
the instance tracking to fail.

.gitignore

@@ -11,8 +11,9 @@ pids
 # Directory for instrumented libs generated by jscoverage/JSCover
 lib-cov
 
-# Coverage directory used by tools like istanbul
+# Coverage directories used by tools like nyc and codecov
 coverage
+.nyc_output
 
 # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
 .grunt
@@ -22,17 +23,21 @@ coverage
 
 # Compiled binary addons (http://nodejs.org/api/addons.html)
 build/Release
+dist/
+
+# Autogenerated docs output
+/docs/api
 
 node_modules/*
-!node_modules/_app
 
 Factorio*
 /factorio
 /instances
 /sharedMods
 /database
-database/items.db
-config.json
+/config-master.json
+/config-control.json
+/config-slave.json
 package-lock.json
 secret-api-token.txt
 mod_cache.json
@@ -40,15 +45,18 @@ static/pictures/spritesheet.js
 static/pictures/spritesheet.png
 static/external
 saves
+/temp/
 
 # weird pattern to ignore sharedPlugins except not ignore some core modules
 !sharedPlugins/
 sharedPlugins/*
 !sharedPlugins/inventoryImports/
-!sharedPlugins/UPSdisplay/
-!sharedPlugins/remoteMap/
-!sharedPlugins/remoteCommands/
 !sharedPlugins/researchSync/
-!sharedPlugins/globalChat/
+!sharedPlugins/clusterioMod/
 
-\.cache-require-paths\.json
+# File used for marking plugins as disabled
+sharedPlugins/*/DISABLED
+
+# IDE
+.idea/
+.vs/

.travis.yml

@@ -1,10 +1,11 @@
 language: node_js
 node_js:
-  - "node"
+  - 10
 before_install:
-  - sudo apt-get -qq update
-  - sudo apt-get install -y curl git tar
   - curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64
-  - tar -xvf factorio.tar.gz
+  - tar -xf factorio.tar.gz
+  - npm install -g codecov
 script:
-  - npm run cover
+  - npm run-script ci-cover
+  - cat temp/test/integration/log.txt
+  - codecov -f coverage/lcov.info

CHANGELOG.md

@@ -0,0 +1,235 @@
+Changelog
+=========
+
+Version 2.0.0
+-------------
+
+### Major Features
+
+- Replaced Hotpatch with save patching.  Removed the Hotpatch scenario and
+  the depency on it for getting code into the game.  Added a save patcher than
+  runs before starting up Factorio that patches in lua modules based on the
+  event_loader lib into the savegame.  Regular freeplay games can now be
+  used with Clusterio and will be compatible without having to convert them
+  to Hotpatch.
+- Daemonized slaves.  Slaves now have the ability to run multiple Factorio
+  instances, starting and stopping them individually.  To manage this the
+  local command line interface has been replaced with a remote interface on
+  the master server that can be accessed through the clusterctl cli tool.
+- Rewritten the communication between slaves and the master from scratch.  The
+  new system is based on a WebSocket connection between the slaves and the
+  master server and provides efficient validated bi-directonal communication.
+- Rewritten the plugin system from scratch.  Plugins now inherit from
+  a base class and use the same WebSocket connection Clusterio uses to
+  communicate.
+- New configuration system with support for initializing configs when needed,
+  modifying config entries using built-in commands, and updating instance
+  configuration remotely.
+
+### Features
+
+- Added export of pollution statitics.
+- Reconnection logic that esures no data is dropped talking to the master
+  server provided the session can be resumed.
+- Added support for having multiple Factorio installs and selecting which
+  version to use on a per instance basis.
+- Added config to specify which path the master interface is accessed under,
+  allowing it to be proxied behind web-server.
+- Added WebSocket usage statistics.
+- Added commands sent statistic.
+
+### Changes
+
+- Fixed sslCert and sslPrivKey entries being ignored in the config.
+- Changed ssl key creation to be done at startup instead of on npm install.
+- Added error handling during master startup.
+- Factorio game and rcon port now defaults to a random port above 49151.
+- Removed unimplemented mods update command.
+- Fixed rcon password being generated with Math.random().
+- Added plugins directory to the views path.  This makes it possible for
+  plugins to render their own ejs views or pages in their own folders by
+  using paths of the format "pluginName/path/to/page-or-view".
+- Replaced the per instance copy of the shared Factorio mods with
+  symlinks.  On Windows hard links are used instead due to the
+  privileges requirements of symlinks.
+- Changed the per instance scenario folder to be linked instead of
+  copied on instance creation.
+- Instance id is no longer derived from the rcon password, instead it's
+  generated upon instance creation and stored in the instance config.
+- The game port, RCON port and RCON password instance config entries are are
+  now null by default, indicating that a random one will be generated every
+  time the instance is started.
+- Removed the broken client download command as this would download
+  pre-releases.
+- Moved the item database and HTTP definitions for /api/place, /api/remove,
+  /api/inventory, /api/inventoryAsObject and the web interface view for the
+  storage page into the clusterioMod plugin.  If you disable this plugin
+  then these things will not be available.
+- Removed undocumented --port and --rcon-port arguments to client.
+- Removed undocumented FACTORIOPORT and RCONPORT enviroment variable
+  handling in client.
+- Removed redundant call to /api/slaves in globalChat plugin.
+- Removed broken serverManager plugin.
+- Removed factorio_version from config.  The version installed is auto
+  detected and used instead.
+- Removed the playerManager specific command CLI tools/delete_player.js.
+- Creating an instance, assigning it to a slave, creating a save for an
+  instance and starting an instance is now four separate commands.
+- Removed oddball limits to slaves.json size.
+- Moved slave specific and instance specific configuration into their own
+  configuration files.
+- Removed unused binary option from plugin config.
+- Removed info and shout command from globalChat plugin
+- Removed mirrorAllChat and enableCrossServerShout configuration options for
+  globalChat plugin.
+- Removed UPSdisplay plugin.  UPS statistics is exported by the statistics
+  exporter plugin.
+- Master server now defaults to hosting on https on port 8443.
+- Renamed client to slave.
+
+### Breaking Changes
+
+- lib/authenticate no longer requires config to be passed to it.  Breaks
+  playerManager.
+- The masterIP and masterPort config entries has been merged into masterURL.
+  Breaks discordChat, playerManager, serverSelect, trainTeleports and
+  clusterioModel.
+- Instance names can no longer be invalid Windows file names.
+- Removed config management from the command line and the server manager.
+- Moved ejs templates into views folder and changed their extension to
+  .ejs.  Breaks playerManager.
+- Mods are no longer copied from the per instance instanceMods directory.
+  If you need per instance mods you can now place them directly in the mods
+  directory inside the instance folder.
+- Hotpatch scenarios and code loading is no longer compatible with Clusterio.
+  Breaks playerManager, serverSelect, and tranTeleports.
+- Removed getLua and getCommand from lib/clusterTools.  If you need to run
+  more than the most trivial of code in commands use the save patcher and
+  add in a remote interface.
+- Removed mod uploading and distributing from the HTTP interface.  Breaks
+  the old unmaintained and no longer needed factorioClusterioClient.
+- Removed the remoteCommands plugin and the old runCommand interface.  Breaks
+  playerManager and external tools sending commands.
+- Removed broken serverManager plugin.
+- Removed fields info, time, rconPort, rconPassword, serverPort, unique,
+  mods, instanceName, playerCount, mac, and meta from the slaves in the
+  slave database.
+- Removed getInstanceName and lib/clusterTools.  Breaks playerManager,
+  and discordChat.
+- Removed the /api/rconPasswords, and /api/slaves endpoints.  Breaks web
+  interface, trainTeleports, and discordChat.
+- Removed the /api/getSlaveMeta and /api/editSlaveMeta endpoints.  Breaks
+  researchSync, and UPSdisplay.
+- Removed the hello event from the socket connection handshake.  Breaks
+  playerManager, trainTeleports, serverSelect, and discordChat
+- Changed the format of database/slaves.json.
+- Removed the output file subscription system.  Breaks inventoryImports,
+  playerManager, trainTeleports, serverSelect and researchSync.
+- Removed the factorioOutput hook from instance plugins.  The onOutput
+  hook provides parsed output instead.
+- Removed the onLoadFinish hook from master plugins.
+- Moved plugins from the sharedPlugins directory to plugins directory.
+- Implemented a new plugin system that replaces the old.  Breaks all plugins.
+- Removed express metric http_request_duration_milliseconds metric.
+- Removed socket.io metrics socket_io_connected, socket_io_connect_total,
+  socket_io_disconnect_total, socket_io_events_received_total,
+  socket_io_events_sent_total, socket_io_recieve_bytes, and
+  socket_io_transmit_bytes,
+- Removed clusterio_connected_instaces_gauge and added
+  clusterio_master_connected_clients_count in its place.
+- Renamed clusterio_player_count_gauge to
+  clusterio_statistics_exporter_instance_player_count
+- Removed clusterio_UPS_gauge and added
+  clusterio_statistics_exporter_instance_game_ticks_total in its place.
+- Renamed clusterio_endpoint_hit_gauge to clusterio_http_enpoint_hits_total
+- Renamed clusterio_statistics_gauge to
+  clusterio_statistics_exporter_instance_force_flows
+- Renamed clusterio_nn_dole_gauge to clusterio_subspace_storage_nn_dole_gauge.
+- Renamed clusterio_dole_factor_gauge to
+  clusterio_subspace_storage_dole_factor_gauge.
+- Renamed clusterio_import_gauge to clusterio_subspace_storage_import_total.
+- Renamed clusterio_export_gauge to clusterio_subspace_storage_export_total.
+- Renamed clusterio_master_inventory_gauge to
+  clusterio_subspace_storage_master_inventory.
+- Removed clusterioMod plugin specific config options logItemTransfers,
+  disableFairItemDistribution useNeuralNetDoleDivider, autosaveInterval, and
+  disableImportsOfEverythingExceptElectricity from the master config.
+- Removed msBetweenCommands config option.  The RCON is instead limited to 5
+  concurrent commands.
+- Removed allowRemoteCommandExecution config option.  Remote commands are
+  always allowed with the move to master managing slaves/instances.
+- Removed `--databaseDirectory`, `--masterPort`, and `--sslPort` command line
+  arguments from the master server.
+- Implemented a new config system that replaces the old.  Breaks all plugins.
+- Removed usage of socket.io entirely in favor of a plain WebSocket connection.
+- Renamed clusterioMod plugin to subspace_storage.
+
+
+Version 1.2.4
+-------------
+
+- Removed broken remote combinator signaling.
+- Fixed research sync endlessly updating already researched technologies
+- Removed obsolete item/fluid statistics from clusterioMod
+- Removed per mod upload logging when config.uploadModsToMaster disabled
+- Fixed bcrypt failing to install on windows due to the new version not
+  having Windows binaries.
+
+
+Version 1.2.3
+-------------
+
+- Disabled uploading mods to the master server by default as this is mostly
+  just a waste of bandwidth with the mod portal being integrated into the
+  game now.
+- Fixed researchSync breaking with heavily modded games due to the tech tree
+  exceeding the default 100kb limit on JSON payloads.
+- Fixed the command inventoryImport sends referencing player instead of
+  character, and trying to count the non-existant quick bar slots.
+- Fixed inventoryImport never receiving the script output after the default
+  mode for reading script output changed to tail mode.
+- Fixed desync caused by the mods and loaded mods arrays in Hotpatch getting
+  out of sync when plugins update their scenario mod code.  To fix existing
+  games you will need copy lib/scenarios/Hotpatch/hotpatch/mod-tools.lua
+  over the existing mod-tools.lua in the save's hotpatch folder.
+- Updated the Windows install instructions.
+
+
+Version 1.2.2
+-------------
+
+- Fixed possible crash with modded technologies named the same as a built-in
+  Object prototype property in researchSync.
+- Fixed progress of a current infinite tech carrying over to the next one
+  when researching it and another node completes it in researchSync.
+- Fixed progress of a previous infinite tech from another node being applied
+  to the current one in researchSync.
+- Fixed crash in researchSync when modded technologies are present only on some
+  nodes.
+- Fixed install failing due to bcrypt version less than 3 not being supported
+  on node v10.
+- Reordered install instructions to avoid problem with npm creating files owned
+  by root in the home directory.
+- Swapped curl out with wget in the install instructions as the latter comes
+  pre-installed on Ubuntu.
+
+### Breaking Changes
+
+- Node.js versions below 10 are no longer supported.
+
+
+Version 1.2.1
+-------------
+
+- Updated node-factorio-api to v0.3.8 to fix mod downloads randomly breaking
+  ([#229][#229].)
+- Fixed SIGINT being sent twice to Factorio server when interrupted by CTRL+C
+  on Linux ([#217][#217].)
+- Fixed package.json incorrectly reporting the license as ISC.
+
+[#217]: https://github.com/clusterio/factorioClusterio/issues/217
+[#229]: https://github.com/clusterio/factorioClusterio/issues/229
+
+### Breaking Changes
+
+- Added authentication to the socket.io server running on the master.

Dockerfile

@@ -1,19 +1,23 @@
-FROM node:latest
-RUN apt-get update && apt install git -y
-
+# Warning: This is currently broken as a result of
+# changes done in the 2.0 refactoring.
+FROM node:12
+RUN apt-get update && apt install git curl tar -y
 RUN mkdir factorioClusterio
-RUN git clone -b dev https://github.com/Danielv123/factorioClusterio.git && cd factorioClusterio && npm install && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xvzf factorio.tar.gz
+
+RUN git clone -b master https://github.com/clusterio/factorioClusterio.git && cd factorioClusterio && npm install --only=production
+RUN cd factorioClusterio && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz
 
 WORKDIR factorioClusterio
-RUN mkdir instances sharedMods sharedPlugins
+RUN mkdir instances sharedMods
+RUN cp config.json.dist config.json
 
-RUN node client.js download
+RUN node client manage shared mods add clusterio
 
-LABEL maintainer "danielv@live.no"
+LABEL maintainer "Sir3lit@gmail.com"
 
 EXPOSE 8080 34167
 VOLUME /factorioClusterio/instances
 VOLUME /factorioClusterio/sharedMods
 VOLUME /factorioClusterio/sharedPlugins
 
-CMD RCONPORT="$RCONPORT" FACTORIOPORT="$FACTORIOPORT" MODE="$MODE" node $MODE\.js start $INSTANCE
+CMD MODE="$MODE" node $MODE start $INSTANCE

LICENSE.md

@@ -1,4 +1,4 @@
-MIT License
+This product uses the MIT License
 
 Copyright (c) 2016 Daniel Vestøl
 
@@ -19,3 +19,8 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+The icons for angels modded items located in /static/pictures/ are licensed
+by https://mods.factorio.com/user/Arch666Angel under CC BY-NC-ND 4.0
+https://creativecommons.org/licenses/by-nc-nd/4.0/
+

README.md

@@ -1,22 +1,46 @@
+<img src="./logo.svg" width="100%" align="right">
+
+<br/>
+<br/>
+<br/>
+
 # factorioClusterio
-[![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2FDanielv123%2FfactorioClusterio.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2FDanielv123%2FfactorioClusterio?ref=badge_shield)
 
-| Branch | Coverage | Build |
-|--------|----------| ----- |
-| Master | [![codecov](https://codecov.io/gh/Danielv123/factorioClusterio/branch/master/graph/badge.svg)](https://codecov.io/gh/Danielv123/factorioClusterio) | [![Build Status](https://travis-ci.org/Danielv123/factorioClusterio.svg?branch=master)](https://travis-ci.org/Danielv123/factorioClusterio) |
-| Dev    | [![codecov](https://codecov.io/gh/Danielv123/factorioClusterio/branch/dev/graph/badge.svg)](https://codecov.io/gh/Danielv123/factorioClusterio) | [![Build Status](https://travis-ci.org/Danielv123/factorioClusterio.svg?branch=dev)](https://travis-ci.org/Danielv123/factorioClusterio) |
+Discord for development/support/play: https://discord.gg/5XuDkje
 
-If you want to connect to a clusterio cluster, please reffer to the [client](https://github.com/Danielv123/factorioClusterioClient)
+## Important notice
+
+**WARNING: The master branch is currently broken, do not use it!**
+
+This is the development branch for factorioClusterio 2.0 which is currently undergoing heavy
+restructuring and refactoring.  Expect plugins and existing installations to frequently break when
+using this branch.  If you don't want to be an alpha tester for 2.0 please use the stable
+[1.2.x branch][1.2.x] or
+[latest stable release](https://github.com/clusterio/factorioClusterio/releases/latest).
+
+Installation instructions below are for the unstable master branch.  Go to the page for
+the [1.2.x branch][1.2.x] for instructions on how to install the stable version.
+
+[1.2.x]: https://github.com/clusterio/factorioClusterio/tree/1.2.x
+
+### Ways to support me/the project:
+
+* Contribute with code/documentation. See [Contributing](docs/contributing.md) for how to make pull
+  requests.  Always nice to move the project forward.
+
+* Support me monetarily on [patreon](https://www.patreon.com/danielv123) or paypal: danielv@live.no
 
 ### Table of contents
 
 * [Introduction & methodology](#introduction)
-
 * [Ubuntu setup](#ubuntu-setup)
-
 * [Windows setup](#windows-setup)
-
-* [Command cheatsheet](#cheatsheet)
+* [Running Clusterio](#running-clusterio)
+  * [Master Server](#master-server)
+  * [Slaves](#slaves)
+  * [Instances](#instances)
+* [Optional plugins](#Plugins)
+* [Common problems](#Common-problems)
 
 ## Introduction
 
@@ -32,7 +56,16 @@ Features:
 
 - Inventory combinator to display item levels in the cluster (and epoch time)
 
-- Reporting of graphs and UPS on master interface
+- Reporting of graphs and UPS on master interface (Also has extensive Prometheus reporting)
+
+Optional extras (see [Plugins](#Plugins))
+
+- Have your inventory synchronize across servers
+
+- Teleport trans from the border of one world to te next
+
+- Show in-game chat in discord
+
 
 Connection diagram:
 
@@ -59,82 +92,64 @@ Clusterio exploits one of the games logging features, game.write_file and RCON t
 
 7. client.js on server2: We were allowed to import x of item y, run command /c remote.call("clusterio", "importMany", "{'copper-plate':120}")
 
-This process works the same for both items and liquids, independant on what mods are used. Yes, modded items are fully supported.
+This process works the same for both items and liquids, independent on what mods are used. Yes, modded items are fully supported.
 
-Clusterio also handles a few other neat things, such as giving you access to epoch time, transmitting combinator signals between worlds (and any other application who wants to) and 
-creating graphs on the master web interface.
+Clusterio can also do a few other neat things, such as giving you access to epoch time, syncing player inventories between servers, keeping track of playtime (playerManager plugin), teleporting trains between servers (trainTeleports) and exporting tons of factorio related statistics to Prometheus for graphing in grafana.
 
 ## Ubuntu setup
 
-NodeJS does not support EOL ubuntu releases. Make sure you are on the most recent LTS release or newer.
+**Warning**: These instructions are for the unstable master version and is not
+recommended for use, see [the 1.2.x branch][1.2.x] for how to install the stable version.
+
+Clusterio runs on Node.js v12 and up, v11.13.0+ and v10.16.0+.  Node.js itself is not
+supported on EOL Ubuntu releases so make sure you're on a recent release of Ubuntu.
 
 Master and all slaves:
 
-    sudo curl -sL https://deb.nodesource.com/setup_9.x | sudo -E bash -
-    sudo apt install -y nodejs python-dev git wget curl tar build-essential
-    sudo npm install pm2 -g
-    git clone https://github.com/Danielv123/factorioClusterio.git
+    wget -qO - https://deb.nodesource.com/setup_12.x | sudo -E bash -
+    sudo apt install -y nodejs python-dev git build-essential
+    git clone -b master https://github.com/clusterio/factorioClusterio.git
     cd factorioClusterio
-    curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64
+    wget -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64
     tar -xf factorio.tar.gz
-    npm install
-    node ./lib/npmPostinstall.js
+    npm install --only=production
 
-downloads and installs nodejs, pm2, git and clusterio. To specify a version, change "latest" in the link to a version number like 0.14.21.
-
-*Copy config.json.dist to config.json* - Otherwise it will crash, as you'd expect.
-
-**Master**
-
-    pm2 start master.js --name master
-    
-**Server Host**
-    
-To download the mod for all its non vanilla features and items, (optional, but very recommended)
-
-    node client.js manage shared mods add clusterio
-	
-Add the master API key, found in the masters install directory (secret-api-token.txt) to your config.json masterAuthToken field
-
-To create a new instance (its own save, set of mods and config files)
-
-    node client.js start [instancename]
-
-To launch an instance with pm2
-
-    pm2 start --name [instancename] client.js -- start [instancename]
-
-use `nano config.json` to change settings.
+downloads and installs nodejs, git and clusterio. To specify a version, change "latest" in the link to a version number like 0.14.21.
 
 **Ubuntu with Docker**
 
-Clusterio has limited support for using docker.
+The Docker support for Clusterio is curently broken.  If you need it
+open an issue about it.
+
+<!--
+Clusterio has *very* limited support for using docker.
 
     sudo docker build -t clusterio --no-cache --force-rm factorioClusterio
-	
+
 	sudo docker run --name master -e MODE=master -p 1234:8080 -d -it --restart=unless-stopped danielvestol/clusterio
-	
-	sudo docker run --name slave -e MODE=client -e INSTANCE=world1 -v /srv/clusterio/instances:/factorioClusterio/instances -p 1235:34167 -it --restart=unless-stopped danielvestol/clusterio
+
+	sudo docker run --name slave -e MODE=slave -e INSTANCE=world1 -v /srv/clusterio/instances:/factorioClusterio/instances -p 1235:34167 -it --restart=unless-stopped danielvestol/clusterio
 
 The -v flag is used to specify the instance directory. Your instances (save files etc) will be stored there.
+-->
 
 ## Windows setup
 
+**Warning**: These instructions are for the unstable master version and is not
+recommended for use, see [the 1.2.x branch][1.2.x] for how to install the stable version.
+
 Clusterio is built up of multiple parts. Here is a quick guide:
 
 Master = master.js
 
-Server host (Slave) = client.js + factorio server
+Server host (Slave) = slave.js + factorio server
 
 Game Client = The people connecting to the server
 
-All Server Hosts (Slaves) AND Game Clients need to be running the clusterio mod located at [github](https://github.com/Danielv123/factorioClusterioMod) Install it by dropping it into either the sharedMods folder or instances/[instanceName]/instanceMods folder.
-
-There are no requirements for other mods, they can be ran in any configuration allowed by the base game.
-
 **Requirements**
 
-download and install nodeJS 8 or 9 from http://nodejs.org
+download and install nodeJS 12 from http://nodejs.org.  Clusterio runs on Node.js v12 and
+up, v11.13.0+ and v10.16.0+.
 
 download and install git from https://git-scm.com/
 
@@ -142,68 +157,138 @@ reboot when you are done, then proceed to the next steps. *reboots matter*
 
 **Master**
 
-1. download and run https://puu.sh/toFHl/01eebbb333.bat
+1. Open PowerShell or Command prompt in the directory you want to install to and run the following commands.
 
-2. Copy config.json.dist to config.json
+        git clone -b master https://github.com/clusterio/factorioClusterio
+        cd factorioClusterio
+        npm install --only=production
 
-3. Follow the instructions in the bat file
+2. Obtain Factorio by either of these two methods:
 
-4. Run `node client.js manage shared mods add clusterio`
+    - Via the stand alone version on from their website
 
-5. type `node master.js` to start the server
+        1.  Create a new folder named "factorio" in the the factorioClusterio
+            folder.
 
-**Server Host**
+        2.  Download the MS Windows (64-bit zip package) from
+            https://www.factorio.com/download .
 
-1. download and run https://puu.sh/toFHl/01eebbb333.bat
+        3.  Open the zip file and drag the folder called "Factorio_0.x.y" into
+            the factorio folder created in step 1.
 
-2. Copy config.json.dist to config.json
+    - Via steam installation
 
-3. Follow the instructions given
+        1.  Create a new folder named "factorio" in the the factorioClusterio
+            folder.
 
-4. Type `node client.js start [instancename]` to create a new instance.
+        2.  Locate the game files by right clicking the game in steam,
+            selecting properties, then Local Files, then Browse local files.
 
-To connect to a master server running on a remote machine, open config.json with your favourite text editor (notepad++). You can also set it up to use the official server browser.
+        3.  Go to the parent folder of the folder that Steam opened and copy
+            the Factorio folder into the factorio folder created in step 1.
 
-Change masterIP to something like 31.152.123.14 (provided by master server owner)
-e war
-Change masterPort to something like 8080 (provided by master server owner)
 
-Repeat step 3 for more servers on one machine. You should be able to find its port by looking at the slave section on master:8080 (the web interface)
+## Running Clusterio
 
-**GameClient**
+After following the installation instructions you can use the following
+commands to run Clusterio.
 
-Fancy game client that does the following steps automatically, but is really old so be warned: [clusterioClient](https://github.com/Danielv123/factorioClusterioClient)
 
-1. Download the same version of the mod as the slave is running from [the mod portal](https://mods.factorio.com/mods/Danielv123/clusterio) or [github](https://github.com/Danielv123/factorioClusterioMod
+### Master Server
 
-2. Drop it into ./factorio/mods
+It's necessary to run the master server in order for anything to work.
+Once you've completed the setup run the following command to start it
+up:
 
-3. Run factorio and connect to slave as a normal MP game. You will find the port number to connect to at http://[masterAddress]:8080
+    node master run
 
-## Cheatsheet
 
-**To create a new instance/start it**
+### Slaves
 
-    node client.js start [instanceName]
+Slaves connect to the master server and are managed remotely from the
+master server.  In order for slaves to connect to the master server they
+need an authentication token from the master server.  This token is
+written to secret-api-token.txt on the master server when it is started
+up.
 
-**Other instance management tools:**
-```
-node client.js delete [instanceName]
-node client.js list
-```
-**To update clusterio to the latest version:**
+To set up the configuration for a new local slave run the following.
 
-1. Download the latest zip version of factorio for your platform manually from factorio.com. Place it in the project root folder and call it "factorio" (folder name is specified in config.json)
+    node slave config set slave.name "Local"
+    node slave config set slave.master_token "<token>"
 
-2. Grab the latest version of the repo
+This will write a new `config-slave.json` file in the current directory
+(you can change the location with the `--config` option) with the name
+and token provided.  If you are connecting to a remote master server you
+will also need to set the `slave.master_url` option to that url.
 
-```
-git pull
+You can list the config of a slave with the `node slave config list`
+command.  Use `node slave config --help` for more information.
 
-npm install
-```
+Once the config is set up run the slave with
 
-3. Download the latest version of the factorioClusterioMod from its github repo
-```
-node client.js manage shared mods add clusterio
-```
+    node slave run
+
+
+### Instances
+
+Instances are created, managed and started from the master server.  For
+now the only interface available is the `clusterctl` command line tool
+included in Clusterio.  You can run this tool from any slave, or the
+master server without having to set up a config, if you want to manage
+the cluster from somewhere else you will need to set the
+`control.master_url` and `control.master_token` options with the  `node
+clusterctl control-config set` command:
+
+The basic operations to start a new instance is the following
+
+    node clusterctl create-instance --name "My instance"
+    node clusterctl assign-instance --instance "My instance" --slave "Local"
+    node clusterctl create-save --instance "My instance"
+    node clusterctl start-instance --instance "My instance"
+
+The first line creates the instance configuration on the master server.
+The second assigns the instance to a slave which creates the instance
+directory and files needed to run the instance on the given slave.  The
+third line creates a new savegame for the instance.  You could also
+upload your own save to the instance directory instead.  And finally the
+fourth line starts the instance.
+
+There are many more commands available with clusterctl.  See
+`node clusterctl --help` for a full list of them.
+
+
+## Plugins
+Here are the known Clusterio plugins in the wild:
+1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta)
+2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook.
+3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers.
+
+## Common problems
+
+### EACCESS [...] LISTEN 443
+
+Some systems don't let non root processes listen to ports below 1000. Either run with `sudo` or change config.json to use higher port numbers.
+
+According to [this link](https://askubuntu.com/questions/839520/open-port-443-for-a-node-web-app) if you manually installed node.js following the above instructions, you may need to run the following command to fix this issue:
+
+    sudo setcap 'cap_net_bind_service=+ep' $(readlink -f $(which node))
+
+### Portforwarding doesn't work on the master server when running under WSL
+
+If you follow the ubuntu guide on WSL (Windows Subsystem for Linux, Bash on Ubuntu on Windows specifically), you will find that the website works on localhost and on your local ip, but not on the global ip. This is also true when you correctly port-forwarded the correct ports. Even when routing this server through nginx in WSL, the issue persists. Then, on a hunch, I tried to run nginx from windows itself and found that this DID work. It came to me that the only usage difference between the 2 versions of nginx is that I got a Windows Firewall popup.
+
+TLDR: the tested fix is:
+
+- open your windows firewall and go to advanced settings
+
+- click on inbound rules and click on new rule...
+
+- select port and click next >
+
+- select TCP and select specific local ports and type in the ports that you want to open (comma separated) and click next > 3 times
+
+- give the rule a name (like 'web server' or something), give it a description (optionally) and click finish
+
+### Other fixes for other potential problems:
+
+Sometimes the install fails. Try `node ./lib/npmPostinstall` to complete it.

build-mod.js

@@ -0,0 +1,92 @@
+"use strict";
+const events = require("events");
+const fs = require("fs-extra");
+const JSZip = require("jszip");
+const klaw = require("klaw");
+const path = require("path");
+const yargs = require("yargs");
+
+
+// Location of mod source files
+const sourceDir = "mod";
+
+// Paths to additional files to put into the mod's folder.
+const additionalFiles = [
+	path.join("modules", "clusterio", "serialize.lua"),
+];
+
+async function main() {
+	const args = yargs
+		.scriptName("build")
+		.options({
+			'clean': { describe: "Remove previous builds", type: 'boolean', default: false },
+			'build': { describe: "Build mod", type: 'boolean', default: true },
+			'pack': { describe: "Pack into zip file", type: 'boolean', default: true },
+			'output-dir': { describe: "Path to output built mod", nargs: 1, type: 'string', default: "dist" },
+		})
+		.argv
+	;
+
+	let info = JSON.parse(await fs.readFile(path.join(sourceDir, "info.json")));
+
+	if (args.clean) {
+		let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/
+		for (let entry of await fs.readdir(args.outputDir)) {
+			let match = splitter.exec(entry);
+			if (match) {
+				let [, name, version] = match;
+				if (name === info.name) {
+					let modPath = path.join(args.outputDir, entry);
+					console.log(`Removing ${modPath}`);
+					await fs.remove(modPath);
+				}
+			}
+		}
+	}
+
+	if (args.build) {
+		await fs.ensureDir(args.outputDir);
+		let modName = `${info.name}_${info.version}`;
+
+		if (args.pack) {
+			let zip = new JSZip();
+			let walker = klaw(sourceDir)
+				.on('data', item => {
+					if (item.stats.isFile()) {
+						// On Windows the path created uses backslashes as the directory sepparator
+						// but the zip file needs to use forward slashes.  We can't use the posix
+						// version of relative here as it doesn't work with Windows style paths.
+						let basePath = path.relative(sourceDir, item.path).replace(/\\/g, "/");
+						zip.file(path.posix.join(modName, basePath), fs.createReadStream(item.path));
+					}
+				})
+			;
+			await events.once(walker, 'end');
+
+			for (let filePath of additionalFiles) {
+				zip.file(path.posix.join(modName, path.basename(filePath)), fs.createReadStream(filePath));
+			}
+
+			let modPath = path.join(args.outputDir, `${modName}.zip`);
+			console.log(`Writing ${modPath}`);
+			let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath));
+			await events.once(writeStream, 'finish');
+
+		} else {
+			let modDir = path.join(args.outputDir, modName);
+			if (await fs.exists(modDir)) {
+				console.log(`Removing existing build ${modDir}`);
+				await fs.remove(modDir);
+			}
+			console.log(`Building ${modDir}`);
+			await fs.copy(sourceDir, modDir);
+			for (let filePath of additionalFiles) {
+				await fs.copy(filePath, path.join(modDir, path.basename(filePath)));
+			}
+		}
+	}
+}
+
+if (module === require.main) {
+	main().catch(err => { console.log(err) });
+}

client.js

@@ -1,999 +0,0 @@
-const fs = require('fs-extra');
-const https = require('follow-redirects').https;
-const needle = require("needle");
-const child_process = require('child_process');
-const path = require('path');
-const syncRequest = require('sync-request');
-const request = require("request");
-const ncp = require('ncp').ncp;
-const Rcon = require('rcon-client').Rcon;
-const hashFiles = require('hash-files');
-const _ = require('underscore');
-const deepmerge = require("deepmerge");
-const getMac = require('getmac').getMac;
-const rmdirSync = require('rmdir-sync');
-const ioClient = require("socket.io-client");
-
-// internal libraries
-const objectOps = require("./lib/objectOps.js");
-const fileOps = require("_app/fileOps");
-const stringUtils = require("./lib/stringUtils.js");
-const modManager = require("./lib/manager/modManager.js");
-const configManager = require("./lib/manager/configManager.js");
-
-// require config.json
-var config = require('./config');
-var global = {};
-
-if (!fs.existsSync(config.instanceDirectory)) {
-	fs.mkdirSync(config.instanceDirectory);
-}
-if (!fs.existsSync("./sharedPlugins/")) {
-	fs.mkdirSync("sharedPlugins");
-}
-if (!fs.existsSync("./sharedMods/")) {
-	fs.mkdirSync("sharedMods");
-}
-const instance = process.argv[3];
-const instancedirectory = config.instanceDirectory + '/' + instance;
-const command = process.argv[2];
-
-// Set the process title, shows up as the title of the CMD window on windows
-// and as the process name in ps/top on linux.
-process.title = "clusterioClient "+instance;
-
-// make sure we have the master access token (can't write to master without it since clusterio 2.0)
-if(!config.masterAuthToken || typeof config.masterAuthToken !== "string"){
-	console.log("ERROR invalid config!");
-	console.log("Master server now needs an access token for write operations. As clusterio slaves depends \
-	upon this, please add your token to config.js in the field named masterAuthToken. \
-	You can retrieve your auth token from the master in secret-api-token.txt after running it once.");
-}
-const needleOptionsWithTokenAuthHeader = {
-	headers: {
-		'x-access-token': config.masterAuthToken
-	},
-};
-
-var instanceInfo = {};
-var commandBuffer=[];
-// messageInterface Management
-setInterval(function(){
-	let command=commandBuffer.shift();
-	if(command){
-		messageInterfaceInternal(command[0], command[1], command[2], command[3]);
-	}
-},config.msBetweenCommands || 50);
-
-// function to handle sending commands into the game
-async function messageInterfaceInternal(command, callback, resolve, reject) {
-	// try to save us if you send a buffer instead of string
-	if(typeof command == "object") {
-		command = command.toString('utf8');
-	}
-	
-	if(false && process.platform == "linux" && typeof command == "string" && serverprocess) {
-		/*
-			to send to stdin, use:
-			serverprocess.stdin.write("/c command;\n")
-		*/
-		serverprocess.stdin.write(command+"\n");
-		if(typeof callback == "function"){
-			callback();
-		}
-		resolve();
-	} else if(typeof command == "string" && client && client.send && typeof client.send == "function") {
-		try {
-			let str = await client.send(command+"\n");
-			if(typeof callback == "function") callback(str)
-			resolve(str)
-		} catch (err) {
-			console.log("RCON failed, server might be paused or shutting down");
-			// console.log(err);
-			// reject(err);
-			if(typeof callback == "function"){
-				callback();
-			}
-		}
-	}
-}
-function messageInterface(command, callback) {
-	return new Promise((resolve,reject) => {
-		commandBuffer.push([command,callback, resolve, reject]);
-	});
-}
-
-
-// handle commandline parameters
-if (!command || command == "help" || command == "--help") {
-	console.error("Usage: ");
-	console.error("node client.js start [instance name]");
-	console.error("node client.js list");
-	console.error("node client.js delete [instance]");
-	console.error("To download the latest version of the Clusterio lua mod, do");
-	console.error("node client.js manage shared mods download clusterio");
-	console.error("For more management options, do");
-	console.error("node client.js manage");
-	process.exit(1);
-} else if (command == "list") {
-	let instanceNames = fileOps.getDirectoriesSync(config.instanceDirectory);
-	instanceNames.unshift("Name:");
-	let longestInstanceName = 0;
-	// determine longest instance name
-	instanceNames.forEach(function(instance){
-		if(instance.length > longestInstanceName) longestInstanceName = instance.length;
-	});
-	let displayLines = [];
-	// push name coloumn to array
-	instanceNames.forEach(function(instance){
-		while(instance.length < longestInstanceName+1){
-			instance += " ";
-		}
-		displayLines.push("| "+ instance + "|");
-	});
-	// create port colloumn
-	let factorioPorts = [];
-	instanceNames.forEach(function(instance){
-		let factorioPort
-		
-		if(instance.includes("Name:")){
-			factorioPort = "Port:"
-		} else {
-			factorioPort = require(config.instanceDirectory+"/"+instance+"/config").factorioPort;
-		}
-		factorioPorts.push(factorioPort);
-	});
-	factorioPorts.forEach((port, index) => {
-		let longestPort = 0;
-		factorioPorts.forEach((port, index) => {
-			if(port.toString().length > longestPort) longestPort = port.toString().length;
-		});
-		while(port.toString().length < longestPort){
-			port += " ";
-		}
-		factorioPorts[index] = port;
-	});
-	instanceNames.forEach(function(instance, index){
-		displayLines[index] += " " + factorioPorts[index] + " |";
-	});
-	
-	displayLines.forEach(line => console.log(line));
-	process.exit(0);
-} else if (command == "manage"){
-	// console.log("Launching mod manager");
-	//const fullUsage = 'node client.js manage [instance, "shared"] ["mods", "config"] ...';
-	function usage(instance, tool, action){
-		if(tool && tool == "mods"){
-			console.log('node client.js manage '+instance+' '+tool+' ["list", "search", "add", "remove", "update"]');
-		} else if(tool && tool == "config") {
-			console.log('node client.js manage '+instance+' '+tool+' ["list", "edit"]');
-		} else {
-			console.log('node client.js manage '+(instance || '[instance, "shared"]') +' '+ (tool || '["mods", "config"]') + ' ...');
-		}
-	}
-	const tool = process.argv[4] || "";
-	const action = process.argv[5] || "";
-	if(instance){
-		if(tool == "mods"){
-			(async function(){try{
-				// allow managing mods
-				if(action == "list"){
-					console.log(await modManager.listMods(instance));
-				} else if(action == "search"){
-					console.log(await modManager.findMods(process.argv[6]));
-				} else if(action == "add" || action == "download"){
-					await modManager.addMod(process.argv[6], instance);
-				} else if(action == "remove" || action == "rm" || action == "delete"){
-					await modManager.removeMod(process.argv[6], instance);
-				} else if(action == "update"){
-					await modManager.updateAllMods();
-				} else {
-					usage(instance, tool);
-				}
-				process.exit(0);
-			}catch(e){
-				console.log("Got error from modManager:")
-				console.log(e);
-			}})();
-		} else if(tool == "config"){
-			// allow managing the config
-			if(action == "list" || action == "show" || action == "display"){
-				configManager.displayConfig(instance);
-			} else if(action == "edit"){
-				let newConfigValue = "";
-				process.argv.forEach((arg, i)=>{
-					if(i >= 8){
-						newConfigValue += " "+arg;
-					} else if(i >= 7){
-						newConfigValue += arg;
-					}
-				});
-				configManager.editConfig(instance, process.argv[6], newConfigValue);
-			} else {
-				usage(instance, tool);
-			}
-		} else {
-			usage(instance);
-		}
-	} else {
-		console.log('Usage:');
-		usage(instance);
-	}
-	// process.exit(0);
-} else if (command == "delete") {
-	if (!process.argv[3]) {
-		console.error("Usage: node client.js delete [instance]");
-		process.exit(1);
-	} else if (typeof process.argv[3] == "string" && fs.existsSync(config.instanceDirectory+"/" + process.argv[3]) && process.argv[3] != "/" && process.argv[3] != "") {
-		fileOps.deleteFolderRecursiveSync(config.instanceDirectory+"/" + process.argv[3]);
-		console.log("Deleted instance " + process.argv[3]);
-		process.exit(0);
-	} else {
-		console.error("Instance not found: " + process.argv[3]);
-		process.exit(0);
-	}
-} else if (command == "download") {
-	console.log("Downloading mods...");
-	// get JSON data about releases
-	let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
-	let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
-	let name = JSON.parse(res.getBody())[0].assets[0].name;
-	if(url) {
-		console.log(url);
-		let file = fs.createWriteStream("sharedMods/"+name);
-		https.get(url, function(response) {
-			response.pipe(file);
-			console.log("Downloaded "+name);
-			process.exit(0);
-		});
-	}
-} else if (command == "start" && instance === undefined) {
-	console.log("ERROR: No instanceName provided!");
-	console.log("Usage: node client.js start [instanceName]");
-	process.exit(0);
-} else if (command == "start" && typeof instance == "string" && instance != "/" && !fs.existsSync(instancedirectory)) {
-	// if instance does not exist, create it
-	console.log("Creating instance...");
-	fs.mkdirSync(instancedirectory);
-	fs.mkdirSync(instancedirectory + "/script-output/");
-	fs.mkdirSync(instancedirectory + "/saves/");
-	fs.writeFileSync(instancedirectory + "/script-output/output.txt", "");
-	fs.writeFileSync(instancedirectory + "/script-output/orders.txt", "");
-	fs.writeFileSync(instancedirectory + "/script-output/txbuffer.txt", "");
-	fs.mkdirSync(instancedirectory + "/mods/");
-	fs.mkdirSync(instancedirectory + "/instanceMods/");
-	// fs.symlinkSync('../../../sharedMods', instancedirectory + "/mods", 'junction') // This is broken because it can only take a file as first argument, not a folder
-	fs.writeFileSync(instancedirectory + `/config.ini`, `[path]\r\n
-read-data=${ path.resolve(config.factorioDirectory, "data") }\r\n
-write-data=${ path.resolve(config.instanceDirectory, instance) }\r\n
-	`);
-	
-	// this line is probably not needed anymore but Im not gonna remove it
-	fs.copySync('sharedMods', instancedirectory + "/mods");
-	let instconf = {
-		"factorioPort": process.env.FACTORIOPORT || Math.floor(Math.random() * 65535),
-		"clientPort": process.env.RCONPORT || Math.floor(Math.random() * 65535),
-		"clientPassword": Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 8),
-	}
-	console.log("Clusterio | Created instance with settings:")
-	console.log(instconf);
-	
-	// create instance config
-	fs.writeFileSync(instancedirectory + "/config.json", JSON.stringify(instconf, null, 4));
-	
-	let name = "Clusterio instance: " + instance;
-	if (config.username) {
-		name = config.username + "'s clusterio " + instance;
-	}
-	let serversettings = {
-		"name": name,
-		"description": config.description,
-		"tags": ["clusterio"],
-		"max_players": "20",
-		"visibility": config.visibility,
-		"username": config.username,
-		"token": config.token,
-		"game_password": config.game_password,
-		"verify_user_identity": config.verify_user_identity,
-		"admins": [config.username],
-		"allow_commands": config.allow_commands,
-		"autosave_interval": 10,
-		"autosave_slots": 5,
-		"afk_autokick_interval": 0,
-		"auto_pause": config.auto_pause,
-	}
-	fs.writeFileSync(instancedirectory + "/server-settings.json", JSON.stringify(serversettings, null, 4));
-	let createSave = child_process.spawnSync(
-		'./' + config.factorioDirectory + '/bin/x64/factorio', [
-			'-c', instancedirectory + '/config.ini',
-			'--create', instancedirectory + '/saves/save.zip',
-		]
-	);
-	console.log("Instance created!");
-	process.exit(0);
-} else if (command == "start" && typeof instance == "string" && instance != "/" && fs.existsSync(instancedirectory)) {
-	// Exit if no instance specified (it should be, just a safeguard);
-	if(instancedirectory != config.instanceDirectory+"/undefined"){
-		var instanceconfig = require(path.resolve(instancedirectory,'config'));
-		instanceconfig.unique = stringUtils.hashCode(instanceconfig.clientPassword);
-		if(process.env.FACTORIOPORT){
-			instanceconfig.factorioPort = process.env.FACTORIOPORT;
-		}
-		if(process.env.RCONPORT){
-			instanceconfig.rconPort = process.env.RCONPORT;
-		}
-	} else {
-		process.exit(1);
-	}
-	console.log("Deleting .tmp.zip files");
-	let savefiles = fs.readdirSync(instancedirectory + "/saves/");
-	for(i = 0; i < savefiles.length; i++){
-		if(savefiles[i].substr(savefiles[i].length - 8, 8) == ".tmp.zip") {
-			fs.unlinkSync(instancedirectory + "/saves/" + savefiles[i]);
-		}
-	}
-	console.log("Deleting logs");
-	// clean old log file to avoid crash
-	// file exists, delete so we don't get in trouble
-	try {
-		fs.unlinkSync(instancedirectory+'/factorio-current.log');
-	} catch (err){
-		if(err){
-			console.log(err);
-		} else {
-			console.log("Clusterio | Deleting old logs...");
-		}
-	}
-	
-	// move mods from ./sharedMods to the instances mod directory
-	try{fs.mkdirSync(instancedirectory + "/instanceMods/");}catch(e){}
-	try{rmdirSync(instancedirectory + "/mods");}catch(e){}
-	try {
-		// mods directory that will be emptied (deleted) when closing the server to facilitate seperation of instanceMods and sharedMods
-		fs.mkdirSync(instancedirectory + "/mods/");
-	} catch(e){}
-	console.log("Clusterio | Moving shared mods from sharedMods/ to instance/mods...");
-	fs.copySync('sharedMods', instancedirectory + "/mods");
-	console.log("Clusterio | Moving instance specific mods from instance/instanceMods to instance/mods...");
-	fs.copySync(instancedirectory + "/instanceMods", instancedirectory + "/mods");
-
-	process.on('SIGINT', function () {
-		console.log("Caught interrupt signal, sending /quit");
-		messageInterface("/quit");
-	});
-
-	// Spawn factorio server
-	//var serverprocess = child_process.exec(commandline);
-	fileOps.getNewestFile(instancedirectory + "/saves/", fs.readdirSync(instancedirectory + "/saves/"),function(err, latestSave) {
-		if(err) {
-			console.error("ERROR!")
-			console.error("Your savefile seems to be missing. This might because you created an instance without having factorio\
- installed and configured properly. Try installing factorio and adding your savefile to instances/[instancename]/saves/\n");
-			throw err;
-		}
-		// implicit global
-		serverprocess = child_process.spawn(
-			'./' + config.factorioDirectory + '/bin/x64/factorio', [
-				'-c', instancedirectory + '/config.ini',
-				'--start-server', latestSave.file,
-				'--rcon-port', Number(process.env.RCONPORT) || instanceconfig.clientPort,
-				'--rcon-password', instanceconfig.clientPassword,
-				'--server-settings', instancedirectory + '/server-settings.json',
-				'--port', Number(process.env.FACTORIOPORT) || instanceconfig.factorioPort
-			], {
-				'stdio': ['pipe', 'pipe', 'pipe']
-			}
-		);
-
-		serverprocess.on('close', (code) => {
-			console.log(`child process exited with code ${code}`);
-			process.exit();
-		});
-		serverprocess.stdout.on("data", (data) => {
-			// log("Stdout: " + data);
-			if(data.toString('utf8').includes("Couldn't parse RCON data: Maximum payload size exceeded")){
-				console.error("ERROR: RCON CONNECTION BROKE DUE TO TOO LARGE PACKET!");
-				console.error("Attempting reconnect...");
-				client.close();
-				client.connect();
-			}
-			// we have to do this to make logs visible on linux and in powershell. Causes log duplication for people with CMD.
-			console.log('Fact: ' + data);
-		});
-		serverprocess.stderr.on('data', (chunk) => {
-			console.log('ERR: ' + chunk);
-		});
-
-		// connect to the server with rcon
-		if(true || process.platform != "linux"){
-			// IP, port, password
-			client = new Rcon();
-			
-			// check the logfile to see if the RCON interface is running as there is no way to continue without it
-			// we read the log every 2 seconds and stop looping when we start connecting to factorio
-			function checkRcon() {
-				fs.readFile(instancedirectory+"/factorio-current.log", function (err, data) {
-					// if (err) console.log(err);
-					if(data && data.indexOf('Starting RCON interface') > 0){
-						client.connect({
-							host: 'localhost',
-							port: Number(process.env.RCONPORT) || instanceconfig.clientPort,
-							password: instanceconfig.clientPassword,
-							timeout: 5000
-						});
-					} else {
-						setTimeout(function(){
-							checkRcon();
-						},5000);
-					}
-				});
-			}
-			setTimeout(checkRcon, 5000);
-		
-			client.onDidAuthenticate(() => {
-				console.log('Clusterio | RCON Authenticated!');
-				instanceManagement(); // start using rcons
-			});
-			client.onDidConnect(() => {
-				console.log('Clusterio | RCON Connected!');
-				// getID();
-			});
-			client.onDidDisconnect(() => {
-				console.log('Clusterio | RCON Disconnected!');
-				process.exit(0); // exit because RCON disconnecting is undefined behaviour and we rather just wanna restart now
-			});
-		} else if(process.platform == "linux"){
-			// don't open an RCON connection and just use stdio instead, does not work on windows.
-			instanceManagement();
-		}
-
-		// set some globals
-		confirmedOrders = [];
-		lastSignalCheck = Date.now();
-	});
-}
-
-// ensure instancemanagement only ever runs once
-_.once(instanceManagement);
-function instanceManagement() {
-	/* Open websocket connection to master */
-	var socket = ioClient("http://"+config.masterIP+":"+config.masterPort);
-	socket.on("hello", data => {
-		console.log("SOCKET | registering slave!");
-		socket.emit("registerSlave", {
-			instanceID: instanceconfig.unique,
-		});
-		setInterval(B=> socket.emit("heartbeat"), 10000);
-	});
-	
-	console.log("Started instanceManagement();");
-	// load plugins and execute onLoad event
-	let pluginDirectories = fileOps.getDirectoriesSync("./sharedPlugins/");
-	let plugins = [];
-	for(let i=0; i<pluginDirectories.length; i++) {
-		let I = i
-		let log = function(message) {
-			console.log("Clusterio | "+ pluginDirectories[I] + " | " + message);
-		}
-		// these are our two config files. We need to send these in case plugin
-		// wants to contact master or know something.
-		let combinedConfig = deepmerge(instanceconfig,config,{clone:true})
-		let pluginConfig = require("./sharedPlugins/" + pluginDirectories[i] + "/config.js");
-		
-		if(!global.subscribedFiles) {
-			global.subscribedFiles = {};
-		}
-		if(pluginConfig.binary == "nodePackage"){
-			// require index.js.main() of plugin and execute it as a class
-			let pluginClass = require("./sharedPlugins/" + pluginDirectories[I] + "/index.js");
-			plugins[I] = new pluginClass(combinedConfig, async function(data, callback){
-				if(data.toString('utf8')[0] != "/") {
-					log("Stdout: " + data.toString('utf8'));
-					return true;
-				} else {
-					return messageInterface(data.toString('utf8'), callback);
-				}
-			}, { // extra functions to pass in object. Should have done it like this from the start, but won't break backwards compat.
-				socket, // socket.io connection to master (and ES6 destructuring, yay)
-			});
-			if(plugins[I].factorioOutput && typeof plugins[I].factorioOutput === "function"){
-				// when factorio logs a line, send it to the plugin. This includes things like autosaves, chat, errors etc
-				serverprocess.stdout.on("data", data => plugins[I].factorioOutput(data.toString()));
-			}
-			if(pluginConfig.scriptOutputFileSubscription && typeof pluginConfig.scriptOutputFileSubscription == "string"){
-				if(global.subscribedFiles[pluginConfig.scriptOutputFileSubscription]) {
-					// please choose a unique file to subscribe to. If you need plugins to share this interface, set up a direct communication
-					// between those plugins instead.
-					throw "FATAL ERROR IN " + pluginDirectories[i] + " FILE ALREADY SUBSCRIBED " + pluginConfig.scriptOutputFileSubscription;
-				}
-				
-				if (!fs.existsSync(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription)) {
-					// Do something
-					fs.writeFileSync(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription, "");
-				}
-				global.subscribedFiles[pluginConfig.scriptOutputFileSubscription] = true;
-				console.log("Clusterio | Registered file subscription on "+instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription);
-				fs.watch(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription, fileChangeHandler);
-				// run once in case a plugin wrote out information before the plugin loaded fully
-				fileChangeHandler(false, pluginConfig.scriptOutputFileSubscription);
-				
-				// send file contents to plugin for processing
-				function fileChangeHandler(eventType, filename) {
-					if(filename != null){
-						setTimeout(
-							()=>{
-								// get array of lines in file
-								let stuff = fs.readFileSync(instancedirectory + "/script-output/" + filename, "utf8").split("\n");
-
-								// if you found anything, reset the file
-								if (stuff[0]) {
-									fs.writeFileSync(instancedirectory + "/script-output/" + filename, "");
-								}
-								for(let i = 0; i < stuff.length; i++) {
-									if(stuff[i] && !stuff[i].includes('\u0000\u0000')) {
-										try{
-											plugins[I].scriptOutput(stuff[i]);
-										}catch(e){console.log(e)}
-									}
-								}
-							},
-							pluginConfig.fileReadDelay || 0
-						);
-					}
-				}
-			}
-			console.log("Clusterio | Loaded plugin " + pluginDirectories[i]);
-		} else if(pluginConfig.binary != "nodePackage" && pluginConfig.binary){
-			// handle as fragile executable plugin
-			let args = pluginConfig.args || [];
-			plugins[I]=child_process.spawn(pluginConfig.binary, args, {
-				cwd: "./sharedPlugins/"+pluginDirectories[i],
-				stdio: ['pipe', 'pipe', 'pipe'],
-			});
-			
-			/*
-				to send to stdin, use:
-				spawn.stdin.write("text\n");
-			*/
-			// If plugin has subscribed to a file, send any text appearing in that file to stdin
-			if(pluginConfig.scriptOutputFileSubscription && typeof pluginConfig.scriptOutputFileSubscription == "string") {
-				if(global.subscribedFiles[pluginConfig.scriptOutputFileSubscription]) {
-					// please choose a unique file to subscribe to. If you need plugins to share this interface, set up a direct communication
-					// between those plugins instead.
-					throw "FATAL ERROR IN " + pluginDirectories[i] + " FILE ALREADY SUBSCRIBED " + pluginConfig.scriptOutputFileSubscription;
-				}
-				
-				if (!fs.existsSync(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription)) {
-					// Do something
-					fs.writeFileSync(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription, "");
-				}
-				global.subscribedFiles[pluginConfig.scriptOutputFileSubscription] = true;
-				fs.watch(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription, function (eventType, filename) {
-					// get array of lines in file
-					// TODO async
-					let stuff = fs.readFileSync(instancedirectory + "/script-output/" + filename, "utf8").split("\n");
-					// if you found anything, reset the file
-					if (stuff[0]) {
-						fs.writeFileSync(instancedirectory + "/script-output/" + filename, "");
-					}
-					for (let i = 0; i < stuff.length; i++) {
-						if (stuff[i]) {
-							plugins[I].stdin.write(stuff[i]);
-						}
-					}
-				});
-			}
-			// these are our two config files. We need to send these in case plugin
-			// wants to contact master or know something.
-			// send through script-output file, maybe more compat?
-			fs.writeFileSync(instancedirectory + "/script-output/" + pluginConfig.scriptOutputFileSubscription, JSON.stringify(combinedConfig)+"\r\n");
-			// send directly through stdin
-			// plugins[i].stdin.write(JSON.stringify(combinedConfig)+"\n");
-			
-			console.log("Clusterio | Loaded plugin " + pluginDirectories[i]);
-			plugins[i].stdout.on("data", (data) => {
-				if(data.toString('utf8')[0] != "/") {
-					log("Stdout: " + data.toString('utf8'))
-				} else {
-					messageInterface(data.toString('utf8'));
-				}
-			});
-			plugins[i].stderr.on("data", (data) => {
-				log("STDERR: " + data);
-			});
-			plugins[i].on('close', (code) => {
-				log(`child process exited with code ${code}`);
-			});
-		} else {
-			// This plugin doesn't have a client portion
-		}
-	}
-	
-	// world IDs ------------------------------------------------------------------
-	hashMods(instance, function(modHashes){
-		setInterval(getID, 10000);
-		getID();
-		function getID() {
-			messageInterface("/silent-command rcon.print(#game.connected_players)", function(playerCount) {
-				var payload = {
-					time: Date.now(),
-					rconPort: instanceconfig.clientPort,
-					rconPassword: instanceconfig.clientPassword,
-					serverPort: instanceconfig.factorioPort,
-					unique: instanceconfig.unique,
-					publicIP: config.publicIP, // IP of the server should be global for all instances, so we pull that straight from the config
-					mods:modHashes,
-					instanceName: instance,
-					playerCount:playerCount.replace(/(\r\n\t|\n|\r\t)/gm, ""),
-				}
-				
-				function callback(err, mac) {
-					if (err) {
-						mac = "unknown";
-						console.log("##### getMac crashed, but we don't really give a shit because we are probably closing down #####");
-					}
-					payload.mac = mac;
-					console.log("Registered our precense with master "+config.masterIP+" at " + payload.time);
-					needle.post(config.masterIP + ":" + config.masterPort + '/api/getID', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) {
-						if (err && err.code != "ECONNRESET"){
-							console.error("We got problems, something went wrong when contacting master");
-							console.error(err);
-						} else if (response && response.body) {
-							// In the future we might be interested in whether or not we actually manage to send it, but honestly I don't care.
-							console.log(response.body);
-						}
-					});
-				}
-				if(global.mac){
-					callback(undefined, global.mac);
-				} else {
-					getMac(callback);
-				}
-			});
-		}
-	});
-	// Mod uploading and management -----------------------------------------------
-	// get mod names and hashes
-	// string: instance, function: callback
-	
-	setTimeout(function(){hashMods(instance, uploadMods)}, 5000);
-	function uploadMods(modHashes) {
-		// [{modName:string,hash:string}, ... ]
-		for(let i=0;i<modHashes.length;i++){
-			let payload = {
-				modName: modHashes[i].modName,
-				hash: modHashes[i].hash,
-			}
-			needle.post(config.masterIP + ":" + config.masterPort + '/api/checkMod', payload, needleOptionsWithTokenAuthHeader, function (err, response, body) {
-				if(err) console.log(new Error("Unable to contact master server! Please check your config.json."));
-				if(response && body && body == "found") {
-					console.log("master has mod "+modHashes[i].modName);
-				} else if (response && body && typeof body == "string") {
-					let mod = response.body;
-					if(config.uploadModsToMaster){
-						console.log("Sending mod: " + mod);
-						// Send mods master says it wants
-						// response.body is a string which is a modName.zip
-						var req = request.post({url: "http://"+config.masterIP + ":" + config.masterPort + '/api/uploadMod',
-							headers: {
-								"x-access-token": config.masterAuthToken,
-							},
-						}, function (err, resp, body) {
-							if (err) {
-								console.log(new Error("Unable to contact master server! Please check your config.json."));
-							} else {
-								console.log('URL: ' + body);
-							}
-						});
-						var form = req.form();
-						form.append('file', fs.createReadStream(config.instanceDirectory+"/"+instance+"/mods/"+mod));
-					} else {
-						console.log("Not sending mod: " + mod + " to master because config.uploadModsToMaster is not enabled")
-					}
-				}
-			});
-		}
-	}
-	
-	// flow/production statistics ------------------------------------------------------------
-	oldFlowStats = false
-	setInterval(function(){
-		fs.readFile(instancedirectory + "/script-output/flows.txt", {encoding: "utf8"}, function(err, data) {
-			if(!err && data) {
-				let timestamp = Date.now();
-				data = data.split("\n");
-				let flowStats = [];
-				for(let i = 0; i < data.length; i++) {
-					// try catch to remove any invalid json
-					try{
-						flowStats[flowStats.length] = JSON.parse(data[i]);
-					} catch (e) {
-						// console.log(" invalid json: " + i);
-						// some lines of JSON are invalid but don't worry, we just filter em out
-					}
-				}
-				// fluids
-				let flowStat1 = flowStats[flowStats.length-1].flows.player.input_counts
-				// items
-				let flowStat2 = flowStats[flowStats.length-2].flows.player.input_counts
-				// merge fluid and item flows
-				let totalFlows = {};
-				for(let key in flowStat1) totalFlows[key] = flowStat1[key];
-				for(let key in flowStat2) totalFlows[key] = flowStat2[key];
-				if(oldFlowStats && totalFlows && oldTimestamp) {
-					let payload = objectOps.deepclone(totalFlows);
-					// change from total reported to per time unit
-					for(let key in oldFlowStats) {
-						// get production per minute
-						payload[key] = Math.floor((payload[key] - oldFlowStats[key])/(timestamp - oldTimestamp)*60000);
-						if(payload[key] < 0) {
-							payload[key] = 0;
-						}
-					}
-					for(let key in payload) {
-						if(payload[key] == '0') {
-							delete payload[key];
-						}
-					}
-					console.log("Recorded flows, copper plate since last time: " + payload["copper-plate"]);
-					needle.post(config.masterIP + ":" + config.masterPort + '/api/logStats', {timestamp: timestamp, instanceID: instanceconfig.unique,data: payload}, needleOptionsWithTokenAuthHeader, function (err, response, body) {
-						// we did it, keep going
-					});
-				}
-				oldTimestamp = timestamp;
-				oldFlowStats = totalFlows;
-				fs.writeFileSync(instancedirectory + "/script-output/flows.txt", "");
-			}
-		});
-		// we don't need to update stats quickly as that could be expensive
-	}, 60000*5);
-	
-	// provide items --------------------------------------------------------------
-	// trigger when something happens to output.txt
-	fs.watch(instancedirectory + "/script-output/output.txt", function (eventType, filename) {
-		// get array of lines in file
-		let items = fs.readFileSync(instancedirectory + "/script-output/output.txt", "utf8").split("\n");
-		// if you found anything, reset the file
-		if (items[0]) {
-			fs.writeFileSync(instancedirectory + "/script-output/output.txt", "");
-		}
-		for (let i = 0; i < items.length; i++) {
-			if (items[i]) {
-				let g = items[i].split(" ");
-				g[0] = g[0].replace("\u0000", "");
-				// console.log("exporting " + JSON.stringify(g));
-				// send our entity and count to the master for him to keep track of
-				needle.post(config.masterIP + ":" + config.masterPort + '/api/place', {
-					name: g[0],
-					count: g[1],
-					instanceName: instance, // name of instance
-					instanceID: instanceconfig.unique, // a hash computed from the randomly generated rcon password
-				}, needleOptionsWithTokenAuthHeader, function (err, resp, body) {
-					if(body == "failure") console.error("#### Export failed! Lost: "+g[1]+" "+g[0]);
-					if(config.logItemTransfers){
-						if(body == "success") console.log(`Exported ${g[1]} ${g[0]} to master`);
-					}
-				});
-			}
-		}
-	});
-	// request items --------------------------------------------------------------
-	setInterval(function () {
-		// get array of lines in file
-		let items = fs.readFileSync(instancedirectory + "/script-output/orders.txt", "utf8").split("\n");
-		// if we actually got anything from the file, proceed and reset file
-		if (items[0]) {
-			fs.writeFileSync(instancedirectory + "/script-output/orders.txt", "");
-			// prepare a package of all our requested items in a more tranfer friendly format
-			var preparedPackage = {};
-			for (let i = 0; i < items.length; i++) {
-				(function (i) {
-					if (items[i]) {
-						items[i] = items[i].split(" ");
-						items[i][0] = items[i][0].replace("\u0000", "");
-						items[i][0] = items[i][0].replace(",", "");
-						if (preparedPackage[items[i][0]]) {
-							// if we have buffered some already, sum the new items
-							if (typeof Number(preparedPackage[items[i][0]].count) == "number" && typeof Number(items[i][1]) == "number") {
-								preparedPackage[items[i][0]] = {
-									"name": items[i][0],
-									"count": Number(preparedPackage[items[i][0]].count) + Number(items[i][1]),
-									"instanceName":instance,
-									"instanceID":instanceconfig.unique,
-								};
-							// else just add em in without summing
-							} else if (typeof Number(items[i][1]) == "number") {
-								preparedPackage[items[i][0]] = {
-									"name": items[i][0],
-									"count": Number(items[i][1]),
-									"instanceName":instance,
-									"instanceID":instanceconfig.unique,
-								};
-							}
-						// this condition will NEVER be triggered but we know how that goes
-						} else if (typeof Number(items[i][1]) == "number") {
-							preparedPackage[items[i][0]] = {
-								"name": items[i][0],
-								"count": Number(items[i][1]),
-								"instanceName":instance,
-								"instanceID":instanceconfig.unique,
-							};
-						}
-					}
-				})(i);
-			}
-			// request our items, one item at a time
-			for (let i = 0; i < Object.keys(preparedPackage).length; i++) {
-				// console.log(preparedPackage[Object.keys(preparedPackage)[i]]);
-				needle.post(config.masterIP + ":" + config.masterPort + '/api/remove', preparedPackage[Object.keys(preparedPackage)[i]], needleOptionsWithTokenAuthHeader, function (err, response, body) {
-					if (response && response.body && typeof response.body == "object") {
-						// buffer confirmed orders
-						confirmedOrders[confirmedOrders.length] = {name:response.body.name,count:response.body.count}
-						if(config.logItemTransfers){
-							console.log(`Imported ${response.body.count} ${response.body.name} from master`);
-						}
-					}
-				});
-			}
-			// if we got some confirmed orders
-			// console.log("Importing " + confirmedOrders.length + " items! " + JSON.stringify(confirmedOrders));
-			//if (!(confirmedOrders.length>0)){return;}
-			let cmd="local t={";
-			for(let i=0;i<confirmedOrders.length;i++)
-			{
-			    cmd+='["'+confirmedOrders[i].name+'"]='+confirmedOrders[i].count+',';
-			    if(cmd.length>320) // Factorio max packet size is 508
-			    {
-			        messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')");
-			        cmd="local t={";
-			    }
-			}
-			if (!(cmd==="local t={")){
-				messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+cmd.slice(0, -1)+"}"+ " for k, item in pairs(t) do GiveItemsToStorage(k, item) end')");
-			}
-			confirmedOrders=[];
-		}
-	}, 1000);
-	// COMBINATOR SIGNALS ---------------------------------------------------------
-	// get inventory from Master and RCON it to our slave
-	setInterval(function () {
-		needle.get(config.masterIP + ":" + config.masterPort + '/api/inventory', function (err, response, body) {
-			if(err){
-				console.log("Unable to get JSON master/api/inventory, master might be unaccessible");
-			} else if (response && response.body) {
-				// Take the inventory we (hopefully) got and turn it into the format LUA accepts
-				if(Buffer.isBuffer(response.body)) {console.log(response.body.toString("utf-8")); throw new Error();} // We are probably contacting the wrong webserver
-				try {
-					var inventory = JSON.parse(response.body);
-					var inventoryFrame = {};
-					for (let i = 0; i < inventory.length; i++) {
-						inventoryFrame[inventory[i].name] = Number(inventory[i].count);
-						if(inventoryFrame[inventory[i].name] >= Math.pow(2, 31)){
-							inventoryFrame[inventory[i].name] = Math.pow(2, 30); // set it waaay lower, 31 -1 would probably suffice
-						}
-					}
-					inventoryFrame["signal-unixtime"] = Math.floor(Date.now()/1000);
-					// console.log("RCONing inventory! " + JSON.stringify(inventoryFrame));
-					let first = true;
-					let cmd="local s={";
-					for (let key in inventoryFrame)
-					{
-						cmd+='["'+key+'"]='+inventoryFrame[key]+",";
-						if(first && cmd.length>300 || !first && cmd.length > 320) // Factorio max packet size is 508
-						{
-					       		messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')");
-					       		cmd="local s={";
-					       		first = false;
-						}
-					}
-					if (!(cmd==="local s={")){
-						messageInterface("/silent-command remote.call('clusterio', 'runcode', '"+(first ? 'global.ticksSinceMasterPinged=0 ':'')+cmd.slice(0, -1)+"}"+ " for name,count in pairs(s) do global.invdata[name]=count end')");
-					}
-					messageInterface("/silent-command remote.call('clusterio', 'runcode', 'UpdateInvCombinators()')");
-				} catch (e){
-					console.log(e);
-				}
-			}
-		});
-	}, 1550);
-	// Make sure world has its worldID
-	setTimeout(function(){
-		messageInterface("/silent-command remote.call('clusterio','setWorldID',"+instanceconfig.unique+")")
-	}, 20000);
-	/* REMOTE SIGNALLING
-	 * send any signals the slave has been told to send
-	 * Fetch combinator signals from the server
-	*/
-	socket.on("processCombinatorSignal", circuitFrameWithMeta => {
-		if(circuitFrameWithMeta && typeof circuitFrameWithMeta == "object" && circuitFrameWithMeta.frame && Array.isArray(circuitFrameWithMeta.frame)){
-			messageInterface("/silent-command remote.call('clusterio', 'receiveFrame', '"+JSON.stringify(circuitFrameWithMeta.frame)+"')");
-		}
-	});
-	// get outbound frames from file and send to master
-	// get array of lines in file, each line should correspond to a JSON encoded frame
-	let signals = fs.readFileSync(instancedirectory + "/script-output/txbuffer.txt", "utf8").split("\n");
-	// if we actually got anything from the file, proceed and reset file
-	let readingTxBufferSoon = false;
-	let txBufferClearCounter = 0;
-	fs.watch(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (eventType, filename) => {
-		if(!readingTxBufferSoon){ // use a 100ms delay to avoid messing with rapid sequential writes from factorio (I think that might be a problem maybe?)
-			readingTxBufferSoon = true;
-			setTimeout(()=>{
-				txBufferClearCounter++;
-				fs.readFile(instancedirectory + "/script-output/txbuffer.txt", "utf-8", (err, signals) => {
-					signals = signals.split("\n");
-					if (signals[0]) {
-						//if(txBufferClearCounter > 500){
-							fs.writeFileSync(instancedirectory + "/script-output/txbuffer.txt", "");
-						//	txBufferClearCounter = 0;
-						//}
-						
-						// loop through all our frames
-						for (let i = 0; i < signals.length; i++) {
-							if (signals[i] && objectOps.isJSON(signals[i])) {
-								// signals[i] is a JSON array called a "frame" of signals. We timestamp it for storage on master
-								// then we unpack and RCON in this.frame to the game later.
-								let framepart = JSON.parse(signals[i]);
-								let doneframe = {
-									time: Date.now(),
-									frame: framepart, // thats our array of objects(single signals);
-								}
-								// send to master using socket.io, opened at the top of instanceManagement()
-								socket.emit("combinatorSignal", doneframe);
-							} else {
-								// console.log("Invalid jsony: "+typeof signals[i])
-							}
-						}
-					}
-				});
-				readingTxBufferSoon = false;
-			},100);
-		}
-	});
-} // END OF INSTANCE START ---------------------------------------------------------------------
-
-// string, function
-// returns [{modName:string,hash:string}, ... ]
-function hashMods(instanceName, callback) {
-	if(!callback) {
-		throw new Error("ERROR in function hashMods NO CALLBACK");
-	}
-	function callback2(hash, modName){
-		hashedMods[hashedMods.length] = {
-			modName: modName,
-			hash: hash,
-		}
-		// check if this callback has ran once for each mod
-		if(hashedMods.length == instanceMods.length) {
-			callback(hashedMods);
-		}
-		//console.log(modname);
-	}
-	let hashedMods = [];
-	/*let mods = fs.readdirSync("./sharedMods/")*/
-	let instanceMods = fs.readdirSync(config.instanceDirectory+"/"+instanceName+"/mods/");
-	if(instanceMods.length == 0){
-		callback({}); // there are no mods installed, return an empty object
-	}
-	for(let o=0;o<instanceMods.length;o++) {
-		if(path.extname(instanceMods[o]) != ".zip") {
-			instanceMods.splice(instanceMods.indexOf(instanceMods[o]), 1); // remove element from array, we can't hash unzipped mods
-		}
-	}
-	for(let i=0; i<instanceMods.length; i++){
-		let path = config.instanceDirectory+"/"+instanceName+"/mods/"+instanceMods[i];
-		let name = instanceMods[i];
-		let options = {
-			files:path,
-		}
-		// options {files:[array of paths]}
-		hashFiles(options, function(error, hash) {
-			// hash will be a string if no error occurred
-			if(!error){
-				callback2(hash, name);
-			} else {
-				throw error;
-			}
-		});
-	}
-}

clusterctl.js

@@ -0,0 +1,566 @@
+const jwt = require("jsonwebtoken");
+const fs = require("fs-extra");
+const yargs = require("yargs");
+const version = require("./package").version;
+const asTable = require("as-table").configure({delimiter: ' | '});
+const chalk = require("chalk");
+const events = require("events");
+
+const link = require("lib/link");
+const errors = require("lib/errors");
+const config = require("lib/config");
+
+
+/**
+ * Format a parsed Factorio output message with colors
+ *
+ * Formats a parsed Factorio output from lib/factorio into a readable
+ * colorized output using terminal escape codes that can be printed.
+ */
+function formatOutputColored(output) {
+	let time = "";
+	if (output.format === 'seconds') {
+		time = chalk.yellow(output.time.padStart(8)) + ' ';
+	} else if (output.format === 'date') {
+		time = chalk.yellow(output.time) + ' ';
+	}
+
+	let info = "";
+	if (output.type === 'log') {
+		let level = output.level;
+		if (level === "Info") {
+			level = chalk.bold.blueBright(level);
+		} else if (output.level === "Warning") {
+			level = chalk.bold.yellowBright(level);
+		} else if (output.level === "Error") {
+			level = chalk.bold.redBright(level);
+		}
+
+		info = level + ' ' + chalk.gray(output.file) + ': ';
+
+	} else if (output.type === 'action') {
+		info = '[' + chalk.yellow(output.action) + '] ';
+	}
+
+	return time + info + output.message;
+}
+
+/**
+ * Represents a command that can be runned by clusterctl
+ */
+class Command {
+	constructor({ definition, handler }) {
+		this.name = definition[0];
+		this._definition = definition;
+		this._handler = handler;
+	}
+
+	register(yargs) {
+		yargs.command(...this._definition);
+	}
+
+	async run(args, control) {
+		await this._handler.call(this, args, control);
+	}
+}
+
+/**
+ * Resolve a string to an instance ID
+ *
+ * Resolves a string with either an instance name or an id into an integer
+ * with the instance ID.
+ *
+ * @param client - link to master server to query instance on.
+ * @param instanceName - string with name or id of instance.
+ * @returns {number} instance ID.
+ */
+async function resolveInstance(client, instanceName) {
+	let instanceId;
+	if (/^-?\d+$/.test(instanceName)) {
+		instanceId = parseInt(instanceName, 10);
+	} else {
+		let response = await link.messages.listInstances.send(client);
+		for (let instance of response.list) {
+			if (instance.name === instanceName) {
+				instanceId = instance.id;
+				break;
+			}
+		}
+
+		if (instanceId === undefined) {
+			throw new errors.CommandError(`No instance named ${instanceName}`);
+		}
+	}
+
+	return instanceId;
+}
+
+/**
+ * Resolve a string into a slave ID
+ *
+ * Resolves a string with either an slave name or an id into an integer with
+ * the slave ID.
+ *
+ * @param client - link to master server to query slave on.
+ * @param slaveName - string with name or id of slave.
+ * @returns {number} slave ID.
+ */
+async function resolveSlave(client, slaveName) {
+	let slaveId;
+	if (/^-?\d+$/.test(slaveName)) {
+		slaveId = parseInt(slaveName, 10);
+	} else {
+		let response = await link.messages.listSlaves.send(client);
+		for (let slave of response.list) {
+			if (slave.name === slaveName) {
+				slaveId = slave.id;
+				break;
+			}
+		}
+
+		if (slaveId === undefined) {
+			throw new errors.CommandError(`No slave named ${slaveName}`);
+		}
+	}
+
+	return slaveId;
+}
+
+let commands = [];
+commands.push(new Command({
+	definition: ["list-slaves", "List slaves connected to the master"],
+	handler: async function(args, control) {
+		let response = await link.messages.listSlaves.send(control);
+		console.log(asTable(response.list));
+	},
+}));
+
+commands.push(new Command({
+	definition: ["list-instances", "List instances known to the master"],
+	handler: async function(args, control) {
+		let response = await link.messages.listInstances.send(control);
+		console.log(asTable(response.list));
+	}
+}));
+
+commands.push(new Command({
+	definition: ['create-instance', "Create an instance", (yargs) => {
+		// XXX TODO: set any specific options?
+		yargs.option('name', {
+			type: "string",
+			nargs: 1,
+			describe: "Instance name",
+			group: "Instance Config:",
+		});
+		yargs.option('id', {
+			type: "number",
+			nargs: 1,
+			describe: "Instance ID",
+			group: "Instance Config:",
+		});
+		yargs.option('base', {
+			type: "boolean",
+			nargs: 0,
+			describe: "Specify this is a base config",
+			group: "Instance Config:"
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceConfig = new config.InstanceConfig();
+		await instanceConfig.init();
+		if (args.id) {
+			instanceConfig.set("instance.id", args.id);
+		}
+		if (args.name) {
+			instanceConfig.set("instance.name", args.name);
+		}
+		let serialized_config = instanceConfig.serialize();
+		let response = await link.messages.createInstance.send(control, { serialized_config });
+	},
+}));
+
+commands.push(new Command({
+	definition: ["list-instance-config", "List configuration for an instance", (yargs) => {
+		yargs.option({
+			"instance": { describe: "Instance to list config for", nargs: 1, type: "string", demandOption: true },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await resolveInstance(control, args.instance);
+		let response = await link.messages.getInstanceConfig.send(control, { instance_id: instanceId });
+
+		for (let group of response.serialized_config.groups) {
+			for (let [name, value] of Object.entries(group.fields)) {
+				console.log(`${group.name}.${name} ${JSON.stringify(value)}`);
+			}
+		}
+	},
+}));
+
+commands.push(new Command({
+	definition: ["set-instance-config", "Set field in instance config", (yargs) => {
+		yargs.option({
+			"instance": { describe: "Instance to set config on", nargs: 1, type: "string", demandOption: true },
+			"field": { describe: "Field to set", nargs: 1, type: "string", demandOption: true },
+			"value": { describe: "Value to set", nargs: 1, type: "string", demandOption: true },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await resolveInstance(control, args.instance);
+		await link.messages.setInstanceConfigField.send(control, {
+			instance_id: instanceId,
+			field: args.field,
+			value: args.value,
+		});
+	},
+}));
+
+commands.push(new Command({
+	definition: ['assign-instance', "Assign instance to a slave", (yargs) => {
+		yargs.options({
+			'instance': { describe: "Instance to assign", nargs: 1, type: 'string', demandOption: true },
+			'slave': { describe: "Slave to assign to", nargs: 1, type: 'string', demandOption: true },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await resolveInstance(control, args.instance);
+		let slaveId = await resolveSlave(control, args.slave);
+		await link.messages.assignInstanceCommand.send(control, {
+			instance_id: instanceId,
+			slave_id: slaveId,
+		});
+	},
+}));
+
+commands.push(new Command({
+	definition: ['create-save', "Create a new save on an instance", (yargs) => {
+		yargs.options({
+			'instance': { describe: "Instance to create on", nargs: 1, type: 'string', demandOption: true },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await resolveInstance(control, args.instance);
+		await link.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
+		let response = await link.messages.createSave.send(control, {
+			instance_id: instanceId,
+		});
+	},
+}));
+
+commands.push(new Command({
+	definition: ['start-instance', "Start instance", (yargs) => {
+		yargs.options({
+			'instance': { describe: "Instance to start", nargs: 1, type: 'string', demandOption: true },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await resolveInstance(control, args.instance);
+		await link.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
+		let response = await link.messages.startInstance.send(control, {
+			instance_id: instanceId,
+		});
+	},
+}));
+
+commands.push(new Command({
+	definition: ['stop-instance', "Stop instance", (yargs) => {
+		yargs.options({
+			'instance': { describe: "Instance to stop", nargs: 1, type: 'string', demandOption: true },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await resolveInstance(control, args.instance);
+		await link.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
+		let response = await link.messages.stopInstance.send(control, {
+			instance_id: instanceId,
+		});
+	},
+}));
+
+commands.push(new Command({
+	definition: ['delete-instance', "Delete instance", (yargs) => {
+		yargs.options({
+			'instance': { describe: "Instance to delete", nargs: 1, type: 'string', demandOption: true },
+		});
+	}],
+	handler: async function(args, control) {
+		let response = await link.messages.deleteInstance.send(control, {
+			instance_id: await resolveInstance(control, args.instance),
+		});
+	},
+}));
+
+commands.push(new Command({
+	definition: ['send-rcon', "Send RCON command", (yargs) => {
+		yargs.options({
+			'instance': { describe: "Instance to sent to", nargs: 1, type: 'string', demandOption: true },
+			'command': { describe: "command to send", nargs: 1, type: 'string', demandOption: true },
+		});
+	}],
+	handler: async function(args, control) {
+		let response = await link.messages.sendRcon.send(control, {
+			instance_id: await resolveInstance(control, args.instance),
+			command: args.command
+		})
+
+		// Factorio includes a newline in it's response output.
+		process.stdout.write(response.result);
+	},
+}));
+
+// Convert to mapping from name to command instance
+commands = new Map([...commands.map(command => [command.name, command])]);
+
+
+/**
+ * Connector for control connection to master server
+ */
+class ControlConnector extends link.WebSocketClientConnector {
+	constructor(url, reconnectDelay, token) {
+		super(url, reconnectDelay);
+		this._token = token;
+	}
+
+	register() {
+		console.log("SOCKET | registering control");
+		this.sendHandshake("register_control", {
+			token: this._token,
+			agent: "clusterctl",
+			version: version,
+		});
+	}
+}
+
+/**
+ * Handles running the control
+ *
+ * Connects to the master server over WebSocket and sends commands to it.
+ */
+class Control extends link.Link {
+
+	// I don't like God classes, but the alternative of putting all this state
+	// into global variables is not much better.
+	constructor(connector) {
+		super('control', 'master', connector);
+		link.attachAllMessages(this);
+	}
+
+	async instanceOutputEventHandler(message) {
+		let { instance_id, output } = message.data;
+		console.log(formatOutputColored(output));
+	}
+
+	async shutdown() {
+		await link.messages.shutdownConnection.send(this);
+		this.connector.close(1001, "Control Quit")
+		await events.once(this.connector, "close");
+	}
+}
+
+async function findCredentials(controlConfig) {
+	// Try looking in the default master config location
+	try {
+		let masterConfig = new config.MasterConfig();
+		await masterConfig.load(JSON.parse(await fs.readFile("config-master.json")));
+
+		let token = jwt.sign({ id: "api" }, masterConfig.get("master.auth_secret"), { expiresIn: 600 });
+		controlConfig.set("control.master_token", token);
+
+		let url;
+		if (masterConfig.get("master.https_port")) {
+			url = `https://localhost:${masterConfig.get("master.https_port")}`;
+		} else {
+			url = `http://localhost:${masterConfig.get("master.http_port")}`;
+		}
+		controlConfig.set("control.master_url", url);
+
+		return;
+
+	} catch (err) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+	}
+
+	// Try looking in the default slave config location
+	// In the future it's likely that tokens will be assigned more granular
+	// authorizations, meaning that you cannot use a slave token to control
+	// the cluster.
+	try {
+		let slaveConfig = new config.SlaveConfig();
+		await slaveConfig.load(JSON.parse(await fs.readFile("config-slave.json")));
+
+		controlConfig.set("control.master_token", slaveConfig.get("slave.master_token"));
+		controlConfig.set("control.master_url", slaveConfig.get("slave.master_url"));
+		return;
+
+	} catch (err) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+	}
+
+	throw new errors.StartupError("Couldn't find credentials to connect to the master server with");
+}
+
+
+class ControlGroup extends config.ConfigGroup {}
+ControlGroup.groupName = "control";
+ControlGroup.define({
+	name: "master_url",
+	description: "URL to connect to the master server at",
+	type: "string",
+	optional: true,
+});
+ControlGroup.define({
+	name: "master_token",
+	description: "Token to authenticate to master server with.",
+	type: "string",
+	optional: true,
+});
+ControlGroup.define({
+	name: "reconnect_delay",
+	title: "Reconnect Delay",
+	description: "Maximum delay to wait before attempting to reconnect WebSocket",
+	type: "number",
+	initial_value: 2,
+});
+ControlGroup.finalize();
+
+class ControlConfig extends config.Config { }
+ControlConfig.registerGroup(ControlGroup);
+ControlConfig.finalize();
+
+
+async function startControl() {
+	yargs
+		.scriptName("clusterctl")
+		.usage("$0 <command> [options]")
+		.option('config', {
+			nargs: 1,
+			describe: "config file to get credentails from",
+			default: "config-control.json",
+			defaultDescription: "auto",
+			type: 'string',
+		})
+		.command("control-config", "Manage Control config", config.configCommand)
+		.wrap(yargs.terminalWidth())
+	;
+
+	for (let command of commands.values()) {
+		command.register(yargs);
+	}
+
+	const args = yargs.demandCommand(1, "You need to specify a command to run")
+		.strict()
+		.argv
+	;
+
+	config.finalizeConfigs();
+
+	console.log(`Loading config from ${args.config}`);
+	let controlConfig = new ControlConfig();
+	try {
+		await controlConfig.load(JSON.parse(await fs.readFile(args.config)));
+
+	} catch (err) {
+		if (err.code === 'ENOENT') {
+			console.log("Config not found, initializing new config");
+			await controlConfig.init();
+
+		} else {
+			throw err;
+		}
+	}
+
+	let commandName = args._[0];
+	if (commandName === "control-config") {
+		await config.handleConfigCommand(args, controlConfig, args.config);
+		return;
+	}
+
+	// The remaining commands require connecting to the master server.
+	if (!controlConfig.get("control.master_url") || !controlConfig.get("control.master_token")) {
+		await findCredentials(controlConfig);
+	}
+
+	let controlConnector = new ControlConnector(
+		controlConfig.get("control.master_url"),
+		controlConfig.get("control.reconnect_delay"),
+		controlConfig.get("control.master_token")
+	);
+	let control = new Control(controlConnector);
+	try {
+		await controlConnector.connect();
+	} catch(err) {
+		if (err instanceof errors.AuthenticationFailed) {
+			throw new errors.StartupError(err.message);
+		}
+		throw err;
+	}
+
+	if (commands.has(commandName)) {
+		command = commands.get(commandName);
+
+		try {
+			await command.run(args, control)
+
+		} catch (err) {
+			if (err instanceof errors.CommandError) {
+				console.error(`Error running command: ${err.message}`);
+				process.exitCode = 1;
+
+			} else if (err instanceof errors.RequestError) {
+				console.error(`Error sending request: ${err.message}`);
+				process.exitCode = 1;
+
+			} else {
+				throw err;
+			}
+
+		} finally {
+			await control.shutdown();
+		}
+	}
+
+	//XXX control.close("done");
+}
+
+module.exports = {
+	// for testing only
+	_formatOutputColored: formatOutputColored,
+	_resolveInstance: resolveInstance,
+	_Control: Control,
+
+	_commands: commands,
+}
+
+
+if (module === require.main) {
+	console.warn(`
++==========================================================+
+I WARNING:  This is the development branch for the 2.0     I
+I           version of clusterio.  Expect things to break. I
++==========================================================+
+`
+	);
+	startControl().catch(err => {
+		if (!(err instanceof errors.StartupError)) {
+			console.error(`
++----------------------------------------------------------------+
+| Unexpected error occured while starting control, please report |
+| it to https://github.com/clusterio/factorioClusterio/issues    |
++----------------------------------------------------------------+`
+			);
+		} else {
+			console.error(`
++-------------------------------+
+| Unable to to start clusterctl |
++-------------------------------+`
+			);
+		}
+
+		console.error(err);
+		process.exitCode = 1;
+	});
+}

config.json.dist

@@ -1,52 +0,0 @@
-{
-	"__comment": "Where factorio folder. Should be in the same folder as this config file.",
-	"factorioDirectory": "factorio",
-	"instanceDirectory": "instances",
-	
-	"__comment1": "IP players should connect to to join your servers, should be portforwarded and stuff",
-	"publicIP": "localhost",
-	
-	"__comment2": "IP the master web server runs at, can be ignored on the master system",
-	"masterIP": "localhost",
-	
-	"__commentMasterPort": "HTTP port for the master to listen to, set to 0 or an empty string to not listen for HTTP requests",
-	"masterPort": 8080,
-	
-	"__commentsslPort": "HTTPS port for the master to listen to. Set to 0 or an empty string to not use HTTPS.",
-	"sslPort": 443,
-	"sslCert": "database/certificates/cert.crt",
-	"sslPrivKey": "database/certificates/cert.key",
-	
-	"__commentMasterAuthToken": "Authentication token to connect to the master server. Find it in secret-api-token.txt on the master.",
-	"masterAuthToken": "",
-	
-	"__commentAuthSecret": "This is a long random string used for generating authentication tokens. Do not share. Regenerates when secret-api-token.txt is deleted.",
-	"masterAuthSecret": "",
-	
-	"__comment3": "Your factorio.com username for factorio MP services, also username will be admin on new instances",
-	"username": "",
-	"__comment4": "Token from your factorio.com profile page. Required for MP and mod manager services.",
-	"token": "",
-	"__comment7": "Used for communication with the mod portal, not needed if you manage mods by hand",
-	"factorio_version": "0.16.41",
-	
-	"__comment5": "General user settings",
-	"verify_user_identity": true,
-	"auto_pause": false,
-	"allow_commands": "admins-only",
-	"game_password": "",
-	"description": "This server runs Clusterio",
-	"visibility": {
-		"public": true,
-		"lan": true
-	},
-	"__comment6": "Statistic gathering statistics, may affect server performance (master memory usage especially)",
-	"itemStats": {
-		"maxEntries":60,
-		"entriesPerSecond": 0.01666666667
-	},
-	"logItemTransfers": false,
-	"uploadModsToMaster": true,
-	"msBetweenCommands": 10,
-	"allowRemoteCommandExecution": true
-}

docs/assets/connection_flow.dot

@@ -0,0 +1,20 @@
+digraph "Client Connection Flow" {
+    node [shape=box];
+    graph [newrank=true];
+
+    start [label="Start"];
+    wait_hello [label="Wait for hello"];
+    new_session [label="New session"];
+    wait_ready [label="Wait for ready"];
+    connected [label="Connected"];
+    abort [label="Abort"];
+    lost_connection [label="Connection lost or\nunexpectedly closed\nwhile connecting" style="dotted"];
+
+
+    lost_connection -> start [label="Wait random delay"];
+    start -> wait_hello [label="Connect to /api/socket"];
+    wait_hello -> new_session [label="Receive hello"];
+    new_session -> wait_ready [label="Send register_*"];
+    wait_ready -> connected [label="Receive ready"];
+    wait_ready -> abort [label="Connection closed with 4003"];
+}

docs/assets/connection_flow.svg

@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.42.3 (0)
+ -->
+<!-- Title: Client Connection Flow Pages: 1 -->
+<svg width="371pt" height="496pt"
+ viewBox="0.00 0.00 371.00 496.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 492)">
+<title>Client Connection Flow</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-492 367,-492 367,4 -4,4"/>
+<!-- start -->
+<g id="node1" class="node">
+<title>start</title>
+<polygon fill="none" stroke="black" points="135,-384 81,-384 81,-348 135,-348 135,-384"/>
+<text text-anchor="middle" x="108" y="-362.3" font-family="Times,serif" font-size="14.00">Start</text>
+</g>
+<!-- wait_hello -->
+<g id="node2" class="node">
+<title>wait_hello</title>
+<polygon fill="none" stroke="black" points="164.5,-297 51.5,-297 51.5,-261 164.5,-261 164.5,-297"/>
+<text text-anchor="middle" x="108" y="-275.3" font-family="Times,serif" font-size="14.00">Wait for hello</text>
+</g>
+<!-- start&#45;&gt;wait_hello -->
+<g id="edge2" class="edge">
+<title>start&#45;&gt;wait_hello</title>
+<path fill="none" stroke="black" d="M108,-347.8C108,-336.16 108,-320.55 108,-307.24"/>
+<polygon fill="black" stroke="black" points="111.5,-307.18 108,-297.18 104.5,-307.18 111.5,-307.18"/>
+<text text-anchor="middle" x="187.5" y="-318.8" font-family="Times,serif" font-size="14.00">Connect to /api/socket</text>
+</g>
+<!-- new_session -->
+<g id="node3" class="node">
+<title>new_session</title>
+<polygon fill="none" stroke="black" points="161,-210 55,-210 55,-174 161,-174 161,-210"/>
+<text text-anchor="middle" x="108" y="-188.3" font-family="Times,serif" font-size="14.00">New session</text>
+</g>
+<!-- wait_hello&#45;&gt;new_session -->
+<g id="edge3" class="edge">
+<title>wait_hello&#45;&gt;new_session</title>
+<path fill="none" stroke="black" d="M108,-260.8C108,-249.16 108,-233.55 108,-220.24"/>
+<polygon fill="black" stroke="black" points="111.5,-220.18 108,-210.18 104.5,-220.18 111.5,-220.18"/>
+<text text-anchor="middle" x="155.5" y="-231.8" font-family="Times,serif" font-size="14.00">Receive hello</text>
+</g>
+<!-- wait_ready -->
+<g id="node4" class="node">
+<title>wait_ready</title>
+<polygon fill="none" stroke="black" points="167.5,-123 48.5,-123 48.5,-87 167.5,-87 167.5,-123"/>
+<text text-anchor="middle" x="108" y="-101.3" font-family="Times,serif" font-size="14.00">Wait for ready</text>
+</g>
+<!-- new_session&#45;&gt;wait_ready -->
+<g id="edge4" class="edge">
+<title>new_session&#45;&gt;wait_ready</title>
+<path fill="none" stroke="black" d="M108,-173.8C108,-162.16 108,-146.55 108,-133.24"/>
+<polygon fill="black" stroke="black" points="111.5,-133.18 108,-123.18 104.5,-133.18 111.5,-133.18"/>
+<text text-anchor="middle" x="164" y="-144.8" font-family="Times,serif" font-size="14.00">Send register_*</text>
+</g>
+<!-- connected -->
+<g id="node5" class="node">
+<title>connected</title>
+<polygon fill="none" stroke="black" points="92,-36 0,-36 0,0 92,0 92,-36"/>
+<text text-anchor="middle" x="46" y="-14.3" font-family="Times,serif" font-size="14.00">Connected</text>
+</g>
+<!-- wait_ready&#45;&gt;connected -->
+<g id="edge5" class="edge">
+<title>wait_ready&#45;&gt;connected</title>
+<path fill="none" stroke="black" d="M64.78,-86.86C57.71,-82.13 51.29,-76.25 47,-69 43.04,-62.32 41.73,-54.22 41.69,-46.49"/>
+<polygon fill="black" stroke="black" points="45.19,-46.55 42.31,-36.36 38.21,-46.13 45.19,-46.55"/>
+<text text-anchor="middle" x="97.5" y="-57.8" font-family="Times,serif" font-size="14.00">Receive ready</text>
+</g>
+<!-- abort -->
+<g id="node6" class="node">
+<title>abort</title>
+<polygon fill="none" stroke="black" points="198,-36 142,-36 142,0 198,0 198,-36"/>
+<text text-anchor="middle" x="170" y="-14.3" font-family="Times,serif" font-size="14.00">Abort</text>
+</g>
+<!-- wait_ready&#45;&gt;abort -->
+<g id="edge6" class="edge">
+<title>wait_ready&#45;&gt;abort</title>
+<path fill="none" stroke="black" d="M131.28,-86.66C137.25,-81.45 143.3,-75.4 148,-69 153.19,-61.93 157.5,-53.45 160.88,-45.5"/>
+<polygon fill="black" stroke="black" points="164.14,-46.78 164.56,-36.19 157.63,-44.21 164.14,-46.78"/>
+<text text-anchor="middle" x="259.5" y="-57.8" font-family="Times,serif" font-size="14.00">Connection closed with 4003</text>
+</g>
+<!-- lost_connection -->
+<g id="node7" class="node">
+<title>lost_connection</title>
+<polygon fill="none" stroke="black" stroke-dasharray="1,5" points="188.5,-488 27.5,-488 27.5,-435 188.5,-435 188.5,-488"/>
+<text text-anchor="middle" x="108" y="-472.8" font-family="Times,serif" font-size="14.00">Connection lost or</text>
+<text text-anchor="middle" x="108" y="-457.8" font-family="Times,serif" font-size="14.00">unexpectedly closed</text>
+<text text-anchor="middle" x="108" y="-442.8" font-family="Times,serif" font-size="14.00">while connecting</text>
+</g>
+<!-- lost_connection&#45;&gt;start -->
+<g id="edge1" class="edge">
+<title>lost_connection&#45;&gt;start</title>
+<path fill="none" stroke="black" d="M108,-434.95C108,-422.36 108,-407.17 108,-394.35"/>
+<polygon fill="black" stroke="black" points="111.5,-394.24 108,-384.24 104.5,-394.24 111.5,-394.24"/>
+<text text-anchor="middle" x="175.5" y="-405.8" font-family="Times,serif" font-size="14.00">Wait random delay</text>
+</g>
+</g>
+</svg>

docs/assets/generate.sh

@@ -0,0 +1,4 @@
+for x in $(dirname $0)/*.dot; do
+    echo $x
+    dot $x -Tsvg -o ${x/.dot/.svg}
+done

docs/assets/reconnection_flow.dot

@@ -0,0 +1,23 @@
+digraph "Client Reconnection Flow" {
+    node [shape=box];
+    graph [newrank=true];
+
+    reconnect [label="Reconnect"];
+    wait_hello [label="Wait for hello"];
+    resume_session [label="Resume session"];
+    new_session [label="New session" style="dashed"];
+    wait_continue [label="Wait for continue"];
+    connected [label="Connected"];
+    lost_connection [label="Connection lost or\nunexpectedly closed\nwhile reconnecting" style="dotted"];
+    abort [label="Abort"];
+
+    { rank="same"; wait_continue; new_session };
+
+    lost_connection -> reconnect [label="Wait random delay"];
+    reconnect -> wait_hello [label="Connect to /api/socket"];
+    wait_hello -> resume_session [label="Receive hello"];
+    resume_session -> wait_continue [label="Send resume"];
+    wait_continue -> abort [label="Connection closed with 4003"];
+    wait_continue -> new_session [label="Receive invalidate"];
+    wait_continue -> connected [label="Receive ready"];
+}

docs/assets/reconnection_flow.svg

@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.42.3 (0)
+ -->
+<!-- Title: Client Reconnection Flow Pages: 1 -->
+<svg width="441pt" height="496pt"
+ viewBox="0.00 0.00 441.00 496.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 492)">
+<title>Client Reconnection Flow</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-492 437,-492 437,4 -4,4"/>
+<!-- reconnect -->
+<g id="node1" class="node">
+<title>reconnect</title>
+<polygon fill="none" stroke="black" points="153.5,-384 62.5,-384 62.5,-348 153.5,-348 153.5,-384"/>
+<text text-anchor="middle" x="108" y="-362.3" font-family="Times,serif" font-size="14.00">Reconnect</text>
+</g>
+<!-- wait_hello -->
+<g id="node2" class="node">
+<title>wait_hello</title>
+<polygon fill="none" stroke="black" points="164.5,-297 51.5,-297 51.5,-261 164.5,-261 164.5,-297"/>
+<text text-anchor="middle" x="108" y="-275.3" font-family="Times,serif" font-size="14.00">Wait for hello</text>
+</g>
+<!-- reconnect&#45;&gt;wait_hello -->
+<g id="edge2" class="edge">
+<title>reconnect&#45;&gt;wait_hello</title>
+<path fill="none" stroke="black" d="M108,-347.8C108,-336.16 108,-320.55 108,-307.24"/>
+<polygon fill="black" stroke="black" points="111.5,-307.18 108,-297.18 104.5,-307.18 111.5,-307.18"/>
+<text text-anchor="middle" x="187.5" y="-318.8" font-family="Times,serif" font-size="14.00">Connect to /api/socket</text>
+</g>
+<!-- resume_session -->
+<g id="node3" class="node">
+<title>resume_session</title>
+<polygon fill="none" stroke="black" points="173.5,-210 42.5,-210 42.5,-174 173.5,-174 173.5,-210"/>
+<text text-anchor="middle" x="108" y="-188.3" font-family="Times,serif" font-size="14.00">Resume session</text>
+</g>
+<!-- wait_hello&#45;&gt;resume_session -->
+<g id="edge3" class="edge">
+<title>wait_hello&#45;&gt;resume_session</title>
+<path fill="none" stroke="black" d="M108,-260.8C108,-249.16 108,-233.55 108,-220.24"/>
+<polygon fill="black" stroke="black" points="111.5,-220.18 108,-210.18 104.5,-220.18 111.5,-220.18"/>
+<text text-anchor="middle" x="155.5" y="-231.8" font-family="Times,serif" font-size="14.00">Receive hello</text>
+</g>
+<!-- wait_continue -->
+<g id="node5" class="node">
+<title>wait_continue</title>
+<polygon fill="none" stroke="black" points="178,-123 38,-123 38,-87 178,-87 178,-123"/>
+<text text-anchor="middle" x="108" y="-101.3" font-family="Times,serif" font-size="14.00">Wait for continue</text>
+</g>
+<!-- resume_session&#45;&gt;wait_continue -->
+<g id="edge4" class="edge">
+<title>resume_session&#45;&gt;wait_continue</title>
+<path fill="none" stroke="black" d="M108,-173.8C108,-162.16 108,-146.55 108,-133.24"/>
+<polygon fill="black" stroke="black" points="111.5,-133.18 108,-123.18 104.5,-133.18 111.5,-133.18"/>
+<text text-anchor="middle" x="155" y="-144.8" font-family="Times,serif" font-size="14.00">Send resume</text>
+</g>
+<!-- new_session -->
+<g id="node4" class="node">
+<title>new_session</title>
+<polygon fill="none" stroke="black" stroke-dasharray="5,2" points="433,-123 327,-123 327,-87 433,-87 433,-123"/>
+<text text-anchor="middle" x="380" y="-101.3" font-family="Times,serif" font-size="14.00">New session</text>
+</g>
+<!-- wait_continue&#45;&gt;new_session -->
+<g id="edge6" class="edge">
+<title>wait_continue&#45;&gt;new_session</title>
+<path fill="none" stroke="black" d="M178.05,-105C220.94,-105 275.39,-105 316.94,-105"/>
+<polygon fill="black" stroke="black" points="316.94,-108.5 326.94,-105 316.94,-101.5 316.94,-108.5"/>
+<text text-anchor="middle" x="252.5" y="-111.8" font-family="Times,serif" font-size="14.00">Receive invalidate</text>
+</g>
+<!-- connected -->
+<g id="node6" class="node">
+<title>connected</title>
+<polygon fill="none" stroke="black" points="92,-36 0,-36 0,0 92,0 92,-36"/>
+<text text-anchor="middle" x="46" y="-14.3" font-family="Times,serif" font-size="14.00">Connected</text>
+</g>
+<!-- wait_continue&#45;&gt;connected -->
+<g id="edge7" class="edge">
+<title>wait_continue&#45;&gt;connected</title>
+<path fill="none" stroke="black" d="M61.23,-86.99C53.95,-82.28 47.4,-76.37 43,-69 39.01,-62.33 38.18,-54.22 38.78,-46.49"/>
+<polygon fill="black" stroke="black" points="42.27,-46.77 40.32,-36.36 35.35,-45.72 42.27,-46.77"/>
+<text text-anchor="middle" x="93.5" y="-57.8" font-family="Times,serif" font-size="14.00">Receive ready</text>
+</g>
+<!-- abort -->
+<g id="node8" class="node">
+<title>abort</title>
+<polygon fill="none" stroke="black" points="198,-36 142,-36 142,0 198,0 198,-36"/>
+<text text-anchor="middle" x="170" y="-14.3" font-family="Times,serif" font-size="14.00">Abort</text>
+</g>
+<!-- wait_continue&#45;&gt;abort -->
+<g id="edge5" class="edge">
+<title>wait_continue&#45;&gt;abort</title>
+<path fill="none" stroke="black" d="M131.28,-86.66C137.25,-81.45 143.3,-75.4 148,-69 153.19,-61.93 157.5,-53.45 160.88,-45.5"/>
+<polygon fill="black" stroke="black" points="164.14,-46.78 164.56,-36.19 157.63,-44.21 164.14,-46.78"/>
+<text text-anchor="middle" x="260.5" y="-57.8" font-family="Times,serif" font-size="14.00">Connection closed with 4003</text>
+</g>
+<!-- lost_connection -->
+<g id="node7" class="node">
+<title>lost_connection</title>
+<polygon fill="none" stroke="black" stroke-dasharray="1,5" points="188.5,-488 27.5,-488 27.5,-435 188.5,-435 188.5,-488"/>
+<text text-anchor="middle" x="108" y="-472.8" font-family="Times,serif" font-size="14.00">Connection lost or</text>
+<text text-anchor="middle" x="108" y="-457.8" font-family="Times,serif" font-size="14.00">unexpectedly closed</text>
+<text text-anchor="middle" x="108" y="-442.8" font-family="Times,serif" font-size="14.00">while reconnecting</text>
+</g>
+<!-- lost_connection&#45;&gt;reconnect -->
+<g id="edge1" class="edge">
+<title>lost_connection&#45;&gt;reconnect</title>
+<path fill="none" stroke="black" d="M108,-434.95C108,-422.36 108,-407.17 108,-394.35"/>
+<polygon fill="black" stroke="black" points="111.5,-394.24 108,-384.24 104.5,-394.24 111.5,-394.24"/>
+<text text-anchor="middle" x="175.5" y="-405.8" font-family="Times,serif" font-size="14.00">Wait random delay</text>
+</g>
+</g>
+</svg>

docs/assets/server_flow.dot

@@ -0,0 +1,21 @@
+digraph "Server Connection Flow" {
+    node [shape=box];
+    graph [newrank=true];
+
+    new_connection [label="New connection"];
+    wait_handshake [label="Wait for handshake"];
+    validate_register [label="Validate register"];
+    validate_resume [label="Validate resume"];
+    connected [label="Connected"];
+    close [label="Close connection"];
+
+    new_connection -> wait_handshake [label="Send hello"];
+    wait_handshake -> validate_register [label="Receive register_*"];
+    wait_handshake -> validate_resume [label="Receive resume"];
+    wait_handshake -> close [label="Bad handshake\nTimeout"];
+    validate_register -> close [label="Bad token\nBad register"];
+    validate_register -> connected [label="Valid register:\nSend ready"];
+    validate_resume -> close [label="Bad token\nBad resume"];
+    validate_resume -> wait_handshake [label="Bad session:\nsend invalidate"];
+    validate_resume -> connected [label="Valid resume:\nSend continue"];
+}

docs/assets/server_flow.svg

@@ -0,0 +1,118 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
+ "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<!-- Generated by graphviz version 2.42.3 (0)
+ -->
+<!-- Title: Server Connection Flow Pages: 1 -->
+<svg width="603pt" height="335pt"
+ viewBox="0.00 0.00 603.00 335.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 331)">
+<title>Server Connection Flow</title>
+<polygon fill="white" stroke="transparent" points="-4,4 -4,-331 599,-331 599,4 -4,4"/>
+<!-- new_connection -->
+<g id="node1" class="node">
+<title>new_connection</title>
+<polygon fill="none" stroke="black" points="355.5,-327 224.5,-327 224.5,-291 355.5,-291 355.5,-327"/>
+<text text-anchor="middle" x="290" y="-305.3" font-family="Times,serif" font-size="14.00">New connection</text>
+</g>
+<!-- wait_handshake -->
+<g id="node2" class="node">
+<title>wait_handshake</title>
+<polygon fill="none" stroke="black" points="367.5,-240 212.5,-240 212.5,-204 367.5,-204 367.5,-240"/>
+<text text-anchor="middle" x="290" y="-218.3" font-family="Times,serif" font-size="14.00">Wait for handshake</text>
+</g>
+<!-- new_connection&#45;&gt;wait_handshake -->
+<g id="edge1" class="edge">
+<title>new_connection&#45;&gt;wait_handshake</title>
+<path fill="none" stroke="black" d="M290,-290.8C290,-279.16 290,-263.55 290,-250.24"/>
+<polygon fill="black" stroke="black" points="293.5,-250.18 290,-240.18 286.5,-250.18 293.5,-250.18"/>
+<text text-anchor="middle" x="328" y="-261.8" font-family="Times,serif" font-size="14.00">Send hello</text>
+</g>
+<!-- validate_register -->
+<g id="node3" class="node">
+<title>validate_register</title>
+<polygon fill="none" stroke="black" points="136,-138 0,-138 0,-102 136,-102 136,-138"/>
+<text text-anchor="middle" x="68" y="-116.3" font-family="Times,serif" font-size="14.00">Validate register</text>
+</g>
+<!-- wait_handshake&#45;&gt;validate_register -->
+<g id="edge2" class="edge">
+<title>wait_handshake&#45;&gt;validate_register</title>
+<path fill="none" stroke="black" d="M212.45,-216.73C161.73,-212.32 101.57,-203.5 83,-186 72.82,-176.41 68.91,-161.41 67.62,-148.23"/>
+<polygon fill="black" stroke="black" points="71.11,-147.99 67.07,-138.2 64.12,-148.37 71.11,-147.99"/>
+<text text-anchor="middle" x="148.5" y="-167.3" font-family="Times,serif" font-size="14.00">Receive register_*</text>
+</g>
+<!-- validate_resume -->
+<g id="node4" class="node">
+<title>validate_resume</title>
+<polygon fill="none" stroke="black" points="356,-138 224,-138 224,-102 356,-102 356,-138"/>
+<text text-anchor="middle" x="290" y="-116.3" font-family="Times,serif" font-size="14.00">Validate resume</text>
+</g>
+<!-- wait_handshake&#45;&gt;validate_resume -->
+<g id="edge3" class="edge">
+<title>wait_handshake&#45;&gt;validate_resume</title>
+<path fill="none" stroke="black" d="M241.15,-203.81C233.91,-199.14 227.42,-193.3 223,-186 216.09,-174.6 216.09,-167.4 223,-156 225.69,-151.55 229.16,-147.65 233.09,-144.22"/>
+<polygon fill="black" stroke="black" points="235.23,-146.98 241.15,-138.19 231.04,-141.38 235.23,-146.98"/>
+<text text-anchor="middle" x="280" y="-167.3" font-family="Times,serif" font-size="14.00">Receive resume</text>
+</g>
+<!-- close -->
+<g id="node6" class="node">
+<title>close</title>
+<polygon fill="none" stroke="black" points="450.5,-36 313.5,-36 313.5,0 450.5,0 450.5,-36"/>
+<text text-anchor="middle" x="382" y="-14.3" font-family="Times,serif" font-size="14.00">Close connection</text>
+</g>
+<!-- wait_handshake&#45;&gt;close -->
+<g id="edge4" class="edge">
+<title>wait_handshake&#45;&gt;close</title>
+<path fill="none" stroke="black" d="M367.54,-213.17C403.99,-207.82 442.06,-199.24 454,-186 493.43,-142.29 497.55,-103.05 465,-54 461.92,-49.36 458.09,-45.33 453.78,-41.84"/>
+<polygon fill="black" stroke="black" points="455.76,-38.95 445.55,-36.1 451.76,-44.69 455.76,-38.95"/>
+<text text-anchor="middle" x="540.5" y="-123.8" font-family="Times,serif" font-size="14.00">Bad handshake</text>
+<text text-anchor="middle" x="540.5" y="-108.8" font-family="Times,serif" font-size="14.00">Timeout</text>
+</g>
+<!-- connected -->
+<g id="node5" class="node">
+<title>connected</title>
+<polygon fill="none" stroke="black" points="127,-36 35,-36 35,0 127,0 127,-36"/>
+<text text-anchor="middle" x="81" y="-14.3" font-family="Times,serif" font-size="14.00">Connected</text>
+</g>
+<!-- validate_register&#45;&gt;connected -->
+<g id="edge6" class="edge">
+<title>validate_register&#45;&gt;connected</title>
+<path fill="none" stroke="black" d="M34.78,-101.9C28.43,-97.02 22.64,-91.06 19,-84 12.9,-72.14 12.28,-65.51 19,-54 21.52,-49.69 24.78,-45.87 28.5,-42.49"/>
+<polygon fill="black" stroke="black" points="30.9,-45.05 36.66,-36.16 26.61,-39.52 30.9,-45.05"/>
+<text text-anchor="middle" x="70" y="-72.8" font-family="Times,serif" font-size="14.00">Valid register:</text>
+<text text-anchor="middle" x="70" y="-57.8" font-family="Times,serif" font-size="14.00">Send ready</text>
+</g>
+<!-- validate_register&#45;&gt;close -->
+<g id="edge5" class="edge">
+<title>validate_register&#45;&gt;close</title>
+<path fill="none" stroke="black" d="M136,-111.34C178.72,-105.47 229.03,-96.29 247,-84 260.03,-75.09 253.43,-63.55 266,-54 277.12,-45.55 290.27,-39.13 303.66,-34.26"/>
+<polygon fill="black" stroke="black" points="305.06,-37.48 313.42,-30.97 302.83,-30.84 305.06,-37.48"/>
+<text text-anchor="middle" x="311" y="-72.8" font-family="Times,serif" font-size="14.00">Bad token</text>
+<text text-anchor="middle" x="311" y="-57.8" font-family="Times,serif" font-size="14.00">Bad register</text>
+</g>
+<!-- validate_resume&#45;&gt;wait_handshake -->
+<g id="edge8" class="edge">
+<title>validate_resume&#45;&gt;wait_handshake</title>
+<path fill="none" stroke="black" d="M321.89,-138.33C327.98,-143.2 333.53,-149.09 337,-156 342.98,-167.92 342.98,-174.08 337,-186 335.05,-189.89 332.44,-193.45 329.44,-196.69"/>
+<polygon fill="black" stroke="black" points="326.86,-194.32 321.89,-203.67 331.61,-199.46 326.86,-194.32"/>
+<text text-anchor="middle" x="395.5" y="-174.8" font-family="Times,serif" font-size="14.00">Bad session:</text>
+<text text-anchor="middle" x="395.5" y="-159.8" font-family="Times,serif" font-size="14.00">send invalidate</text>
+</g>
+<!-- validate_resume&#45;&gt;connected -->
+<g id="edge9" class="edge">
+<title>validate_resume&#45;&gt;connected</title>
+<path fill="none" stroke="black" d="M223.7,-112.81C197.28,-107.96 167.61,-99.33 144,-84 129.91,-74.85 132.49,-66.26 121,-54 117.46,-50.23 113.56,-46.43 109.61,-42.79"/>
+<polygon fill="black" stroke="black" points="111.86,-40.11 102.07,-36.06 107.2,-45.33 111.86,-40.11"/>
+<text text-anchor="middle" x="195.5" y="-72.8" font-family="Times,serif" font-size="14.00">Valid resume:</text>
+<text text-anchor="middle" x="195.5" y="-57.8" font-family="Times,serif" font-size="14.00">Send continue</text>
+</g>
+<!-- validate_resume&#45;&gt;close -->
+<g id="edge7" class="edge">
+<title>validate_resume&#45;&gt;close</title>
+<path fill="none" stroke="black" d="M336.9,-101.98C345.46,-97.2 353.66,-91.27 360,-84 369.17,-73.47 374.56,-58.96 377.7,-46.32"/>
+<polygon fill="black" stroke="black" points="381.18,-46.78 379.85,-36.27 374.33,-45.32 381.18,-46.78"/>
+<text text-anchor="middle" x="418" y="-72.8" font-family="Times,serif" font-size="14.00">Bad token</text>
+<text text-anchor="middle" x="418" y="-57.8" font-family="Times,serif" font-size="14.00">Bad resume</text>
+</g>
+</g>
+</svg>

docs/communication.md

@@ -0,0 +1,58 @@
+Communication Layer
+===================
+
+<sub>This document describes the communication layer in Clusterio and
+unless you're developing Clusterio it's probably not going to be very
+useful for you.</sub>
+
+Communication is facilitated by the lib/link library and is based on
+three core abstractions: links, connectors and messages.  Links
+represents one side of a two way communication pipe, connectors deal
+with connecting and establishing communication, and messages are the
+content sent over the link.
+
+
+Link
+----
+
+The Link class represents one side of a two way communication channel
+where the actual communication is done by a connector associated with
+the link.  The link also knows what the endpoint types it's connected to
+are via the source and target attributes.
+
+The endpoint types currently implemented are master, slave, instance,
+and control.  Although slave and instances run in the same Node.js
+program they use virtual links to communicate between them in order to
+simplify the communication architecture.
+
+Upon creation the link will register with the connector in order to
+receive and process messages from it.  The messages received are
+validated using the validator registered for the type of the message.  A
+message for which there is no validator for is considered an error.
+
+
+Connector
+---------
+
+The connector used with a link is responsible for establishing and/or
+maintain the connection that messages are sent over.  Reconnect logic is
+expected to be implemented in the connector, though this has not yet
+been done.
+
+There's also the virtual connector which lets two links in the same
+program be connected to each other without having to establish a
+network connection.
+
+
+Message
+-------
+
+There's two types of messages currently implemented apart from the
+Link builtin close message and that's Request and Event.  The builtin
+messages are defined in [lib/link/messages.js](../lib/link/messages.js)
+and are attached to links via the attachAllMessages function, which is
+usally called in the Link subclass constructor.
+
+The messages define which links they are valid on and only attaches
+handlers for those links.  There's also a forwarding mechanism for when
+a request or event is to be forwarded another hop.

docs/config-system.md

@@ -0,0 +1,102 @@
+Configuration System
+====================
+
+<sub>This document describes the implementation and inner working of the
+configuration system in Clusterio, and unless you're developing
+Clusterio it's probably not going to be very useful for you.</sub>
+
+Clusterio uses a group based configuration system to manage settings for
+the master server, instances, slaves, and the clusterctl utility.
+Plugins get their own config group for the master server and instances
+which they can add their own fields to.  The fields are defined early in
+the startup, and the groups and their fields for disabled plugins will
+still be created.
+
+The built-in config groups and their fields are defined in
+[/lib/config/definitions.js](lib/config/definitions.js) while plugins
+can add their own config groups to the `MasterConfigGroup` and
+`InstanceConfigGroup` properties of the `info.js` export using
+subclasses of `PluginConfigGroup`.  The plugin group has the enabled
+field pre-defined on it, and it's used to enable/disable plugins.
+
+
+Defing Fields
+-------------
+
+Fields are defined using the `define` class method on group classes
+after the groupName have been set and before the class has been
+finalized.  The `define` method takes an object as argument with the
+following properties:
+
+**type**:
+    The type of config value this field will support.  The supported
+    values are boolean, string, number, and object, and if an attept is
+    made to set the field to a value that is not of the right type it
+    will be reject unless it's a string that can be converted to the
+    right type.
+
+**name**:
+    Name of the configuration field, this is the primary way the field
+    is accessed, both in code and on the command line.  As such it
+    should follow the lower\_case\_underscore style and not contain any
+    spaces or dots.
+
+**title** (optional):
+    Text used to identify the config field in user interfaces.
+
+**description** (optional):
+    Text used to describe the config field in user interfaces.
+
+**enum** (optional):
+    Array of acceptable values this field can have.  Attempts to set a
+    value not in this array will be rejected, which makes it useful for
+    making enumareted list of choices.
+
+**optional** (optional):
+    True if this field can be set to null.  Defaults to false.
+
+**initial_value** (optional if optional is true):
+    Value this config field should take in a newly initialized config.
+    This can also be an async function returning the value to use.
+
+If the value for a field stored in on disk somehow ends up being invalid
+the field will take on the initial\_value instead when the config is
+loaded.
+
+
+Accessing Fields
+----------------
+
+On the Config instance there's two ways to access fields:  through the
+set and get methods using dot notation.  E.g.:
+
+    let value = configInstance.get("group.field");
+
+Or by getting the group first and then calling set or get on that.
+E.g.:
+
+    let value = configInstance.group("group").get("field")
+
+The set method takes an extra argument, which is the value to set the
+field to.  If the value is not valid this will throw an InvalidValue
+exception, if the group or field doesn't exist it will throw an
+InvalidField exception.
+
+
+Config Lifecycle
+----------------
+
+Values for all config fields are created when the config is initialized,
+as well as for any fields with missing values when a config is loaded.
+Once a field has a value it is never automatically removed, even if the
+field or group it was part of no longer exists.  This ensures that
+plugin configuration is not lost should the server be started up without
+the plugins that defined the configuration installed.
+
+For instance configs, they are kept on the master server and serialized
+and sent over to the relevant slaves on startup.  This makes it possible
+to edit instance configs when the slave the instance is on is offline.
+A copy is also stored in the instance directory on the slave, and this
+config will be deliverd to the master server on slave startup, which
+enables instances to be copied between clusters as long as their IDs are
+unique.

docs/contributing.md

@@ -0,0 +1,221 @@
+Contributing to Clusterio
+=========================
+
+You are probably reading this because you want to contribute code to
+Clusterio.  Great!  We need people like you.  Though we have some
+conventions and workflows that we kindly ask you to follow.
+
+
+Contents
+--------
+
+- [Workflow](#workflow)
+    - [Starting a Feature Branch](#starting-a-feature-branch)
+    - [Submitting a Pull Request](#submitting-a-pull-request)
+    - [Editing your Feature Branch](#editing-your-feature-branch)
+    - [Resolving Merge Conflicts](#resolving-merge-conflicts)
+    - [Updating Your GitHub Fork](#updating-your-github-fork)
+- [Changelog](#changelog)
+- [Supported Node.js and Factorio Version](#supported-node.js-and-factorio-version)
+- [Code Style](#code-style)
+    - [Indenting](#indenting)
+    - [Line Length](#line-length)
+
+
+Workflow
+--------
+
+While we don't require you to follow this workflow, it makes things much
+simpler for both you and us if you choose to do so.  Especially when it
+comes to making the commit history reflect what happened and not be a
+giant ball of mess.  GitHub's git workflow is less than ideal for
+dealing with small feature-branches so you might see things being done
+here a little diffrent than the usual.
+
+It is recommended that you use the command line git version for doing
+the operations described here.  GUI wrappers tend to have different
+ideas of what is what and you might find it difficult to make it do the
+things described here.  For doing the actual commits though, your
+faviorit git GUI should work just fine.
+
+If you pressed the fork button on GitHub and used `git clone` _on your
+own fork_ you can skip to the next paragraph.  Otherwise you will have
+to click the fork button and clone your fork.  If you already cloned the
+main GitHub repository you can change your clone to point at your own
+fork using the `git remote` command, but it's probably easier to delete
+your local clone and clone it again.
+
+To keep your local repository up to to date with the main repository add
+the main repository as a remote.
+
+    git remote add upstream https://github.com/clusterio/factorioClusterio
+
+After this running `git fetch upstream` will get the updated branches
+from the main repository.  Do not run `git pull upstream` it will mess
+thing up!
+
+
+### Starting a Feature Branch
+
+Development is done exclusively in feature branches.  This means the
+first thing to do before starting any development at all is to make a
+new branch starting from the master or 1.2.x branch of the main
+repository.  To do this check out that branch from the main repository
+(substitute master for 1.2.x if you're working on that branch.)
+
+    git fetch upstream
+    git checkout upstream/master
+
+This will give a warning about detached HEAD which means you did it
+correctly.  Now you can make a feature branch based on this with.
+
+    git checkout -b my-feature-branch
+
+The name of the branch is not important, as long as it's not a name
+you've used before.  But it's nice to have its name be somewhat
+descriptive of what it implements.
+
+
+### Submitting a Pull Request
+
+Once you've commited changes to your feature branch that you want to
+have included into the main repository push it to you fork with
+
+    git push origin my-feature-branch
+
+and make a pull request through the GitHub web interface.  You can also
+pass the `--set-upstream` (or `-u`) flag to make `git push` do the same
+thing for this branch.  After your feature branch have been merged you
+should consider it final.  If there are more changes you want to have
+merged or mistakes you need to fix you must start a new feature branch
+and submit a new pull request
+
+
+### Editing your Feature Branch
+
+Until your feature branch is merged you can edit it as you see fit by
+using the rebase tool.  If your feature branch has been merged however
+it's final and you will have to start a new feature branch and submit
+that in a new pull request.  The base usage of rebase assuming you are
+checked out on your feature branch is the following command (substitute
+master with 1.2.x if your feature branch is based on that)
+
+    git rebase -i upstream/master
+
+This will open up an editor with the commits of your feature branch and
+let you reorder, edit, and remove commits as you see fit.
+
+If the main repository has been updated and you have fetched down those
+updates with `git fetch upstream` this will also move your feature
+branch so that it is based on the latest commit from the main
+repository.
+
+If you have already pushed your feature branch to your fork a simple
+`git push origin my-feature-branch` will no longer work as the histories
+are different, instead you will have to run
+
+    git push origin +my-feature-branch
+
+The + sign signifies a force push and should be used with care.  If you
+have a pull request that has not yet been merged submitted for this
+feature branch this will update the pull request too.
+
+
+### Resolving Merge Conflicts
+
+The best way to resolve merge conflicts in feature braches is to rebase
+it on top of the updated upstream.  Run the following commands
+(substitute master with 1.2.x if your feature branch is based on that)
+
+    git fetch upstream
+    git rebase upstream/master
+
+This will stop on the commit(s) that caused the merge conflict and let
+you resolve them manually before continuing.  Once you're done force
+push your updated feature branch with
+
+    git push origin +my-feature-branch
+
+
+### Updating Your GitHub Fork
+
+No.  This is pointless busywork that accomplishes nothing.  Fetch the
+main repository directly as described above and base your feature
+branches on the refs from it.  GitHub has no interface or functionality
+for keeping the branches in your fork up to date with the branches in
+the main repository and there is no point in doing this for our feature
+branch based workflow.
+
+
+Changelog
+---------
+
+There's a [Changelog](../CHANGELOG.md) in the root of the project.  If
+you make changes that are visible to users of this project you should
+add an entry to the changelog describing what has changed.  The format
+of this log should be self explanatory, please follow it carefully.
+
+
+Supported Node.js and Factorio Versions
+---------------------------------------
+
+Clusterio run on Node.js v12 and up, v11.13.0+ and v10.16.0+, make sure
+to check that the Node builtin API's and npm packages you use are
+supported on these versions.  The Travis CI tests are runned on whatever
+version of Node.js v10 is the default, which at the time of writing is
+v10.18.0.
+
+For Factorio Clusterio 2.0 aims to support version 0.17.69 and up,
+including the latest experimental release.  It's recommended that you
+use the lua API reference for 0.17.69, as there's no information on what
+version Factorio API's were introduced in.  The Travis CI tests are
+runned against latest experimental release.
+
+
+Code Style
+----------
+
+The style of the code is a bit of a mixed bag at the moment.  But there
+are at least a few things that have been agreed upon.
+
+
+### Strings
+
+JavaScript and Lua code should use double quoted strings (`"string"`)
+for all strings with the only exception that strings that contain double
+quotes may use single quotes (i.e., `'String with "quotes"'`.)  For
+JavaScript template literals may also be used where appropriate.
+
+There are places in the codebase that are using `'single'` quoted
+strings.  Should you be modifying such code use double quotes on new and
+modified lines and leave the rest untouched.
+
+
+### Indenting
+
+JavaScript code should be indented with tabs.  There are places in the
+codebase that are mixing tabs and spaces or uses only spaces for
+indents.  Should you be modifiying such code that is mixing spaces and
+tabs as indending you should update the indenting to use tabs _on the
+code you change_ while leaving the rest of the code untouched.
+
+Lua code should be indented with 4 spaces.
+
+
+### Line Length
+
+Lines are limited to 120 characters where tabs count as 4 characters.
+If the argument list for a function or expressino for compound statement
+gets very long break them out on their own line(s).  For example:
+
+    function foo(
+        really, long, argument, list,
+        over, multiple, lines
+    ) {
+        if (
+            really && long && expression
+            || over && multiple && lines
+        ) {
+            // code
+        }
+    })

docs/developing-for-clusterio.md

@@ -0,0 +1,389 @@
+Developing for Clusterio
+========================
+
+This document describes the various different types of content that can
+be created for Factorio and/or Clusterio and how this is made compatible
+with Clusterio.
+
+
+Contents
+--------
+
+- [Factorio Mods](#factorio-mods)
+- [Factorio Scenarios](#factorio-scenarios)
+    - [event_handler interface](#event_handler-interface)
+- [Communicating with Clusterio](#communicating-with-clusterio)
+- [Clusterio Modules](#clusterio-modules)
+- [Clusterio Lua API](#clusterio-lua-api)
+- [Clusterio Plugins](#clusterio-plugins)
+
+
+Factorio Mods
+-------------
+
+Factorio mods that do not interact with Clusterio should not have any
+conciderations that need to be taken into account for them to work with
+Clusterio.
+
+
+Factorio Scenarios
+------------------
+
+Clusterio patches saves and because of this scenarios have a few
+limitations and must explicitly mark themselves as compatible with
+Clusterio to work.  The main limitation is that the
+[event_handler](https://github.com/wube/factorio-data/blob/master/core/lualib/event_handler.lua)
+lib must be used, and control.lua cannot contain any code other than
+calls to event_handler loading the relevant libs of the scenario.
+This is because the control.lua file will be overwritten by the save
+patcher.  Scenarios also cannot use the event registring functions
+exposed by `script` such as `script.on_event` or `script.on_nth_tick` as
+using these will overwrite the handlers registered by the
+event_handler lib.
+
+A breif description of the usage of the event_handler library is
+provided in the [event_handler interface](#event_handler-interface)
+section.
+
+Once a scenario is made using the event_handler it should have a
+control.lua file that looks something like this:
+
+```lua
+-- control.lua
+local event_handler = require("event_handler")
+event_handler.add_lib(require("module_foo"))
+event_handler.add_lib(require("module_bar"))
+```
+
+To make Clusterio recognize and correctly handle this scenario it
+needs to include a clusterio.json file with following content:
+
+```json
+{
+    "scenario": {
+        "name": "example-scenario",
+        "modules": [
+            "module_foo",
+            "module_bar"
+        ]
+    }
+}
+```
+
+This tells Clusterio that it should load both module_foo and module_bar
+by passing the result of requiring them to the  `add_lib` function when
+writing a new control.lua file to the save.
+
+
+### event_handler interface
+
+<sub>**Note:** At the time of writing the event_handler lib has been
+a part of Factorio since at least 0.17.4, and was moved from base to
+core in 0.17.63 but is still not documented anywhere.  A brief
+description is provided here for this reason.</sub>
+
+The event_handler lib provides a simple interface for registring
+multiple callbacks to the same events without having to be concerned
+with callback chaining or overwriting callbacks defined elsewhere.  It
+works by taking over the task of registring the actual callbacks with
+Factorio, providing its own interface for the rest of the code to use.
+
+The sole function of interest exported by event_handler is `add_lib`
+which accepts a table with event handler callbacks definitions that it
+will register.  The following entries are recognized by `add_lib`, all
+of which are optional:
+
+- `on_init`:
+    Callback called when the callback of `script.on_init` is invoked.
+- `on_load`:
+    Callback called when the callback of `script.on_load` is invoked.
+- `on_configuration_changed`:
+    Callback called when the callback of
+    `script.on_configuration_changed` is invoked.
+- `events`:
+    Table mapping event ids as defined in `defines.events` to callbacks.
+    For example the table `{ [defines.events.on_player_died] = foo }`
+    will call bar with the usual `event` table as argument every time a
+    player dies.
+- `on_nth_tick`:
+    Table mapping the nth tick number to a callback.  For example the
+    table `{ [30] = bar }` will call bar every 30th tick.
+- `add_remote_interface`:
+    Callback called before on_init/on_load callbacks<sup>[1]</sup>.  It
+    has no special meaning and receives no arguments but should be used
+    for registring remote interfaces.
+- `add_commands`:
+    Callback called before on_init/on_load callbacks<sup>[1]</sup>.  It
+    has no special meaning and receives no arguments but should be used
+    for registring commands.
+
+<sub>1: Before 0.17.69 these callbacks were called after
+on_init/on_load.</sub>
+
+The usual way to use `add_lib` is to define the table of events to
+register in a separate file and return it, then load it in control.lua
+via `require` before passing it to `add_lib`.  For example a module
+could be defined as
+
+```lua
+-- example.lua
+local function my_handler(event)
+    local name = game.player[event.player_index].name
+    game.print("It appears that " .. name .. " has died")
+end
+
+return {
+    on_init = function() game.print("example_lib init") end,
+    events = {
+        [defines.events.on_player_died] = my_handler,
+    },
+}
+```
+
+And then in control.lua the following used to load it with `add_lib`:
+
+```lua
+-- control.lua
+local event_handler = require("event_handler")
+event_handler.add_lib(require("example"))
+```
+
+Because the event_handler lib registers events itself you may not use
+`script.on_load` or `script.on_init` at all in your code and any usage
+of `script.on_configuration_changed`, `script.on_nth_tick` and/or
+`script.on_event` will cause the corresponding events registered with
+event_handler to break and should therefore not be used.
+
+There's also the `add_libraries` function exported by event_handler,
+which accepts a table and calls `add_lib` for each value in the table.
+
+
+Communicating with Clusterio
+----------------------------
+
+There's a poorly to virtuarly undocumented HTTP interface that can be
+used to communicate with Clusterio to do things like get the status of
+instances, grab items from the store, run commands on instances, etc,
+etc.  For now you can read the source code, and maybe ask on the Discord
+for some pointers if you get stuck.
+
+
+Clusterio Modules
+-----------------
+
+Modules are primarily used by plugins to inject code into Factorio games
+with the save patcher, though it's also possible to make stand alone
+modules that are loaded into the game if you don't need the capabilites
+of the plugin system.  The save patcher puts modules into the `modules`
+folder of the Factorio save and adds code to `control.lua` to load the
+module according to the `load` and `require` options to the module.json
+file.
+
+Stand alone modules are placed into the modules folder of Clusterio,
+plugin modules are located in the module folder of the plugin.  In
+either case a `module.json` file is required and has the following
+structure:
+
+```json
+{
+    "name": "my_module",
+    "version": "1.2.0",
+    "dependencies": {
+        "clusterio": "*",
+        "foo": ">=0.4.2"
+    },
+    "require": ["bar.lua"],
+    "load": ["foo.lua"]
+}
+```
+
+The following entries are supported in the module.json file:
+
+- `name`:
+    Name of the module, must match the folder the module is located in
+    for stand alone modules or the name of the plugin the module is
+    located in.
+- `version`:
+    Version of the module.  Must be compatible with [Semantic Versioning
+    2.0.0](https://semver.org/).  In plugin modules this defaults to the
+    version of the plugin otherwise it's required.
+- `dependencies`:
+    Optional mapping of modules this module depends on and their
+    version.  In the example `>=0.4.2` means that the foo module must be
+    at least version 0.4.2.  See the [Ranges
+    syntax](https://www.npmjs.com/package/semver#ranges) of the
+    node-semver package for a full description of what operators are
+    supported.  The events for dependencies is invoked before the events
+    of the dependent, and starting an instance will fail if the
+    dependencies cannot be satisfied.  If not specified it will default
+    to depending on `clusterio`.  Make sure to add `clusterio` to your
+    dependencies if you add other dependencies and depend on the Lua API.
+- `require`:
+    Lua files to require in `control.lua`.  This should only really be
+    needed if you want to make a global function available for use in
+    Lua commands.  The paths are relative to the module's own folder and
+    should be specified using forward slashes as directory sepparators
+    if it's located in a sub directory in the module.
+- `load`:
+    Lua files to load with the `event_handler` lib, the result of
+    requiring the file will be passed to the `add_lib` function of the
+    `event_handler` lib in `control.lua`.  See the section on the
+    [event_handler interface](#event_handler-interface) for a detailed
+    description on how the `event_handler` lib works.  The paths are
+    relative to the module's own folder and should be specified using
+    forward slashes as directory sepparators if it's located in a sub
+    directory in the module.
+
+It's recommended to use the new style of defining modules in Lua, as
+well as avoid the use of global variables as much as possible.  This
+means always declearing your top level variables and functions local and
+exporting only the things you need in other files.  You can require
+other files in your module by prefixing your require paths with
+`modules/your_module_name/`.  It's also possible to require files from
+other modules this way too.
+
+The global variables and data stored in the global table is shared by
+all Clusterio modules as well as the scenario, so it's important that
+you use unique names.  It's recommended to prefix the global variables
+and data in the global table that your module has with the name of your
+module.
+
+Because modules can be patched into an existing game you cannot rely on
+the `on_init` callback to be called in Clusterio Modules.  Nor can you
+rely on the `on_configuration_changed` callback, as this is not called
+when level code changes.  The Clusterio Lua API provides the custom
+`on_server_startup` event that can be used as a substitute, see the next
+section.
+
+
+Clusterio Lua API
+-----------------
+
+Clusterio provides a save patched module as well as a regular Factorio
+mod for interfacing with Clusterio.  This includes tools sending data to
+plugins, script events to listen for, and an item serialization module.
+
+
+### clusterio_api library
+
+**require path**  
+From a module  
+`local clusterio_api = require("modules/clusterio/api")`  
+From a mod  
+`local clusterio_api = require("__clusterio_lib__/api")`
+
+Provides the main interface to Clusterio from within the game.  When
+using the mod version you must call `init` before it'll function
+properly.
+
+#### init
+
+<sub>Mod version only</sub>
+
+Initialize the API.  This should be called from both the `on_init` and
+`on_load` events in your mod, and before this is called certain features
+will not be available, most notably the events table.
+
+#### clusterio_api.events
+
+Table of events raised by clusterio.  If the game is started outside of
+Clusterio then none of these events will be raised.
+
+In the mod version this table does not exist before the `init` function
+is called.
+
+##### on_instance_updated
+
+Raised after the name and id of an instance has been updated.  This may
+occur even if the id and name didn't change.
+
+Event data:
+- `instance_id`: The id of the instance.
+- `instance_name`: the name of the instance.
+
+##### on_server_startup
+
+Raised when Clusterio is starting up the server.  It can be used to
+initialize clusterio related features in a mod, and as a stand-in for
+the `on_init` and `on_configuration_changed` event for modules.  It is
+invoked on the first tick the server runs after the save has been
+patched, before most other events.
+
+Use this event to initialize and and/or migrate the data structures
+you need.  Keep in mind that both mods and Clusterio modules can switch
+from any version to any version so it should be able to handle both
+forwards and backwards migrations gracefully.
+
+#### clusterio_api.send_json(channel, data)
+
+Send json data to Clusterio over the given channel.  The `data` argument
+should be table that can be serialized with `game.table_to_json`.
+Clusterio plugins can listen to channels and will receive an event with
+the data sent here.  See the [Communicating with Factorio
+section](writing-plugins.md#communicating-with-factorio) in the Writing
+Plugins document for more information.
+
+If the game is started outside of Clusterio the data will be sent, but
+since there's no code following stdout to pick it up it will be lost.
+
+**Note**: Payloads greater than 4 MB will cause stuttering in the game.
+
+**Note**: This is not a binary safe way of sending data.  Strings
+embedded into the tables sent must be valid UTF-8 text.
+
+Parameters:
+- `channel`: string identifying which channel to send it on.
+- `data`: table that can be converted to JSON with game.table_to_json
+
+
+### serialize library
+
+**require path**  
+From a module  
+`local serialize = require("modules/clusterio/serialize")`  
+From a mod  
+`local serialize = require("__clusterio_lib__/serialize")`
+
+Serializes inventories and item stacks into a table that in turn can be
+converted into JSON with `game.table_to_json`.  Properly handles the
+extra data for most items.
+
+#### serialize.serialize_equipment_grid(LuaEquipmentGrid)
+
+Returns a table containing a serialized representation of the equipment
+grid passed.
+
+#### serialize.deserialize_equipment_grid(LuaEquipmentGrid, serialized_grid)
+
+Deserialize a previously serialized equipment grid into the target
+LuaEquipmentGrid.  Overwrites all content in the target.
+
+#### serialize.serialize_item_stack(LuaItemStack, destination_table)
+
+Serialize the LuaItemStack passed into the target destination_table.
+The destination_table should be a new empty table.
+
+#### serialize.deserialize_item_stack(LuaItemStack, serialized_stack)
+
+Deserialize a previously serialized item stack into the target
+LuaItemStack.  Overwrites the content of the item stack.
+
+#### serialize.serialize_inventory(LuaInventory)
+
+Returns a table containing a serialized representation of the inventory
+passed.
+
+#### serialize.deserialize_inventory(LuaInventory, serialized_inventory)
+
+Deserialize a previously serialized inventory into the target
+LuaInventory.  Overwrites slots that has content in the serialized
+inventory.
+
+
+Clusterio Plugins
+-----------------
+
+The plugin interface is undergoing a major rewrite, and will be
+documented once that is done.  See
+[issue #220](https://github.com/clusterio/factorioClusterio/issues/220)
+for details.

docs/how-it-works.md

@@ -0,0 +1,15 @@
+Save Patching
+-------------
+
+In order to efficiently get data in and out of running Factorio servers
+and to provide custom interfaces Clusterio uses save
+patching<sup>[1]</sup> to add code to the game.  The save patching is
+based on the built in
+[event_handler](https://github.com/wube/factorio-data/blob/master/core/lualib/event_handler.lua)
+library to Factorio and requires scenarios to be coded specifically for
+it.  Mods on the other hand work without modifications.
+
+<sub>1: Before version 2.0 of Clusterio this was done at runtime
+with a patcher called Hotpatch, but due to the difficulty in supporting
+run time patching and the lack of documentation it was replace it with
+save patching instead.</sub>

docs/jsdoc.json

@@ -0,0 +1,23 @@
+{
+    "plugins": [
+        "plugins/markdown"
+    ],
+    "tags": {
+        "allowUnknownTags": false
+    },
+    "opts": {
+        "destination": "docs/api",
+        "recurse": true
+    },
+    "source": {
+        "include": [
+            "master.js",
+            "slave.js",
+            "clusterctl.js",
+            "lib/",
+            "plugins/",
+            "routes/",
+            "routes.js"
+        ]
+    }
+}

docs/socket.md

@@ -0,0 +1,280 @@
+WebSocket protocol
+==================
+
+<sub>This document describes the protocol used between the master and
+slaves, and unless you're developing Clusterio it's probably not going
+to be very useful for you.</sub>
+
+The protocol is built on WebSocket and uses JSON encoded text messages
+to communicate to and from the master server.  These messages contain
+a JSON encoded object with the following three properties:
+
+- seq - integer|null - The sequence number of the message
+- type - string - The type of message
+- data - object - Message data
+
+For the handshake and heartbeats the seq property is null.
+
+
+Handshake
+---------
+
+Upon connecting to the WebSocket endpoint the server will respond with a
+`hello` message containing the master server version as the data.  The
+client is then expected to send either `register_*` or a `resume`
+message to set up the connection with the master.  The exact flow is
+described below.
+
+
+### Client connection flow
+
+For a client making a new connecting to the master server the first
+course of action after obtaining a token is to connect with WebSocket to
+the /api/socket endpoint and wait for the server hello.  After the
+hello, the client sends a register\_\* message corresponding to the type
+of the client.  If successful the server will reply with a ready message
+and the connection has been established.  See the flowchart below.
+
+![Client connection flowchart](assets/connection_flow.svg)
+
+If for any reason the connection is closed or lost during the connection
+attempt the process is started over after a random delay, with the
+exception of the connection being closed with a 4003 code which means
+authentication failed and the client should stop trying to connect.  The
+wait for hello and wait for ready states should also have a timeout,
+after which the connection is closed if the hello/reply wasn't received
+in time.
+
+
+### Client reconnection flow
+
+If the connection is lost after the client has received the ready
+message from the server it should attempt a reconnect instead.  This is
+very similar to a new connection, but instead of a register\_\* messsage
+the client sends a resume message and the server replies with a continue
+message if resuming the session was successful.  This continue message
+has sequence number of the last message the master received and the
+client is expected to imediately send all messages from its send buffer
+that occured after the sequence number given.
+
+The server will respond with an invalidate message if resuming the
+session is not possible.  In this case the client should continue from
+the new session state of the connection flow (see the previous section).
+
+![Client reconnection flowchart](assets/reconnection_flow.svg)
+
+Like the connection flow the process starts over after a delay if the
+connection is closed or lost.  The same exception for connection closed
+with 4003 applies as with the connection flow.  Additionally after an
+invalidate message has been received and the connection flow fails, it
+should start over from the connection flow.
+
+
+### Server connection flow
+
+From the master server side new connections and reconnection attempts
+start out the same, a new connection has been made and the server sends
+the hello messoge to the client.  If a vaild register\_\* message is
+received a new connection is set up for the client and the ready message
+is sent in reply.  If a valid resume message is received and the session
+that's attempted to be resumed is active it'll signal that the session
+is resumed with a continue message and the client upon receiving the
+continue message should send the messages from its send buffer from
+after the last\_seq given in the continue message.
+
+If the session token used to resume is not valid it'll send an invalidate
+message and expect the client to continue with a registr\_\* message.  If an
+invalid or unexpected message is received during the handshake the connection
+is closed.  See the flowchart below.
+
+![Client connection flowchart](assets/server_flow.svg)
+
+For authentication failure the connection is closed with code 4003,
+which the connection client should interpret as connecting will never
+succeed and stop trying to reconnect.
+
+
+### Connection tracking and heartbeats
+
+Once connected as indicated by the ready and continue messages from the
+master server the client must start sending heartbeats at the interval
+given in the ready/continue message, and storing messages sent over the
+link in a sending side buffer.  The heartbeats sent must have the seq of
+the last received message on the link and acts as an ack for received
+messages.  The master server also sends heartbeats in the same manner
+and the client should remove messages up to and including the sequence
+number received in the heartbeat from its send buffer.
+
+
+Handshake messages
+------------------
+
+### `hello`
+
+Sent by the master server when the WebSocket connection has been opened.
+
+- version - string - The version of the master, e.g. "2.0.0".
+
+### `register_slave`
+
+Slave handshake for establishing a new connection session.
+
+- token - string - Authentication token.
+- agent - string - Human readable string of the software connecting.
+- version - string - The protocol version of the slave, e.g. "2.0.0".
+- name - string - Name of the slave.
+- id - integer - ID of the slave.
+
+### `register_control`
+
+Control handshake for enstablishing a new connection session.
+
+- token - string - Authentication token.
+- agent - string - Human readable string of the software connecting.
+- version - string - The protocol version of the control, e.g. "2.0.0".
+
+### `resume`
+
+Handshake for resuming an existing connection session where the
+connection had dropped unexpectedly.
+
+- session_token - string - Session token to resume session with.
+- last_seq - integer|null - Last message seq received by the client.
+
+### `invalidate`
+
+Sent by the master server in reply to a resume for a session that cannot
+be resumed.  The client should start a new session with a register\_\*
+message in response.
+
+### `continue`
+
+Sent by the master server to signal resuming the session was successful.
+
+- last_seq - integer|null - Last message seq received by the server.
+- heartbeat_interval - number - Rate client should send heartbeats at.
+
+### `ready`
+
+Sent by the master server to signal the connection has been established
+with a new session.
+
+- session_token - string - Token to resume the session with.
+- heartbeat_interval - number - Rate client should send heartbeats at.
+
+### `heartbeat`
+
+Sent by both the master server and clients at the interval given in the
+continue/ready messages.
+
+- last_seq - integer - Last message seq received over the link.
+
+
+Example handshake
+-----------------
+
+Server hello after opening the WebSocket.
+
+```json
+{
+    "seq": null,
+    "type": "hello",
+    "data": {
+        "version": "2.0.0"
+    }
+}
+```
+
+Slave registering a new connection session.
+
+```json
+{
+    "seq": null,
+    "type": "register_slave",
+    "data": {
+        "token": "xyzabc123",
+        "agent": "Clusterio Slave",
+        "version": "2.0.0",
+        "name": "Foo",
+        "id": 123
+    }
+}
+```
+
+Master server reply signaling the session is established and ready.
+
+```json
+{
+    "seq": null,
+    "type": "ready",
+    "data": {
+        "session_token": "xyz.azy.bzy",
+        "heartbeat_interval": 45
+    }
+}
+```
+
+
+Events
+------
+
+Events are messages send in one direction over the link, that the
+receiver is expected to act upon, but not make any replies (for that
+there's [Requests](#requests).  By convention message types ending in
+`*_event` are events.  The data payload of an event consists solely of
+event specific properties.
+
+See [lib/link/messages.js](/lib/link/messages.js) for the recognized
+events and their contents.
+
+
+Requests
+--------
+
+Requests are messages which expects a response in return.  They function
+a lot like HTTP requests except that both parties of a connection can
+inniate a request that is expected to be responded to.  By convention
+message types ending in `*_request` are requests, and is expected to be
+replied to with a corresponding `*_response` message.  If an error
+occured while processing the request an error response containing an
+error property in the data payload is sent instead.
+
+See [lib/link/messages.js](/lib/link/messages.js) for the recognized
+requests and their contents.
+
+### `*_response`
+
+- seq - integer - The seq number of the request message.
+- additional properties may be present depending on the type of request.
+
+### `*_response` error
+
+- seq - integer - The seq number of the request message.
+- error - string - Human readable text message describing the error.
+
+
+Graceful shutdown
+-----------------
+
+To ensure no data is lost when closing the link it's important to
+send connection shutdown request and wait for it to complete before
+closing the WebSocket.  Either party of the link can send this request
+and the party sending the request should be the one that initates the
+close of the WebSocket.
+
+### `connection_shutdown_request`
+
+Request graceful shutdown of the connection.  The receiving party should
+stop tasks that send events and requests over the link and wait for
+pending requests it has over the link to complete, but it should not
+stop responding to requests sent to it.  Only after this process is done
+should the response be sent.
+
+### `connection_shutdown_response`
+
+Signals the shutdown is complete and that the connection can be closed.
+The party that initiatied the shutdown request may still send other
+requests/events over the link to finish its part of the shutdown.
+
+If the error response variant is received it should be logged if
+possible and treated as a success.

docs/writing-plugins.md

@@ -0,0 +1,497 @@
+Writing Plugins
+===============
+
+Plugins for Clusterio are classes written in JavaScript that run under
+Node.js.  The plugin classes have pre-defined hooks that are called
+during various stages and operations of Clusterio.
+
+
+Contents
+--------
+
+- [Plugin Structure](#plugin-structure)
+- [Defining the plugin class](#defining-the-plugin-class)
+- [Plugin Configuration](#plugin-configuration)
+- [Communicating with Factorio](#communicating-with-factorio)
+- [Defining Link Messages](#defining-link-messages)
+    - [Defining Events](#defining-events)
+    - [Defining Requests](#defining-requests)
+- [Sending Link Messages](#sending-link-messages)
+- [Collecting Statistics](#collecting-statistics)
+
+
+Plugin Structure
+----------------
+
+The basic file structure of a plugin is the following.
+
+    plugin_name/
+      +- info.js
+      +- package.json
+      +- master.js
+      +- instance.js
+      +- test/
+      |  +- plugin.js
+      +- module/
+         +- module.json
+         +- plugin.lua
+
+The most important file is the `info.js` file.  Without it the plugin
+will not recognized by Clusterio.  Here's an example of it:
+
+    const link = require('lib/link'); // For messages
+
+    module.exports = {
+        name: "foo_frobber",
+        title: "Foo Frobber",
+        description: "Does advance frobnication",
+        version: "0.8.1",
+        instanceEntrypoint: "instance",
+        masterEntrypoint: "master",
+        messages: {
+            /* See below */
+        },
+    };
+
+The following properties are recognized:
+
+**name**:
+    Internal name of the plugin.  Must be the same as the plugin's
+    directory name
+
+**title**:
+    Name of the plugin as shown to users.  Currently not used.
+
+**description**:
+    Brief description of what the plugin does.  Currently not used.
+
+**version**:
+    Semver compatible version string of the plugin.  Currently only
+    servers as the default for the Clusterio module embedded in the
+    plugin, if any.
+
+**instanceEntrypoint**:
+    Path to a Node.js module relative to the plugin directory which
+    contains the InstancePlugin class definition for this plugin.  This
+    is an optional paramater.  A plugin can be for instances only but
+    for it to be able to send any messages to other instances it will
+    still have to be loaded on the master server.
+
+**InstanceConfigGroup**:
+    Subclass of `PluginConfigGroup` to defining the per instance
+    configuration fields for this plugin.  See [Plugin
+    Configuration](#plugin-configuration)
+
+**masterEntrypoint**:
+    Path to a Node.js module relative to the plugin directory which
+    contains the MasterPlugin class definiton for this plugin.  This is
+    an optional parameter.  A plugin can be made that only runs on the
+    master server.
+
+**MasterConfigGroup**:
+    Subclass of `PluginConfigGroup` to defining the master server
+    configuration fields for this plugin.  See [Plugin
+    Configuration](#plugin-configuration)
+
+**messages**:
+    Object with link messages definitions for this plugin.  See guide
+    for [defining link messages](#defining-link-messages) below.
+
+The optional module folder contains a Clusterio module that will be
+patched into the save when the plugin is enabled.  See the section on
+[Clusterio Modules](developing-for-clusterio.md) in the Developing for
+Clusterio document.  The only restriction imposed on modules embedded
+into plugins is that they must be named the same as the plugin.
+
+While there is no standard for how to organize a plugin it's recommended
+to put the MasterPlugin class definition into master.js and the
+InstancePlugin class definition into instance.js.  You can put them into
+whatever file you want (even the same one for both)
+
+For both instanceEntrypoint and masterEntrypoint the path should not end
+with .js and it should use forward slashes for directory sepparators if
+any.
+
+
+Defining the plugin class
+-------------------------
+
+The plugin class should derive from its respective base class defined in
+`lib/plugin`.  For example to define a MasterPlugin class the following
+code can be used:
+
+    const plugin = require('lib/plugin');
+
+    class MasterPlugin extends plugin.BaseMasterPlugin {
+        async init() {
+            this.foo = 42;
+            await this.startFrobnication();
+        }
+
+        // ...
+    }
+
+    module.exports = {
+        MasterPlugin,
+    }
+
+For the instance plugin it's exactly the same except "Master" is
+replaced with "Instance".  The available hooks that you can override is
+documented on the base class [in lib/plugin.js](../lib/plugin.js).
+
+It's best to avoid defining a constructor, but if you insist on defining
+one forward all arguments to be the base class.  E.g.:
+
+        constructor(...args) {
+            super(...args);
+
+            // Code here
+        }
+
+The arguments passed may change, and attempting to modify them will
+result in unpredicatable behaviour.  The async init method always called
+immediatly after the constructor so there's little reason to do this.
+
+
+Defining Configuration
+----------------------
+
+Clusterio provides a configuration system that handles storing,
+distributing, editing and validating config fields for you.  You can
+take advantage of it by subclassing `PluginConfigGroup`, setting the
+`groupName` to your plugin name, defining fields on it, finalizing it,
+and passing it as either `MasterConfigGroup` or `InstanceConfigGroup` in
+the `info.js` export.  For example in info.js:
+
+    const config = require("lib/config");
+
+    class MasterConfigGroup extends config.PluginConfigGroup { }
+    MasterConfigGroup.groupName = "foo_frobber";
+    MasterConfigGroup.define({
+        name: "level",
+        description: "Level of frobnication done",
+        type: "number",
+        initial_value: 2,
+    });
+    MasterConfigGroup.finalize();
+
+    module.exports = {
+        ...
+        MasterConfigGroup: MasterConfigGroup,
+    };
+
+Code inside the `MasterPlugin` class will then be able to access the
+level config field through the `Config` object at `this.master.config`,
+for example in the MasterPluginClass:
+
+    async init() {
+        let level = this.master.config.get("foo_frobber.level");
+        console.log("I got a frobnication level of", level);
+    }
+
+The same applies for instance configs, replace "master" with "instance"
+where appropriate.  See [Configuration System](configuration-system.md)
+for more details on how this system works.
+
+
+Communicating with Factorio
+---------------------------
+
+For pushing data into Factorio there's RCON which lets you send
+arbitrary Lua command to invoke whatever code you want in the game.
+This is done by calling sendRcon method on the instance's server object.
+For example:
+
+    async onStart() {
+        let response = await this.instance.server.sendRcon(
+            "/sc rcon.print('data')"
+        );
+
+        // Do stuff with response.
+    }
+
+
+Because data into Factorio is streamed at a rate of 3-6 kB/s by default
+it is recommended to avoid sending large commands as much as possible,
+and to strip down the data on the ones you send to only what's strictly
+necessary.  You can have lua code injected into the game via the module
+system and call that from RCON to avoid having to send code through the
+commands.
+
+For getting data out from Factorio there's both RCON and the `send_json`
+API of the Clusterio module.  Returning data via RCON is prefered if the
+action is initiated from the Node.js side.  The `send_json` API allows
+sending JSON payloads on channels that plugins can listen to.  From a
+plugin you listen for an event named ipc-channel_name in order to get
+data sent by `send_json`.  For example in the plugin code:
+
+    async init() {
+        this.instance.server.on('ipc-my_plugin_foo', content =>
+            this.handleFoo(content).catch(err => console.log(
+                "Error handling foo:", foo
+            ))
+        );
+    }
+
+    async handleFoo(content) {
+        // Do stuff with content
+    }
+
+And then in the module for the plugin:
+
+    local clusterio_api = require('modules/clusterio/api')
+
+    -- inside some event handler
+    clusterio_api.send_json('my_plugin_foo', { data = 123 })
+
+It's recommended to either use the plugin name as the channel name or to
+prefix the channel name with the name of the plugin if you need multiple
+channels.  It's also important to catch any errors that might occur as
+they will otherwise be propogated to the instance code and kill the
+server.
+
+Data out from Factorio does not have the same limits as data into
+Factorio, RCON responses can be in 100kB range without causing issues,
+and payloads to the `send_json` API can be in the 4MB range provided the
+server has a fast enough storage system.
+
+**Note:** both `send_json` and RCON can operate out of order.  For
+`send_json` it's possible that payloads greater than 4kB are received
+after payloads that was sent at a later point in time.  For RCON,
+commands longer than 50 characters may end up being executed after
+shorter commands sent after it.
+
+
+Defining Link Messages
+----------------------
+
+You will most likely have to communicate with the master or other
+instances in your plugin for it to do anything useful.  For this there's
+a WebSocket communication channel established between the slaves and the
+master server that plugins can define their own messages to send over
+it.  This channel is bi-directional and all messages send over it are
+validated with a JSON schema (see [this guide][guide] for an
+introduction to writing JSON schema.)
+
+[guide]: https://json-schema.org/learn/getting-started-step-by-step.html
+
+There are currently two kinds of messages that can be defined.  Events
+and requests.  Events are simple one way notifications that invoke a
+handler on target it's sent to.  Requests are pairs of request and
+response messages where the request is sent to the target and the
+response is the reply back from the target.  The requests are similar to
+HTTP requests only that both parties of a link may innitiate one.
+
+
+### Defining Events
+
+Events are defined as properties of the messages object exported by
+`info.js` that map to instances of the `Event` class from `lib/link`.
+The name of the property correspond to the handler invoked on the plugin
+class.  The Event constructor takes an object of properties that define
+the event, for example the following could be defined in `info.js`:
+
+    messages: {
+        startFrobnication: new link.Event({
+            type: 'foo_frobber:start_frobnication',
+            links: ['master-slave', 'slave-instance'],
+            forwardTo: 'instance',
+            eventProperties: {
+                "frobnication_type": { type: "string" },
+            },
+        }),
+    },
+
+This specifies an event that can be sent from the master to a slave,
+and from a slave to an instance.  And that the event must contain the
+property `frobnication_type` with a string value in the data payload.
+It will also be forwarded by slaves to a specific instance.
+
+The following properties are recognized by the Event constructor:
+
+**type**:
+    The message type sent over the wire.  This can be any string but
+    it must be unique across all plugins and it's recommended that it is
+    of format `plugin_name:message_name`.  The suffix `_event` will be
+    appened to the type.
+
+**links**:
+    An array of strings describing which links this event can be sent
+    over.  Direction matters, `'master-slave'` means the event can be
+    sent from the master to slave, but can't be sent back the other way
+    unless `'slave-master'` is also present in the links array.
+
+**forwardTo**:
+    Target to forward event to.  Can be either `'master'` to indicate a
+    slave should forward it to the master server or `'instance'` to
+    indicate it should be forwarded to the instances specified by the
+    `instance_id` event property.  This works by using a default handler
+    for the event at the links that forward it.
+
+**broadcastTo**:
+    Target to broadcast this message towards.  Currently only
+    `'instance'` is supported and means the event will be broadcast to
+    all instances downstream of the target it's sent to, but not back
+    from where it came from.  This means that sending an event to a
+    slave from an instance will cause it to be broadcast to all
+    instances of that slave except for the instance it came from.
+
+**eventProperties**:
+    Object with properties mapping to a JSON schema of that property
+    that specifies what's valid to send in the event.  This is
+    equivalent to using the `properties` keyword in JSON schema except
+    that the properties specified are implicitly required and additional
+    properties are not allowed.  See [this guide][guide] for an
+    introduction to writing JSON schemas
+
+The forwardTo and broadcastTo can be combined such that specifying
+`'master'` as the forwardTo value and `'instance'` as the broadcastTo
+value will cause the event to be broadcast to all instances in the
+cluster.  For this to work you will need to specify instance-slave,
+slave-master and master-slave, slave-instance as the links.
+
+Keep in mind when forwarding events that if the target an event is being
+forwarded to is not online the event will be dropped.  If you need a
+confirmation that the message was received use a request.
+
+
+### Definining Requests
+
+Requests are defined as properties of the messages object exported by
+`info.js` that map to instances of the `Request` class from `lib/link`.
+The name of the property correspond to the handler invoked on the plugin
+class.  The Request constructor takes an object of properties that define
+the event, for example the following could be defined in `info.js`:
+
+    messages: {
+        reportFrobnication: new link.Request({
+            type: 'foo_frobber:report_frobnication',
+            links: ['master-slave', 'slave-instance'],
+            forwardTo: 'instance',
+            requestProperties: {
+                "verbosity": { type: "integer" },
+            },
+            responseProperties: {
+                "report": {
+                    type: "array",
+                    items: { type: "string" },
+                },
+            },
+        }),
+    },
+
+This specifies a request that can be sent from the master to a slave,
+and from a slave to an instance.  The request data must contain the
+property `verbosity` with an integer number as the value as well as the
+`instance_id` property (implied by `forwardTo: 'instance'`), and the
+response sent must contain a `report` property mapping to an array of
+strings.  When received by a slave it will also be forwarded to the
+instance specified by `instance_id`.
+
+The following properties are recognized by the Request constructor:
+
+**type**:
+    The message type sent over the wire.  This can be any string but
+    it must be unique across all plugins and it's recommended that it is
+    of format `plugin_name:message_name`.  The suffix `_request` will be
+    appened to the type for the request message sent, and the suffix
+    `_response` will be appended to the type for the response.
+
+**links**:
+    An array of strings describing which links this request can be sent
+    over.  Direction matters, `'master-slave'` means the request can be
+    sent from the master to slave and the slave can reply to the master,
+    but the slave can't sent a request to the master unless
+    `'slave-master'` is also present in the links array.
+
+**forwardTo**:
+    Target to forward request to.  Can be either `'master'` to indicate
+    a slave should forward it to the master server when receiving it
+    from an instance or `'instance'` to indicate it should be forwarded
+    to the instances specified by the `instance_id` request property.
+    This works by using a default handler for the request at the links
+    that forward it.
+
+**requestProperties**:
+    Object with properties mapping to a JSON schema of that property
+    that specifies what's valid to send in the request.  This is
+    equivalent to using the `properties` keyword in JSON schema except
+    that the properties specified are implicitly required and additional
+    properties are not allowed.  See [this guide][guide] for an
+    introduction to writing JSON schemas
+
+**responseProperties**:
+    Same as the requestProperties only for the response sent back by the
+    target.
+
+
+Sending Link Messages
+---------------------
+
+Link messages are sent by calling the `.send()` method on the
+Event/Request instance with the link you want to send it over and the
+data to send.  For instance plugins the link is the instance itself
+which is accessible through the `.instance` property on the plugin.  The
+`.info` property of the plugin class exposes the data exported from the
+plugin's `info.js` module.  In other words:
+
+    // In an InstancePlugin class
+    async frobnicate() {
+        this.info.messages.exampleEvent.send(this.instance, { foo: "bar" });
+    }
+
+For the Request class the send method is async and returns the response
+data received from the target it was sent to, or throws an error if the
+request failed.
+
+
+Collecting Statistics
+---------------------
+
+Clusterio comes with it's own Prometheus client implementation, one part
+due to Not Invented Here and another part due to collectors in
+prom-client being difficult to get to work nicely with collecting data
+from plugins optionally loaded at runtime on different computers.
+
+In its simplest form collecting data from a plugins consists of defining
+the metric and updating it somewhere in the plugin code.  For example:
+
+    const { Counter } = require('lib/prometheus');
+
+    const fooMetric = new Counter(
+        'clusterio_foo_frobber_foo_metric', "Measures the level of foo",
+    );
+
+    // Somewhere in the master plugin code
+    fooMetric.inc();
+
+This works for master plugins, and the metric will be automatically
+available through the /metric HTTP endpoint.  It's recommended that
+plugin metrics follow `clusterio_<plugin_name>_<metric_name>` as the
+naming scheme.
+
+For metrics that are per-instance you must define an `instance_id` label and
+set it accordingly, for example:
+
+    const { Counter } = require('lib/prometheus');
+
+    const barMetric = new Gauge(
+        'clusterio_foo_frobber_bar_metric', "Bar instance level",
+        { labels: ['instance_id'] }
+    );
+
+    // Somewhere in the instance plugin code
+    barMetric.labels(String(this.instance.config.get("instance.id"))).set(someValue);
+
+Metrics are automatically registered to the default registry, and this
+default registry is automatically polled by the master server on slaves.
+This means that it's important that you place the definition of the
+metric at module level so that it's not created more than once over the
+lifetime of a slave.  Since the metrics remember their values and would
+continue to be exported after an instance is shutdown there's code at
+instance shutdown that removes all the values where the `instance_id`
+label matches the id of the instance shut down.
+
+For statistics you need to update on collection there's an `onMetrics`
+hook on both master and instance plugins that is runned before the
+metrics in the default registry is collected.

documentation.yml

@@ -1,10 +0,0 @@
-toc:
- - name: Master Server API
- - clusterioMaster
- - name: Type declarations
- - itemStack
- - name: Internal modules
-   description: |
-     Here are some helper functions for navigation.
- - modManager
- - getFactorioLocale

lib/authenticate.js

@@ -1,15 +1,48 @@
 const jwt = require('jsonwebtoken');
 
-const config = require("./../config");
+let authSecret = null;
+function setAuthSecret(secret) {
+	authSecret = secret;
+}
+
+function middleware(req, res, next) {
+	// This should not happen
+	if (!authSecret) {
+		return res.status(500).send({ auth: false, message: 'Secret not set', endpoint: req.route.path });
+	}
 
-var middleware = function (req, res, next) {
 	var token = req.headers['x-access-token'];
-	if(!token) return res.status(401).send({ auth: false, message: 'No token provided.', endpoint: req.route.path});
-	
-	jwt.verify(token, config.masterAuthSecret, function(err, decoded) {
-		if(err) return res.status(500).send({ auth: false, message: 'Failed to authenticate token.', endpoint: req.route.path});
+	if (!token) {
+		return res.status(401).send({ auth: false, message: 'No token provided.', endpoint: req.route.path });
+	}
+
+	jwt.verify(token, authSecret, function(err, decoded) {
+		if (err) {
+			return res.status(401).send({ auth: false, message: 'Failed to authenticate token.', endpoint: req.route.path });
+		}
 		next();
 	});
 }
 
-module.exports = {middleware};
+async function check(token) {
+	return new Promise((resolve, reject) => {
+		// This should not happen
+		if (!authSecret) {
+			resolve({ ok: false, msg: 'Secret not set' });
+		}
+
+		jwt.verify(token, authSecret, function(err, decoded) {
+			if (err) {
+				resolve({ok:false, msg:err})
+			} else {
+				resolve({ok:true, msg:"Successfully authenticated"});
+			}
+		});
+	});
+}
+
+module.exports = {
+	setAuthSecret,
+	middleware,
+	check,
+};

lib/config/classes.js

@@ -0,0 +1,579 @@
+/**
+ * @file Configuration classes
+ * @author Hornwitser
+ */
+"use strict";
+
+const { basicType } = require("lib/helpers");
+
+
+/**
+ * Invalid Value exception
+ *
+ * Exception class for when invalid values are attempted to be set on a
+ * field.
+ */
+class InvalidValue extends Error { };
+
+/**
+ * Invalid Field exception
+ *
+ * Exception class for when a group or field that does not exist is
+ * attempted to be accessed.
+ */
+class InvalidField extends Error { };
+
+/**
+ * Split the given string on the first instance of separator
+ *
+ * Splits @p string on the first instance of @p separator and returns an
+ * array consisting of the string up to the separator and the string
+ * after the separator.  Returns an array with the string and an empty
+ * string if the separator is not present.
+ *
+ * @returns {Array} string split on separator.
+ */
+function splitOn(separator, string) {
+	let index = string.indexOf(separator);
+	if (index === -1) {
+		return [string, ''];
+	}
+	return [string.slice(0, index), string.slice(index + separator.length)];
+}
+
+/**
+ * Collection of config groups
+ *
+ * Represents a collection of ConfigGroup instances that can be
+ * managed as one.
+ */
+class Config {
+	constructor() {
+		if (!this.constructor._finalized) {
+			throw new Error(`Cannot instantiate incomplete Config class ${this.constructor.name}`)
+		}
+
+		this._groups = new Map();
+		this._unknownGroups = new Map();
+	}
+
+	async init() {
+		for (let [name, GroupClass] of this.constructor._groups) {
+			if (!this._groups.has(name)) {
+				let group = new GroupClass();
+				await group.init();
+				this._groups.set(name, group);
+			}
+		}
+		this._initialized = true;
+	}
+
+	_validate(serializedConfig) {
+		if (basicType(serializedConfig) !== "object") {
+			throw new Error(`Expected object, not ${basicType(serializedConfig)} for config`);
+		}
+
+		if (basicType(serializedConfig.groups) !== "array") {
+			throw new Error(`Expected groups to be an array, not ${basicType(serializedConfig.groups)}`);
+		}
+	}
+
+	async load(serializedConfig) {
+		this._validate(serializedConfig);
+		for (let serializedGroup of serializedConfig.groups) {
+			let GroupClass = this.constructor._groups.get(serializedGroup.name);
+			if (!GroupClass) {
+				this._unknownGroups.set(serializedGroup.name, serializedGroup);
+				continue;
+			}
+
+			let group = new GroupClass();
+			await group.load(serializedGroup);
+			this._groups.set(GroupClass.groupName, group);
+		}
+
+		await this.init();
+	}
+
+	/**
+	 * Serialize the config to a plain JavaScript object
+	 */
+	serialize() {
+		let groups = [...this._unknownGroups.values()];
+		for (let group of this._groups.values()) {
+			groups.push(group.serialize());
+		}
+
+		return { groups };
+	}
+
+	/**
+	 * Update the configuration based on a serialized config
+	 *
+	 * Updates all groups in the config with groups in the serialized
+	 * config passed.
+	 */
+	update(serializedConfig) {
+		this._validate(serializedConfig);
+		for (let serializedGroup of serializedConfig.groups) {
+			let group = this._groups.get(serializedGroup.name);
+			if (group) {
+				group.update(serializedGroup);
+			} else {
+				this._unknownGroups.set(serializedGroup.name, serializedGroup);
+			}
+		}
+	}
+
+	/**
+	 * Get the value for a config field
+	 */
+	get(name) {
+		let [group, field] = splitOn('.', name);
+		return this.group(group).get(field);
+	}
+
+	/**
+	 * Set the value for a config field
+	 */
+	set(name, value) {
+		let [group, field] = splitOn('.', name);
+		return this.group(group).set(field, value);
+	}
+
+	/**
+	 * Get the config group instance with the given name
+	 */
+	group(name) {
+		if (!this._initialized) {
+			throw new Error(`${this.constructor.name} instance is uninitialized`)
+		}
+		let group = this._groups.get(name);
+		if (!group) {
+			throw new InvalidField(`No config group named '${name}'`);
+		}
+		return group;
+	}
+
+	static _initSubclass() {
+		// These properties are initialized here in order to ensure that
+		// they end up on the sub-class.
+		if (!this._initialized) {
+			this._initialized = true;
+			this._finalized = false;
+			this._groups = new Map();
+		}
+	}
+
+	/**
+	 * Mapping of group name to group class
+	 *
+	 * Returns the internal mapping of group names to group classes that
+	 * mave been registered with this config.  Note that mutating this
+	 * object will lead to unexpected behaviour.
+	 *
+	 * @returns {Map<string, GroupClass>} config groups
+	 */
+	static get groups() {
+		this._initSubclass();
+		return this._groups;
+	}
+
+	/**
+	 * Locks this Config class from adding more groups.
+	 */
+	static finalize() {
+		this._initSubclass();
+		this._finalized = true;
+	}
+
+	/**
+	 * Register a ConfigGroup with this Config
+	 *
+	 * Adds the ConfigGroup to this config as a group that will get
+	 * initialized, loaded and serialized with this config class.
+	 */
+	static registerGroup(group) {
+		this._initSubclass();
+
+		if (this._finalized) {
+			throw new Error("Cannot register group on finalized config");
+		}
+
+		if (!group._finalized) {
+			throw new Error("Group must be finalized before it can be registered");
+		}
+
+		if (this._groups.has(group.groupName)) {
+			throw new Error(`${group.groupName} has already been registered`);
+		}
+
+		this._groups.set(group.groupName, group);
+	}
+}
+
+
+/**
+ * Collection of related config entries
+ */
+class ConfigGroup {
+	/**
+	 * Creates a new config group.
+	 *
+	 * After the creation of the new config group you have to call
+	 * either .init() or .load() on it in order to fully initialize it.
+	 */
+	constructor(serializedGroup) {
+		if (!this.constructor._finalized) {
+			throw new Error(`Cannot instantiate incomplete ConfigGroup class ${this.constructor.name}`)
+		}
+
+		this._fields = new Map();
+		this._unknownFields = new Map();
+	}
+
+	/**
+	 * Initialize new config group
+	 *
+	 * Computes and assigns the initial values for all fields of the
+	 * config group.
+	 */
+	async init() {
+		for (let [name, def] of this.constructor._definitions) {
+			if (this._fields.has(name)) {
+				continue;
+			}
+
+			let value = null;
+			if (typeof def.initial_value === "function") {
+				value = await def.initial_value()
+
+			} else if (def.initial_value !== undefined) {
+				value = def.initial_value;
+			}
+
+			this._fields.set(name, value);
+		}
+	}
+
+	/**
+	 * Load from serialized group
+	 *
+	 * Loads all the values from the passed serialized version of the
+	 * group and then initializes any possible missing fields from it to
+	 * their default values.
+	 */
+	async load(serializedGroup) {
+		this.update(serializedGroup);
+		await this.init();
+	}
+
+	/**
+	 * Get value for field
+	 */
+	get(name) {
+		if (!this.constructor._definitions.has(name)) {
+			throw new InvalidField(`No field named '${name}'`);
+		}
+
+		return this._fields.get(name);
+	}
+
+	/**
+	 * Set value of field
+	 *
+	 * The field must be defined for the config group and the value is
+	 * type checked against the field definition.
+	 *
+	 * @throws {InvalidValue} if value is not allowed for the field.
+	 */
+	set(name, value) {
+		let def = this.constructor._definitions.get(name);
+		if (!def) {
+			throw new Error(`No field named '${name}'`);
+		}
+
+		// Empty strings are treates as null
+		if (value === "") {
+			value = null;
+		}
+
+		if (def.enum && !def.enum.includes(value)) {
+			throw new InvalidValue(`Expected one of [${def.enum.join(', ')}], not ${value}`);
+		}
+
+		if (basicType(value) === "string") {
+			if (def.type === "boolean") {
+				if (value === "true") { value = true; }
+				else if (value === "false") { value = false; }
+
+			} else if (def.type === "number") {
+				let numberRegExp = /^[+-]?(Infinity|\d+\.(\d+)?([eE][+-]?\d+)?|\.?\d+([eE][+-]?\d+)?)$/;
+				if (numberRegExp.test(value.trim())) {
+					value = Number.parseFloat(value);
+				}
+
+			} else if (def.type === "object" ) {
+				try {
+					value = JSON.parse(value);
+				} catch (err) {
+					throw new InvalidValue(`Error parsing value for ${name}: ${err.message}`)
+				}
+			}
+		}
+
+		if (value === null) {
+			if (!def.optional) {
+				throw new InvalidValue(`Field ${name} cannot be null`)
+			}
+
+		} else {
+			if (basicType(value) !== def.type) {
+				throw new InvalidValue(`Expected type of ${name} to be ${def.type}, not ${basicType(value)}`);
+			}
+		}
+
+		this._fields.set(name, value);
+	}
+
+	/**
+	 * Update a config group from a serialized group
+	 *
+	 * Overwrites the values of all the fields in this config group that
+	 * has a valid value in the passed serialized group with that value.
+	 */
+	update(serializedGroup) {
+		if (basicType(serializedGroup) !== "object") {
+			throw new Error(`Expected object, not ${basicType(serializedGroup)} for ConfigGroup`);
+		}
+
+		if (serializedGroup.name !== this.constructor.groupName) {
+			throw new Error(`Expected group name ${this.constructor.groupName}, not ${serializedGroup.name}`);
+		}
+
+		if (basicType(serializedGroup.fields) !== "object") {
+			throw new Error(`Expected fields to be an object, not ${basicType(serializedGroup.fields)}`);
+		}
+
+		for (let [name, value] of Object.entries(serializedGroup.fields)) {
+			let def = this.constructor._definitions.get(name);
+			if (!def) {
+				this._unknownFields.set(name, value);
+				continue;
+			}
+
+			if (def.optional) {
+				if (value !== null && basicType(value) !== def.type) {
+					continue;
+				}
+			} else {
+				if (basicType(value) !== def.type) {
+					continue;
+				}
+			}
+
+			if (def.enum && !def.enum.includes(value)) {
+				continue;
+			}
+
+			this._fields.set(name, value);
+		}
+	}
+
+	/**
+	 * Serialize the config to a plain JavaScript object
+	 */
+	serialize() {
+		let fields = {};
+		for (let [name, value] of this._fields) {
+			fields[name] = value;
+		}
+
+		for (let [name, value] of this._unknownFields) {
+			fields[name] = value;
+		}
+
+		return {
+			name: this.constructor.groupName,
+			fields,
+		};
+	}
+
+	static _initSubclass() {
+		// These properties are initialized here in order to ensure that
+		// they end up on the sub-class.
+		if (!this._initialized) {
+			this._initialized = true;
+			this._definitions = new Map();
+		}
+	}
+
+	/**
+	 * Mapping of config name to field meta data
+	 *
+	 * Returns the internal meta data for the fields of this class.
+	 * Note that mutating this object will lead to unexpected behaviour.
+	 *
+	 * @returns {Map<string, Object>} field meta data
+	 */
+	static get definitions() {
+		this._initSubclass();
+		return this._definitions;
+	}
+
+	/**
+	 * Define a new configuration field
+	 *
+	 * The following properties are supported:
+	 * - type:
+	 *     string declaring the type of the config value, supports
+	 *     boolean, string, number and object.
+	 * - name:
+	 *     Name of the configuration field.  Should follow the
+	 *     lower_case_underscore style.
+	 * - title:
+	 *     Text used to identify the config field in user interfaces.
+	 * - description:
+	 *     Text used to describe the config field in user interfaces.
+	 * - enum (optional):
+	 *     Array of all values that are acceptible.  Useful for making
+	 *     enumerated values.
+	 * - optional (optional):
+	 *     True if this field can be set to null.  Defaults to false.
+	 * - initial_value (optional):
+	 *     Value this config field should take in a newly initialized
+	 *     config.  This can also be an async function returning the
+	 *     value to use.
+	 */
+	static define(item) {
+		if (this._finalized) {
+			throw new Error(`Cannot define field for ConfigGroup class ${this.name} after it has been finalized`)
+		}
+
+		if (basicType(this.groupName) !== "string") {
+			throw new Error(`Expected ConfigGroup class ${this.name} to have the groupName property set to a string`)
+		}
+
+		this._initSubclass();
+
+		let itemKeys = Object.keys(item);
+
+		// Check for required properties
+		["name", "type"].forEach(prop => {
+			if (!itemKeys.includes(prop)) {
+				throw new Error(`${prop} is required when defining an field`);
+			}
+		})
+
+		// Validate properties
+		let validTypes = ["boolean", "string", "number", "object"];
+		Object.keys(item).forEach(key => {
+			let value = item[key];
+			let valueType = basicType(value);
+			switch (key) {
+				case "type":
+					if (!validTypes.includes(value)) {
+						throw new Error(`${value} is not a valid type`);
+					}
+					break;
+
+				case "enum":
+					if (valueType !== "array") {
+						throw new Error("enum must be an array");
+					}
+					break;
+
+				case "name":
+					if (valueType !== "string") {
+						throw new Error("name must be a string");
+					}
+					break;
+
+				case "title":
+					if (valueType !== "string") {
+						throw new Error("title must be a string");
+					}
+					break;
+
+				case "description":
+					if (valueType !== "string") {
+						throw new Error("description must be a string");
+					}
+					break;
+
+				case "optional":
+					if (valueType !== "boolean") {
+						throw new Error("optional must be a boolean");
+					}
+					break;
+
+				case "initial_value":
+					if (valueType !== item.type && valueType !== "function") {
+						throw new Error("initial_value must match the type or be a function");
+					}
+					break;
+
+				default:
+					throw new Error(`Unknown property ${key}`);
+			}
+		});
+
+		if (this._definitions.has(item.name)) {
+			throw new Error(`Config field ${item.name} has already been defined`);
+		}
+
+		let computed = Object.assign({
+			optional: false,
+			fullName: `${this.groupName}.${item.name}`,
+		}, item)
+
+		if (!computed.optional && computed.initial_value === undefined) {
+			throw new Error(`Non-optional field ${item.name} needs an initial_value`);
+		}
+
+		this._definitions.set(item.name, computed);
+	}
+
+	/**
+	 * Locks this ConfigGroup class from adding more definitons.
+	 */
+	static finalize() {
+		if (basicType(this.groupName) !== "string") {
+			throw new Error(`Expected ConfigGroup class ${this.name} to have the groupName property set to a string`)
+		}
+
+		this._initSubclass();
+		this._finalized = true;
+	}
+}
+
+/**
+ * Config group for plugins
+ *
+ * Grouping that includes per plugin fields clusterio uses to manage
+ * plugins.  This is currently only the enabled field.  It is otherwise
+ * identical to the ConfigGroup class.
+ */
+class PluginConfigGroup extends ConfigGroup {
+	static _initSubclass() {
+		if (!this._initialized) {
+			super._initSubclass();
+			this.define({
+				name: "enabled",
+				description: "Enable plugin",
+				type: "boolean",
+				initial_value: true,
+			});
+		}
+	}
+}
+
+
+module.exports = {
+	InvalidField,
+	InvalidValue,
+
+	Config,
+	ConfigGroup,
+	PluginConfigGroup,
+}

lib/config/definitions.js

@@ -0,0 +1,345 @@
+/**
+ * @file Core definitions for the configuration system
+ * @author Hornwitser
+ */
+"use strict";
+
+const util = require("util");
+const crypto = require("crypto");
+const path = require("path");
+const fs = require("fs-extra");
+
+const classes = require("./classes");
+
+
+class MasterGroup extends classes.ConfigGroup { }
+MasterGroup.groupName = "master"
+MasterGroup.define({
+	name: "database_directory",
+	title: "Database directory",
+	description: "Directory where item and configuration data is stored.",
+	type: "string",
+	initial_value: "database",
+});
+MasterGroup.define({
+	name: "factorio_directory",
+	description: "Path to directory to look for factorio installs",
+	type: "string",
+	initial_value: "factorio"
+});
+MasterGroup.define({
+	name: "http_port",
+	title: "HTTP Port",
+	description: "Port to listen for HTTP connections on, set to null to not listen for HTTP connections.",
+	type: "number",
+	optional: true
+});
+MasterGroup.define({
+	name: "https_port",
+	title: "HTTPS Port",
+	description: "Port to listen for HTTPS connection on, set to null to not listen for HTTPS connections.",
+	type: "number",
+	optional: true,
+	initial_value: 8443,
+});
+MasterGroup.define({
+	name: "web_root",
+	title: "Web Root",
+	description: "Sub-path to where the web interface is hosted.  Needed when proxying the web interface.",
+	type: "string",
+	initial_value: "/",
+});
+MasterGroup.define({
+	name: "tls_certificate",
+	title: "TLS Certificate",
+	description: "Path to the certificate to use for HTTPS.",
+	type: "string",
+	optional: true,
+	initial_value: "database/certificates/cert.crt",
+});
+MasterGroup.define({
+	name: "tls_private_key",
+	title: "TLS Private Key",
+	description: "Path to the private key to use for HTTPS.",
+	type: "string",
+	optional: true,
+	initial_value: "database/certificates/cert.key",
+});
+MasterGroup.define({
+	name: "auth_secret",
+	title: "Master Authentication Secret",
+	description:
+		"Secret used to generate and verify authentication tokens."
+		+"  Should be a long string of random letters and numbers."
+		+"  Do not share this.",
+	type: "string",
+	initial_value: async function() {
+		console.log("Generating new master authentication secret");
+		let asyncRandomBytes = util.promisify(crypto.randomBytes);
+		let bytes = await asyncRandomBytes(256);
+		return bytes.toString("base64");
+	},
+});
+MasterGroup.define({
+	name: "heartbeat_interval",
+	title: "Heartbeat Interval",
+	description: "Interval heartbeats are sent out on WebSocket connections",
+	type: "number",
+	initial_value: 60,
+});
+MasterGroup.finalize();
+
+class MasterConfig extends classes.Config { }
+MasterConfig.registerGroup(MasterGroup);
+
+
+class SlaveGroup extends classes.ConfigGroup {}
+SlaveGroup.groupName = "slave";
+SlaveGroup.define({
+	name: "name",
+	description: "Name of the slave",
+	type: "string",
+	initial_value: "New Slave",
+});
+SlaveGroup.define({
+	name: "id",
+	description: "ID of the slave",
+	type: "number",
+	initial_value: () => Math.random() * 2**31 | 0,
+});
+SlaveGroup.define({
+	name: "factorio_directory",
+	description: "Path to directory to look for factorio installs",
+	type: "string",
+	initial_value: "factorio"
+});
+SlaveGroup.define({
+	name: "instances_directory",
+	description: "Path to directory to store instances in.",
+	type: "string",
+	initial_value: "instances",
+});
+SlaveGroup.define({
+	name: "master_url",
+	description: "URL to connect to the master server at",
+	type: "string",
+	initial_value: "https://localhost:8443/",
+});
+SlaveGroup.define({
+	name: "master_token",
+	description: "Token to authenticate to master server with.",
+	type: "string",
+	initial_value: "enter token here",
+});
+SlaveGroup.define({
+	name: "public_address",
+	description: "Public facing address players should connect to in order to join instances on this slave",
+	type: "string",
+	initial_value: "localhost",
+});
+SlaveGroup.define({
+	name: "reconnect_delay",
+	title: "Reconnect Delay",
+	description: "Maximum delay to wait before attempting to reconnect WebSocket",
+	type: "number",
+	initial_value: 5,
+});
+SlaveGroup.finalize();
+
+class SlaveConfig extends classes.Config { }
+SlaveConfig.registerGroup(SlaveGroup);
+
+
+class InstanceGroup extends classes.ConfigGroup { }
+InstanceGroup.groupName = "instance"
+InstanceGroup.define({
+	name: "name",
+	type: "string",
+	initial_value: "New Instance",
+});
+InstanceGroup.define({
+	name: "id",
+	description: "ID of the slave",
+	type: "number",
+	initial_value: () => Math.random() * 2**31 | 0,
+});
+InstanceGroup.define({
+	name: "assigned_slave",
+	type: "number",
+	optional: true,
+});
+InstanceGroup.finalize();
+
+class FactorioGroup extends classes.ConfigGroup { }
+FactorioGroup.groupName = "factorio";
+FactorioGroup.define({
+	name: "version",
+	description: "Version of the game to run, use latest to run the latest installed version",
+	type: "string",
+	initial_value: "latest",
+});
+FactorioGroup.define({
+	name: "game_port",
+	description: "UDP port to run game on, uses a random port if null",
+	type: "number",
+	optional: true,
+});
+FactorioGroup.define({
+	name: "rcon_port",
+	description: "TCP port to run RCON on, uses a random port if null",
+	type: "number",
+	optional: true,
+});
+FactorioGroup.define({
+	name: "rcon_password",
+	description: "Password for RCON, randomly generated if null",
+	type: "string",
+	optional: true,
+});
+FactorioGroup.define({
+	name: "settings",
+	description: "Settings overridden in server-settings.json",
+	type: "object",
+	initial_value: {
+		"tags": ["clusterio"],
+		"auto_pause": false,
+	},
+});
+FactorioGroup.finalize();
+
+class InstanceConfig extends classes.Config { }
+InstanceConfig.registerGroup(InstanceGroup);
+InstanceConfig.registerGroup(FactorioGroup);
+
+
+function validateGroup(pluginInfo, groupName) {
+	if (!pluginInfo[groupName] instanceof classes.PluginConfigGroup) {
+		throw new Error(
+			`Expected ${groupName} for ${pluginInfo.name} to be an instance of PluginConfigGroup`
+		);
+	}
+
+	if (pluginInfo[groupName].groupName !== pluginInfo.name) {
+		throw new Error(
+			`Expected ${groupName} for ${pluginInfo.name} to be named after the plugin`
+		);
+	}
+}
+
+/**
+ * Registers the config groups for the provided plugin infos
+ */
+function registerPluginConfigGroups(pluginInfos) {
+	for (let pluginInfo of pluginInfos) {
+		if (pluginInfo.MasterConfigGroup) {
+			validateGroup(pluginInfo, "MasterConfigGroup");
+			MasterConfig.registerGroup(pluginInfo.MasterConfigGroup);
+
+		} else {
+			class MasterConfigGroup extends classes.PluginConfigGroup { }
+			MasterConfigGroup.groupName = pluginInfo.name;
+			MasterConfigGroup.finalize();
+			MasterConfig.registerGroup(MasterConfigGroup);
+		}
+
+		if (pluginInfo.instanceEntrypoint) {
+			if (pluginInfo.InstanceConfigGroup) {
+				validateGroup(pluginInfo, "InstanceConfigGroup");
+				InstanceConfig.registerGroup(pluginInfo.InstanceConfigGroup);
+
+			} else {
+				class InstanceConfigGroup extends classes.PluginConfigGroup { }
+				InstanceConfigGroup.groupName = pluginInfo.name;
+				InstanceConfigGroup.finalize();
+				InstanceConfig.registerGroup(InstanceConfigGroup);
+			}
+		}
+	}
+}
+
+/**
+ * Lock configs from adding more groups and make them usable
+ */
+function finalizeConfigs() {
+	MasterConfig.finalize();
+	SlaveConfig.finalize();
+	InstanceConfig.finalize();
+}
+
+/**
+ * Yargs config command
+ *
+ * Can be passed to yargs.command to implement a config command.  Use
+ * handleConfigCommand to do the requested action.
+ */
+function configCommand(yargs) {
+	yargs
+		.command("set <field> [value]", "Set config field")
+		.command("show <field>", "Show value of the given config field")
+		.command("list", "List all configuration fields and their values")
+		.demandCommand(1, "You need to specify a command to run")
+		.strict()
+	;
+}
+
+/**
+ * Handle yargs command
+ *
+ * Handle the actions that are made available by configCommand.
+ */
+async function handleConfigCommand(args, instance, configPath) {
+	let command = args._[1];
+
+	if (command === "list") {
+		for (let GroupClass of instance.constructor.groups.values()) {
+			for (let def of GroupClass.definitions.values()) {
+				let value = instance.get(def.fullName);
+				console.log(`${def.fullName} ${JSON.stringify(value)}`);
+			}
+		}
+
+	} else if (command === "show") {
+		try {
+			console.log(instance.get(args.field));
+		} catch (err) {
+			if (err instanceof classes.InvalidField) {
+				console.error(err.message);
+			} else {
+				throw err;
+			}
+		}
+
+	} else if (command === "set") {
+		if (args.value === undefined) {
+			args.value = null;
+		}
+
+		try {
+			instance.set(args.field, args.value);
+			await fs.outputFile(configPath, JSON.stringify(instance.serialize(), null, 4));
+		} catch (err) {
+			if (err instanceof classes.InvalidField || err instanceof classes.InvalidValue) {
+				console.error(err.message);
+			} else {
+				throw err;
+			}
+		}
+	}
+}
+
+
+module.exports = {
+	MasterGroup,
+	SlaveGroup,
+	InstanceGroup,
+	FactorioGroup,
+
+	MasterConfig,
+	SlaveConfig,
+	InstanceConfig,
+
+	registerPluginConfigGroups,
+	finalizeConfigs,
+	configCommand,
+	handleConfigCommand,
+}

lib/config/index.js

@@ -0,0 +1,9 @@
+/**
+ * @file Configuration System
+ * @author Hornwitser
+ */
+
+module.exports = {
+	...require("./classes"),
+	...require("./definitions"),
+}

lib/database.js

@@ -0,0 +1,290 @@
+/**
+ * Implementation of master databases
+ *
+ * @module
+ * @author Hornwitser
+ */
+
+"use strict";
+const fs = require("fs-extra");
+
+const { basicType } = require("lib/helpers");
+
+
+/**
+ * Converts a Map with only string keys to a JavaScript object.
+ *
+ * @param {Map} map - the Map to convert to an object.
+ * @return {object} - Object with the mapping's key-values as properties.
+ * @throws {Error} if there are non-string keys in map.
+ */
+function mapToObject(map) {
+	let obj = {};
+	for (let [key, value] of map) {
+		if (typeof key !== 'string') {
+			throw new Error(
+				`Expected all keys to be string but got ${typeof key}`
+			);
+		}
+
+		obj[key] = value;
+	}
+	return obj;
+}
+
+/**
+ * Load JSON object file as a Map
+ *
+ * Loads the JSON file specified by path containing an object with
+ * properties as key value pairs and converts it into a JavaScript Map.
+ * Returns an empty Map if the file does not exist.
+ *
+ * @param {string} filePath - The path to the JSON file to load.
+ * @throws {Error} If JSON file did not contain an object.
+ * @throws {Error} If an error occured reading the file.
+ */
+async function loadJsonAsMap(filePath) {
+	try {
+		let content = await fs.readFile(filePath);
+		let parsed = JSON.parse(content);
+
+		if (basicType(parsed) !== "object") {
+			throw new Error(`Expected object but got ${basicType(parsed)}`);
+		}
+
+		return new Map(Object.entries(parsed));
+
+	} catch (err) {
+		if (err.code == "ENOENT") {
+			// File does not exist, return empty map
+			return new Map();
+
+		} else {
+			throw err;
+		}
+	}
+}
+
+/**
+ * Save a Map as a JSON object file
+ *
+ * Save the content of a key-value Map with only string keys as a JSON file.
+ * If the directory the file is to be saved into doesn't exist it will be
+ * created.
+ *
+ * @param {string} filePath - The path to the JSON file that will be saved.
+ * @param {Map} map - Map with only string keys to save.
+ * @throws {Error} if there are non-string keys in map.
+ * @throws {Error} if an error occured writing to the file.
+ */
+async function saveMapAsJson(filePath, map) {
+	let obj = mapToObject(map);
+	await fs.outputFile(filePath, JSON.stringify(obj, null, 4));
+}
+
+/**
+ * Load JSON array file as a Map
+ *
+ * Loads the JSON file specified by path containing an array of objects that
+ * each have an id property into a Map from the id to the object.
+ *
+ * @param {string} filePath - The path to the JSON file to load.
+ * @throws {Error} If JSON file did not contain an array.
+ * @throws {Error} If there are objects in the array without an id property.
+ * @throws {Error} If an error occured reading the file.
+ */
+async function loadJsonArrayAsMap(filePath) {
+	try {
+		let content = await fs.readFile(filePath);
+		let parsed = JSON.parse(content);
+
+		if (basicType(parsed) !== "array") {
+			throw new Error(`Expected array but got ${basicType(parsed)}`);
+		}
+
+		let map = new Map();
+		for (let element of parsed) {
+			if (basicType(element) !== "object") {
+				throw new Error(`Expected all elements to be objects`);
+			}
+
+			if (element.id === undefined) {
+				throw new Error(`Expected all elements to have an id property`);
+			}
+
+			map.set(element.id, element);
+		}
+		return map;
+
+	} catch (err) {
+		if (err.code == "ENOENT") {
+			// File does not exist, return empty map
+			return new Map();
+
+		} else {
+			throw err;
+		}
+	}
+}
+
+/**
+ * Save a Map as a JSON array file
+ *
+ * Save the values of a Map with objects containing an id property that is
+ * equal to the key the objects are stored at into a JSON file.
+ *
+ * @param {string} filePath - The path to the JSON file that will be saved.
+ * @param {Map} map -
+ *     Map with objects containing an id property equal to the key they are
+ *     stored under.
+ * @throws {Error} if an error occured writing to the file.
+ */
+async function saveMapAsJsonArray(filePath, map) {
+	await fs.outputFile(filePath, JSON.stringify([...map.values()], null, 4));
+}
+
+
+
+function checkName(name) {
+	if (typeof name !== "string") {
+		throw new Error("name must be a string");
+	}
+}
+
+function checkCount(count) {
+	if (typeof count !== "number" || isNaN(count)) {
+		throw new Error("count must be a number");
+	}
+}
+
+
+/**
+ * Item database
+ *
+ * Stores counts of items for the master server.  Items that have not been
+ * stored in the database are treated as having a stored count of zero.  When
+ * serialized the database discards entries with a zero counts, and when
+ * deserialized the content is verified.
+ */
+class ItemDatabase {
+
+	/**
+	 * Create a new item database
+	 *
+	 * @param {object|undefined} serialized - An object from a previous call to
+	 *     {@link module:lib/database~ItemDatabase#serialize} to restore the
+	 *     database from.  An empty database will be created if this parameter
+	 *     is left undefined.
+	 */
+	constructor(serialized) {
+		this._items = new Map();
+
+		// Verify the content of the serialized database
+		if (serialized !== undefined) {
+			for (let [name, count] of Object.entries(serialized)) {
+				checkName(name);
+				checkCount(count);
+
+				this._items.set(name, count);
+			}
+		}
+	}
+
+	/**
+	 * Serialize item database
+	 *
+	 * Serialize the item database into a plain JavaScript object that can be
+	 * turned inta a string with JSON.stringify().
+	 *
+	 * @returns {object} Serialized representation of the database
+	 */
+	serialize() {
+		let obj = {};
+		for (let [name, count] of this._items) {
+			if (count !== 0) {
+				obj[name] = count;
+			}
+		}
+		return obj;
+	}
+
+	/**
+	 * Approximate size of the items database.
+	 *
+	 * Does not account for entries being zero in the item database.
+	 */
+	get size() {
+		return this._items.size;
+	}
+
+
+	/**
+	 * Get the count of an item stored in the database
+	 *
+	 * Returns the count of a given item that has been stored in to the
+	 * database.  If the item has not been previously stored into the database
+	 * then 0 is returned.
+	 *
+	 * @param {string} name - The name of the item to get the count of.
+	 * @returns {number} The count of the item stored.
+	 */
+	getItemCount(name) {
+		checkName(name);
+
+		if (!this._items.has(name)) {
+			return 0
+		}
+
+		return this._items.get(name);
+	}
+
+	/**
+	 * Adds count of an item to the database
+	 *
+	 * Add count copies of the item specified by name to the item database.
+	 *
+	 * @param {string} name - The name of the item to add.
+	 * @param {number} count - The count of item to remove.
+	 */
+	addItem(name, count) {
+		checkName(name);
+		checkCount(count);
+
+		if (this._items.has(name)) {
+			count += this._items.get(name);
+		}
+
+		this._items.set(name, count);
+	};
+
+	/**
+	 * Removes count of an item from the database.
+	 *
+	 * Remove count copies of the item specified by name from the item database
+	 * If count is greater than the currently stored count the stored item
+	 * count will become negative.
+	 *
+	 * @param {string} name - The name of the item to remove.
+	 * @param {number} count - The count of items to remove.
+	 */
+	removeItem(name, count) {
+		checkName(name);
+		checkCount(count);
+
+		count = -count;
+		if (this._items.has(name)) {
+			count += this._items.get(name);
+		}
+
+		this._items.set(name, count);
+	};
+}
+
+module.exports = {
+	mapToObject,
+	loadJsonAsMap,
+	saveMapAsJson,
+	loadJsonArrayAsMap,
+	saveMapAsJsonArray,
+	ItemDatabase,
+}

lib/errors.js

@@ -0,0 +1,43 @@
+/**
+ * Errors thrown in Clusterio
+ * @module
+ */
+
+class CommandError extends Error { }
+class RequestError extends Error { }
+
+// Signal for messages that fail validation
+class InvalidMessage extends Error {
+	constructor(msg, errors) {
+		super(msg);
+		this.errors = errors || null;
+	}
+}
+
+// Thrown when WebSocket authentication failed
+class AuthenticationFailed extends Error { }
+
+// Errror class for known errors occuring during startup
+class StartupError extends Error { }
+
+// Errors outside of our control.
+class EnvironmentError extends Error { }
+
+// Errors caused by plugins
+class PluginError extends Error {
+	constructor(pluginName, original) {
+		super(`PluginError: ${original.message}`);
+		this.pluginName = pluginName;
+		this.original = original;
+	}
+}
+
+module.exports = {
+	CommandError,
+	RequestError,
+	InvalidMessage,
+	AuthenticationFailed,
+	StartupError,
+	EnvironmentError,
+	PluginError,
+}

lib/factorio/index.js

@@ -0,0 +1,5 @@
+module.exports = {
+	...require('lib/factorio/locale'),
+	...require('lib/factorio/patch'),
+	...require('lib/factorio/server'),
+};

lib/factorio/locale.js

@@ -0,0 +1,28 @@
+/**
+ * Function for parsing factorio locale from a local factorio install
+ */
+
+"use strict";
+const fs = require("fs-extra");
+const path = require("path");
+const ini = require("ini");
+
+/**
+ * Gets factorios locale as an object, does not respect mods.
+ *
+ * @param {string} factorioDirectory
+ * @param {string} languageCode
+ * @param {function} callback
+ */
+async function getLocale(dataDir, languageCode) {
+	if (typeof dataDir !== "string") throw new TypeError("dataDir must be a string");
+	if (typeof languageCode !== "string") throw new TypeError("languageCode must be a string");
+
+	let localeFilePath = path.join(dataDir, "base", "locale", languageCode, "base.cfg");
+	let content = await fs.readFile(localeFilePath, "utf8");
+	return ini.parse(content);
+}
+
+module.exports = {
+	getLocale,
+};

lib/factorio/patch.js

@@ -0,0 +1,290 @@
+/**
+ * Library for patching Factorio saves with scenario code.
+ * @author Hornwitser
+ */
+
+"use strict";
+const events = require("events");
+const fs = require("fs-extra");
+const jszip = require("jszip");
+const path = require("path");
+const semver = require("semver");
+
+const hash = require("lib/hash");
+
+
+const knownScenarios = {
+	// First seen in 0.17.63
+	"4e866186ebe297f1038fd325b09df1a1f5e2fdd1": {
+		"name": "freeplay",
+		"modules": [
+			"freeplay",
+			"silo-script",
+		],
+	},
+};
+
+/**
+ * Returns the root folder in the zip file
+ *
+ * Returns the name of the folder that all files in the zip file is
+ * contained in.  Throws an error if there are multiple such folders.
+ *
+ * @returns {string} name of the root folder.
+ */
+function findRoot(zip) {
+	let root = null;
+	zip.forEach((relativePath, file) => {
+		let index = relativePath.indexOf("/");
+		if (index === -1) {
+			throw new Error(`Zip contains file '${relativePath}' in root dir`);
+		}
+
+		let pathRoot = relativePath.slice(0, index);
+		if (root === null) {
+			root = pathRoot;
+		} else if (root !== pathRoot) {
+			throw new Error(`Zip contains multiple root folders`);
+		}
+	});
+
+	if (root === null) {
+		throw new Error(`Empty zip file`);
+	}
+
+	return root;
+}
+
+
+/**
+ * Generates control.lua code for loading the Clusterio modules
+ *
+ * @param {Object} patchInfo - The patch info files's json content
+ */
+function generateLoader(patchInfo) {
+	let lines = [
+		'-- Auto generated scenario module loader created by Clusterio',
+		'-- Modifications to this file will be lost when loaded in Clusterio',
+		`clusterio_patch_number = ${patchInfo.patch_number}`,
+		'',
+		'local event_handler = require("event_handler")',
+		'',
+		'-- Scenario modules',
+	];
+
+	for (let moduleName of patchInfo["scenario"]["modules"]) {
+		lines.push('event_handler.add_lib(require("'+moduleName+'"))');
+	}
+
+	lines.push(...[
+		'',
+		'-- Clusterio modules',
+	]);
+
+	for (let module of patchInfo["modules"]) {
+		for (let file of module["files"]) {
+			let requirePath = `modules/${module.name}/${file["path"].slice(0, -4)}`;
+			if (file["load"]) {
+				lines.push('event_handler.add_lib(require("'+requirePath+'"))');
+			}
+			if (file["require"]) {
+				lines.push('require("'+requirePath+'")');
+			}
+		}
+	}
+
+	// End last line with a newline
+	lines.push('');
+
+	return lines.join('\n');
+}
+
+/**
+ * Reorders modules to satisfy their dependencies
+ *
+ * Looks through and reorders the array of module definitions in order to
+ * satisfy the property that dependencies are earlier in the array than
+ * their dependents.  Throws an error if this is not possible.
+ *
+ * @param {Array<Object>} modules - Array of modules to reorder
+ */
+function reorderDependencies(modules) {
+	let index = 0;
+	let present = new Map();
+	let hold = new Map();
+	reorder: while (index < modules.length) {
+		let module = modules[index];
+		if (semver.valid(module.version) === null) {
+			throw new Error(`Invalid version '${module.version}' for module ${module.name}`);
+		}
+
+		for (let [dependency, requirement] of Object.entries(module.dependencies)) {
+			if (semver.validRange(requirement) === null) {
+				throw new Error(
+					`Invalid version range '${requirement}' for dependency ${dependency} on module ${module.name}`
+				);
+			}
+
+			if (present.has(dependency)) {
+				if (!semver.satisfies(present.get(dependency), requirement)) {
+					throw new Error(`Module ${module.name} requires ${dependency} ${requirement}`);
+				}
+
+			// We have an unmet dependency, take it out and continue
+			} else {
+				if (hold.has(dependency)) {
+					hold.get(dependency).push(module);
+				} else {
+					hold.set(dependency, [module]);
+				}
+				modules.splice(index, 1);
+				continue reorder;
+			}
+		}
+
+		// No unmet dependencies, record and continue
+		present.set(module.name, module.version);
+		index++;
+
+		if (hold.has(module.name)) {
+			modules.splice(index, 0, ...hold.get(module.name));
+			hold.delete(module.name);
+		}
+	}
+
+	if (!hold.size) {
+		return;
+	}
+
+	// There are three reasons for a module to end up being held: The module depends
+	// on a module that is missing, the module is part of a dependency loop, or the
+	// the module depends on a module that satisfy any of these conditions.
+
+	let remaining = new Map();
+	for (let modules of hold.values()) {
+		for (let module of modules) {
+			remaining.set(module.name, module);
+		}
+	}
+
+	// Start with a random module from the remaining modules
+	for (let module of remaining.values()) {
+		let cycle = [];
+		while (true) {
+			// Find an unmet dependency
+			let dependency = Object.keys(module.dependencies).find(name => !present.has(name));
+
+			if (!remaining.has(dependency)) {
+				// There's no module being held up by this dependency, the
+				// dependency is missing.
+				throw new Error(`Missing dependency ${dependency} for module ${module.name}`);
+			}
+
+			if (cycle.includes(module.name)) {
+				cycle.push(module.name);
+				cycle.splice(0, cycle.indexOf(module.name))
+				throw new Error(`Module dependency loop detected: ${cycle.join(" -> ")}`);
+			}
+
+			cycle.push(module.name);
+			module = remaining.get(dependency);
+		}
+	}
+}
+
+/**
+ * Patch a save with the given modules
+ *
+ * Adds the modules given by the modules parameter to the save located
+ * at savePath and rewrites the control.lua in the save to load the
+ * modules that were added.  Will also remove any previous module
+ * located in the save.
+ *
+ * @param {string} savePath - Path to the Factorio save to patch.
+ * @param {Array<Object>} modules - Description of the modules to patch.
+ */
+async function patch(savePath, modules) {
+	let zip = await jszip.loadAsync(await fs.readFile(savePath));
+	let root = zip.folder(findRoot(zip));
+
+	let patchInfoFile = root.file("clusterio.json");
+	let patchInfo;
+	if (patchInfoFile !== null) {
+		let content = await patchInfoFile.async("string");
+		patchInfo = JSON.parse(content);
+
+	// No info file present, try to detect if it's a known compatible scenario.
+	} else {
+		let controlStream = root.file("control.lua").nodeStream("nodebuffer");
+		let controlHash = await hash.hashStream(controlStream);
+
+		if (controlHash in knownScenarios) {
+			patchInfo = {
+				"scenario": knownScenarios[controlHash],
+			};
+		} else {
+			throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
+		}
+	}
+
+	// Increment patch number
+	patchInfo["patch_number"] = (patchInfo["patch_number"] || 0) + 1;
+
+	// Remove any existing modules from the save
+	patchInfo["modules"] = [];
+	for (let file of root.file(/^modules\//)) {
+		zip.remove(file.name);
+	}
+
+	reorderDependencies(modules);
+
+	// Add the modules to the save.
+	for (let module of modules) {
+		let moduleDir = root.folder(`modules/${module.name}`);
+		let moduleInfo = {
+			"name": module.name,
+			"files": [],
+		}
+
+		let dirs = [[module.path, '']];
+		while (dirs.length) {
+			let [dir, relativeDir] = dirs.pop();
+			for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
+				let fsPath = path.join(dir, entry.name);
+				let relativePath = path.posix.join(relativeDir, entry.name);
+
+				if (entry.isFile()) {
+					moduleDir.file(relativePath, await fs.readFile(fsPath));
+					moduleInfo["files"].push({
+						"path": relativePath,
+						"load": module.load.includes(relativePath),
+						"require": module.require.includes(relativePath),
+					});
+
+				} else if (entry.isDirectory()) {
+					dirs.push([fsPath, relativePath]);
+				}
+			}
+		}
+
+		patchInfo["modules"].push(moduleInfo);
+	}
+
+	// Add loading code and patch info
+	root.file("control.lua", generateLoader(patchInfo));
+	root.file("clusterio.json", JSON.stringify(patchInfo, null, 4));
+
+	// Write back the save
+	let stream = zip.generateNodeStream({compression: "DEFLATE"});
+	let pipe = stream.pipe(fs.createWriteStream(savePath));
+	await events.once(pipe, "finish");
+}
+
+module.exports = {
+	patch,
+
+	// For testing only
+	_findRoot: findRoot,
+	_generateLoader: generateLoader,
+	_reorderDependencies: reorderDependencies,
+}

lib/factorio/server.js

@@ -0,0 +1,932 @@
+/**
+ * Factorio interfaces
+ *
+ * @module
+ */
+
+"use strict";
+const fs = require("fs-extra");
+const child_process = require("child_process");
+const path = require("path");
+const events = require("events");
+const util = require("util");
+const crypto = require("crypto");
+
+const ini = require("ini");
+const rconClient = require("rcon-client");
+const errors = require("lib/errors");
+
+
+/**
+ * Determines the version of Factorio the datadir is pointing to by
+ * reading the changelog.txt in it.
+ */
+async function getVersion(changelogPath) {
+	let changelog;
+	try {
+		changelog = await fs.readFile(changelogPath, "utf-8");
+	} catch (err) {
+		if (err.code === "ENOENT") {
+			return null;
+		}
+		throw err;
+	}
+
+	for (let line of changelog.split(/[\r\n]+/)) {
+		let index = line.indexOf(":");
+		if (index !== -1) {
+			let name = line.slice(0, index).trim();
+			if (name.toLowerCase() === "version") {
+				return line.slice(index + 1).trim();
+			}
+		}
+	}
+
+	return null;
+}
+
+/**
+ * Comparison function for sorting Factorio version strings
+ */
+function versionOrder(a, b) {
+	let aParts = a.split(".").map(s => Number.parseInt(s, 10));
+	let bParts = b.split(".").map(s => Number.parseInt(s, 10));
+
+	for (let i = 0; i < aParts.length; i++) {
+		if (aParts[i] === bParts[i]) {
+			continue;
+		} else if (aParts[i] < bParts[i]) {
+			return 1;
+		} else {
+			return -1;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * Find factorio data directory of the given version
+ *
+ * Searches the given factorio dir for an installation of Factorio of
+ * the target version.
+ */
+async function findVersion(factorioDir, targetVersion) {
+
+	// There are two supported setups: having the factorio dir be the actual
+	// install directory, and having the factorio dir be a folder containing
+	// multiple install directories
+
+	let simpleVersion = await getVersion(path.join(factorioDir, "data", "changelog.txt"));
+	if (simpleVersion !== null) {
+		if (simpleVersion === targetVersion || targetVersion === "latest") {
+			return [path.join(factorioDir, "data"), simpleVersion];
+		}
+
+		throw new Error(
+			`Factorio version ${targetVersion} was requested, but install directory contains ${simpleVersion}`
+		);
+	}
+
+	let versions = new Map();
+	for (let entry of await fs.readdir(factorioDir, { withFileTypes: true })) {
+		if (!entry.isDirectory()) {
+			continue;
+		}
+
+		let version = await getVersion(path.join(factorioDir, entry.name, "data", "changelog.txt"));
+		if (version === null) {
+			continue;
+		}
+
+		if (version === targetVersion) {
+			return [path.join(factorioDir, entry.name, "data"), version];
+		}
+
+		versions.set(version, entry.name);
+	}
+
+	if (!versions.size) {
+		throw new Error(`Unable to find any Factorio install in ${factorioDir}`);
+	}
+
+	if (targetVersion === "latest") {
+		let latest = [...versions.keys()].sort(versionOrder)[0];
+		return [path.join(factorioDir, versions.get(latest), "data"), latest];
+	}
+
+	throw new Error(`Unable to find Factorio version ${targetVersion}`);
+}
+
+/**
+ * Give a random dynamic port
+ *
+ * Returns a random port number in the Dynamic Ports range as defined by
+ * RFC 6335.
+ *
+ * @return {number} a number in the range 49152 to 65535.
+ */
+function randomDynamicPort() {
+	const start = 49152;
+	const end = 65535 + 1;
+
+	return Math.floor(Math.random() * (end - start) + start)
+}
+
+/**
+ * Generate a secure random password of the given length
+ *
+ * Uses crypto.randomBytes to generate a secure alphanumeric password of
+ * the given length.
+ *
+ * @param {number} length - the length of the password to generate.
+ * @return {string} password of the given length
+ */
+async function generatePassword(length) {
+	function validChar(byte) {
+		const ranges = ['az', 'AZ', '09'];
+		return ranges.some(range =>
+			range.codePointAt(0) <= byte && byte <= range.codePointAt(1)
+		);
+	}
+	let randomBytesAsync = util.promisify(crypto.randomBytes);
+
+	let password = "";
+	while (true) {
+		let bytes = await randomBytesAsync((length - password.length) * 3);
+		for (let byte of bytes) {
+
+			// Crop to ASCII values only
+			byte = byte & 0x7f;
+
+			if (validChar(byte)) {
+				password += String.fromCharCode(byte);
+				if (password.length == length) {
+					return password;
+				}
+			}
+		}
+	}
+}
+
+/**
+ * Line splitter for chunked data
+ *
+ * Splits a stream of bytes into lines terminated by either line feed or
+ * carriage return followed by linefeed.  Each line sepparated is passed
+ * to the callback one at a time without the line terminator included.
+ *
+ * Data is streamed via the data method, and once the stream ends the
+ * end method should be called.  This ensures an unterminated line at
+ * the end of the stream is passed to the callback.
+ */
+class LineSplitter {
+
+	/**
+	 * Create a line splitter
+	 *
+	 * @param {function(Buffer)} callback - Function called for every line.
+	 */
+	constructor(callback) {
+		this._callback = callback;
+		this._partial = null
+	}
+
+	/**
+	 * Stream in bytes
+	 *
+	 * Input bytes to split lines over.  Will invoke the callback for every
+	 * line found in buf and keep any potential partial line at the end of
+	 * buf for the next invocation.
+	 *
+	 * @param {Buffer} buf - Data to stream in.
+	 */
+	data(buf) {
+		if (this._partial) {
+			buf = Buffer.concat([this._partial, buf]);
+			this._partial = null;
+		}
+
+		while (buf.length) {
+			let end = buf.indexOf("\n");
+			if (end === -1) {
+				this._partial = buf;
+				break;
+			}
+
+			let next = end + 1;
+			// Eat carriage return as well if present
+			if (end >= 1 && buf[end-1] === "\r".charCodeAt(0)) {
+				end -= 1;
+			}
+
+			let line = buf.slice(0, end);
+			buf = buf.slice(next);
+			this._callback(line);
+		}
+	}
+
+	/**
+	 * Mark end of stream
+	 *
+	 * Signal the end of the bytestream.  If the previous invocation of the
+	 * data method ended with a partial line this will invoke the callback
+	 * with this line.
+	 */
+	end() {
+		if (this._partial) {
+			this._callback(this._partial);
+			this._partial = null;
+		}
+	}
+}
+
+/**
+ * Interpret lines of output from Factorio
+ *
+ * Parses a line of output from Factorio and interprets the content
+ * based on the broad categories of output that it produces.
+ *
+ * TODO document output format.
+ *
+ * @param {string} line - A line of output not including the line terminator
+ * @param {string} source - Passed into the output structure as source
+ *
+ * @returns {object} - An object with interpeted data.
+ */
+function parseOutput(line, source) {
+	let output = {
+		source,
+		received: Date.now(),
+	}
+
+	// There are three broad categories of output from Factorio, the first kind
+	// starts with the seconds since the server started and has a format of
+	// "   0.704 message"
+	const secRegex = /^ {0,3}(\d+\.\d+) (.*)$/
+	let secMatch = secRegex.exec(line);
+	if (secMatch) {
+		output.format = "seconds";
+		output.time = secMatch[1];
+		let secContent = secMatch[2];
+
+		// A seconds output has two different kinds of formats: The first is a
+		// log level and source location and has a format of
+		// "Level File.cpp:123: message"
+		const secLogRegex = /^(\w+) (\w+\.cpp:\d+): (.*)$/
+		let secLogMatch = secLogRegex.exec(secContent);
+		if (secLogMatch) {
+			output.type = "log";
+			output.level = secLogMatch[1];
+			output.file = secLogMatch[2];
+			output.message = secLogMatch[3];
+
+		// The other possibility is that the content is a generic message
+		} else {
+			output.type = "generic";
+			output.message = secContent;
+		}
+
+	// The second category of output starts with a date stamp of the format
+	// "yyyy-mm-dd hh:mm:ss message"
+	} else {
+		const dateRegex = /^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d) (.*)$/
+		let dateMatch = dateRegex.exec(line);
+		if (dateMatch) {
+			output.format = "date";
+			output.time = dateMatch[1];
+			let dateContent = dateMatch[2];
+
+			// A date output has two general formats.  The first is an action
+			// followed by a message and has a format of "[ACTION] message"
+			const dateActionRegex = /^\[(\w+)\] (.*)$/
+			let dateActionMatch = dateActionRegex.exec(dateContent);
+			if (dateActionMatch) {
+				output.type = "action";
+				output.action = dateActionMatch[1];
+				output.message = dateActionMatch[2];
+
+			// The other format is a generic message
+			} else {
+				output.type = "generic";
+				output.message = dateContent;
+			}
+
+		// The last category of output is simply a generic message with no
+		// formating.
+		} else {
+			output.format = "none";
+			output.type = "generic";
+			output.message = line;
+		}
+	}
+
+	return output;
+}
+
+// These are filters applied to the output from Factorio.  The filter
+// syntax is a set of properties to test for in the parsed output.  If
+// the property is a RegExp it's tested using the RegExp.  If it's an
+// array it's tested using the includes method of the array.  Otherwise
+// the property must compare strictly equal.
+//
+// If the filter matches the action is called with the FactorioServer
+// instance bound as this.
+const outputHeuristics = [
+	// Message indicating the RCON interface has started
+	{
+		filter: { type: 'log', message: /^Starting RCON interface/ },
+		action: function(output) {
+			this._startRcon().catch((err) => { this.emit('error', err); });
+		},
+	},
+
+	// Message indicating the server is done starting up
+	{
+		filter: { type: 'log', message: /^updateTick\(\d+\) changing state from\(CreatingGame\) to\(InGame\)$/ },
+		action: function(output) {
+			this._notifyGameReady().catch((err) => { this.emit('error', err); });
+		}
+	},
+
+	// Message indicating the server has finished saving
+	{
+		filter: {
+			type: 'log',
+			message: [
+				"Saving finished",
+
+				// Outputted when saving failed from --server-start-load-scenario
+				"Can't save to default location: Default location not known",
+			],
+		},
+		action: function(output) {
+			this.emit('_saved');
+		}
+	},
+
+	// Message indicating the server is shutting down
+	{
+		filter: {
+			type: 'generic',
+			message: /^Quitting: /
+		},
+		action: function(output) {
+			this.emit('_quitting');
+		}
+	},
+
+	// Messages that might be tha cause of an unexpected shutdown.
+	{
+		filter: {
+			type: 'log',
+			level: 'Error',
+			message: [
+				"MultiplayerManager failed: \"Binding IPv4 socket failed: Permission denied\"",
+				"MultiplayerManager failed: Host address is already in use.",
+			],
+		},
+		action: function(output) {
+			this._unexpected.push(`Factorio failed to bind to game port: ${output.message}`);
+		},
+	},
+	{
+		filter: {
+			type: 'log',
+			level: 'Error',
+			message: [
+				"Can't bind socket: Address already in use",
+				"Can't bind socket: Permission denied",
+			],
+		},
+		action: function(output) {
+			this._unexpected.push(`Factorio failed to bind to RCON port: ${output.message}`);
+		},
+	},
+];
+
+
+/**
+ * Factorio Server interface
+ *
+ * Handles the interactions with a Factorio server, including running,
+ * stopping and sending commands to the server.  It does not deal with
+ * creating or managing servers, or downloading Factorio.
+ *
+ * This is an events emitter with the following events:
+ * stdout - invoked when Factorio outputs a line to stdout
+ * stderr - invoked when Factorio outputs a line to stderr
+ * output - invoked with parsed output from Factorio
+ * rcon-ready - invoked when the RCON client has connected
+ * game-ready - invoked when the server is finished starting up
+ * exit - invoked when the sterver has exited
+ */
+class FactorioServer extends events.EventEmitter {
+	/**
+	 * Create a Factorio server interface
+	 *
+	 * @param {string} factorioDir - Directory of the Factorio install(s).
+	 * @param {string} writeDir - Directory to write runtime data to.
+	 * @param {object} options - Optional parameters.
+	 */
+	constructor(factorioDir, writeDir, options) {
+		super();
+
+		this._factorioDir = factorioDir;
+		this._writeDir = writeDir;
+
+		// Resolved in init
+		this._version = null;
+		this._dataDir = null;
+
+		this._targetVersion = options.version || "latest";
+		this.gamePort = options.gamePort || randomDynamicPort();
+		this.rconPort = options.rconPort || randomDynamicPort();
+		this.rconPassword = options.rconPassword;
+		this._state = "new";
+		this._server = null;
+		this._rconClient = null;
+		this._rconReady = false;
+		this._gameReady = false;
+
+		// Array of possible causes for an unexpected shutdown
+		this._unexpected = [];
+
+		this._stdout = new LineSplitter((line) => { this._handleOutput(line, 'stdout'); });
+		this._stderr = new LineSplitter((line) => { this._handleOutput(line, 'stderr'); });
+	}
+
+	_check(expectedStates) {
+		if (!expectedStates.includes(this._state)) {
+			throw new Error(
+				`Expected state ${expectedStates} but state is ${this._state}`
+			);
+		}
+	}
+
+	async _handleIpc(line) {
+		let channelEnd = line.indexOf('?');
+		if (channelEnd === -1) {
+			throw new Error(`Malformed IPC line "${line.toString()}"`);
+		}
+
+		let channel = line
+			.slice(6, channelEnd)
+			.toString('utf-8')
+			.replace(/\\x([0-9a-f]{2})/g, (match, p1) =>
+				String.fromCharCode(parseInt(p1, 16))
+			)
+		;
+
+		let type = line.slice(channelEnd + 1, channelEnd + 2).toString('utf-8');
+		let content;
+		if (type === 'j') {
+			content = JSON.parse(line.slice(channelEnd + 2).toString('utf-8'));
+
+		} else if (type === 'f') {
+			let fileName = line.slice(channelEnd + 2).toString('utf-8');
+			let filePath = this.writePath('script-output', fileName);
+
+			// Prevent malicious names
+			if (/[\\/\0]/.test(fileName)) {
+				throw new Error(`Invalid IPC file name '${fileName}'`)
+			}
+
+			if (fileName.endsWith('.json')) {
+				content = JSON.parse(await fs.readFile(filePath, 'utf-8'));
+				await fs.unlink(filePath);
+
+			} else {
+				let format = fileName.slice(fileName.lastIndexOf('.') + 1);
+				throw new Error(`Unknown IPC file format '${format}'`)
+			}
+
+		} else {
+			throw new Error(`Unknown IPC type '${type}'`);
+		}
+
+		if (!this.emit(`ipc-${channel}`, content)) {
+			console.warn(`Warning: Unhandled ipc-${channel}`, content);
+		}
+	}
+
+	_handleOutput(line, source) {
+		if (line.slice(0, 6).equals(Buffer.from("\f$ipc:"))) {
+			this._handleIpc(line).catch(err => this.emit('error', err));
+			return;
+		}
+
+		this.emit(source, line);
+
+		let output = parseOutput(line.toString('utf-8'), source);
+		heuristicLoop: for (let heuristic of outputHeuristics) {
+			for (let [name, expected] of Object.entries(heuristic.filter)) {
+				if (!Object.hasOwnProperty.call(output, name)) {
+					continue heuristicLoop;
+				}
+
+				if (expected instanceof RegExp) {
+					if (!expected.test(output[name])) {
+						continue heuristicLoop;
+					}
+				} else if (expected instanceof Array) {
+					if (!expected.includes(output[name])) {
+						continue heuristicLoop;
+					}
+				} else {
+					if (expected !== output[name]) {
+						continue heuristicLoop;
+					}
+				}
+			}
+
+			// If we get here the filter matched the output
+			heuristic.action.call(this, output);
+		}
+
+		this.emit('output', output);
+	}
+
+	async _startRcon() {
+		if (this._state === "stopping") {
+			return;
+		}
+
+		if (this._rconClient !== null) {
+			throw Error("RCON client is already started");
+		}
+
+		let config = {
+			host: "localhost",
+			port: this.rconPort,
+			password: this.rconPassword,
+			timeout: 200000, // 200s, should allow for commands up to 1250kB in length
+			maxPending: 5,
+		};
+
+		this._rconClient = new rconClient.Rcon(config);
+		this._rconClient.on('error', (err) => this.emit('error', err));
+		this._rconClient.on('authenticated', () => { this._rconReady = true; this.emit('rcon-ready'); });
+		this._rconClient.on('end', () => {/* XXX TODO */});
+		await this._rconClient.connect();
+	}
+
+	/**
+	 * Initialize class instance
+	 *
+	 * Must be called before instances of this class can be used.
+	 */
+	async init() {
+		this._check(["new"]);
+		[this._dataDir, this._version] = await findVersion(this._factorioDir, this._targetVersion);
+		this.rconPassword = this.rconPassword || await generatePassword(10);
+		this._state = "init";
+	}
+
+	/**
+	 * The version of Factorio that was detected
+	 */
+	get version() {
+		return this._version;
+	}
+
+	_attachStdio() {
+		this._server.stdout.on('data', chunk => { this._stdout.data(chunk); });
+		this._server.stdout.on('close', () => { this._stdout.end(); });
+		this._server.stderr.on('data', chunk => { this._stderr.data(chunk); });
+		this._server.stderr.on('close', () => { this._stderr.end(); });
+	}
+
+	_watchExit() {
+		this._server.on('exit', (code, signal) => {
+			if (this._state !== "stopping") {
+				if (this._unexpected.length === 0) {
+					if (signal === 'SIGKILL') {
+						this.emit('error', new errors.EnvironmentError(
+							"Factorio server was unexpectedly killed, is the system low on memory?"
+						));
+
+					} else {
+						let msg;
+						if (code !== null) {
+							msg = `Factorio server unexpectedly shut down with code ${code}`
+						} else {
+							msg = `Factorio server was unexpectedly shut by down by signal ${signal}`;
+						}
+
+						this.emit('error', new Error(msg));
+					}
+
+				} else if (this._unexpected.length === 1) {
+					this.emit('error', new Error(this._unexpected[0]));
+
+				} else {
+					this.emit('error', new Error(
+						"Factorio server unexpectedly shut down. Possible causes:\n- "+
+						this._unexpected.join("\n- ")
+					));
+				}
+			}
+
+			// Reset server state
+			this._state = "init";
+			this._server = null;
+			this._rconClient = null;
+			this._rconReady = false;
+			this._gameReady = false;
+			this._unexpected = [];
+
+			this.emit("exit");
+		});
+	}
+
+	async _notifyGameReady() {
+		this._gameReady = true;
+		this.emit('game-ready');
+	}
+
+	async _waitForReady() {
+		if (!this._gameReady) {
+			await events.once(this, 'game-ready');
+		}
+	}
+
+	/**
+	 * Create a new save
+	 *
+	 * Spawns the Factorio server with the --create argument to create a new
+	 * map save with the given name.
+	 *
+	 * @param {string} name -
+	 *     Name of the save to create.  Should end with ".zip".
+	 */
+	async create(name) {
+		this._check(["init"]);
+		this._state = "create";
+
+		await this._writeConfigIni();
+		this._server = child_process.spawn(
+			this.dataPath('..', 'bin', 'x64', 'factorio'),
+			[
+				'--config', this.writePath('config.ini'),
+				'--create', this.writePath('saves', name),
+			],
+			{
+				detached: true,
+				stdio: 'pipe',
+			}
+		);
+
+		this._attachStdio();
+		this._server.once("exit", () => this.emit("exit"));
+
+		try {
+			let [code, signal] = await events.once(this._server, 'exit');
+			if (code !== 0) {
+				throw new Error(`Factorio exited with status ${code}`);
+			}
+		} finally {
+			this._state = "init";
+		}
+	}
+
+	/**
+	 * Start server
+	 *
+	 * Spawn the Factorio server with the --start-server argument to
+	 * start the given save
+	 *
+	 * @param {string} save - Name of the save to run.
+	 */
+	async start(save) {
+		this._check(["init"]);
+		this._state = "running";
+
+		await this._writeConfigIni();
+		this._server = child_process.spawn(
+			this.dataPath('..', 'bin', 'x64', 'factorio'),
+			[
+				'--config', this.writePath('config.ini'),
+				'--start-server', this.writePath('saves', save),
+				'--port', this.gamePort,
+				'--rcon-port', this.rconPort,
+				'--rcon-password', this.rconPassword,
+			],
+			{
+				detached: true,
+				stdio: 'pipe',
+			}
+		);
+
+		this._attachStdio();
+		this._watchExit();
+		await this._waitForReady();
+	}
+
+	/**
+	 * Start scenario
+	 *
+	 * Spawn the Factorio server with the --start-server-load-scenario
+	 * argument to start the given scenario.
+	 *
+	 * @param {string} scenario - Name of the scenario to run.
+	 */
+	async startScenario(scenario) {
+		this._check(["init"]);
+		this._state = "running";
+
+		await this._writeConfigIni();
+		this._server = child_process.spawn(
+			this.dataPath('..', 'bin', 'x64', 'factorio'),
+			[
+				'--config', this.writePath('config.ini'),
+				'--start-server-load-scenario', scenario,
+				'--port', this.gamePort,
+				'--rcon-port', this.rconPort,
+				'--rcon-password', this.rconPassword,
+			],
+			{
+				detached: true,
+				stdio: 'pipe',
+			}
+		);
+
+		this._attachStdio();
+		this._watchExit();
+		await this._waitForReady();
+	}
+
+
+	/**
+	 * Send message over RCON
+	 *
+	 * If the rcon connection hasn't been established yet, this will
+	 * wait until it is establied and then send the message.
+	 *
+	 * @param {string} message - message to send to server over RCON.
+	 * @param {boolean} expectEmpty -
+	 *     if true throw if the response is not empty.  Useful for detecting
+	 *     errors that might have been sent in response.
+	 * @returns {string} response from server.
+	 */
+	async sendRcon(message, expectEmpty) {
+		this._check(["running"])
+		if (!this._rconReady) {
+			await events.once(this, 'rcon-ready');
+		}
+
+		let response = await this._rconClient.send(message);
+		if (expectEmpty && response !== '') {
+			throw new Error(`Expected empty response but got "${response}"`);
+		}
+		return response;
+	}
+
+	/**
+	 * Stop the server
+	 *
+	 * Send stop signal to the server process and wait for it to shut
+	 * down.
+	 */
+	async stop() {
+		this._check(["running"]);
+
+		// On Windows we need to save the map as there's no graceful shutdown.
+		if (this._rconClient && process.platform === "win32") {
+			let saved = events.once(this, '_saved');
+			await this.sendRcon("/server-save");
+			await saved;
+		}
+
+		this._state = "stopping";
+		if (this._rconClient) {
+
+			// If RCON is not yet fully connected that operation needs to
+			// complete before the RCON connection can be closed cleanly.
+			if (!this._rconReady) {
+				await events.once(this, 'rcon-ready');
+			}
+
+			await this._rconClient.end();
+		}
+
+		// On linux this sends SIGTERM to the process.  On windows this
+		// will terminate the process with no cleanup.  You can send your
+		// complaint for the lack of graceful shutdown to the Factorio
+		// developers.  Rcon does not recognize /quit, stdin is not
+		// recognized, and there's no "send CTRL+C to process" on Windows.
+		this._server.kill();
+
+		// There appears to be an race condition where sending SIGTERM
+		// immediatly before the RCON interface comes online causes the
+		// Factorio server to hang.  It's also possible the server doesn't
+		// stop if it's stuck with an infinite lua code loop.  In either
+		// case there's no recovering from it.
+		if (process.platform !== "win32") {
+			let hanged = true;
+			function setAlive(output) { hanged = false; }
+			this.on('_quitting', setAlive);
+
+			let timeoutId = setTimeout(() => {
+				if (hanged) {
+					console.error("Factorio appears to have hanged, sending SIGKILL");
+					this._server.kill('SIGKILL');
+				}
+			}, 5000);
+
+			await events.once(this._server, 'exit');
+
+			clearTimeout(timeoutId);
+			this.off('_quitting', setAlive);
+
+		// On windows the process is terminated immediately, but to keep
+		// ordering wait until after the exit event here.
+		} else {
+			await events.once(this._server, 'exit');
+		}
+	}
+
+	/**
+	 * Disable achievements on the running server
+	 *
+	 * Ensures achievements are disabled on the save that's running.  This is
+	 * necessary in order to run any commands at all.
+	 *
+	 * @returns {boolean}
+	 *     True if acheivements got disabled and false if they already where
+	 *     disabled.
+	 */
+	async disableAchievements() {
+		this._check(["running"]);
+
+		let check = await this.sendRcon("/sc rcon.print('disabled')");
+		if (check === 'disabled\n') {
+			return false;
+		}
+
+		// Factorio will print a warning to the console and ask for the command
+		// to be runned again before disabling achievements and allowing commands.
+		check = await this.sendRcon("/sc rcon.print('disabled')");
+		if (check === 'disabled\n') {
+			return true;
+		}
+
+		throw new Error("An error occured trying to disable acheivements");
+	}
+
+	/**
+	 * Return path in data directory
+	 *
+	 * Creates a path using path.join with the given parts that's relative to
+	 * the data directory of the Factorio server.  Not valid before init has
+	 * been called.
+	 */
+	dataPath(...parts) {
+		return path.join(this._dataDir, ...parts);
+	}
+
+	/**
+	 * Return path in write directory
+	 *
+	 * Creates a path using path.join with the given parts that's relative to
+	 * the write directory of the Factorio server.
+	 */
+	writePath(...parts) {
+		return path.join(this._writeDir, ...parts);
+	}
+
+	/**
+	 * Get example server settings
+	 *
+	 * Loads server-settings.example.json from the data dir.
+	 *
+	 * @returns {object} the parsed server-settings.
+	 */
+	async exampleSettings() {
+		return JSON.parse(await fs.readFile(this.dataPath("server-settings.example.json"), "utf-8"));
+	}
+
+	async _writeConfigIni() {
+		let content = ini.encode({
+			path: {
+				"read-data": this.dataPath(),
+				"write-data": this.writePath(),
+			}
+		});
+		await fs.outputFile(this.writePath("config.ini"), content);
+	}
+}
+
+
+module.exports = {
+	FactorioServer,
+
+	// For testing only
+	_getVersion: getVersion,
+	_versionOrder: versionOrder,
+	_findVersion: findVersion,
+	_randomDynamicPort: randomDynamicPort,
+	_generatePassword: generatePassword,
+	_LineSplitter: LineSplitter,
+	_parseOutput: parseOutput,
+};

lib/fileOps/fileOps.js

@@ -0,0 +1,33 @@
+const fs = require("fs-extra");
+const path = require("path");
+
+/**
+ * Returns the newest file in a directory
+ *
+ * @param {string} directory - The directory to look for the newest file in.
+ * @param {function(string): boolean} filter -
+ *     Optional function to filter out files to ignore.  Should return true
+ *     if the file is to be considered.
+ * @returns {?string}
+ *     Name of the file with the newest timestamp or null if the
+ *     directory contains no files.
+ */
+async function getNewestFile(directory, filter = (name) => true) {
+	let newestTime = new Date(0);
+	let newestFile = null;
+	for (let entry of await fs.readdir(directory, { withFileTypes: true })) {
+		if (entry.isFile()) {
+			let stat = await fs.stat(path.join(directory, entry.name));
+			if (filter(entry.name) && stat.mtime > newestTime) {
+				newestTime = stat.mtime;
+				newestFile = entry.name;
+			}
+		}
+	}
+
+	return newestFile;
+}
+
+module.exports = {
+	getNewestFile,
+}

lib/generateSSLcert.js

@@ -1,49 +1,44 @@
-var forge = require("node-forge");
-forge.options.usePureJavaScript = true;
-var fs = require("fs");
-var mkdirp = require("mkdirp");
+const forge = require("node-forge");
+const fs = require("fs-extra");
+const util = require("util");
 
-var pki = forge.pki;
-var rsa = forge.pki.rsa;
-var startTime, endTime;
+const hrtime = process.hrtime.bigint;
 
 
-
-module.exports = function({
+module.exports = async function({
 	bits = 2048,
-	certificatePath = "database/certificate",
+	sslCertPath,
+	sslPrivKeyPath,
 	doLogging = false,
 } = {}){
-	function log(t){
+	function log(t) {
 		if(doLogging) console.log("lib/generateSSLcert.js | "+t);
 	}
-	log("Generating SSL certificate with "+bits+" bits at "+certificatePath);
+	log("Generating SSL certificate with "+bits+" bits at "+sslCertPath);
 
-	startTime = Date.now();
-	var keypair = rsa.generateKeyPair({bits: bits, e: 0x10001});
-	endTime = Date.now();
-	log("Generated 2048bit keypair in: "+(endTime-startTime)+" ms");
+	let startNs = hrtime();
+	let keypair = await util.promisify(forge.pki.rsa.generateKeyPair)({bits: bits, e: 0x10001});
+	log("Generated 2048bit keypair in: "+(Number(hrtime()-startNs) / 1e6)+" ms");
 
-	startTime = Date.now();
-	var cert = pki.createCertificate();
+	startNs = hrtime();
+	let cert = forge.pki.createCertificate();
 	cert.publicKey = keypair.publicKey;
 	cert.serialNumber = "01";
 
 	cert.validity.notBefore = new Date();
 	cert.validity.notAfter = new Date(Date.now()+(1000*60*60*24*365)); // valid for one year
 
+	// XXX This should probably be limited to the actual hostname
+	let attrs = [{ name: "commonName", value: "*" }];
+	cert.setSubject(attrs);
+	cert.setIssuer(attrs);
+
 	// self-sign certificate
 	cert.sign(keypair.privateKey);
 
-	// Convert certificate to pem type and write it to a file
-	if(certificatePath[certificatePath.length-1] != "/") certificatePath += "/";
-	if(!fs.existsSync(certificatePath)) {
-		mkdirp.sync(certificatePath);
-		console.log("Created directory: "+certificatePath);
-	}
-	fs.writeFileSync(certificatePath+ "cert.key", pki.privateKeyToPem(keypair.privateKey));
-	fs.writeFileSync(certificatePath+ "cert.crt", pki.certificateToPem(cert));
+	// Write certificate and private key to pem files
+	await fs.outputFile(sslPrivKeyPath, forge.pki.privateKeyToPem(keypair.privateKey));
+	await fs.outputFile(sslCertPath, forge.pki.certificateToPem(cert));
 
-	endTime = Date.now();
-	log("Generated certificate in: "+(endTime-startTime)+" ms");
+	log("Generated certificate in: "+(Number(hrtime()-startNs) / 1e6)+" ms");
 }

lib/generateSSLcert.spec.js

@@ -1,25 +0,0 @@
-var assert = require("assert");
-var fs = require("fs");
-
-var fileOps = require("_app/fileOps");
-var generateSSLcert = require("./generateSSLcert.js")
-
-describe("generateSSLcert.js(options)", ()=>{
-	it("Creates a folder with a .crt and .key file in it", ()=>{
-		let certPath = "lib/certTestPlsDelete";
-		assert(!fs.existsSync(certPath)); // if it already exsits, the test might pass because of leftover data (which is bad)
-		
-		generateSSLcert({
-			// bits:512, // supported, but I just use the default of 2048 for this test
-			certificatePath:certPath,
-			doLogging: false,
-		});
-		
-		assert(fs.existsSync(certPath));
-		assert(fs.existsSync(certPath+"/cert.key")); // private key
-		assert(fs.existsSync(certPath+"/cert.crt")); // certificate file
-		
-		// remove certificates and data from the test
-		fileOps.deleteFolderRecursiveSync(certPath);
-	}).timeout(10000);
-});

lib/getFactorioLocale.js

@@ -1,53 +0,0 @@
-/**
-Module for grabbing factorio locale from a local factorio install
-@module getFactorioLocale
-*/
-
-const fs = require("fs");
-const ini = require("ini");
-
-const cache = {};
-
-/**
-Gets raw factorio locale from the .ini file.
-Not available outside of module through exports.
-
-@memberof getFactorioLocale
-@inner
-@param {string} factorioDirectory
-@param {string} languageCode
-@param {function} callback
-*/
-function getLocale(factorioDirectory, languageCode, callback){
-	var localeFile = factorioDirectory + '/data/base/locale/' + languageCode + '/base.cfg';
-	fs.readFile(localeFile, "utf8", (err, rawLocale) => {
-		callback(err, rawLocale);
-	});
-}
-
-/**
-Gets factorios locale as an object, does not respect mods.
-
-@param {string} factorioDirectory
-@param {string} languageCode
-@param {function} callback
-*/
-module.exports.asObject = function getLocaleAsObject(factorioDirectory, languageCode, callback){
-	if(typeof factorioDirectory != "string" || typeof languageCode != "string" || typeof callback != "function"){
-		throw new Error("Error: wrong parameters provided");
-	}
-	
-	if(!cache[languageCode]){
-		getLocale(factorioDirectory, languageCode, (err, rawLocale) => {
-			if(err){
-				callback(err);
-			} else {
-				let processedLocale = ini.parse(rawLocale);
-				cache[languageCode] = processedLocale;
-				callback(undefined, processedLocale);
-			}
-		});
-	} else {
-		callback(undefined, cache[languageCode]);
-	}
-}

lib/getFactorioLocale.spec.js

@@ -1,74 +0,0 @@
-const assert = require("assert");
-const getFactorioLocale = require("./getFactorioLocale");
-
-describe("getFactorioLocale.js", () => {
-	describe("asObject()", () => {
-		it("requires 2 strings and a function as parameters", () => {
-			let validate = (err) => {
-				return err == "Error: wrong parameters provided"
-			}
-			let message = "getFactorioLocale is responding incorrectly to invalid arguments"
-			let callback = (err,object) => true;
-			
-			assert.throws(() => getFactorioLocale.asObject(), validate, message);
-			assert.throws(() => getFactorioLocale.asObject("hello", "I forgot my callback"), validate, message);
-			assert.throws(() => getFactorioLocale.asObject(undefined, undefined, callback), validate, message);
-			
-			// the proper way to do it
-			assert.doesNotThrow(() => getFactorioLocale.asObject("factorio_0.15.27", "en", callback), validate, message);
-		});
-		it("returns a nested object with base game locale information", (done) => {
-			getFactorioLocale.asObject("factorio", "en", callback);
-			
-			function callback(err, object){
-				assert.equal(typeof object, "object");
-				assert.equal(object["entity-name"]["fish"], "Fish");
-				assert.equal(object["entity-name"]["small-lamp"], "Lamp");
-				Object.keys(object).forEach(key => {
-					// first level of the nested object is always an object
-					assert.equal(typeof object[key], "object");
-					
-					Object.keys(object[key]).forEach(key2 => {
-						// second level of the nested object is always a string, nearly always truthy
-						if(key2 != "so-long-and-thanks-for-all-the-fish"){
-							assert.ok(object[key][key2]);
-							assert.equal(typeof object[key][key2], "string");
-						}
-					});
-				});
-				done();
-			}
-		});
-		it("has some duplicate keys", (done) => {
-			getFactorioLocale.asObject("factorio", "en", callback);
-			
-			function callback(err, object){
-				let arrayOfKeys = [];
-				// collect all keys in a single array
-				Object.keys(object).forEach(key => {
-					arrayOfKeys.push(key);
-					Object.keys(object[key]).forEach(key2 => {
-						arrayOfKeys.push(key2);
-					});
-				});
-				
-				// check for duplicates
-				let duplicateKeys = [];
-				let i = 0;
-				arrayOfKeys.forEach(key => {
-					let o = 0;
-					arrayOfKeys.forEach(key2 => {
-						if(i != o && key == key2){
-							duplicateKeys.push(key);
-						}
-						o++
-					});
-					i++;
-				});
-				// we got 704 but this is a very unexact number
-				assert(duplicateKeys.length > 10)
-				done();
-			}
-		});
-	});
-});

lib/getInstances.js

@@ -1,15 +0,0 @@
-const fileOps = require("_app/fileOps");
-const config = require("./../config");
-
-module.exports = {
-	sync: function(){
-		let instances;
-		try {
-			instances = fileOps.getDirectoriesSync(config.instanceDirectory);
-		} catch (e) {
-			// there are no instances created yet, probably missing the folder
-			instances = [];
-		}
-		return instances;
-	}
-}

lib/hash.js

@@ -0,0 +1,39 @@
+const crypto = require('crypto');
+const fs = require('fs');
+
+
+/**
+ * Returns a promise that resolves to the SHA1 hash of the stream given
+ *
+ * @param stream - Node stream of the content to hash.
+ */
+function hashStream(stream) {
+	return new Promise(function(resolve, reject) {
+		let hasher = crypto.createHash('sha1')
+		hasher.setEncoding('hex')
+		hasher.on('finish', function() {
+			resolve(hasher.read());
+		});
+
+		stream.on('error', function(error) {
+			// the docs doesn't say anything about what's passed on error
+			reject(error);
+		});
+
+		stream.pipe(hasher);
+	});
+}
+
+/**
+ * Returns a promise that resolves to the SHA1 hash of the file given by path
+ *
+ * @param {string} path - Path to the file to hash.
+ */
+function hashFile(path) {
+	return hashStream(fs.createReadStream(path));
+}
+
+module.exports = {
+	hashStream,
+	hashFile,
+}

lib/helpers.js

@@ -0,0 +1,23 @@
+/**
+ * Collection of small utilites that are useful in multiple places.
+ * @module
+ */
+
+/**
+ * Return a string describing the type of the value passed
+ *
+ * Works the same as typeof, excpet that null and array types get their
+ * own string.
+ *
+ * @param value - value to return the type of.
+ * @returns {string} basic type of the value passed.
+ */
+function basicType(value) {
+	if (value === null) { return "null"; }
+	if (value instanceof Array) { return "array"; }
+	return typeof value;
+}
+
+module.exports = {
+	basicType,
+}

lib/link/connectors.js

@@ -0,0 +1,403 @@
+"use strict";
+const events = require("events");
+const WebSocket = require("ws");
+
+const schema = require("lib/schema");
+const errors = require("lib/errors");
+
+
+class WebSocketBaseConnector extends events.EventEmitter {
+	constructor() {
+		super();
+
+		this._socket = null;
+		this._seq = 1;
+		this._lastHeartbeat = null;
+		this._heartbeatId = null;
+		this._heartbeatInterval = null;
+		this._lastReceivedSeq = null;
+		this._sendBuffer = [];
+	}
+
+	_dropSendBufferSeq(seq) {
+		if (seq === null) {
+			return;
+		}
+
+		let dropCount = 0;
+		for (let index = 0; index < this._sendBuffer.length; index++) {
+			if (this._sendBuffer[index].seq <= seq) {
+				dropCount++;
+			} else {
+				break;
+			}
+		}
+		this._sendBuffer.splice(0, dropCount);
+	}
+
+	_doHeartbeat() {
+		this._check(["connected"]);
+		if (Date.now() - this._lastHeartbeat > 2000 * this._heartbeatInterval) {
+			console.log("SOCKET | closing after heartbeat timed out");
+			this._socket.close(1008, "Heartbeat timeout");
+			clearInterval(this._heartbeatId);
+			this._heartbeatId = null;
+
+		} else {
+			this._socket.send(JSON.stringify({
+				seq: null,
+				type: "heartbeat",
+				data: { seq: this._lastReceivedSeq },
+			}));
+		}
+	}
+
+	_processHeartbeat(message) {
+		if (!schema.heartbeat(message)) {
+			console.log("SOCKET | closing after received invalid heartbeat:", message);
+			this._socket.close(1002, "Invalid heartbeat");
+			console.log(schema.heartbeat.errors);
+			return;
+		}
+
+		this._lastHeartbeat = Date.now();
+		this._dropSendBufferSeq(message.data.seq);
+	}
+
+	/**
+	 * Start sending heartbeats
+	 */
+	startHeartbeat() {
+		this._lastHeartbeat = Date.now();
+		this._heartbeatId = setInterval(() => {
+			this._doHeartbeat();
+		}, this._heartbeatInterval * 1000);
+	}
+
+	/**
+	 * Stop sending heartbeats
+	 */
+	stopHeartbeat() {
+		if (this._heartbeatId) {
+			clearInterval(this._heartbeatId);
+			this._heartbeatId = null;
+		}
+	}
+
+	/**
+	 * Send a message over the socket
+	 *
+	 * @returns the sequence number of the message sent
+	 */
+	send(type, data) {
+		this._check(["handshake", "connected", "closing"]);
+		let message = { seq: this._seq, type, data };
+		this._sendBuffer.push(message);
+		if (["connected", "closing"].includes(this._state)) {
+			this._socket.send(JSON.stringify(message));
+		}
+		return this._seq++;
+	}
+
+	/**
+	 * Set connection state to closing
+	 *
+	 * Signal the connection is about tho close and that once the close
+	 * frame is received it should be considered finished.
+	 */
+	setClosing() {
+		this._state = "closing";
+	}
+}
+
+/**
+ * Connector for master server clients
+ */
+class WebSocketClientConnector extends WebSocketBaseConnector {
+	constructor(url, reconnectDelay) {
+		super();
+
+		this._url = url;
+
+		// The following states are used in the client connector
+		// new: Not connected
+		// handshake: Attempting to (re)connect to server.
+		// connected: Connection is online
+		// closing: Connection is in the process of being closed.
+		this._state = "new";
+		this._reconnectId = null;
+		this._sessionToken = null;
+		this._connected = false;
+		this._reconnectDelay = reconnectDelay;
+	}
+
+	_check(expectedStates) {
+		if (!expectedStates.includes(this._state)) {
+			throw new Error(
+				`Expected state ${expectedStates} but state is ${this._state}`
+			);
+		}
+	}
+
+	/**
+	 * Send a handshake message over the socket
+	 *
+	 * Used in the register function in order to send the register handshake
+	 * message over the WebSocket.
+	 */
+	sendHandshake(type, data) {
+		this._check(["handshake"]);
+		this._socket.send(JSON.stringify({ seq: null, type, data }));
+	}
+
+	/**
+	 * Close the connection with the given reason.
+	 *
+	 * Sends a close frame and disconnects the connector.
+	 */
+	close(code, reason) {
+		if (this._state === "new") {
+			return;
+		}
+
+		if (this._reconnectId) {
+			clearTimeout(this._reconnectId);
+			this._reconnectId = null;
+		}
+
+		this.stopHeartbeat();
+		this._state = "closing";
+		this._socket.close(code, reason);
+	}
+
+	/**
+	 * Connect to the master server
+	 */
+	async connect() {
+		this._check(["new"]);
+		this._state = "handshake";
+
+		this._doConnect();
+
+		// Wait for the connection to be ready
+		await events.once(this, "connected");
+	}
+
+	async _doConnect() {
+		let url = new URL(this._url);
+		url.pathname += "api/socket";
+
+		// Open WebSocket to master
+		console.log(`SOCKET | connecting to ${url}`)
+
+		this._socket = new WebSocket(url, {
+			// For now we do not verify TLS certificates since the default setup is
+			// to create a self-signed certificate.
+			rejectUnauthorized: false,
+		});
+
+		this._attachSocketHandlers();
+	}
+
+	/**
+	 * Attempt re-establishing the connecting again
+	 */
+	reconnect() {
+		if (this._reconnectId !== null) {
+			return;
+		}
+
+		let delay = Math.random() * this._reconnectDelay;
+		console.log(`SOCKET | waiting ${delay.toLocaleString("en", { maximumFractionDigits: 2 })} seconds for reconnect`)
+		this._reconnectId = setTimeout(() => {
+			this._reconnectId = null;
+			this._doConnect();
+		}, delay * 1000);
+	}
+
+	_attachSocketHandlers() {
+		this._socket.on("close", (code, reason) => {
+			console.log(`SOCKET | Close (code: ${code}, reason: ${reason})`);
+			// Authentication failed
+			if (code === 4003) {
+				this.emit("error", new errors.AuthenticationFailed("Authentication failed"));
+				this._state = "closing";
+			}
+
+			if (this._state === "closing") {
+				this._lastReceivedSeq = null;
+				this._sessionToken = null;
+				this._sendBuffer.length = 0;
+				this._state = "new";
+
+				if (this._connected) {
+					this.emit("close");
+					this._connected = false;
+				}
+
+			} else {
+				this._state = "handshake";
+				this.reconnect();
+
+				if (this._connected) {
+					this.emit("dropped");
+					this._connected = false;
+				}
+			}
+
+			this.stopHeartbeat();
+		});
+
+		this._socket.on("error", err => {
+			// It's assumed that close is always called by ws
+			console.error("SOCKET | Error:", err);
+		});
+
+		/*
+		this._socket.on("unexpected-response", (req, res) => {
+			let data = "";
+			res.on('data', (chunk) => { data += chunk; });
+			res.on('end', () => {
+				console.error(`SOCKET | Unexpeced Response ${res.statusCode}`);
+				console.error(data);
+				this._socket.emit("error", new Error(`Unexpected Response ${res.statusCode}`));
+			});
+		});
+		*/
+
+		this._socket.on("open", () => {
+			console.log("SOCKET | Open");
+		});
+		this._socket.on("ping", data => {
+			console.log(`SOCKET | Ping (data: ${data}`);
+		});
+		this._socket.on("pong", data => {
+			console.log(`SOCKET | Pong (data: ${data}`);
+		});
+
+		// Handle messages
+		this._socket.on("message", data => {
+			let message = JSON.parse(data);
+			if (this._state === "handshake") {
+				this._processHandshake(message);
+
+			} else if (["connected", "closing"].includes(this._state)) {
+				if (message.seq !== null) {
+					this._lastReceivedSeq = message.seq;
+				}
+
+				if (message.type === "heartbeat") {
+					this._processHeartbeat(message);
+
+				} else {
+					this.emit("message", message);
+				}
+
+			} else {
+				throw new Error(`Received message in unexpected state ${this._state}`);
+			}
+		});
+	}
+
+	_processHandshake(message) {
+		if (!schema.serverHandshake(message)) {
+			console.log("SOCKET | closing after received invalid handshake:", message);
+			this._socket.close(1002, "Invalid handshake");
+			return;
+		}
+
+		let { seq, type, data } = message;
+		if (type == 'hello') {
+			console.log(`SOCKET | received hello from master version ${data.version}`);
+			if (this._sessionToken) {
+				console.log("SOCKET | Attempting resume");
+				this.sendHandshake("resume", {
+					session_token: this._sessionToken,
+					last_seq: this._lastReceivedSeq,
+				});
+			} else {
+				this.register();
+			}
+
+		} else if (type === 'ready') {
+			console.log("SOCKET | received ready from master");
+			this._state = "connected";
+			this._sessionToken = data.session_token;
+			this._heartbeatInterval = data.heartbeat_interval;
+			this.startHeartbeat();
+			this._connected = true;
+			this.emit("connected");
+
+		} else if (type === "continue") {
+			console.log("SOCKET | resuming existing session");
+			this._state = "connected";
+			this._heartbeatInterval = data.heartbeat_interval;
+			this.startHeartbeat();
+			this._dropSendBufferSeq(data.last_seq);
+			for (let message of this._sendBuffer) {
+				this._socket.send(JSON.stringify(message));
+			}
+			this._connected = true;
+			this.emit("connected");
+
+		} else if (type === "invalidate") {
+			this._heartbeatInterval = null;
+			this._lastReceivedSeq = null;
+			this._sessionToken = null;
+			this._sendBuffer.length = 0;
+			this.emit("invalidate");
+			this.register();
+		}
+	}
+
+	/**
+	 * Register the link with the server
+	 *
+	 * This function is expected to be overriden by sub-classes and send the
+	 * register message over the socket when invoked.
+	 */
+	register() {
+		throw Error("Abstract function");
+	}
+}
+
+class VirtualConnector extends events.EventEmitter {
+	constructor() {
+		super();
+
+		this.other = this;
+		this._seq = 1;
+	}
+
+	/**
+	 * Create a pair of virtual connector hooked into each other
+	 *
+	 * Creates two virtual connectors that are liked to each othes such that
+	 * a message sent on one is received by the other.
+	 *
+	 * @returns {Array} two virtuarl connectors.
+	 */
+	static makePair() {
+		let first = new this();
+		let second = new this();
+		first.other = second;
+		second.other = first;
+		return [first, second];
+	}
+
+	/**
+	 * Send a message to the other end of the connector
+	 *
+	 * @returns the sequence number of the message sent
+	 */
+	send(type, data) {
+		this.other.emit('message', { seq: this._seq, type, data });
+		return this._seq++;
+	}
+}
+
+module.exports = {
+	WebSocketBaseConnector,
+	WebSocketClientConnector,
+	VirtualConnector,
+};

lib/link/index.js

@@ -0,0 +1,5 @@
+module.exports = {
+	...require('lib/link/link'),
+	...require('lib/link/messages'),
+	...require('lib/link/connectors'),
+};

lib/link/link.js

@@ -0,0 +1,234 @@
+"use strict";
+
+const schema = require("lib/schema");
+const errors = require("lib/errors");
+const messages = require("lib/link/messages");
+
+// Some definitions for the terminology used here:
+// link: Either side of a master - client connection
+// connector: Adapter between a link and a socket connection
+// connection: the master server side of a link
+// client: the side that is not the master server of a link
+// message: the complete object sent using the 'message' event
+// data: the data property of a message, essentially the payload.
+// type: the type property of a message, identifies what to expect in data.
+
+/**
+ * Common interface for server and client connections
+ */
+class Link {
+	constructor(source, target, connector) {
+		this.source = source;
+		this.target = target;
+		this.connector = connector;
+
+		this._waiters = new Map();
+		this._handlers = new Map();
+		this._validators = new Map();
+
+		// Process messages received by the connector
+		connector.on('message', payload => {
+			try {
+				this.processMessage(payload);
+			} catch (err) {
+				if (err instanceof errors.InvalidMessage) {
+					console.error(`Invalid message on ${this.source}-${this.target} link: ${err.message}`)
+					if (err.errors) {
+						console.error(err.errors);
+					}
+				} else {
+					this.connector.emit('error', err);
+				}
+			}
+		});
+
+		connector.on("invalidate", () => {
+			for (let waiterType of this._waiters.values()) {
+				for (let waiter of waiterType) {
+					waiter.reject(new Error("Session lost"));
+				}
+			}
+
+			this._waiters.clear();
+		});
+	}
+
+	/**
+	 * Process a received message on the link
+	 *
+	 * Validates and invokes the handler and/or waiters for a message that has
+	 * been received.  An unhandled message is considered to be an error.
+	 *
+	 * @throws {InvalidMessage}
+	 *     if validation failed, is missing, the message is invalid, or no
+	 *     handlers/waiters were invoked to handle the message.
+	 */
+	processMessage(message) {
+		if (!schema.message(message)) {
+			throw new errors.InvalidMessage("Malformed", schema.message.errors);
+		}
+
+		let validator = this._validators.get(message.type);
+		if (!validator) {
+			throw new errors.InvalidMessage(`No validator for ${message.type} on ${this.source}-${this.target}`);
+		}
+
+		if (!validator(message)) {
+			throw new errors.InvalidMessage(`Validation failed for ${message.type}`, validator.errors);
+		}
+
+		let hadHandlers = this._processHandler(message);
+		let hadWaiters = this._processWaiters(message);
+
+		if (!hadWaiters && !hadHandlers) {
+			// XXX console.error(`Unhandled message: ${JSON.stringify(message, null, 4)}`);
+			throw new errors.InvalidMessage(`Unhandled message ${message.type}`);
+		}
+	}
+
+	/**
+	 * Invoke the handler for this message if it exists
+	 *
+	 * @returns {Boolean} true if a handler was invoked.
+	 */
+	_processHandler(message) {
+		let handler = this._handlers.get(message.type);
+		if (handler) {
+			handler(message);
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	/**
+	 * Invokes the waiters for this message if any
+	 *
+	 * @returns {Boolean} true if a waiter was triggered.
+	 */
+	_processWaiters(message) {
+		let { seq, type, data } = message;
+		let waiters = this._waiters.get(type);
+		if (!waiters || !waiters.length) {
+			return false;
+		}
+
+		let matched = [];
+		waitersLoop: for (let index = 0; index < waiters.length; index++) {
+			let waiter = waiters[index];
+			for (let [key, value] of Object.entries(waiter.data)) {
+				if (data[key] !== value) {
+					continue waitersLoop;
+				}
+			}
+
+			waiter.resolve(message);
+			matched.push(index);
+		}
+
+		matched.reverse();
+		for (let index of matched) {
+			waiters.splice(index, 1);
+		}
+
+		return matched.length > 0;
+	}
+
+	/**
+	 * Set handler for a message type
+	 *
+	 * Set function called upon receiving a message of the given type.
+	 *
+	 * @param {string} type - Type of message to listen to
+	 * @param {function} handler -
+	 *     Callack to invoke on messages of the given type.
+	 * @param {function} validator -
+	 *     Function validating the message have the correct format.
+	 */
+	setHandler(type, handler, validator) {
+		if (this._handlers.has(type)) {
+			throw new Error(`${type} already has a handler`);
+		}
+
+		if (!validator) {
+			throw new Error("validator is required");
+		}
+
+		this.setValidator(type, validator);
+		this._handlers.set(type, handler);
+	}
+
+	/**
+	 * Set validator for a message type
+	 *
+	 * Set function called upon receiving a message of the given type in
+	 * order to validate the message.  The function should return true if
+	 * the message is valid and false otherwise.
+	 *
+	 * @param {string} type - Type of message to validate
+	 * @param {function} validator - Validation function.
+	 */
+	setValidator(type, validator) {
+		if (this._validators.has(type)) {
+			throw new Error(`${type} already has a validator`);
+		}
+
+		this._validators.set(type, validator);
+	}
+
+	/**
+	 * Wait for a message matching given type and data
+	 *
+	 * Waits for a message over the link that matches the type given as well
+	 * as any properties specified in data.  For example specifying { foo:
+	 * 21 } as the data will cause it to wait until a message with the right
+	 * type and a foo property value of 21 in the data payload.
+	 */
+	async waitFor(type, data) {
+		if (!this._validators.has(type)) {
+			throw new Error(`No validator for ${type} on ${this.source}-${this.target}`);
+		}
+
+		let waiter = { data };
+		waiter.promise = new Promise((resolve, reject) => {
+			waiter.resolve = resolve;
+			waiter.reject = reject;
+			if (!this._waiters.has(type)) {
+				this._waiters.set(type, [waiter]);
+			} else {
+				this._waiters.get(type).push(waiter);
+			}
+		});
+		return waiter.promise;
+	}
+
+	/**
+	 * Handle ping requests
+	 */
+	async pingRequestHandler() { }
+
+	/**
+	 * Prepare connection for close
+	 *
+	 * Waits for all pending requests on the connection to resolve.
+	 * Sub-classes should implement this handler with code that prevents
+	 * additional requests to be sent out before calling the super class
+	 * method.
+	 */
+	async shutdownConnectionRequestHandler() {
+		let promises = [];
+		for (let waiterType of this._waiters.values()) {
+			for (let waiter of waiterType) {
+				promises.push(waiter.promise.then(() => {}, () => {}));
+			}
+		}
+
+		await Promise.all(promises);
+		this.connector.setClosing();
+	}
+}
+
+
+module.exports = {
+	Link,
+}

lib/link/messages.js

@@ -0,0 +1,558 @@
+"use strict";
+const schema = require("lib/schema");
+const errors = require("lib/errors");
+
+/**
+ * Represents a request sent over the link
+ *
+ * A request-response message that can be innitated by either party of the
+ * link.
+ */
+class Request {
+
+	/**
+	 * Creates a request
+	 *
+	 * @param {string} type -
+	 *     message type used.  Will have suffix '_request' for the request
+	 *     message and suffix '_response' for the response message.
+	 * @param {Array<string>} links -
+	 *     Links this request is valid on.  Takes an array of strings
+	 *     containing "<source>-<target>" specifications.
+	 * @param {?string} forwardTo -
+	 *     Optional target to forward this request to.  'instance' add
+	 *     instance_id into the requestProperties and forward to the given
+	 *     instance.  'master' forwards it to the master server.
+	 * @param {Object<string, Object>} requestProperties -
+	 *     Mapping of property values to JSON schema specifications for the
+	 *     properties that are valid in the data payload of this requst.
+	 * @param {Object<string, Object>} responseProperties -
+	 *     Mapping of property values to JSON schema specifications for the
+	 *     properties that are valid in the data payload of the response to
+	 *     this requst.
+	 */
+	constructor({
+		type, links, forwardTo = null, requestProperties = {}, responseProperties = {}
+	}) {
+		this.type = type;
+		this.links = links;
+		this.forwardTo = forwardTo;
+
+		this.requestType = type + '_request';
+		this.responseType = type + '_response';
+
+		if (forwardTo === 'instance') {
+			requestProperties = Object.assign(
+				{"instance_id": { type: "integer" }}, requestProperties
+			);
+
+		} else if (forwardTo !== "master" && forwardTo !== null) {
+			throw new Error(`Invalid forwardTo value ${forwardTo}`);
+		}
+
+		this._requestValidator = schema.compile({
+			$schema: "http://json-schema.org/draft-07/schema#",
+			properties: {
+				"type": { const: this.requestType },
+				"data": {
+					additionalProperties: false,
+					required: Object.keys(requestProperties),
+					properties: requestProperties,
+				},
+			},
+		});
+
+		this._responseValidator = schema.compile({
+			$schema: "http://json-schema.org/draft-07/schema#",
+			properties: {
+				"type": { const: this.responseType },
+				"data": {
+					anyOf: [
+						{
+							additionalProperties: false,
+							required: ["seq", ...Object.keys(responseProperties)],
+							properties: {
+								"seq": { type: "integer" },
+								...responseProperties,
+							},
+						},
+						{
+							additionalProperties: false,
+							required: ["seq", "error"],
+							properties: {
+								"seq": { type: "integer" },
+								"error": { type: "string" },
+							},
+						},
+					],
+				},
+			}
+		});
+	}
+
+	/**
+	 * Attach to a link
+	 *
+	 * Set a the validator for the response on the given link if it's a
+	 * source and sets the handler for the request if it is a target.
+	 *
+	 * If forwardTo was set to 'instance' the `forwardRequestToInstance`
+	 * method on the link is used as the default handler.
+	 *
+	 * If forwardTo was set to 'master' the `forwardRequesToMaster` method
+	 * on the link is used as the default handler.
+	 *
+	 * Does nothing if the link is neither a source nor a target for this
+	 * request.
+	 *
+	 * @param {Link} link - The link to attach to.
+	 * @param {Function} handler -
+	 *    Async function to invoke with link set as this to handle the
+	 *    request request.  Only used on the target side.
+	 * @throws {Error} if the handler is needed and not defined.
+	 */
+	attach(link, handler) {
+		// Source side.
+		if (this.links.includes(`${link.source}-${link.target}`)) {
+			link.setValidator(this.responseType, this._responseValidator);
+		}
+
+		// Target side.  Note: Source and target is reversed here.
+		if (this.links.includes(`${link.target}-${link.source}`)) {
+
+			// Use forwarder if handler is not present.
+			if (!handler) {
+				if (this.forwardTo === 'instance') {
+					handler = link.forwardRequestToInstance;
+				} else if (this.forwardTo === "master") {
+					handler = link.forwardRequestToMaster;
+				}
+			}
+
+			if (!handler) {
+				throw new Error(`Missing handler for ${this.requestType} on ${link.source}-${link.target} link`);
+			}
+
+			link.setHandler(this.requestType, message => {
+				handler.call(link, message, this).then(response => {
+					if (response === undefined) {
+						// XXX Should we allow implicit responses like this?
+						response = {};
+					}
+
+					if (!this._responseValidator({ seq: 0, type: this.responseType, response})) {
+						console.error(this._responseValidator.errors);
+						throw new Error(`Validation failed responding to ${this.requestType}`);
+					}
+
+					link.connector.send(this.responseType, { ...response, seq: message.seq });
+
+				}).catch(err => {
+					if (!(err instanceof errors.RequestError)) {
+						console.error(`Unexpected error while responding to ${this.requestType}`, err);
+					}
+					link.connector.send(this.responseType, { seq: message.seq, error: err.message });
+				});
+			}, this._requestValidator);
+		}
+	}
+
+	/**
+	 * Send request over the given link
+	 *
+	 * Sends the given data over the link as the request payload and waits
+	 * for the response.
+	 *
+	 * @returns {object} response data
+	 */
+	async send(link, data = {}) {
+		// XXX validate link target/source?
+		if (!this._requestValidator({ seq: 0, type: this.requestType, data})) {
+			console.error(this._requestValidator.errors);
+			throw new Error(`Validation failed sending ${this.requestType}`);
+		}
+		let seq = link.connector.send(this.requestType, data);
+		let responseMessage = await link.waitFor(this.responseType, { seq });
+		if (responseMessage.data.error) {
+			throw new errors.RequestError(responseMessage.data.error);
+		}
+		return responseMessage.data;
+	}
+}
+
+
+let messages = {}
+
+// Connection requests
+let wsLinks = ["master-control", "control-master", "master-slave", "slave-master"];
+messages.shutdownConnection = new Request({
+	type: "shutdown_connection",
+	links: wsLinks,
+});
+
+messages.ping = new Request({
+	type: "ping",
+	links: wsLinks,
+});
+
+// Management requests
+messages.listSlaves = new Request({
+	type: 'list_slaves',
+	links: ['control-master'],
+	responseProperties: {
+		"list": {
+			type: "array",
+			items: {
+				additionalProperties: false,
+				required: ["agent", "version", "name", "id", "connected"],
+				properties: {
+					"agent": { type: "string" },
+					"version": { type: "string" },
+					"name": { type: "string" },
+					"id": { type: "integer" },
+					"connected": { type: "boolean" },
+				},
+			},
+		},
+	},
+});
+
+messages.listInstances = new Request({
+	type: 'list_instances',
+	links: ['control-master'],
+	responseProperties: {
+		"list": {
+			type: "array",
+			items: {
+				additionalProperties: false,
+				required: ["name", "id", "assigned_slave"],
+				properties: {
+					"name": { type: "string" },
+					"id": { type: "integer" },
+					"assigned_slave": { type: ["null", "integer"] },
+				},
+			},
+		},
+	},
+});
+
+messages.setInstanceOutputSubscriptions = new Request({
+	type: 'set_instance_output_subscriptions',
+	links: ['control-master'],
+	requestProperties: {
+		"instance_ids": {
+			type: "array",
+			items: { type: "integer" },
+		},
+	},
+});
+
+messages.createInstance = new Request({
+	type: 'create_instance',
+	links: ['control-master'],
+	requestProperties: {
+		"serialized_config": { type: "object" },
+	},
+});
+
+messages.getInstanceConfig = new Request({
+	type: "get_instance_config",
+	links: ["control-master"],
+	requestProperties: {
+		"instance_id": { type: "integer" },
+	},
+	responseProperties: {
+		"serialized_config": { type: "object" },
+	},
+});
+
+messages.setInstanceConfigField = new Request({
+	type: "set_instance_config_field",
+	links: ["control-master"],
+	requestProperties: {
+		"instance_id": { type: "integer" },
+		"field": { type: "string" },
+		"value": { type: "string" },
+	},
+});
+
+messages.assignInstanceCommand = new Request({
+	type: 'assign_instance_command',
+	links: ['control-master'],
+	requestProperties: {
+		"instance_id": { type: "integer" },
+		"slave_id": { type: "integer" },
+	},
+});
+
+messages.assignInstance = new Request({
+	type: 'assign_instance',
+	links: ['master-slave'],
+	requestProperties: {
+		"serialized_config": { type: "object" },
+	},
+	forwardTo: 'instance',
+})
+
+messages.startInstance = new Request({
+	type: 'start_instance',
+	links: ['control-master', 'master-slave', 'slave-instance'],
+	requestProperties: {
+		"instance_id": { type: "integer" },
+	},
+	forwardTo: 'instance',
+});
+
+messages.createSave = new Request({
+	type: 'create_save',
+	links: ['control-master', 'master-slave', 'slave-instance'],
+	forwardTo: 'instance',
+});
+
+messages.stopInstance = new Request({
+	type: 'stop_instance',
+	links: ['control-master', 'master-slave', 'slave-instance'],
+	forwardTo: 'instance',
+});
+
+messages.deleteInstance = new Request({
+	type: 'delete_instance',
+	links: ['control-master', 'master-slave'],
+	forwardTo: 'instance',
+});
+
+messages.sendRcon = new Request({
+	type: 'send_rcon',
+	links: ['control-master', 'master-slave', 'slave-instance'],
+	forwardTo: 'instance',
+	requestProperties: {
+		"command": { type: "string" },
+	},
+	responseProperties: {
+		"result": { type: "string" },
+	}
+});
+
+messages.getMetrics = new Request({
+	type: 'statistics_exporter:get_metrics',
+	links: ['master-slave', 'slave-instance'],
+	responseProperties: {
+		"results": { type: "array" },
+	},
+});
+
+
+/**
+ * Represents an event sent over the link
+ *
+ * A one way message without any response or recipt confirmation that
+ * can be innitiated by either party of the link.
+ */
+class Event {
+
+	/**
+	 * Creates an event
+	 *
+	 * @param {string} type -
+	 *     message type used.  Will have suffix '_event' appended to it for
+	 *     the messages sent through this event.
+	 * @param {Array<string>} links -
+	 *     Links this event is valid on.  Takes an array of strings
+	 *     containing "<source>-<target>" specifications.
+	 * @param {?string} forwardTo -
+	 *     Optional target to forward this request to.  Supported values are
+	 *     'master' and 'instance'.  If set to 'instance' an instance_id
+	 *     property is implicity added to eventProperties.
+	 * @param {?string} broadcastTo -
+	 *     Optional target to broadcast this event to.  Only 'instance' is
+	 *     currently supported.  An event will only be broadcast towards
+	 *     links that are downstream, so sending an event set to instance
+	 *     broadcast that's sent to a slave will only be broadcast to that
+	 *     slave's instances.
+	 * @param {Object<string, Object>} eventProperties -
+	 *     Mapping of property values to JSON schema specifications for the
+	 *     properties that are valid in the data payload of this event.
+	 */
+	constructor({ type, links, forwardTo = null, broadcastTo = null, eventProperties = {} }) {
+		this.type = type;
+		this.links = links;
+		this.forwardTo = forwardTo;
+		this.broadcastTo = broadcastTo;
+
+		if (forwardTo === 'instance') {
+			eventProperties = Object.assign(
+				{"instance_id": { type: "integer" }}, eventProperties
+			);
+
+		} else if (forwardTo !== 'master' && forwardTo !== null) {
+			throw new Error(`Invalid forwardTo value ${forwardTo}`);
+		}
+
+		if (broadcastTo !== 'instance' && broadcastTo !== null) {
+			throw new Error(`Invalid broadcastTo value ${broadcastTo}`);
+		}
+
+		this.eventType = type + '_event';
+		this._eventValidator = schema.compile({
+			$schema: "http://json-schema.org/draft-07/schema#",
+			properties: {
+				"type": { const: this.eventType },
+				"data": {
+					additionalProperties: false,
+					required: Object.keys(eventProperties),
+					properties: eventProperties,
+				},
+			},
+		});
+	}
+
+	/**
+	 * Attach to a link
+	 *
+	 * Set a the handler for the event on the given link if it's a target
+	 * for the event.
+	 *
+	 * If forwardTo was set to 'master' the `forwardEventToMaster` method on
+	 * the link is used as the default handler.  If it was set to 'instance'
+	 * the `forwardEventToInstance` method on the link is used by default.
+	 *
+	 * If broadcastTo was set to 'instance' the `broadcastEventToInstance`
+	 * method on the link is invoked before the handler.
+	 *
+	 * Does nothing if the link is not a target for this event.
+	 *
+	 * @param {Link} link - The link to attach to.
+	 * @param {Function} handler -
+	 *    Async function to invoke with link set as this to handle the
+	 *    event.  Only used on the target side.
+	 * @throws {Error} if the handler is needed and not defined.
+	 */
+	attach(link, handler) {
+		// Target side.  Note: Source and target is reversed here.
+		if (this.links.includes(`${link.target}-${link.source}`)) {
+			// Use forwarder if handler is not present.
+			if (!handler) {
+				if (this.forwardTo === 'instance') {
+					handler = link.forwardEventToInstance;
+				} else if (this.forwardTo === 'master') {
+					handler = link.forwardEventToMaster;
+				}
+			}
+
+			if (
+				this.broadcastTo === 'instance'
+				&& [
+					'instance-slave', 'slave-master', 'control-master', 'master-slave'
+				].includes(`${link.target}-${link.source}`)
+			) {
+				if (!handler) {
+					handler = link.broadcastEventToInstance;
+				} else {
+					let originalHandler = handler;
+					handler = async (message, event) => {
+						await link.broadcastEventToInstance(message, event);
+						await originalHandler.call(link, message, event);
+					}
+				}
+			}
+
+			if (!handler) {
+				throw new Error(`Missing handler for ${this.eventType} on ${link.source}-${link.target} link`);
+			}
+
+			link.setHandler(this.eventType, message => {
+				// XXX Should event handlers even be allowed to be async?
+				handler.call(link, message, this).catch(err => {
+					console.error(`Unexpected error while handling ${this.eventType}`, err);
+				});
+			}, this._eventValidator);
+		}
+	}
+
+	/**
+	 * Send event over the given link
+	 *
+	 * Sends the given event data over the link.
+	 */
+	send(link, data = {}) {
+		if (!this._eventValidator({ seq: 0, type: this.eventType, data})) {
+			console.error(this._eventValidator.errors);
+			throw new Error(`Validation failed sending ${this.eventType}`);
+		}
+
+		link.connector.send(this.eventType, data);
+	}
+}
+
+messages.instanceStopped = new Event({
+	type: 'instance_stopped',
+	links: ['instance-slave'],
+});
+
+messages.updateInstances = new Event({
+	type: 'update_instances',
+	links: ['slave-master'],
+	eventProperties: {
+		"instances": {
+			type: "array",
+			items: {
+				type: "object",
+				additionalProperties: false,
+				required: ["serialized_config"],
+				properties: {
+					"serialized_config": { type: "object" },
+				},
+			},
+		},
+	},
+});
+
+messages.instanceOutput = new Event({
+	type: 'instance_output',
+	links: ['instance-slave', 'slave-master', 'master-control'],
+	forwardTo: 'master',
+	eventProperties: {
+		"instance_id": { type: "integer" },
+		"output": {
+			type: "object",
+			additionalProperties: false,
+			required: ["source", "received", "format", "type", "message"],
+			properties: {
+				"source": { type: "string" },
+				"received": { type: "number" },
+				"format": { type: "string" },
+				"time": { type: "string" },
+				"level": { type: "string" },
+				"file": { type: "string" },
+				"type": { type: "string" },
+				"action": { type: "string" },
+				"message": { type: "string" },
+			},
+		},
+	},
+});
+
+
+/**
+ * Attaches all requests and events to the given link
+ *
+ * Loops through all builtin messages defined and attaches all of them
+ * to the link.  The handler used for the messageis looked up on the
+ * link instance as the concatenation of the name of the message, the
+ * name of message class, and 'Handler'.
+ */
+function attachAllMessages(link) {
+	for (let [name, message] of Object.entries(messages)) {
+		message.attach(link, link[name + message.constructor.name + 'Handler']);
+	}
+}
+
+
+module.exports = {
+	Request,
+	Event,
+
+	attachAllMessages,
+
+	messages,
+}

lib/luaTools.js

@@ -0,0 +1,17 @@
+/**
+ * Escapes a string for inclusion into a lua string
+ */
+function escapeString(content) {
+	return content
+		.replace(/\\/g, "\\\\")
+		.replace(/"/g, '\\"')
+		.replace(/'/g, "\\'")
+		.replace(/\0/g, "\\0")
+		.replace(/\n/g, "\\n")
+		.replace(/\r/g, "\\r")
+	;
+}
+
+module.exports = {
+	escapeString,
+}

lib/manager/configManager.js

@@ -1,48 +0,0 @@
-/**
-Provides tools for managing both masters and slaves config files.
-@module configManager
-*/
-// require modules
-const asTable = require("as-table").configure({delimiter: ' | '});
-const fs = require("fs");
-
-// internal requires
-var config = require("./../../config");
-const objectOps = require("./../objectOps.js");
-/**
-console.log()s each entry in the config
-
-@param {string} instance Either undefined, "shared" or the instance to show the config for.
-*/
-module.exports.displayConfig = function(instance){
-	if(!instance || instance == "shared"){
-		let configArray = [];
-		Object.keys(config).forEach(key=>{
-			if(key != "__comment"){
-				if(typeof config[key] == "object"){
-					configArray.push({entry: key, value: JSON.stringify(config[key])});
-				} else configArray.push({entry: key, value: config[key]});
-			}
-		});
-		console.log(asTable(configArray));
-	}
-}
-module.exports.editConfig = function(instance, entry, newValue){
-	if(!instance || instance == "shared"){
-		if(config[entry] && newValue){
-			if(objectOps.isJSON(newValue)){
-				config[entry] = JSON.parse(newValue);
-				console.log("Changed config entry "+entry+" from "+JSON.stringify(config[entry])+" to "+JSON.stringify(newValue));
-				console.log("new config file:");
-				console.log(JSON.stringify(config, null, "	"))
-				fs.writeFileSync("config.json", JSON.stringify(config, null, 4));
-			} else {
-				config[entry] = newValue;
-				console.log("Changed config entry "+entry+" from "+config[entry]+" to '"+newValue+"'");
-				console.log("new config file:");
-				console.log(JSON.stringify(config, null, "	"))
-				fs.writeFileSync("config.json", JSON.stringify(config, null, 4));
-			}
-		}
-	}
-};

lib/manager/modManager.js

@@ -1,44 +1,54 @@
 /**
 Provides tools to manage mods in a clusterio install
-@module modManager
+@module
 */
 
-const rightpad = require("right-pad");
 const asTable = require("as-table").configure({delimiter: ' | '});
 const moment = require("moment");
 const syncRequest = require("sync-request");
-const fs = require("fs");
+const fs = require("fs-extra");
 const https = require('follow-redirects').https;
 const factorioAPI = require("node-factorio-api");
 const sLog = require("single-line-log").stdout;
 const query = require('cli-interact').getYesNo;
+const path = require("path");
 
-const fileOps = require("_app/fileOps");
-const config = require("./../../config");
+const fileOps = require("lib/fileOps");
 
-const instances = require("./../getInstances.js").sync();
 
 function ellipse(str, max){
    return str.length > (max - 3) ? str.substring(0,max-3) + '...' : str;
 }
+
 // https://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript
-function formatBytes(a,b){if(0==a)return"0 Bytes";var c=1024,d=b||2,e=["Bytes","KB","MB","GB","TB","PB","EB","ZB","YB"],f=Math.floor(Math.log(a)/Math.log(c));return parseFloat((a/Math.pow(c,f)).toFixed(d))+" "+e[f]};
+function formatBytes(size, decimals) {
+	if (bytes === 0) return '0 Bytes';
+
+	const k = 1024;
+	const dm = decimals < 0 ? 0 : decimals;
+	const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
+
+	const i = Math.floor(Math.log(bytes) / Math.log(k));
+
+	return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
+}
+
 
 /**
 Lists mods for an instance as a table. Warning, handles printing internally.
-@param {string} instance - Name of the instance or "shared"
+@param {Instance} instance - Instance object, see slave.js
 
 @example
 var modManager = require("modManager");
-modManager.listMods("testInstance");
+modManager.listMods(config, instance);
 // console.logs table of mods installed on instance
-modManager.listMods("shared");
+// if instance.name is shared:
 // console.logs mods located in /sharedMods as a nice table
 */
-module.exports.listMods = function(instance){
+async function listMods(config, instance) {
 	return new Promise((resolve, reject) => {
-		if(instance && instances.includes(instance)){
-			factorioAPI.init(false, config.instanceDirectory+'/'+instance+'/instanceMods', config.instanceDirectory+'/'+instance+'/saves', config.factorio_version);
+		if (true) {
+			factorioAPI.init(false, instance.path("instanceMods"), instance.path("saves"), instance.server.version);
 			factorioAPI.authenticate({
 				username: config.username,
 				token: config.token,
@@ -68,7 +78,7 @@ module.exports.listMods = function(instance){
 			});
 		} else {
 			// list shared mods in /sharedMods/
-			factorioAPI.init(false, 'sharedMods/', '', config.factorio_version);
+			factorioAPI.init(false, 'sharedMods/', '', instance.server.version);
 			factorioAPI.authenticate({
 				username: config.username,
 				token: config.token,
@@ -84,22 +94,23 @@ module.exports.listMods = function(instance){
 				reject(new Error(e));
 			});
 		}
-		
+
 		// get JSON data about GH releases
-		/*let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
+		/*let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"factorioClusterio"}});
 		let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
 		let name = JSON.parse(res.getBody())[0].assets[0].name;
 		console.log(JSON.parse(res.getBody()))
 		*/
 	});
 }
+
 /**
 Searches mods.factorio.com for mods matching a search term and prints
 the results to console as a neat looking table
 
 @param {string} searchTerm What to search for
 */
-module.exports.findMods = function(searchTerm){
+async function findMods(config, searchTerm) {
 	return new Promise((resolve, reject) => {
 		factorioAPI.init(false);
 		factorioAPI.authenticate({
@@ -123,23 +134,23 @@ module.exports.findMods = function(searchTerm){
 				// console.log(result);
 			});
 		}).catch(e => {
-			if(e.contains("Insufficient information")) console.log("We are missing some parameters required for logging in! Make sure your factorio username and token are specified in config.json.");
+			if (e.contains("Insufficient information")) console.log("We are missing some parameters required for logging in! Make sure your factorio username and token are specified in config.json.");
 			reject(new Error(e));
 		});
 	});
 }
 
-module.exports.addMod = function(term, instance){
+async function addMod(config, term, instance) {
 	return new Promise((resolve, reject) => {
-		if(instance == "shared"){ // modDirectory has to end in /
-			var modDirectory = "sharedMods/";
+		if (instance.name == "shared") {
+			var modDirectory = "sharedMods";
 		} else {
-		var modDirectory = config.instanceDirectory+"/"+instance+"/instanceMods/";
+			var modDirectory = instance.path("instanceMods");
 		}
-		if(term && term.length > 0){
-			if(term.includes("mods.factorio.com/mods/")){ // https://mods.factorio.com/mods/Danielv123/ChooChoo
+		if (term && term.length > 0) {
+			if (term.includes("mods.factorio.com/mods/")) { // https://mods.factorio.com/mods/Danielv123/ChooChoo
 				// treat it as a link download
-				factorioAPI.init(false, modDirectory, "", config.factorio_version);
+				factorioAPI.init(false, modDirectory, "", instance.server.version);
 				factorioAPI.authenticate({
 					username: config.username,
 					token: config.token,
@@ -153,16 +164,16 @@ module.exports.addMod = function(term, instance){
 						});
 					});
 				});
-			} else if(term.includes("github.com")){ // https://github.com/Danielv123/factorioClusterioMod
+			} else if (term.includes("github.com")) { // https://github.com/Danielv123/factorioClusterioMod
 				// treat it as a github releases download
 				let apiURL = "https://api.github.com/repos/" + term.split("/")[3] + "/" + term.split("/")[4] + "/releases"; // https://api.github.com/repos/Danielv123/factorioClusterioMod/releases
-				
-				let res = syncRequest('GET', apiURL, {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
+
+				let res = syncRequest('GET', apiURL, {"headers":{"User-Agent":"factorioClusterio"}});
 				let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
 				let name = JSON.parse(res.getBody())[0].assets[0].name;
-				if(url) {
+				if (url) {
 					console.log("Downloading " + url);
-					let file = fs.createWriteStream(modDirectory + name);
+					let file = fs.createWriteStream(path.join(modDirectory, name));
 					let bytesDownloaded = 0;
 					let downloadStartTime = Date.now();
 					https.get(url, function(response) {
@@ -184,7 +195,7 @@ module.exports.addMod = function(term, instance){
 				}
 			} else {
 				// treat it as a search and download the top result
-				factorioAPI.init(false, modDirectory, "", config.factorio_version);
+				factorioAPI.init(false, modDirectory, "", instance.server.version);
 				factorioAPI.authenticate({
 					username: config.username,
 					token: config.token,
@@ -205,29 +216,30 @@ module.exports.addMod = function(term, instance){
 		}
 	});
 }
-module.exports.removeMod = function removeMod(term, instance, force = false){
+
+async function removeMod(config, term, instance, force = false) {
 	return new Promise((resolve, reject) => {
-		if(instance == "shared"){ // modDirectory has to end in /
-			var modDirectory = "sharedMods/";
+		if (instance.name == "shared") {
+			var modDirectory = "sharedMods";
 		} else {
-		var modDirectory = config.instanceDirectory+"/"+instance+"/instanceMods/";
+			var modDirectory = instance.path("instanceMods");
 		}
-		if(term && typeof term == "string" && term.length > 0){
+		if (term && typeof term == "string" && term.length > 0) {
 			let mods = fs.readdirSync(modDirectory);
 			let modsMatchingFilter = mods.filter(mod => mod.toLowerCase().includes(term.toLowerCase()));
-			if(modsMatchingFilter.length < 1){
+			if (modsMatchingFilter.length < 1) {
 				console.log("No mods found matching search");
-			} else if(modsMatchingFilter.length == 1){
+			} else if (modsMatchingFilter.length == 1) {
 				console.log("Deleting "+modsMatchingFilter[0]);
-				fs.unlinkSync(modDirectory+modsMatchingFilter[0]);
+				fs.unlinkSync(path.join(modDirectory, modsMatchingFilter[0]));
 				console.log("Deleted.");
-			} else if(modsMatchingFilter.length > 1){
+			} else if (modsMatchingFilter.length > 1) {
 				console.log("Found multiple mods");
 				modsMatchingFilter.forEach(mod => console.log(mod));
-				if(query("Delete all mods matching filter?")){
+				if (query("Delete all mods matching filter?")) {
 					let modsToDelete = modsMatchingFilter.length;
-					for(let i = 0; i < modsToDelete; i++){
-						fs.unlinkSync(modDirectory+modsMatchingFilter[i]);
+					for (let i = 0; i < modsToDelete; i++) {
+						fs.unlinkSync(path.join(modDirectory, modsMatchingFilter[i]));
 						console.log("Deleted "+modsMatchingFilter[i]);
 					}
 				} else console.log("Did not delete mods.")
@@ -246,14 +258,14 @@ the latest release, saving it to /sharedMods
 
 @param {string} url a github repo  URL, for example https://api.github.com/repos/Danielv123/factorioClusterioMod/releases. Must link to /releases
 */
-module.exports.downloadFromGitURL = function(url){
+function downloadFromGitURL(url) {
 	console.log("Downloading mod from "+url);
 	// get JSON data about releases
-	let res = syncRequest('GET', url, {"headers":{"User-Agent":"Please don't require setting a user agent"}});
+	let res = syncRequest('GET', url, {"headers":{"User-Agent":"factorioClusterio"}});
 	let downloadUrl = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
 	let name = JSON.parse(res.getBody())[0].assets[0].name;
 	console.log(JSON.parse(res.getBody()))
-	if(downloadUrl) {
+	if (downloadUrl) {
 		console.log(downloadUrl);
 		let file = fs.createWriteStream("sharedMods/"+name);
 		https.get(downloadUrl, function(response) {
@@ -262,12 +274,21 @@ module.exports.downloadFromGitURL = function(url){
 		});
 	}
 }
+
 /*
-module.exports.findUpdates = function(){
+function findUpdates() {
 	console.log("Downloading mods...");
 	// get JSON data about releases
-	let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"Fuck you for requiring user agents!"}});
+	let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"factorioClusterio"}});
 	let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
 	let name = JSON.parse(res.getBody())[0].assets[0].name;
 	console.log(JSON.parse(res.getBody()))
-}*/
+}*/
+
+module.exports = {
+	listMods,
+	findMods,
+	addMod,
+	removeMod,
+	downloadFromGitURL,
+}

lib/npmPostinstall.js

@@ -1,15 +1,11 @@
-var mkdirp = require("mkdirp");
 var http = require('http');
 var https = require('https');
-var fs = require('fs');
+var fs = require('fs-extra');
 
-var modManager = require("./manager/modManager.js");
-
-// modManager.addMod("clusterio", "shared");
 
 function makeDirectory(path){
 	return new Promise((resolve, reject) => {
-		mkdirp(path, function(err){
+		fs.mkdirp(path, function(err){
 			if(err){
 				console.error("Unable to mkdirp "+path);
 				console.error(err);
@@ -56,22 +52,21 @@ function download(url, file, callback){
 }
 
 var directories = [
-	"database/linvodb",
 	"sharedMods",
 	"instances",
 ];
 let downloadList = [
-	{
-		url: "http://fbpviewer.trakos.pl/images/spritesheet.json",
-		file: "static/pictures/spritesheet.js",
-		callback: function(file){
-			fs.writeFileSync("static/pictures/spritesheet.js", "export default\n"+fs.readFileSync("static/pictures/spritesheet.js"));
-		},
-	},
-	{
-		url: "http://fbpviewer.trakos.pl/images/spritesheet.png",
-		file: "static/pictures/spritesheet.png",
-	},
+	// {
+		// url: "http://fbpviewer.trakos.pl/images/spritesheet.json",
+		// file: "static/pictures/spritesheet.js",
+		// callback: function(file){
+			// fs.writeFileSync("static/pictures/spritesheet.js", "export default\n"+fs.readFileSync("static/pictures/spritesheet.js"));
+		// },
+	// },
+	// {
+		// url: "http://fbpviewer.trakos.pl/images/spritesheet.png",
+		// file: "static/pictures/spritesheet.png",
+	// },
 	{
 		url: "https://code.jquery.com/jquery.js",
 		file: "static/external/jquery.js",
@@ -80,10 +75,6 @@ let downloadList = [
 		url: "https://code.jquery.com/ui/1.12.1/jquery-ui.js",
 		file: "static/external/jquery-ui.js",
 	},
-	{
-		url: "https://cdnjs.cloudflare.com/ajax/libs/canvasjs/1.7.0/canvasjs.js",
-		file: "static/external/canvasjs.js",
-	},
 	{
 		url: "https://momentjs.com/downloads/moment.min.js",
 		file: "static/external/moment.min.js",
@@ -96,14 +87,9 @@ let downloadList = [
 		url: "https://fonts.googleapis.com/css?family=Roboto",
 		file: "static/external/css/Roboto.css",
 	},
-	{
-		url: "https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.0.4/socket.io.js",
-		file: "static/external/socket.io.js",
-	},
 ]
 
 // initiate folder creations and downloads
-processMkdirRecursive(processDownloadRecursive);
 function processMkdirRecursive(callback){
 	let item = directories.shift();
 	makeDirectory(item).then(()=>{
@@ -125,12 +111,6 @@ function processDownloadRecursive(callback){
 	});
 }
 
-// generate SSL certificates if they are missing
-if(!fs.existsSync("database/certificates") || !fs.existsSync("database/certificates/cert.crt") || !fs.existsSync("database/certificates/cert.key")){
-	var generateSSLcert = require("./generateSSLcert.js");
-	
-	generateSSLcert({
-		certificatePath: "database/certificates",
-		doLogging: true,
-	});
-}
+if (module === require.main) {
+	processMkdirRecursive(processDownloadRecursive);
+}

lib/objectOps.js

@@ -1,6 +1,6 @@
 /**
 Helper functions related to objects
-@module objectOps
+@module
 */
 
 /**

lib/package.json

@@ -0,0 +1,5 @@
+{
+    "name": "lib",
+    "description": "Internal library for factorioClusterio"
+}
+

lib/plugin.js

@@ -0,0 +1,192 @@
+/**
+ * Plugin interfaces and utilities.
+ * @module
+ */
+"use strict";
+const fs = require("fs-extra");
+const path = require("path");
+
+const errors = require("lib/errors");
+
+
+/**
+ * Base class for instance plugins
+ */
+class BaseInstancePlugin {
+	constructor(info, instance, slave) {
+		/**
+		 * The plugin's own info module
+		 */
+		this.info = info;
+
+		/**
+		 * Instance the plugin started for
+		 * @type {Instance}
+		 */
+		this.instance = instance;
+
+		/**
+		 * Slave running the instance
+		 *
+		 * With the exepction of accessing the slave's config you should
+		 * avoid ineracting with the slave object directly.
+		 *
+		 * @type {Slave}
+		 */
+		this.slave = slave;
+	}
+
+	/**
+	 * Called immediately after the class is instantiated
+	 */
+	async init() { }
+
+	/**
+	 * Called before collecting Prometheus metrics
+	 *
+	 * Invoked before the default metrics of prometheus is collected.  Note
+	 * that since this is done while replying to /metrics this should not
+	 * take a long time, otherwise the requst will time out.
+	 *
+	 * Although you can return the results of collecting your prometheus
+	 * collectors directly, all gauges in the default registry is already
+	 * automatically collected.  It's therefore recommended to only update
+	 * the Gauges/Counters/etc in onMetric and let Clusterio deal will
+	 * collecting the values.
+	 *
+	 * @returns an async iterator of prometheus metric results or undefined.
+	 */
+	async onMetrics() { }
+
+	/**
+	 * Called after the Factorio server is started
+	 */
+	async onStart() { }
+
+	/**
+	 * Called before the Factorio server is stopped
+	 *
+	 * This will not be called if for example the Factorio server crashes or
+	 * is killed.
+	 */
+	async onStop() { }
+
+	/**
+	 * Called when the Factorio server exited
+	 */
+	onExit() { }
+
+	/**
+	 * Called when the Factorio outputs a line
+	 *
+	 * Called when the Factorio server outputs a line on either stdout or
+	 * stderr.  The output is a parsed form of the line and contains the
+	 * following properties (those marked with ? are optional):
+	 *
+	 * - source: Where the message came from, one of "stdout" and "stderr".
+	 * - format: Timestamp format, one of "date", "seconds" and "none".
+	 * - time?: Timestamp of the message.  Not present if format is "none".
+	 * - type: Type of message, one of "log", "action" and "generic".
+	 * - level?: Log level for "log" type.  i.e "Info" normally.
+	 * - file?: File reported for "log" type.
+	 * - action?: Kind of action for "action" type. i.e "CHAT" for chat.
+	 * - message: Main content of the line.
+	 */
+	async onOutput(output) { }
+}
+
+class BaseMasterPlugin {
+	constructor(info, master, metrics) {
+		this.info = info;
+		this.master = master;
+		this.metrics = metrics;
+	}
+
+	/**
+	 * Called immediately after the class is instantiated
+	 */
+	async init() { }
+
+	/**
+	 * Called before collecting Prometheus metrics
+	 *
+	 * Invoked before the default metrics of prometheus is collected.  Note
+	 * that since this is done while replying to /metrics this should not
+	 * take a long time, otherwise the requst will time out.
+	 *
+	 * Although you can return the results of collecting your prometheus
+	 * collectors directly, all gauges in the default registry is already
+	 * automatically collected.  It's therefore recommended to only update
+	 * the Gauges/Counters/etc in onMetric and let Clusterio deal will
+	 * collecting the values.
+	 *
+	 * @returns an async iterator of prometheus metric results or undefined.
+	 */
+	async onMetrics() { }
+
+	/**
+	 * Called when the master server is shutting down
+	 */
+	async onExit() { }
+}
+
+/**
+ * Load plugin information
+ *
+ * Searches through the plugin directory and loads each plugin's info
+ * module.  Once loaded the info modules will not be reloaded should this
+ * function be called again.
+ *
+ * @returns {Map<string, Object>} mapping of plugin name to info module.
+ */
+async function loadPluginInfos(baseDir) {
+	let plugins = [];
+	for (let pluginDir of await fs.readdir(baseDir)) {
+		let pluginInfo;
+
+		try {
+			// Note: Require path is relative to this module
+			pluginInfo = require(path.posix.join('..', baseDir, pluginDir, 'info'));
+
+		} catch (err) {
+			if (err.code === 'MODULE_NOT_FOUND') {
+				continue;
+			} else {
+				throw new errors.PluginError(pluginDir, err);
+			}
+		}
+
+		if (pluginInfo.name !== pluginDir) {
+			throw new errors.EnvironmentError(
+				`Plugin dir ${baseDir}/${pluginDir} does not match the name of the plugin (${pluginInfo.name})`
+			);
+		}
+
+		plugins.push(pluginInfo);
+	}
+	return plugins;
+}
+
+function attachPluginMessages(link, pluginInfo, plugin) {
+	let messageDefinitions = pluginInfo.messages || [];
+	for (let [name, messageFormat] of Object.entries(messageDefinitions)) {
+		let handler = name + messageFormat.constructor.name + 'Handler';
+		if (plugin === null || !plugin[handler]) {
+			messageFormat.attach(link);
+
+		} else {
+			messageFormat.attach(link, async function(message, format) {
+				return await plugin[handler](message, format, this);
+			});
+		}
+	}
+}
+
+
+module.exports = {
+	BaseInstancePlugin,
+	BaseMasterPlugin,
+
+	loadPluginInfos,
+	attachPluginMessages,
+};

lib/prometheus.js

@@ -0,0 +1,432 @@
+class Metric {
+	constructor(type, name, help, labels = []) {
+		if (typeof name !== 'string') {
+			throw new Error("Expected name to be a string");
+		}
+
+		if (typeof help !== 'string') {
+			throw new Error("Expected help to be a string");
+		}
+
+		if (!/^[a-zA-Z_:][a-zA-Z0-9_:]*$/.test(name)) {
+			throw new Error(`Invalid name '${name}'`);
+		}
+
+		for (let name of labels) {
+			if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
+				throw new Error(`Invalid label '${name}'`);
+			}
+		}
+
+		this.name = name;
+		this.help = help;
+		this.type = type;
+		this.labels = labels;
+	}
+}
+
+let defaultRegistry;
+
+class Collector {
+	constructor(register = true) {
+		if (register) {
+			defaultRegistry.register(this);
+		}
+	}
+
+	async* collect() { }
+}
+
+function escapeLabelValue(value) {
+	return value
+		.replace(/\\/g, "\\\\")
+		.replace(/\n/g, "\\n")
+		.replace(/"/g, "\\\"")
+	;
+}
+
+function labelsToKey(labels, metricLabels) {
+	let items = []
+	if (labels.length === 1 && typeof labels[0] === "object") {
+		labels = labels[0];
+		for (let name of metricLabels) {
+			if (!Object.hasOwnProperty.call(labels, name)) {
+				throw new Error(`Missing label '${name}'`);
+			}
+			if (typeof labels[name] !== "string") {
+				throw new Error(
+					`Expected value for label '${name}' to be a string`
+				);
+			}
+			items.push(`${name}="${escapeLabelValue(labels[name])}"`);
+		}
+
+		for (let name of Object.keys(labels)) {
+			if (!metricLabels.includes(name)) {
+				throw new Error(`Extra label '${name}'`);
+			}
+		}
+
+	} else {
+		if (metricLabels.length > labels.length) {
+			throw new Error(`Missing label '${metricLabels[labels.length]}'`);
+		}
+
+		if (labels.length > metricLabels.length) {
+			throw new Error(`Extra positional label`);
+		}
+
+		for (let i=0; i < metricLabels.length; ++i) {
+			if (typeof labels[i] !== "string") {
+				throw new Error(
+					`Expected value for label '${metricLabels[i]}' to be a string`
+				);
+			}
+			items.push(`${metricLabels[i]}="${escapeLabelValue(labels[i])}"`);
+		}
+	}
+
+	return items.join(',');
+}
+
+function keyToLabels(key) {
+	let labels = new Map();
+
+	if (key !== '') for (let pair of key.split(',')) {
+		let [name, value] = pair.split('=', 2);
+		labels.set(name, value
+			.slice(1, -1)
+			.replace(/\\"/g, "\"")
+			.replace(/\\n/g, "\n")
+			.replace(/\\\\/g, "\\")
+		);
+	}
+
+	return labels;
+}
+
+class ValueCollector extends Collector {
+	constructor(type, name, help, options, childClass) {
+		let labels = [];
+		let register = true;
+		let callback = null;
+		for (let [name, value] of Object.entries(options)) {
+			if (name === 'labels') { labels = value; }
+			else if (name === 'register') { register = value; }
+			else if (name === 'callback') { callback = value; }
+			else { throw new Error(`Unrecognized option '${name}'`); }
+		}
+
+		// Make sure we don't register to the default registry if metric throws.
+		let metric = new Metric(type, name, help, labels);
+
+		super(register);
+
+		this.callback = callback;
+		this.metric = metric;
+		this._values = new Map();
+		this._children = new Map();
+		this._childClass = childClass;
+		this._defaultChild = labels.length ? null : this.labels({});
+	}
+
+	async* collect() {
+		if (this.callback) { await this.callback(this); }
+		yield { metric: this.metric, samples: this._values };
+	}
+
+	get() {
+		return this._defaultChild.get();
+	}
+
+	labels(...labels) {
+		let key = labelsToKey(labels, this.metric.labels);
+		let child = this._children.get(key);
+		if (child === undefined) {
+			child = new this._childClass(this._values, key);
+			this._children.set(key, child);
+		}
+		return child;
+	}
+
+	remove(...labels) {
+		let key = labelsToKey(labels, this.metric.labels);
+		if (key === '') {
+			throw new Error("labels cannot be empty");
+		}
+		this._children.delete(key);
+		this._values.delete(key);
+	}
+
+	removeAll(labels) {
+		if (!Object.keys(labels).length) {
+			throw new Error("labels cannot be empty");
+		}
+
+		function removeMatchingLabels(mapping, labels) {
+			for (let key of mapping.keys()) {
+				let candidate = keyToLabels(key);
+				let hasLabels = Object.entries(labels).every(([name, value]) => {
+					return candidate.get(name) === value;
+				});
+				if (hasLabels) {
+					mapping.delete(key)
+				}
+			}
+		}
+
+		removeMatchingLabels(this._values, labels);
+		removeMatchingLabels(this._children, labels);
+	}
+
+	clear() {
+		if (!this.metric.labels.length) {
+			throw new Error("Cannot clear unlabeled metric");
+		}
+		this._children = new Map();
+		this._values = new Map();
+	}
+}
+
+class ValueCollectorChild {
+	constructor(values, key) {
+		this._values = values;
+		this._key = key;
+
+		this._values.set(key, 0);
+	}
+
+	get() {
+		return this._values.get(this._key);
+	}
+}
+
+class CounterChild extends ValueCollectorChild {
+	inc(value = 1) {
+		// Note: Inverted to also catch NaN
+		if (!(value >= 0)) {
+			throw new Error("Expected value to be a positive number");
+		}
+
+		this._values.set(this._key, this._values.get(this._key) + value)
+	}
+}
+
+class Counter extends ValueCollector {
+	constructor(name, help, options = {}) {
+		super("counter", name, help, options, CounterChild);
+	}
+
+	inc(value = 1) {
+		this._defaultChild.inc(value);
+	}
+}
+
+class GaugeChild extends ValueCollectorChild {
+	inc(value = 1) {
+		this._values.set(this._key, this._values.get(this._key) + value)
+	}
+
+	dec(value = 1) {
+		this._values.set(this._key, this._values.get(this._key) - value)
+	}
+
+	set(value) {
+		this._values.set(this._key, value)
+	}
+
+	setToCurrentTime() {
+		this._values.set(this._key, Date.now() / 1000)
+	}
+}
+
+class Gauge extends ValueCollector {
+	constructor(name, help, options = {}) {
+		super("gauge", name, help, options, GaugeChild);
+	}
+
+	inc(value = 1) {
+		this._defaultChild.inc(value);
+	}
+
+	dec(value = 1) {
+		this._defaultChild.dec(value);
+	}
+
+	set(value) {
+		this._defaultChild.set(value);
+	}
+
+	setToCurrentTime() {
+		this._defaultChild.setToCurrentTime();
+	}
+}
+
+class CollectorRegistry {
+	constructor() {
+		this.collectors = [];
+	}
+
+	async* collect() {
+		for (let collector of this.collectors) {
+			for await (let result of collector.collect()) {
+				yield result;
+			}
+		}
+	}
+
+	register(metric) {
+		let index = this.collectors.lastIndexOf(metric);
+		if (index !== -1) {
+			throw new Error(
+				"Collector is already registered in this registry."
+			);
+		}
+
+		this.collectors.push(metric);
+	}
+
+	unregister(metric) {
+		let index = this.collectors.lastIndexOf(metric);
+		if (index === -1) {
+			throw new Error(
+				"Collector is not registered in this registry."
+			);
+		}
+
+		this.collectors.splice(index, 1);
+	}
+}
+
+defaultRegistry = new CollectorRegistry();
+
+function escapeHelp(help) {
+	return help
+		.replace(/\\/g, "\\\\")
+		.replace(/\n/g, "\\n")
+	;
+}
+
+function formatValue(value) {
+	if (value === Infinity) {
+		return '+Inf';
+	} else if (value === -Infinity) {
+		return '-Inf';
+	} else {
+		return value.toString();
+	}
+}
+
+async function* expositionLines(resultsIterator) {
+	let first = true;
+	for await (let result of resultsIterator) {
+		if (first) {
+			first = false;
+		} else {
+			yield `\n`
+		}
+
+		yield `# HELP ${result.metric.name} ${escapeHelp(result.metric.help)}\n`;
+		yield `# TYPE ${result.metric.name} ${result.metric.type}\n`;
+
+		for (let [key, value] of result.samples) {
+			if (key === '') {
+				yield `${result.metric.name} ${formatValue(value)}\n`;
+			} else {
+				yield `${result.metric.name}{${key}} ${formatValue(value)}\n`;
+			}
+		}
+	}
+}
+
+async function exposition(resultsIterator = defaultRegistry.collect()) {
+	let exposition = '';
+	for await (let line of expositionLines(resultsIterator)) {
+		exposition += line;
+	}
+	return exposition;
+}
+
+// HTTP Content-Type for the exposition format that's implemented
+exposition.contentType = 'text/plain; version=0.0.4';
+
+function serializeResult(result, options = {}) {
+	let addLabels = null;
+	let metricName = result.metric.name;
+	let metricHelp = result.metric.help;
+	for (let [name, value] of Object.entries(options)) {
+		if (name === 'addLabels') { addLabels = value; }
+		else if (name === 'metricName') { metricName = value; }
+		else if (name === 'metricHelp') { metricHelp = value; }
+		else { throw new Error(`Unrecognized option '${name}'`); }
+	}
+
+	let samples;
+	if (addLabels === null) {
+		samples = result.samples;
+		addLabels = {};
+
+	} else {
+		samples = new Map();
+		let key = labelsToKey([addLabels], [...Object.keys(addLabels)]);
+		for (let [labels, value] of result.samples.entries()) {
+			if (labels === '') { samples.set(key, value); }
+			else { samples.set(`${labels},${key}`, value); }
+		}
+	}
+
+	return {
+		metric: {
+			type: result.metric.type,
+			name: metricName,
+			help: metricHelp,
+			labels: [...result.metric.labels, ...Object.keys(addLabels)],
+		},
+		samples: [...samples.entries()],
+	};
+}
+
+function deserializeResult(serializedResult) {
+	return {
+		metric: new Metric(
+			serializedResult.metric.type,
+			serializedResult.metric.name,
+			serializedResult.metric.help,
+			serializedResult.metric.labels,
+		),
+		samples: new Map(serializedResult.samples),
+	}
+}
+
+let defaultCollectors = {};
+defaultCollectors.processCpuSecondsTotal = new Gauge(
+	'process_cpu_seconds', "Total user and system CPU time spent in seconds.",
+	{
+		callback: async function() {
+			let usage = process.cpuUsage();
+			this.set((usage.user + usage.system) / 1e6);
+		},
+	},
+);
+
+defaultCollectors.processStartTimeSeconds = new Gauge(
+	'process_start_time_seconds',
+	"Start time of the process since unix epoch in seconds."
+);
+defaultCollectors.processStartTimeSeconds.setToCurrentTime();
+
+
+module.exports = {
+	Counter,
+	Gauge,
+	CollectorRegistry,
+	Collector,
+	ValueCollector,
+	Metric,
+
+	exposition,
+	defaultRegistry,
+	defaultCollectors,
+	serializeResult,
+	deserializeResult,
+};

lib/schema.js

@@ -0,0 +1,164 @@
+const ajv = new (require('ajv'))({
+	verbose: true,
+	format: "full",
+	extendRefs: "fail",
+	strictDefaluts: true,
+	strictKeywords: true,
+});
+
+
+/**
+ * The base format for any message sent over the link
+ */
+const messageSchema = {
+	$schema: "http://json-schema.org/draft-07/schema#",
+	title: "Message",
+	description: "Format for messages on a link",
+	type: "object",
+	additionalProperties: false,
+	required: ["seq", "type", "data"],
+	properties: {
+		"seq": { type: ["integer", "null"] },
+		"type": { type: "string" },
+		"data": { type: "object" },
+	},
+};
+
+const message = ajv.compile(messageSchema);
+
+const heartbeat = ajv.compile({
+	$schema: "http://json-schema.org/draft-07/schema#",
+	title: "Heartbeat",
+	description: "Heartbeat sent over link",
+	type: "object",
+	additionalProperties: false,
+	required: ["seq", "type", "data"],
+	properties: {
+		"seq": { const: null },
+		"type": { const: "heartbeat" },
+		"data": {
+			type: "object",
+			additionalProperties: false,
+			required: ["seq"],
+			properties: {
+				"seq": { type: ["integer", "null"] },
+			},
+		},
+	},
+});
+
+const serverHandshake = ajv.compile({
+	$schema: "http://json-schema.org/draft-07/schema#",
+	title: "Server Handshake",
+	description: "Message sent from the server to a client during the handshake",
+	allOf: [ messageSchema ],
+	anyOf: [
+		{
+			properties: {
+				"type": { const: "hello" },
+				"data": {
+					additionalProperties: false,
+					required: ["version"],
+					properties: {
+						"version": { type: "string" },
+					}
+				},
+			},
+		},
+		{
+			properties: {
+				"type": { const: "ready" },
+				"data": {
+					additionalProperties: false,
+					required: ["session_token", "heartbeat_interval"],
+					properties: {
+						"session_token": { type: "string" },
+						"heartbeat_interval": { type: "number" },
+					},
+				},
+			},
+		},
+		{
+			properties: {
+				"type": { const: "continue" },
+				"data": {
+					additionalProperties: false,
+					required: ["last_seq", "heartbeat_interval"],
+					properties: {
+						"last_seq": { type: ["integer", "null"] },
+						"heartbeat_interval": { type: "number" },
+					},
+				},
+			},
+		},
+		{
+			properties: {
+				"type": { const: "invalidate" },
+				"data": { const: {} },
+			},
+		},
+	],
+});
+
+const clientHandshake = ajv.compile({
+	$schema: "http://json-schema.org/draft-07/schema#",
+	title: "Client Message",
+	description: "Message sent from a client to the server",
+	allOf: [ messageSchema ],
+	anyOf: [
+		{
+			properties: {
+				"type": { const: "register_slave" },
+				"data": {
+					additionalProperties: false,
+					required: ["token", "agent", "version", "name", "id"],
+					properties: {
+						"token": { type: "string" },
+						"agent": { type: "string" },
+						"version": { type: "string" },
+						"name": { type: "string" },
+						"id": { type: "integer" },
+					},
+				},
+			},
+		},
+		{
+			properties: {
+				"type": { const: "register_control" },
+				"data": {
+					additionalProperties: false,
+					required: ["token", "agent", "version"],
+					properties: {
+						"token": { type: "string" },
+						"agent": { type: "string" },
+						"version": { type: "string" },
+					}
+				},
+			},
+		},
+		{
+			properties: {
+				"type": { const: "resume" },
+				"data": {
+					additionalProperties: false,
+					required: ["session_token", "last_seq"],
+					properties: {
+						"session_token": { type: "string" },
+						"last_seq": { type: ["integer", "null"] },
+					},
+				},
+			},
+		},
+	],
+});
+
+
+module.exports = {
+	compile: ajv.compile.bind(ajv),
+
+	message,
+	heartbeat,
+
+	serverHandshake,
+	clientHandshake,
+};

lib/stringUtils.js

@@ -1,14 +0,0 @@
-function hashCode(string){
-	if(typeof string !== "string") throw "ERROR stringUtils: Not a string";
-	var hash = 0;
-	if (string.length == 0) return hash;
-	for (let i = 0; i < string.length; i++) {
-		char = string.charCodeAt(i);
-		hash = ((hash << 5) - hash) + char;
-		hash = hash & hash; // Convert to 32bit integer
-	}
-	return hash;
-}
-module.exports = {
-	hashCode: hashCode,
-}

lib/stringUtils.spec.js

@@ -1,48 +0,0 @@
-const assert = require("assert");
-const stringUtils = require("./stringUtils");
-
-describe("stringUtils.js", function(){
-	describe("hashCode", function(){
-		it("Produces a 32bit signed int hash from a string", function(){
-			let x = stringUtils.hashCode("whatever");
-			let y = stringUtils.hashCode(" a much longer string to prove this works for long strings as well and also special characters &!¤(#)¤");
-			
-			assert(typeof x == "number", "hashCode should be returning a 32 bit signed int");
-			assert(x.toString() == "1934383586");
-			assert(y.toString() == "1648886132");
-		});
-		let permutations = 1000;
-		it("Rarely produces same hash for different strings ("+permutations+" hashes with collision check)", function(){
-			let keyHashPairs = [];
-			for(let i = 0; i < permutations; i++){
-				let string = Math.random().toString();
-				keyHashPairs.push({
-					key:string,
-					hash:stringUtils.hashCode(string),
-				});
-			}
-			
-			for(let i = 0; i < permutations; i++){
-				for(let o = 0; o < permutations; o++){
-					assert(i == o || keyHashPairs[o].key != keyHashPairs[i].key, "Woha, Math.random() generated 2 equal numbers???");
-					assert(i == o || keyHashPairs[o].hash != keyHashPairs[i].hash, "Hash collision, as long at it doesn't happen again its fine.");
-				}
-			}
-		});
-		it("Returns 0 for 0 length strings", function(){
-			let x = stringUtils.hashCode("");
-			assert.deepEqual(x, 0);
-		});
-		it("Throws when supplied with anything other than a string", function(){
-			assert.throws(function(){
-				stringUtils.hashCode([]);
-			}, "ERROR: Not a string")
-			assert.throws(function(){
-				stringUtils.hashCode({});
-			}, "ERROR: Not a string")
-			assert.throws(function(){
-				stringUtils.hashCode(0);
-			}, "ERROR: Not a string")
-		});
-	});
-});

lib/typedef.js

@@ -1,11 +0,0 @@
-/**
-An item with a count specified
-
-@typedef {object} itemStack
-@property {string} name The name of the item in the stack
-@property {(number|string)} count The amount of items in the stack
-
-@example
-itemStack = {name:"iron-plate", count:"12"}
-itemStack = {name:"copper-plate", count:972}
-*/

logo.svg

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?><!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd"><svg width="100%" height="100%" viewBox="0 0 154 24" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:1.41421;"><path d="M105.009,19.309l-3.12,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68l0,6.2l-0.96,0.96l-7.596,-0.003l0,4.816l-0.96,0.953l0,0.007Zm6.876,0l-3.12,0l0,-3.944l0.96,-0.96l4.8,0l0,2.212l-2.64,2.692Zm-63.405,0l-9.917,0l-1.68,-1.68l0,-10.293l2.64,-2.64l3.118,0l0.002,9.827l2.718,0l0.002,-8.867l0.96,-0.96l3.117,0l1.68,1.68l0,10.293l-2.64,2.64Zm-15.652,0l-9.918,0l-1.68,-1.68l0,-10.293l2.64,-2.64l3.118,0l0.002,9.827l8.478,0.009l0,2.137l-2.64,2.64Zm-15.712,0l-9.996,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-0.96,0.942l0,0.018l-7.596,0l0,5.04l8.556,0l0,2.146l-2.64,2.64Zm47.185,0l-9.997,0l-1.68,-1.68l0,-2.142l0.96,-0.96l7.597,-0.002l0,-0.993l-8.557,0l0,-6.196l1.68,-1.68l0,0l0.96,-0.96c2.029,0 7.282,0 9.997,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-0.96,0.96l-7.597,0.002l0,0.957l8.557,0l0,6.227l-0.96,0.96l0,0l-1.68,1.68Zm12.444,0l-3.12,0l-1.68,-1.68l0,-6.216l0.96,-0.96l6.48,0l0,6.216l-2.64,2.64Zm19.325,0l-9.996,0l-1.68,-1.68l0,-10.293l2.64,-2.64c2.029,0 7.281,0 9.996,0l1.68,1.68c0,1.028 0,2.147 0,2.147l-1.086,0.96l-7.47,0l0,0.959l5.841,0l0,2.13l-0.96,0.96l-4.881,0l0,0.991l8.556,0l0,2.146l-2.64,2.64Zm20.065,-7.896l0.96,-0.96l6.481,0l0,5.953l-2.64,2.903l-3.121,0l-1.68,-1.943l0,-5.953Zm21.411,7.896l-2.872,0l0,-1.178c-1.382,-0.123 -2.749,-0.431 -4.028,-0.977l-0.056,-0.024l-1.307,0.804l-2.03,-1.25l1.277,-0.786c-0.81,-0.666 -1.487,-1.522 -1.761,-2.534l-1.749,0l0,-1.768l0.96,-0.96l1.141,0c0.332,-0.616 0.809,-1.156 1.35,-1.61l-1.198,-0.78l0.94,-0.93l2.03,-1.25l1.227,0.755c0.949,-0.424 2.033,-0.741 3.204,-0.922l0,-0.248l0.013,0l0.947,-0.96l2.872,0l0,1.116c1.505,0.156 2.889,0.535 4.059,1.079l1.332,-0.82l2.03,1.25l-1.12,1.059c0.701,0.671 1.181,1.437 1.377,2.261l1.976,0l0,1.768l-0.96,0.962l-0.002,-0.002l-1.327,0c-0.228,0.624 -0.547,1.236 -1.053,1.681l1.109,0.679l-0.968,0.978l-0.015,-0.004l-2.007,1.236l-1.411,-0.868c-0.958,0.432 -1.978,0.728 -3.02,0.908l0,0.375l-0.96,0.93l0,0.03Zm-15.17,-7.656l-6.001,0l0,5.624l1.55,1.792l2.901,0l1.55,-1.792l0,-5.624Zm-44.191,0l-6,0l0,5.877l1.54,1.539l2.921,0l1.539,-1.539l0,-5.877Zm35.14,3.952l-4.32,0l0,3.464l2.778,0l1.542,-1.59l0,-1.874Zm-96.308,-9.709l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l9.798,0l1.539,-1.539l0,-1.807l-8.556,0l0,-6.48l8.556,0l0,-1.807l-1.539,-1.54Zm8.771,0l-2.778,0l-1.54,1.54l0,10.094l1.54,1.539l9.719,0l1.539,-1.539l0,-1.799l-8.478,-0.008l-0.002,-9.827Zm15.653,0l-2.779,0l-1.539,1.54l0,10.094l1.539,1.539l9.719,0l1.539,-1.539l0,-10.094l-1.539,-1.54l-2.778,0l-0.003,9.827l-4.157,0l-0.002,-9.827Zm54.53,0l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l9.798,0l1.539,-1.539l0,-1.807l-8.556,0l0,-2.431l5.841,0l0,-1.65l-5.841,0l0,-2.399l8.556,0l0,-1.807l-1.539,-1.54Zm-31.77,0l-9.797,0l-0.82,0.82l-0.72,0.72l0,5.856l8.557,0l0,2.432l-8.557,0.003l0,1.803l1.54,1.539l9.797,0l1.54,-1.539l0,-5.888l-8.557,0l0,-2.396l8.557,-0.003l0,-1.807l-1.54,-1.54Zm47.585,0l-9.798,0l-1.539,1.54l0,10.094l1.539,1.539l2.781,0l0,-5.777l8.556,0.004l0,-5.86l-1.539,-1.54Zm28.324,7.453c0.327,-0.363 0.421,-0.95 0.212,-1.53c-0.459,-1.279 -2.339,-2.021 -4.278,-2.122c-0.058,0 -0.058,0 -0.115,0c-2.792,0.201 -3.386,1.31 -3.433,1.781c-0.151,1.505 1.994,2.74 4.45,2.74c1.269,0 2.7,-0.357 3.164,-0.869Zm-31.101,-1.707l0,-2.398l-4.24,0.002l0,2.396l4.24,0Zm-3.04,-2.157l0,0.957l2.8,0l0,-0.958l-2.8,0.001Zm-24.048,0.012l-13.471,0l0,-2.161l2.64,-2.64l10.111,0l1.68,1.68c0,1.034 0,2.161 0,2.161l-0.96,0.96Zm40.606,-0.048l-6.392,0l0,-2.113l2.64,-2.64l4.801,-0.005l0,3.798l-1.049,0.96Zm-42.386,-3.553l-9.912,0l-1.539,1.54l0,1.821l12.991,0l0,-1.821l-1.54,-1.54Zm37.774,0l-1.54,1.54l0,1.773l6.001,0l0,-3.318c-1.487,0.002 -2.974,0.003 -4.461,0.005Z" style="fill:#141414;"/><path d="M73.625,19.309l3.12,0l-3.12,0Zm47.201,-0.24l-2.901,0l-1.55,-1.792l0,-5.624l6.001,0l0,5.624l-1.55,1.792Zm-44.18,0l-2.921,0l-1.54,-1.539l0,-5.877l6,0l0,5.877l-1.539,1.539Zm60.66,0l-2.392,0l0,-1.157l-0.288,-0.026c-1.392,-0.131 -2.761,-0.465 -4.055,-1.026l-1.288,0.792l-1.572,-0.968l1.175,-0.724l0.005,-0.076l-0.25,-0.205c-0.598,-0.5 -1.12,-1.098 -1.438,-1.808c-0.159,-0.355 -0.249,-0.743 -0.25,-0.747l-1.693,0l0,-1.288l1.676,0l0.056,-0.226c0.27,-1.047 1.011,-1.878 1.84,-2.574l-0.005,-0.072l-1.116,-0.688l1.572,-0.968l1.208,0.743l0.228,-0.1c1.335,-0.574 2.76,-0.86 4.195,-0.988l0,-1.072l2.392,0l0,1.092l0.284,0.03c1.394,0.155 2.76,0.518 4.031,1.105l1.316,-0.81l1.572,0.968l-1.259,0.776l0.236,0.276c0.559,0.497 1.039,1.085 1.324,1.77c0.148,0.356 0.225,0.734 0.226,0.738l1.924,0l0,1.288l-1.941,0l-0.059,0.219c-0.367,1.3 -1.768,2.53 -1.768,2.53l1.317,0.811l-1.572,0.968l-1.394,-0.858lLine truncated

master.spec.js

@@ -1,184 +0,0 @@
-var app = require('./master.js'),
-  assert = require('assert'),
-  request = require('supertest');
-const validateHTML = require('html5-validator');
-var parallel = require('mocha.parallel');
-
-
-describe('Master server endpoint testing', function() {
-	describe('#GET /api/getFactorioLocale', function() { 
-		it('should get the basegame factorio locale', function(done) { 
-			request(app).get('/api/getFactorioLocale').end(function(err, res) {
-				assert.equal(res.statusCode, 200);
-				
-				let object = res.body;
-				
-				// test that it is actual factorio locale, copied from getFactorioLocale.spec.js
-				assert.equal(typeof object, "object");
-				assert.equal(object["entity-name"]["fish"], "Fish");
-				assert.equal(object["entity-name"]["small-lamp"], "Lamp");
-				Object.keys(object).forEach(key => {
-					// first level of the nested object is always an object
-					assert.equal(typeof object[key], "object");
-					
-					Object.keys(object[key]).forEach(key2 => {
-						// second level of the nested object is always a string, nearly always truthy
-						if(key2 != "so-long-and-thanks-for-all-the-fish"){
-							assert.ok(object[key][key2]);
-							assert.equal(typeof object[key][key2], "string");
-						}
-					});
-				});
-				done();
-			});
-		});
-	});
-	// describe("#GET /api/")
-	parallel("#GET static website data", function() {
-		this.timeout(6000);
-		it("sends some HTML when accessing /", function(done){
-			this.timeout(6000);
-			request(app).get("/").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
-		it("sends some HTML when accessing /nodes",function(done){
-			this.timeout(6000);
-			request(app).get("/nodes").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
-		it("sends some HTML when accessing /settings",function(done){
-			this.timeout(6000);
-			request(app).get("/settings").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
-		it("sends some HTML when accessing /nodeDetails",function(done){
-			this.timeout(6000);
-			request(app).get("/nodeDetails").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					assert(result.messages.length === 1, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
-		it("sends some HTML when accessing /remoteMap",function(done){
-			this.timeout(6000);
-			request(app).get("/remoteMap").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				validateHTML(res.text).then(result => {
-					// there should be 1 error regarding complaining about me using ES6 modules before they are fully supported
-					assert(result.messages.length === 2, "there are HTML errors on the page, please fix: "+JSON.stringify(result.messages));
-					done();
-				});
-			});
-		});
-	});
-	let persistentMaster = request(app);
-	describe("#POST /api/place", function(){
-		it("adds an itemStack to the masters inventory", function(done){
-			persistentMaster.post("/api/place")
-			.send({
-				name:"steel-plate",
-				count:20,
-				instanceName:"unitTest"
-			})
-			.end(function(err,res){
-				assert(!err)
-				assert.equal(res.text, "success", "something went wrong with the request")
-				done();
-			});
-		});
-	});
-	describe("#GET /api/inventory", function(){
-		it("returns the masters current inventory", function(done){
-			persistentMaster.get("/api/inventory").end(function(err,res){
-				assert.equal(res.statusCode, 200);
-				let inventory = JSON.parse(res.text);
-				assert.equal(typeof inventory, "object", "Inventory should be an object");
-				assert(inventory.length >= 1, "There should be at least 1 entry in the inventory");
-				
-				let contains20SteelPlate = false;
-				inventory.forEach(itemStack => {
-					if(itemStack.name == "steel-plate" && itemStack.count >= 20) contains20SteelPlate = true;
-				});
-				assert(contains20SteelPlate, "Please ensure there are at least 20 steel plate in the inventory")
-				done();
-			});
-		});
-	});
-	parallel("#POST /api/remove", function(){
-		it("returns an itemStack of how many items were removed", function(done){
-			persistentMaster.post("/api/remove")
-			.send({
-				name:"steel-plate",
-				count:10
-			})
-			.end(function(err,res){
-				assert(!err);
-				assert.equal(res.body.count, 10, "Something is wrong with the response, maybe the format changed slightly or you didn't have enough steel?");
-				
-				done();
-			});
-		});
-		it("returns an empty itemStack if you try to request addItem or removeItem", function(done){
-			let casesRan = 0;
-			persistentMaster.post("/api/remove")
-			.send({
-				name:"addItem",
-				count:10
-			})
-			.end(function(err,res){
-				assert(!err);
-				assert.equal(res.text, '{"name":"addItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0");
-				
-				isDone();
-			});
-			
-			persistentMaster.post("/api/remove")
-			.send({
-				name:"removeItem",
-				count:10
-			})
-			.end(function(err,res){
-				assert(!err);
-				assert.equal(res.text, '{"name":"removeItem","count":0}', "When there are none of the item, you should get 0 back. addItem and removeItem should always return 0");
-				
-				isDone();
-			});
-			
-			function isDone(){
-				if(++casesRan == 2) done();
-			}
-		});
-		it("returns an empty itemStack if you don't have any of the item you request", function(done){
-			persistentMaster.post("/api/remove")
-			.send({
-				name:"imaginaryItem",
-				count:999999
-			})
-			.end(function(err,res){
-				assert(!err);//{"name":"imaginaryItem","count":0}
-				assert.equal(res.body.name, "imaginaryItem", "Make sure body.name is the item we asked for");
-				assert.equal(res.body.count, 0, "Count should be 0 since we are asking for something that does not exist anywhere");
-				assert.equal(Object.keys(res.body).length, 2, "name and count should be the only keys on this object");
-				
-				done();
-			});
-		});
-	});
-});

mod/api.lua

@@ -0,0 +1,67 @@
+-- Important: Keep this API in sync with modules/clusterio/api.lua
+local api = {}
+
+
+local initialized = false
+function api.init()
+    if initialized then
+        return
+    end
+    initialized = true
+
+    if remote.interfaces.clusterio_api then
+        api.events = remote.call("clusterio_api", "get_events")
+    else
+        local null_event = script.generate_event_name()
+        api.events = {
+            on_instance_updated = null_event,
+            on_server_startup = null_event,
+        }
+    end
+end
+
+local function call_api(fn, ...)
+    if remote.interfaces.clusterio_api and remote.interfaces.clusterio_api[fn] then
+        return remote.call("clusterio_api", fn, ...)
+    else
+        return nil
+    end
+end
+
+function api.get_instance_name()
+    return call_api("get_instance_name")
+end
+
+function api.get_instance_id()
+    return call_api("get_instance_id")
+end
+
+
+function api.send_json(channel, data)
+    if not remote.interfaces.clusterio_api then
+        return
+    end
+
+    -- Escape bad characters.  The question mark is used for separating the
+    -- channel name from the payload.
+    channel = channel:gsub("([\x00-\x1f?\\])", function(match)
+        return "\\x" .. string.format("%02x", match:byte())
+    end)
+
+    data = game.table_to_json(data)
+
+    -- If there's more than about 4kB of data users running with the Windows
+    -- console open will start to experience stuttering, otherwise we could
+    -- output about 1 MB of data at a time through stdout.
+    if #data < 4000 then
+        print("\f$ipc:" .. channel .. "?j" .. data)
+    else
+        local file_no = remote.call("clusterio_api", "get_file_no")
+        local file_name = "clst_" .. file_no .. ".json"
+        game.write_file(file_name, data, false, 0)
+        print("\f$ipc:" .. channel .. "?f" .. file_name)
+    end
+end
+
+
+return api

mod/changelog.txt

@@ -0,0 +1,5 @@
+---------------------------------------------------------------------------------------------------
+Version: 0.1.0
+  Major Features:
+    - API for mods to interact with Clusterio
+    - Serialization library for converting items to JSON compatible tables.

mod/info.json

@@ -0,0 +1,9 @@
+{
+    "name": "clusterio_lib",
+    "version": "0.1.0",
+    "title": "Clusterio Library (Alpha)",
+    "author": "Clusterio Team",
+    "homepage": "https://github.com/clusterio/factorioClusterio",
+    "description": "Library for interfacing with Clusterio.  Warning: work in progress alpha.",
+    "factorio_version": "0.17"
+}

modules/clusterio/api.lua

@@ -0,0 +1,42 @@
+-- Important: Keep this API in sync with mod/api.lua
+local api = {}
+
+
+api.events = {
+    on_instance_updated = script.generate_event_name(),
+    on_server_startup = script.generate_event_name(),
+}
+
+function api.get_instance_name()
+    return global.clusterio.instance_name
+end
+
+function api.get_instance_id()
+    return global.clusterio.instance_id
+end
+
+function api.send_json(channel, data)
+
+    -- Escape bad characters.  The question mark is used for separating the
+    -- channel name from the payload.
+    channel = channel:gsub("([\x00-\x1f?\\])", function(match)
+        return "\\x" .. string.format("%02x", match:byte())
+    end)
+
+    data = game.table_to_json(data)
+
+    -- If there's more than about 4kB of data users running with the Windows
+    -- console open will start to experience stuttering, otherwise we could
+    -- output about 1 MB of data at a time through stdout.
+    if #data < 4000 then
+        print("\f$ipc:" .. channel .. "?j" .. data)
+    else
+        global.clusterio_file_no = (global.clusterio_file_no or 0) + 1
+        local file_name = "clst_" .. global.clusterio_file_no .. ".json"
+        game.write_file(file_name, data, false, 0)
+        print("\f$ipc:" .. channel .. "?f" .. file_name)
+    end
+end
+
+
+return api

modules/clusterio/impl.lua

@@ -0,0 +1,61 @@
+local api = require('modules/clusterio/api')
+
+local impl = {}
+
+
+impl.events = {}
+impl.events[defines.events.on_tick] = function()
+    if global.clusterio_patch_number ~= clusterio_patch_number then
+        global.clusterio_patch_number = clusterio_patch_number
+        script.raise_event(api.events.on_server_startup, {})
+    end
+end
+
+impl.events[api.events.on_server_startup] = function()
+    if not global.clusterio then
+        global.clusterio = {
+            instance_id = nil,
+            instance_name = nil,
+        }
+    end
+end
+
+-- Internal API
+clusterio_private = {}
+function clusterio_private.update_instance(new_id, new_name)
+    global.clusterio.instance_id = new_id
+    global.clusterio.instance_name = new_name
+    script.raise_event(api.events.on_instance_updated, {
+        instance_id = new_id,
+        instance_name = new_name,
+    })
+end
+
+
+-- This is not part of the add_remote_interface callback to ensure it is
+-- available when the clusterio_lib mod is loaded.  The reason this is
+-- neccessary is that on_init for newly added mods happen before on_load
+-- for existing mods, and the add_remote_interface callback is done in
+-- on_load.  See https://forums.factorio.com/viewtopic.php?f=25&t=81552 for
+-- more details.
+remote.add_interface('clusterio_api', {
+    get_events = function()
+        return api.events
+    end,
+
+    get_instance_id = function()
+        return global.clusterio.instance_id
+    end,
+
+    get_instance_name = function()
+        return global.clusterio.instance_name
+    end,
+
+    get_file_no = function()
+        global.clusterio_file_no = (global.clusterio_file_no or 0) + 1
+        return global.clusterio_file_no
+    end,
+})
+
+
+return impl

modules/clusterio/module.json

@@ -0,0 +1,6 @@
+{
+    "name": "clusterio",
+    "version": "0.0.0",
+    "dependencies": {},
+    "load": ["impl.lua"]
+}

modules/clusterio/serialize.lua

@@ -0,0 +1,262 @@
+-- Library for serializing items in Factorio
+-- Based on code from playerManager and trainTeleports
+local serialize = {}
+
+-- Equipment Grids are serialized into an array of equipment entries
+-- where ench entry is a table with the following fields:
+--   n: name
+--   p: position (array of 2 numbers corresponding to x and y)
+--   s: shield (optional)
+--   e: energy (optional)
+-- If the equipment is a burner the following is also present:
+--   i: burner inventory
+--   r: result inventory
+--   b: curently burning (optional)
+--   f: remaining_burning_fuel (optional)
+function serialize.serialize_equipment_grid(grid)
+    local serialized = {}
+    local processed = {}
+    for y = 0, grid.height - 1 do
+        for x = 0, grid.width - 1 do
+            local equipment = grid.get({x, y})
+            if equipment ~= nil then
+                local pos = equipment.position
+                local combined_pos = pos.x + pos.y * grid.width + 1
+                if not processed[combined_pos] then
+                    processed[combined_pos] = true
+                    local entry = {
+                        n = equipment.name,
+                        p = {pos.x, pos.y},
+                    }
+                    if equipment.shield > 0 then entry.s = equipment.shield end
+                    if equipment.energy > 0 then entry.e = equipment.energy end
+                    -- TODO: Test with Industrial Revolution
+                    if equipment.burner then
+                        local burner = equipment.burner
+                        entry.i = serialize.serialize_inventory(burner.inventory)
+                        entry.r = serialize.serialize_inventory(burner.burnt_result_inventory)
+                        if burner.curently_burning then
+                            entry.b = {}
+                            serialize.serialize_item_stack(burner.curently_burning, entry.b)
+                            entry.f = burner.remaining_burning_fuel
+                        end
+                    end
+                    table.insert(serialized, entry)
+                end
+            end
+        end
+    end
+    return serialized
+end
+
+function serialize.deserialize_equipment_grid(grid, serialized)
+    grid.clear()
+    for _, entry in ipairs(serialized) do
+        local equipment = grid.put({
+            name = entry.n,
+            position = entry.p,
+        })
+        if equipment then
+            if entry.s then equipment.shield = entry.s end
+            if entry.e then equipment.energy = entry.e end
+            if entry.i then
+                if entry.b then serialize.deserialize_item_stack(burner.currently_burning, entry.b) end
+                if entry.f then burner.remaining_burning_fuel = entry.f end
+                serialize.deserialize_inventory(burner.burnt_result_inventory, entry.r)
+                serialize.deserialize_inventory(burner.inventory, entry.i)
+            end
+        end
+    end
+end
+
+-- Item stacks are serialized into a table with the following fields:
+--   n: name
+--   c: count
+--   h: health (optional)
+--   d: durability (optional)
+--   a: ammo count (optional)
+--   l: label (optional)
+--   g: equipment grid (optional)
+--   i: item inventory (optional)
+-- If the item stack is exportable it has the following property instead
+--   e: export string
+-- Label is a table with the following fields:
+--   t: label text (optional)
+--   c: color (optional)
+--   a: allow manual label change
+function serialize.serialize_item_stack(slot, entry)
+    if
+        slot.is_blueprint
+        or slot.is_blueprint_book
+        or slot.is_upgrade_item
+        or slot.is_deconstruction_item
+        or slot.is_item_with_tags
+    then
+        local call_success, call_return = pcall(slot.export_stack)
+        if not call_success then
+            print("Error: '" .. call_return .. "' thrown exporting '" .. slot.name .. "'")
+        else
+            entry.e = call_return
+        end
+
+        return
+    end
+
+    entry.n = slot.name
+    entry.c = slot.count
+    if slot.health < 1 then entry.h = slot.health end
+    if slot.durability then entry.d = slot.durability end
+    if slot.type == "ammo" then entry.a = slot.ammo end
+    if slot.is_item_with_label then
+        local label = {}
+        if slot.label then label.t = slot.label end
+        if slot.label_color then label.c = slot.label_color end
+        label.a = slot.allow_manual_label_change
+        entry.l = label
+    end
+
+    if slot.grid then
+        entry.g = serialize.serialize_equipment_grid(slot.grid)
+    end
+
+    if slot.is_item_with_inventory then
+        local sub_inventory = slot.get_inventory(defines.inventory.item_main)
+        entry.i = serialize.serialize_inventory(sub_inventory)
+    end
+end
+
+function serialize.deserialize_item_stack(slot, entry)
+    if entry.e then
+        local success = slot.import_stack(entry.e)
+        if success == 1 then
+            print("Error: import of '" .. entry.e .. "' succeeded with errors")
+        elseif success == -1 then
+            print("Error: import of '" .. entry.e .. "' failed")
+        end
+
+        return
+    end
+
+    local item_stack = {
+        name = entry.n,
+        count = entry.c,
+    }
+    if entry.h then item_stack.health = entry.h end
+    if entry.d then item_stack.durability = entry.d end
+    if entry.a then item_stack.ammo = entry.a end
+
+    local call_success, call_return = pcall(slot.set_stack, item_stack)
+    if not call_success then
+        print("Error: '" .. call_return .. "' thrown setting stack ".. serpent.line(entry))
+
+    elseif not call_return then
+        print("Error: Failed to set stack " .. serpent.line(entry))
+
+    else
+        if entry.l then
+            -- TODO test this with AAI's unit-remote-control
+            local label = entry.l
+            if label.t then slot.label = label.t end
+            if label.c then slot.label_color = label.c end
+            slot.allow_manual_label_change = label.a
+        end
+        if entry.g then
+            serialize.deserialize_equipment_grid(slot.grid, entry.g)
+        end
+        if entry.i then
+            local sub_inventory = slot.get_inventory(defines.inventory.item_main)
+            serialize.deserialize_inventory(sub_inventory, entry.i)
+        end
+    end
+end
+
+-- Inventories are serialized into a table with the following fields:
+--   i: array of item stack or exportable item entries
+--   b: bar position (optional)
+-- Each item entry has the following fields
+--   s: index (optional, equals to previous plus one if not present)
+--   r: repeat count (optional)
+--   f: slot filter (optional)
+-- Pluss all the fields for item stacks (see deserialize_item_stack)
+-- It's also possible that the slot is empty but has a slot filter.
+function serialize.serialize_inventory(inventory)
+    local serialized = {}
+    if inventory.hasbar() and inventory.getbar() <= #inventory then
+        serialized.b = inventory.getbar()
+    end
+
+    serialized.i = {}
+    local previous_index = 0
+    local previous_serialized = nil
+    for i = 1, #inventory do
+        local item = {}
+        local slot = inventory[i]
+        if inventory.supports_filters() then
+            item.f = inventory.get_filter(i)
+        end
+
+        if slot.valid_for_read then
+            serialize.serialize_item_stack(slot, item)
+        end
+
+        if item.n or item.f or item.e then
+            local item_serialized = game.table_to_json(item)
+            if item_serialized == previous_serialized then
+                local previous_item = serialized.i[#serialized.i]
+                previous_item.r = (previous_item.r or 0) + 1
+                previous_index = i
+
+            else
+                if i ~= previous_index + 1 then
+                    item.s = i
+                end
+
+                previous_index = i
+                previous_serialized = item_serialized
+                table.insert(serialized.i, item)
+            end
+
+        else
+            -- Either an empty slot or serilization failed
+            previous_index = 0
+            previous_serialized = nil
+        end
+    end
+
+    return serialized
+end
+
+function serialize.deserialize_inventory(inventory, serialized)
+    if serialized.b and inventory.hasbar() then
+        inventory.setbar(serialized.b)
+    end
+
+    local last_slot_index = 0
+    for _, entry in ipairs(serialized.i) do
+        local base_index = entry.s or last_slot_index + 1
+
+        local repeat_count = entry.r or 0
+        for offset = 0, repeat_count do
+            -- XXX what if the inventory is smaller on this instance?
+            local index = base_index + offset
+            local slot = inventory[index]
+            if entry.f then
+                local call_success, call_return = pcall(inventory.set_filter, index, entry.f)
+                if not call_success then
+                    print("Error: '" .. call_return .. "' thrown setting filter " .. entry.f)
+
+                elseif not call_return then
+                    print("Error: Failed to set filter " .. entry.f)
+                end
+            end
+
+            if entry.n or entry.e then
+                serialize.deserialize_item_stack(slot, entry)
+            end
+        end
+        last_slot_index = base_index + repeat_count
+    end
+end
+
+
+return serialize

node_modules/_app/factorioServer/index.js

@@ -1,168 +0,0 @@
-/**
-
-class factorioServer
-
-This class has the responsiblity of :
-
-* spawning a factorio server
-* Handling input via stdin or RCON
-
-
-*/
-
-const fs = require("fs");
-const child_process = require("child_process");
-const path = require("path");
-
-const mkdirp = require("mkdirp");
-const appRootPath = require("app-root-path");
-
-const fileOps = require("_app/fileOps");
-
-module.exports = class factorioServer {
-	constructor({
-		factorioDirectory = "factorio",
-		instanceDirectory = "instances/data",
-		instanceName = "Default server name",
-		factorioPort = Math.floor(Math.random() * 65535),
-		serversettings = {},
-		onProgress = function(type, a, b, c){
-			if(type == "progress") console.log("Progress: ",a," of ",b," unit: ",c);
-			if(type == "log") console.log(a);
-		},
-	} = {}){
-		this.factorioDirectory = factorioDirectory;
-		this.instanceDirectory = instanceDirectory;
-		this.instanceName = instanceName;
-		this.factorioPort = factorioPort;
-		this.serversettings = Object.assign({
-			"name": "Clusterio instance: " + this.instanceName,
-			"description": "Public factorio server running clusterio",
-			"tags": ["clusterio"],
-			"max_players": "20",
-			"visibility": "public",
-			"username": "",
-			"token": "",
-			"game_password": "",
-			"verify_user_identity": true,
-			"admins": ["Danielv123"],
-			"allow_commands": true,
-			"autosave_interval": 10,
-			"autosave_slots": 5,
-			"afk_autokick_interval": 0,
-			"auto_pause": false,
-		}, serversettings);
-		this.onProgress = (type, a, b, c) => {
-			if(typeof onProgress == "function") onProgress(type, a, b, c);
-		};
-	}
-	async initialize(){
-		if(!await this.isValidInstance()){
-			await this.createInstance();
-		}
-		return true;
-	}
-	async createInstance(path = this.instanceDirectory){
-		if(await this.isValidInstance(path)){
-			throw new Error("Instance already exists!");
-		}
-		
-		// create folders
-		let folders = [
-			"/script-output/",
-			"/saves/",
-			"/mods/",
-			"/instanceMods/",
-		];
-		mkdirp.sync(path);
-		folders.forEach(folder => mkdirp.sync(path + folder));
-		
-		// create files
-		let files = [
-			{name: "/script-output/output.txt", content: ""},
-			{name: "/script-output/orders.txt", content: ""},
-			{name: "/script-output/txbuffer.txt", content: ""},
-			{name: "/config.ini", content: "[path]\r\n\ read-data=__PATH__executable__/../../data\r\n\ write-data=__PATH__executable__/../../../" + this.instanceDirectory + "\r\n\ "},
-			{name: "/config.json", content: JSON.stringify({
-				factorioPort: this.factorioPort,
-				clientPassword: Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 8),
-			}, null, 4)},
-			{name: "/server-settings.json", content: JSON.stringify(this.serversettings, null, 4)},
-		];
-		files.forEach(file => {
-			fs.writeFile(path + file.name, file.content, doneWritingHandler);
-			// console.log("Created: ",path+file.name);
-		});
-		
-		let filesToWrite = files.length;
-		function doneWritingHandler(){
-			if(!--filesToWrite){
-				return path;
-			}
-		}
-	}
-	async deleteInstance(path = this.instanceDirectory){
-		if(await this.isValidInstance(path)){
-			await fileOps.deleteFolderRecursive(path);
-			return true;
-		} else {
-			return false;
-		}
-	}
-	isValidInstance(path = this.instanceDirectory){
-		return new Promise((resolve, reject) => {
-			fs.stat(path, (err, stat) => {
-				if(err){
-					// console.log(err)
-					resolve(false);
-				} else {
-					// the instance folder exists, do some more checks
-					fs.readdir(path, (err, files) => {
-						if(files && Array.isArray(files)){
-							let foldersThatShouldBeThere = [
-								"script-output",
-								"saves",
-								"mods",
-								"instanceMods",
-							];
-							let instanceValidity = true;
-							foldersThatShouldBeThere.forEach(folder => {
-								if(!files.includes(folder)){
-									instanceValidity = false;
-								}
-							});
-							resolve(instanceValidity);
-						}
-					});
-				}
-			});
-		});
-	}
-	createMap(name = "map.zip"){
-		return new Promise((resolve, reject) => {
-			this.isValidInstance().then(isValid => {
-				if(!isValid){
-					return createInstance();
-				}
-			}).then(()=>{
-				this.onProgress("log", "Starting map creation");
-				// console.log(this.factorioDirectory + '/bin/x64/factorio')
-				let proc = child_process.spawn(
-					path.join(this.factorioDirectory, 'bin', 'x64', 'factorio'), [
-						'-c', path.join(this.instanceDirectory,'config.ini'),
-						'--create', path.join(this.instanceDirectory, 'saves',name),
-						'--server-settings', path.join(this.instanceDirectory, 'server-settings.json'),
-					],{
-						cwd:process.cwd(),
-					}
-				);
-				// console.log(proc.stdout.toString("utf8"));
-				proc.stdout.on("data", data => this.onProgress("log", data.toString("utf8")));
-				proc.on("close", code => {
-					this.onProgress("log", "Map created! "+code);
-					resolve();
-				});
-			});
-		});
-	}
-}

node_modules/_app/factorioServer/index.spec.js

@@ -1,114 +0,0 @@
-const assert = require("assert");
-const fs = require("fs");
-const path = require("path");
-
-const appRootPath = require("app-root-path");
-const parallel = require("mocha.parallel");
-const mkdirp = require("mkdirp");
-
-const factorioServer = require("./index.js");
-
-describe("class factorioServer/index.js", ()=>{
-	it("is a class (or at least a function)", ()=>{
-		assert.equal(typeof factorioServer, "function");
-	});
-	describe(".deleteInstance(instanceName)", ()=>{
-		it("deletes an instance, defaulting to the classes instanceDirectory", async () => {
-			let server = new factorioServer({
-				instanceDirectory: path.resolve(__dirname, "_testData_deleteInstance"),
-				instanceName: "Unit testing instance",
-			});
-			await server.initialize();
-			await server.deleteInstance();
-			assert.equal(await server.isValidInstance(), false);
-			return true;
-		});
-	});
-	describe(".isValidInstance(instanceName)", ()=>{
-		it("tells us whether an instance exists or not", async () => {
-			let server = new factorioServer({
-				instanceDirectory: path.join(__dirname, "_testData_isValidInstance"),
-				instanceName: "Unit testing instance",
-			});
-			await server.deleteInstance();
-			let status = await server.isValidInstance();
-			assert.equal(status, false);
-			let x = await server.initialize(); // creates our instance
-			let status2 = await server.isValidInstance();
-			assert.equal(status2, true);
-			
-			await server.deleteInstance(); // clean up folder
-			return true;
-		});
-		it("does not get tricked by an empty folder", async () => {
-			let server = new factorioServer({
-				instanceDirectory: path.join(__dirname, "_testData_isValidInstance_2"),
-				instanceName: "Unit testing instance",
-			});
-			assert.equal(await server.isValidInstance(), false);
-			
-			mkdirp.sync(path.join(__dirname, "_testData_isValidInstance_2")); // create empty folder simulating instance
-			assert.equal(await server.isValidInstance(), false);
-			
-			await server.initialize(); // create a proper instance
-			assert.equal(await server.isValidInstance(), true);
-			
-			await server.deleteInstance(); // delete the instance and ensure we see that it is in fact deleted
-			assert.equal(await server.isValidInstance(), false);
-			
-			return true;
-		}).timeout(5000);
-	});
-	describe(".initialize()", ()=>{
-		it("creates an instance", done => {
-			let server = new factorioServer({
-				instanceDirectory: path.resolve(__dirname, "_testData_initialize"),
-				instanceName: "Unit testing instance",
-			});
-			server.initialize().then(()=>{
-				return server.deleteInstance();
-			}).then(() => {
-				done();
-			}).catch(e => console.log(e));
-		});
-	});
-	describe(".createMap(mapName.zip)", ()=>{
-		it("creates a map file as instanceDirectory/saves/map.zip", async function(){
-			this.timeout(20000); // these test often takes a lot of time in the map creation step (4500 ms on i3 dev system)
-			let relativePath = __dirname.replace(appRootPath, "").substr(1);
-			let server = new factorioServer({
-				instanceDirectory: path.join(relativePath, "_testData_createMap"),
-				instanceName: "Unit testing instance",
-				onProgress: false,
-			});
-			await server.initialize();
-			let maps = fs.readdirSync(path.join(relativePath, "_testData_createMap", "saves"));
-			assert.equal(maps.length, 0);
-			await server.createMap();
-			let maps2 = fs.readdirSync(path.join(relativePath, "_testData_createMap", "saves"));
-			assert(maps2.includes("map.zip"));
-			
-			await server.deleteInstance();
-			return true;
-		});
-		it("supports custom filenames", async function(){
-			this.timeout(20000);
-			let relativePath = __dirname.replace(appRootPath, "").substr(1);
-			let server = new factorioServer({
-				instanceDirectory: path.join(relativePath, "_testData_createMap_2"),
-				instanceName: "Unit testing instance",
-				onProgress: false,
-			});
-			await server.initialize();
-			
-			let maps = fs.readdirSync(path.join(relativePath, "_testData_createMap_2", "saves"));
-			assert.equal(maps.length, 0);
-			await server.createMap("mapWithCustomName.zip");
-			let maps2 = fs.readdirSync(path.join(relativePath, "_testData_createMap_2", "saves"));
-			assert(maps2.includes("mapWithCustomName.zip"));
-			
-			await server.deleteInstance();
-			return true;
-		});
-	});
-});

node_modules/_app/fileOps/fileOps.js

@@ -1,120 +0,0 @@
-const fs = require("fs");
-const path = require("path");
-
-// gets newest file in a directory
-// dir is directory
-// files is array of filenames
-// callback(err, filename string);
-var getNewestFile = function(dir, files, callback) {
-	if (!callback) throw new Error("Missing callback");
-	if (!files || (files && files.length === 0)) {
-		callback(new Error("No files"));
-		return;
-	}
-	// we are appending the file name to the directory, without the
-	// "/" we would get paths like /folderfile.txt instead of /folder/file.txt
-	if(dir[dir.length-1] != "/"){
-		dir += "/";
-	}
-	var newest = { file: files[0] };
-	var checked = 0;
-	fs.stat(dir + newest.file, function(err, stats) {
-		if(err) {
-			callback(err);
-		} else {
-			newest.mtime = stats.mtime;
-			for (var i = 0; i < files.length; i++) {
-				var file = files[i];
-				(function(file) {
-					fs.stat(file, function(err, stats) {
-						++checked;
-						if (stats.mtime.getTime() > newest.mtime.getTime()) {
-							newest = { file : file, mtime : stats.mtime };
-						}
-						if (checked == files.length) {
-							callback(undefined,newest);
-						}
-					});
-				})(dir + file);
-			}
-		}
-	});
-}
-
-// get names of all directories in directory
-var getDirectoriesSync = function(srcpath) {
-	return fs.readdirSync(srcpath).filter(function (file) {
-		return fs.statSync(path.join(srcpath, file)).isDirectory();
-	});
-}
-// get names of all files in a directory
-var getFileNamesSync = function(srcpath) {
-	return fs.readdirSync(srcpath).filter(function (file) {
-		return !fs.statSync(path.join(srcpath, file)).isDirectory();
-	});
-}
-// does what it says on the tin
-var deleteFolderRecursiveSync = function(path) {
-	if (fs.existsSync(path)) {
-		fs.readdirSync(path).forEach(function (file, index) {
-			var curPath = path + "/" + file;
-			if (fs.lstatSync(curPath).isDirectory()) { // recurse
-				deleteFolderRecursiveSync(curPath);
-			} else { // delete file
-				fs.unlinkSync(curPath);
-			}
-		});
-		fs.rmdirSync(path);
-		return true;
-	} else {
-		return false;
-	}
-};
-var deleteFolderRecursive = function(path){
-	return new Promise((resolve, reject) => {
-		fs.readdir(path, (err, files) => {
-			if(err){
-				reject(err);
-			} else {
-				let thingsToDelete = files.length;
-				files.forEach(file => {
-					let curPath = path + "/" + file;
-					fs.stat(curPath, (err, stat) => {
-						if(!err){
-							if(stat.isDirectory()){
-								// recurse
-								deleteFolderRecursive(curPath).then(() => {
-									whenFilesDeleted();
-								}).catch(e => console.log(e));
-							} else {
-								fs.unlink(curPath, () => {
-									// console.log("Deleted "+curPath);
-									whenFilesDeleted();
-								});
-							}
-						} else {
-							resolve(false);
-						}
-					});
-				});
-				whenFilesDeleted();
-				function whenFilesDeleted(){
-					if(!thingsToDelete--){// var-- instead of --var because we are calling the function once too much and need the delay
-						fs.rmdir(path, () => {
-							resolve();
-							// console.log("Deleted "+path);
-						});
-					}
-				}
-			}
-		});
-	});
-}
-
-module.exports = {
-	getNewestFile,
-	getDirectoriesSync,
-	deleteFolderRecursiveSync,
-	deleteFolderRecursive,
-	getFileNamesSync,
-}

node_modules/_app/fileOps/fileOps.spec.js

@@ -1,135 +0,0 @@
-const assert = require('assert');
-const fs = require('fs');
-const path = require('path');
-
-const fileOps = require('./fileOps');
-
-// create folders for testing
-function setupTestingEnv(){
-	if (!fs.existsSync(path.resolve(__dirname, "database"))){
-		fs.mkdirSync(path.resolve(__dirname, "database"));
-	}
-	if (!fs.existsSync(path.resolve(__dirname, "test"))){
-		fs.mkdirSync(path.resolve(__dirname, "test"));
-	}
-	if (!fs.existsSync(path.resolve(__dirname, "test/folder"))){
-		fs.mkdirSync(path.resolve(__dirname, "test/folder"));
-	}
-	if (!fs.existsSync(path.resolve(__dirname, "test/another folder"))){
-		fs.mkdirSync(path.resolve(__dirname, "test/another folder"));
-	}
-	if (!fs.existsSync(path.resolve(__dirname, "test/file.txt"))){
-		fs.writeFileSync(path.resolve(__dirname, "test/file.txt"), "contents");
-	}
-	if (!fs.existsSync(path.resolve(__dirname, "test/another file.txt"))){
-		fs.writeFileSync(path.resolve(__dirname, "test/another file.txt"), "more contents");
-	}
-	// we need this xfile to test a branch in fileOps.getNewestFile()s propagation
-	if (!fs.existsSync(path.resolve(__dirname, "test/xfile.txt"))){
-		fs.writeFileSync(path.resolve(__dirname, "test/xfile.txt"), "contents");
-	}
-}
-setupTestingEnv();
-setupTestingEnv(); // run twice to get branch coverage
-describe("fileOps.js", function(){
-	describe("fileOps.getDirectoriesSync()", function(){
-		it("gets an array of directory names from a path", function(){
-			let directories = fileOps.getDirectoriesSync(path.resolve(__dirname, "test"));
-			
-			// because there are no arrays for some reason
-			assert(typeof directories == "object");
-			
-			// they are sorted alphabetically because thats the way it is
-			assert(directories[0] == "another folder" && directories[1] == "folder");
-		});
-		it("does not return any filenames", function(){
-			let directories = fileOps.getDirectoriesSync(path.resolve(__dirname, "test"));
-			
-			// this is a function to return directories, not files. So yeah.
-			assert(!directories.includes("file.txt") && !directories.includes("another file.txt"));
-		});
-		it("throws when lacking first argument", function(){
-			assert.throws(function(){
-				let files = fileOps.getDirectoriesSync();
-			}, Error);
-		});
-	});
-	describe("fileOps.getFileNamesSync(path)", function(){
-		it("gets names of files in directory", function(){
-			let files = fileOps.getFileNamesSync(path.resolve(__dirname, "test"));
-			assert(files.length > 0);
-			assert(files.includes("file.txt") && files.includes("another file.txt"));
-		});
-		it("throws when lacking first argument", function(){
-			assert.throws(function(){
-				let files = fileOps.getFileNamesSync();
-			}, Error);
-		});
-	});
-	describe("fileOps.getNewestFile()", function(){
-		it("gets the newest file in a directory async", function(done){
-			fileOps.getNewestFile(path.resolve(__dirname, "test/"), fileOps.getFileNamesSync(path.resolve(__dirname, "test")), function(err, newest){
-				assert(newest && typeof newest.file == "string");
-				done();
-			});
-		});
-		it("throws if callback is missing", function(){
-			assert.throws(function(){
-				fileOps.getNewestFile("test", fileOps.getFileNamesSync(path.resolve(__dirname, "test")));
-			}, Error);
-		});
-		it("executes callback(err) if no files are provided", function(done){
-			fileOps.getNewestFile(path.resolve(__dirname, "test"), [], function(err, newest){
-				assert(err && typeof err == "object");
-				done();
-			});
-		});
-		it("executes callback(err) if launched without a directory", function(done){
-			fileOps.getNewestFile("", fileOps.getFileNamesSync(path.resolve(__dirname, "test")), function(err, newest){
-				assert(err);
-				done();
-			});
-		});
-		it("executes callback(err) when you do stupid shit", function(done){
-			fileOps.getNewestFile({thisWontWork: "but not crash either"}, {lolz:"yeah..."}, function(err, newest){
-				assert(err);
-				// ENOENT: no such file or directory, stat 'C:\**\factorioClusterio\lib\test\undefined'
-				done();
-			});
-		});
-	});
-	describe("fileOps.deleteFolderRecursiveSync(path)", function(){
-		it("deletes a folder and all files and folders in it", function(){
-			let x = fileOps.deleteFolderRecursiveSync(path.resolve(__dirname, "test"));
-			assert.throws(function(){
-				fs.statSync(path.resolve(__dirname, "test"));
-			}, Error);
-			assert(x);
-			// give it some extra time because its sometimes very slow
-		}).timeout(5000);
-		it("returns false if you give it an invalid path", function(){
-			let x = fileOps.deleteFolderRecursiveSync("This isn't a real path");
-			assert(!x);
-		});
-	});
-	describe("fileOps.deleteFolderRecursive(path)", function(){
-		it("deletes a folder and all files and folders in it, just like sync version", async function(){
-			setupTestingEnv();
-			assert(fs.statSync(path.resolve(__dirname, "test")));
-			let x = await fileOps.deleteFolderRecursive(path.resolve(__dirname, "test"));
-			assert.throws(function(){
-				fs.statSync(path.resolve(__dirname, "test"));
-			}, Error);
-			assert(x === undefined, "Promise version of this function does not have a return value upon success");
-			return true;
-		}).timeout(5000);
-		it("throws like a promise does if you give it an invalid path", async function(){
-			let x = await fileOps.deleteFolderRecursive("Look this path is fake!")
-			.catch(e => {
-				assert(e);
-				assert.equal(e.code, "ENOENT");
-				return true;
-			});
-		});
-	});
-});

node_modules/_app/isFactorioCommand/index.js

@@ -1 +0,0 @@
-module.exports = require("./isFactorioCommand.js");

node_modules/_app/isFactorioCommand/isFactorioCommand.js

@@ -1,23 +0,0 @@
-const luamin = require("luamin");
-function isFactorioCommand(command){
-	if(typeof command != "string"){
-		return false;
-	} else if(!command.includes("/silent-command ") && !command.includes("/c ")){
-		return false;
-	} else if(command[0] != "/"){
-		return false;
-	} else {
-		// check that what follows /c is valid LUA
-		let x = command.replace("/silent-command ", "").replace("/c ","");
-		let y
-		try{
-			y = luamin.minify(x);
-		} catch (e) {
-			return false;
-		}
-		
-		// by this point it is surely a LUA command
-		return true;
-	}
-}
-module.exports = isFactorioCommand;

node_modules/_app/isFactorioCommand/isFactorioCommand.spec.js

@@ -1,18 +0,0 @@
-const isFactorioCommand = require("./isFactorioCommand");
-const assert = require("assert");
-
-describe("inventoryImports/isFactorioCommand.js()", function(){
-	it("Returns false if input is not a factorio command", function(){
-		assert(isFactorioCommand({}) == false, "This is not even a string so should return false");
-		assert(isFactorioCommand("Howdy boys!") == false, "This is not a command so should return false");
-		assert(isFactorioCommand("/c this ain't a command because its invalid LUA") == false, "This is not valid LUA so it should return false");
-		assert(isFactorioCommand("Trying to trick the system /c game.print('Hello world!')") == false, "this is still not a command");
-		
-	});
-	it("Returns true if it is a valid command", function(){
-		assert(isFactorioCommand("/c game.print('hello world')"), "This is a valid command, should output true not false")
-	});
-	it("Fails to understand that leading spaces are valid", function(){
-		assert(isFactorioCommand("  /c game.print('Hello world!')") == false, "This should be true but often returns false unintentionally. If you see this you probably fixed the bug so change this test, OK?")
-	});
-});

package.json

@@ -1,77 +1,80 @@
 {
   "name": "clusterio",
-  "version": "1.7.0",
+  "version": "2.0.0-alpha",
   "description": "Interserver communication for factorio",
-  "repository": "https://github.com/Danielv123/factorioClusterio",
+  "repository": "https://github.com/clusterio/factorioClusterio",
   "main": "master.js",
   "scripts": {
-    "test": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- \"./{{,node_modules/_app/**/},!(node_modules)/**/}*.spec.js\" -R spec || echo .",
-    "start": "echo Run node client.js start [instance] to start slaves && node master.js",
-    "cover": "./node_modules/.bin/istanbul cover ./node_modules/mocha/bin/_mocha -- \"./{{,node_modules/_app/**/},!(node_modules)/**/}*.spec.js\" -R spec || echo . && codecov -t f7b38085-36eb-4d72-84e6-64721910171e",
-    "docs": "documentation build index.js --config documentation.yml -f html -o docs",
-    "livedocs": "documentation serve --watch *.js lib/** --config documentation.yml",
-    "install": "node lib/npmPostinstall.js"
+    "test": "mocha --exit --check-leaks -R spec --recursive test plugins/*/test",
+    "fast-test": "FAST_TEST=y npm test",
+    "cover": "nyc npm test",
+    "fast-cover": "FAST_TEST=y nyc npm test",
+    "ci-cover": "nyc --reporter=lcovonly npm run-script test",
+    "docs": "jsdoc --configure docs/jsdoc.json",
+    "install": "node lib/npmPostinstall",
+    "bundle-dependencies": "bundle-dependencies"
+  },
+  "nyc": {
+    "exclude": [
+      "test/**",
+      "plugins/*/test/**"
+    ]
   },
   "keywords": [
     "factorio"
   ],
   "author": "Danielv123",
-  "license": "ISC",
+  "license": "MIT",
+  "engines": {
+    "node": ">=10.16 <11 || >= 11.13"
+  },
   "dependencies": {
-    "app-root-path": "^2.0.1",
     "as-table": "^1.0.31",
-    "averaged-timeseries": "^1.4.0",
     "base64url": "^3.0.0",
-    "bcrypt": "^2.0.1",
+    "bcrypt": "=3.0.6",
     "bcrypt-promise": "^2.0.0",
-    "bcryptjs": "^2.4.3",
     "body-parser": "^1.18.2",
-    "cache-require-paths": "^0.3.0",
+    "chalk": "^3.0.0",
     "cli-interact": "^0.1.9",
-    "codecov": "^3.0.1",
     "compression": "^1.7.2",
-    "deepmerge": "^1.5.2",
+    "cookie-parser": "^1.4.3",
+    "deepmerge": "^4.0.0",
     "ejs": "^2.5.7",
     "express": "^4.16.2",
-    "express-fileupload": "0.0.6",
+    "express-fileupload": "1.1.5",
     "follow-redirects": "^1.5.0",
-    "fs-extra": "^6.0.1",
-    "getmac": "^1.4.0",
-    "hash-files": "^1.1.1",
-    "html5-validator": "^1.0.7",
+    "fs-extra": "^8.1.0",
     "ini": "^1.3.5",
     "jsonwebtoken": "^8.2.1",
-    "leveldown": "^1.9.0",
-    "linvodb3": "^3.25.1",
+    "jszip": "^3.2.2",
+    "lib": "file:./lib",
     "luamin": "^1.0.4",
-    "mkdirp": "^0.5.1",
-    "mkdirp-promise": "^5.0.1",
-    "moment": "^2.20.1",
-    "ncp": "^2.0.0",
-    "nedb": "^1.8.0",
-    "needle": "^1.1.0",
-    "node-factorio-api": "^0.3.1-alpha",
+    "minimist": "^1.2.0",
+    "moment": "^2.22.2",
+    "needle": "^2.4.0",
+    "node-factorio-api": "^0.3.8",
     "node-forge": "^0.7.1",
-    "prom-client": "^10.2.3",
-    "rcon-client": "^3.0.2",
+    "rcon-client": "^4.2.0",
     "request": "^2.83.0",
-    "right-pad": "^1.0.1",
-    "rmdir-sync": "^1.0.1",
     "sanitizer": "^0.1.3",
+    "semver": "^7.1.1",
+    "set-blocking": "^2.0.0",
+    "simple-git": "^1.96.0",
     "single-line-log": "^1.1.2",
-    "socket.io": "^2.0.4",
-    "socket.io-client": "^2.0.4",
     "sync-request": "^4.0.1",
-    "underscore": "^1.8.3"
+    "typescript-collections": "^1.3.2",
+    "ws": "^7.2.3",
+    "yargs": "^14.2.0"
   },
   "devDependencies": {
-    "documentation": "^5.4.0",
-    "istanbul": "^0.4.5",
-    "mocha": "^3.5.3",
-    "mocha-sinon": "^2.0.0",
+    "html5-validator": "^1.2.1",
+    "jsdoc": "^3.6.3",
+    "klaw": "^3.0.0",
+    "mocha": "^6.2.0",
+    "mocha-sinon": "^2.1.0",
     "mocha.parallel": "^0.15.5",
-    "nock": "^9.1.6",
-    "sinon": "^2.4.1",
-    "supertest": "^3.0.0"
+    "nock": "^11.3.0",
+    "nyc": "^14.1.1",
+    "sinon": "^7.4.1"
   }
 }

plugins/global_chat/info.js

@@ -0,0 +1,23 @@
+let link = require("lib/link");
+let plugin = require("lib/plugin");
+
+module.exports = {
+	name: "global_chat",
+	title: "Global Chat",
+	description: "Forwards chat between instances.",
+	version: "2.0.0-alpha",
+	instanceEntrypoint: "instance",
+
+	messages: {
+		chat: new link.Event({
+			type: 'global_chat:chat',
+			links: ['instance-slave', 'slave-master', 'master-slave', 'slave-instance'],
+			forwardTo: 'master',
+			broadcastTo: 'instance',
+			eventProperties: {
+				"instance_name": { type: "string" },
+				"content": { type: "string" },
+			},
+		}),
+	},
+}

plugins/global_chat/instance.js

@@ -0,0 +1,44 @@
+const plugin = require("lib/plugin");
+const luaTools = require("lib/luaTools");
+
+
+/**
+ * Removes gps and train tags from messags
+ */
+function removeTags(content) {
+	return content.replace(/(\[gps=-?\d+,-?\d+\]|\[train=\d+\])/g, "");
+}
+
+class InstancePlugin extends plugin.BaseInstancePlugin {
+	constructor(...args) {
+		super(...args);
+	}
+
+	async init() {
+		// XXX Nothing?
+	}
+
+	async chatEventHandler(message) {
+		// TODO check if cross server chat is enabled
+		let content = `[${message.data.instance_name}] ${removeTags(message.data.content)}`;
+		await this.instance.server.sendRcon(`/sc game.print('${luaTools.escapeString(content)}')`, true);
+	}
+
+	async onOutput(output) {
+		if (output.type === "action" && output.action === "CHAT") {
+			this.info.messages.chat.send(this.instance, {
+				instance_name: this.instance.name,
+				content: output.message,
+			});
+		}
+	}
+
+	// TODO implement info command in lua?
+}
+
+module.exports = {
+	InstancePlugin,
+
+	// For testing only
+	_removeTags: removeTags,
+}

plugins/global_chat/test/plugin.js

@@ -0,0 +1,67 @@
+const assert = require('assert').strict;
+
+const mock = require('../../../test/mock');
+const lines = require('../../../test/lib/factorio/lines');
+const instance = require('../instance');
+const info = require('../info');
+
+
+describe("global_chat plugin", function() {
+	describe("removeTags()", function() {
+		it("should pass through an ordinary string", function() {
+			assert.equal(instance._removeTags("string"), "string");
+		});
+		it("should strip out gps tag", function() {
+			assert.equal(instance._removeTags("Look at [gps=12,-4]"), "Look at ");
+		});
+		it("should strip out train tag", function() {
+			assert.equal(instance._removeTags("Train [train=1235]"), "Train ");
+		});
+	});
+
+	describe("class InstancePlugin", function() {
+		let instancePlugin;
+
+		before(async function() {
+			instancePlugin = new instance.InstancePlugin(info, new mock.MockInstance());
+			await instancePlugin.init();
+		});
+
+		describe(".chatEventHandler()", function() {
+			it("should send received chat as command", async function() {
+				instancePlugin.instance.server.rconCommands = [];
+				await instancePlugin.chatEventHandler({ data: { instance_name: "test", content: "User: message" } });
+				assert.deepEqual(
+					instancePlugin.instance.server.rconCommands,
+					["/sc game.print('[test] User: message')"],
+				);
+			});
+		});
+		describe(".onOutput()", function() {
+			it("should forward chat", async function() {
+				let count = 0;
+				for (let [_, output] of lines.testLines) {
+					if (output.type === "action" && output.action === "CHAT") {
+						instancePlugin.instance.connector.sentMessages = [];
+						await instancePlugin.onOutput(output);
+						assert(instancePlugin.instance.connector.sentMessages.length, "message was not sent");
+						count++;
+					}
+				}
+				assert(count > 0, "no lines were tested");
+			});
+			it("should ignore regular output", async function() {
+				let count = 0;
+				for (let [_, output] of lines.testLines) {
+					if (output.type !== "action" || output.action !== "CHAT") {
+						instancePlugin.instance.connector.sentMessages = [];
+						await instancePlugin.onOutput(output);
+						assert(!instancePlugin.instance.connector.sentMessages.length, "message was sent");
+						count++;
+					}
+				}
+				assert(count > 0, "no lines were tested");
+			});
+		})
+	});
+});

plugins/research_sync/info.js

@@ -0,0 +1,80 @@
+const link = require("lib/link");
+
+
+// schema used for syncing technologies to and from the master
+const technologies = {
+	type: "array",
+	items: {
+		type: "array",
+		minItems: 4,
+		maxItems: 4,
+		items: [
+			{ type: "string" },
+			{ type: "integer" },
+			{ type: ["null", "number"] },
+			{ type: "boolean" },
+		]
+	},
+};
+
+module.exports = {
+	name: "research_sync",
+	title: "Research Sync",
+	description: "Synchronises technology research progress between instances.",
+	version: "2.0.0-alpha",
+	instanceEntrypoint: "instance",
+	masterEntrypoint: "master",
+
+	messages: {
+		contribution: new link.Event({
+			type: "research_sync:contribution",
+			links: ["instance-slave", "slave-master"],
+			forwardTo: "master",
+			eventProperties: {
+				"name": { type: "string" },
+				"level": { type: "integer" },
+				"contribution": { type: "number" },
+			},
+		}),
+		progress: new link.Event({
+			type: "research_sync:progress",
+			links: ["master-slave", "slave-instance"],
+			broadcastTo: "instance",
+			eventProperties: {
+				"technologies": {
+					type: "array",
+					items: {
+						additionalProperties: false,
+						required: ["name", "level", "progress"],
+						properties: {
+							"name": { type: "string" },
+							"level": { type: "integer" },
+							"progress": { type: "number" },
+						},
+					},
+				},
+			},
+		}),
+		finished: new link.Event({
+			type: "research_sync:finished",
+			links: ["instance-slave", "slave-master", "master-slave", "slave-instance"],
+			forwardTo: "master",
+			broadcastTo: "instance",
+			eventProperties: {
+				"name": { type: "string" },
+				"level": { type: "integer" },
+			},
+		}),
+		syncTechnologies: new link.Request({
+			type: "research_sync:sync_technologies",
+			links: ["instance-slave", "slave-master"],
+			forwardTo: "master",
+			requestProperties: {
+				"technologies": technologies,
+			},
+			responseProperties: {
+				"technologies": technologies,
+			},
+		}),
+	},
+}

plugins/research_sync/instance.js

@@ -0,0 +1,81 @@
+const needle = require("needle");
+
+const plugin = require("lib/plugin");
+const luaTools = require("lib/luaTools");
+
+
+function unexpectedError(err) {
+	console.log("Unexpected error in research_sync")
+	console.log("---------------------------------")
+	console.log(err);
+}
+
+class InstancePlugin extends plugin.BaseInstancePlugin {
+	async init() {
+		this.instance.server.on("ipc-research_sync:contribution", (tech) => {
+			this.researchContribution(tech).catch(unexpectedError);
+		});
+		this.instance.server.on("ipc-research_sync:finished", (tech) => {
+			this.researchFinished(tech).catch(unexpectedError);
+		});
+	}
+
+	async researchContribution(tech) {
+		this.info.messages.contribution.send(this.instance, tech);
+	}
+
+	async progressEventHandler(message) {
+		let techsJson = luaTools.escapeString(JSON.stringify(message.data.technologies));
+		await this.instance.server.sendRcon(`/sc research_sync.update_progress("${techsJson}")`, true);
+	}
+
+	async researchFinished(tech) {
+		this.info.messages.finished.send(this.instance, tech);
+	}
+
+	async finishedEventHandler(message) {
+		let { name, level, researched } = message.data;
+		await this.instance.server.sendRcon(
+			`/sc research_sync.research_technology("${luaTools.escapeString(name)}", ${level})`, true
+		);
+	}
+
+	async onStart() {
+		let dumpJson = await this.instance.server.sendRcon("/sc research_sync.dump_technologies()");
+		let techsToSend = [];
+		let instanceTechs = new Map();
+		for (let tech of JSON.parse(dumpJson)) {
+			techsToSend.push([
+				tech.name,
+				tech.level,
+				tech.progress || null,
+				tech.researched,
+			]);
+			instanceTechs.set(tech.name, tech);
+		}
+
+		let response = await this.info.messages.syncTechnologies.send(this.instance, { technologies: techsToSend });
+		let techsToSync = [];
+		for (let masterTech of response.technologies) {
+			let [name, level, progress, researched] = masterTech;
+			let instanceTech = instanceTechs.get(name);
+			if (
+				!instanceTech
+				|| instanceTech.level !== level
+				|| (instanceTech.progress || null) !== progress
+				|| instanceTech.researched !== researched
+			) {
+				techsToSync.push(masterTech);
+			}
+		}
+
+		if (techsToSync.length) {
+			let syncJson = luaTools.escapeString(JSON.stringify(techsToSync));
+			await this.instance.server.sendRcon(`/sc research_sync.sync_technologies("${syncJson}")`, true);
+		}
+	}
+}
+
+module.exports = {
+	InstancePlugin,
+}

plugins/research_sync/master.js

@@ -0,0 +1,184 @@
+const fs = require("fs-extra");
+const path = require("path");
+
+const plugin = require("lib/plugin");
+
+
+class MasterPlugin extends plugin.BaseMasterPlugin {
+	async init() {
+		this.technologies = await loadTechnologies(this.master.config);
+		this.lastProgressBroadcast = Date.now();
+		this.progressBroadcastId = null;
+		this.progressToBroadcast = new Set();
+	}
+
+	async onExit() {
+		if (this.progressBroadcastId) {
+			clearTimeout(this.progressBroadcastId);
+		}
+		await saveTechnologies(this.master.config, this.technologies);
+	}
+
+	registerProgress() {
+		// Rate limit progress broadcasts to one per second
+		if (Date.now() < this.lastProgressBroadcast + 1000) {
+			if (!this.progressBroadcastId) {
+				this.progressBroadcastId = setTimeout(() => {
+					this.broadcastProgress();
+					this.progressBroadcastId = null;
+				}, 2000);
+			}
+
+		} else {
+			if (this.progressBroadcastId) {
+				clearTimeout(this.progressBroadcastId);
+			}
+			this.broadcastProgress();
+		}
+	}
+
+	broadcastProgress() {
+		let techs = []
+		for (let name of this.progressToBroadcast) {
+			let tech = this.technologies.get(name);
+			if (tech.progress) {
+				techs.push({ name, level: tech.level, progress: tech.progress });
+			}
+		}
+		this.progressToBroadcast.clear();
+
+		this.lastProgressBroadcast = Date.now();
+		if (techs.length) {
+			for (let slaveConnection of this.master.slaveConnections.values()) {
+				this.info.messages.progress.send(slaveConnection, { technologies: techs });
+			}
+		}
+	}
+
+	async contributionEventHandler(message) {
+		let { name, level, contribution } = message.data;
+		let tech = this.technologies.get(name);
+		if (!tech) {
+			tech = { level, progress: 0, researched: false };
+			this.technologies.set(name, tech)
+
+		// Ignore contribution to already researched technologies
+		} else if (tech.level > level || tech.level === level && tech.researched) {
+			return;
+		}
+
+		let newProgress = tech.progress + contribution;
+		if (newProgress < 1) {
+			tech.progress = newProgress;
+			this.progressToBroadcast.add(name);
+			this.registerProgress();
+
+		} else {
+			tech.researched = true;
+			tech.progress = null;
+			this.progressToBroadcast.delete(name);
+
+			for (let slaveConnection of this.master.slaveConnections.values()) {
+				this.info.messages.finished.send(slaveConnection, {
+					name,
+					level: tech.level
+				});
+			}
+		}
+	}
+
+	async finishedEventHandler(message) {
+		let { name, level } = message.data;
+		let tech = this.technologies.get(name);
+		if (!tech || tech.level <= level) {
+			this.progressToBroadcast.delete(name);
+			this.technologies.set(name, { level, progress: null, researched: true })
+		}
+	}
+
+	async syncTechnologiesRequestHandler(message) {
+		function baseLevel(name) {
+			let match = /-(\d+)$/.exec(name);
+			if (!match) {
+				return 1;
+			} else {
+				return Number.parseInt(match[1], 10);
+			}
+		}
+
+		for (let instanceTech of message.data.technologies) {
+			let [name, level, progress, researched] =  instanceTech;
+			let tech = this.technologies.get(name);
+			if (!tech) {
+				this.technologies.set(name, { level, progress, researched });
+				if (progress) {
+					this.progressToBroadcast.add(name);
+				} else if (researched || baseLevel(name) != level) {
+					for (let slaveConnection of this.master.slaveConnections.values()) {
+						this.info.messages.finished.send(slaveConnection, { name, level });
+					}
+				}
+
+			} else {
+				if (tech.level > level || tech.level === level && tech.researched) {
+					continue;
+				}
+
+				if (tech.level < level || researched) {
+					tech.level = level;
+					tech.progress = progress;
+					tech.researched = researched;
+					for (let slaveConnection of this.master.slaveConnections.values()) {
+						this.info.messages.finished.send(slaveConnection, { name, level });
+					}
+
+					if (progress) {
+						this.progressToBroadcast.add(name);
+					} else {
+						this.progressToBroadcast.delete(name);
+					}
+
+				} else if (tech.progress < progress) {
+					tech.progress = progress;
+					this.progressToBroadcast.add(name);
+				}
+			}
+		}
+		this.registerProgress();
+
+		let technologies = [];
+		for (let [name, tech] of this.technologies) {
+			technologies.push([ name, tech.level, tech.progress, tech.researched ])
+		}
+
+		return { technologies }
+	}
+}
+
+async function loadTechnologies(masterConfig) {
+	let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json");
+	console.log(`Loading ${filePath}`);
+	try {
+		return new Map(JSON.parse(await fs.readFile(filePath)));
+
+	} catch (err) {
+		if (err.code === "ENOENT") {
+			console.log("Creating new technologies database");
+			return new Map();
+
+		} else {
+			throw err;
+		}
+	}
+}
+
+async function saveTechnologies(masterConfig, technologies) {
+	let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json");
+	console.log(`writing ${filePath}`);
+	await fs.outputFile(filePath, JSON.stringify([...technologies.entries()], null, 4));
+}
+
+
+module.exports = {
+	MasterPlugin
+}

plugins/research_sync/module/module.json

@@ -0,0 +1,4 @@
+{
+    "name": "research_sync",
+    "load": ["sync.lua"]
+}

plugins/research_sync/module/sync.lua

@@ -0,0 +1,215 @@
+local clusterio_api = require("modules/clusterio/api")
+
+local sync = {}
+
+
+local function get_technology_progress(tech)
+    if tech == tech.force.current_research then
+        return tech.force.research_progress
+    else
+        return tech.force.get_saved_technology_progress(tech.name)
+    end
+end
+
+local function set_technology_progress(tech, progress)
+    if tech == tech.force.current_research then
+        tech.force.research_progress = progress
+    else
+        tech.force.set_saved_technology_progress(tech.name, progress)
+    end
+end
+
+sync.events = {}
+sync.events[clusterio_api.events.on_server_startup] = function(event)
+    if not global.research_sync then
+        global.research_sync = {
+            technologies = {},
+        }
+    end
+
+    -- Used when syncing completed technologies from the master
+    global.research_sync.ignore_research_finished = false
+
+    local force = game.forces["player"]
+    for _, tech in pairs(force.technologies) do
+        if tech.enabled then
+            local progress = get_technology_progress(tech)
+            global.research_sync.technologies[tech.name] = {
+                level = tech.level,
+                researched = tech.researched,
+                progress = progress,
+            }
+        end
+    end
+end
+
+local function get_contribution(tech)
+    local progress = get_technology_progress(tech)
+    if not progress then
+        return 0, nil
+    end
+
+    local prev_tech = global.research_sync.technologies[tech.name]
+    if prev_tech.progress and prev_tech.level == tech.level then
+        return progress - prev_tech.progress, progress
+    else
+        return progress, progress
+    end
+end
+
+local function send_contribution(tech)
+    local contribution, progress = get_contribution(tech)
+    if contribution ~= 0 then
+        clusterio_api.send_json("research_sync:contribution", {
+            name = tech.name,
+            level = tech.level,
+            contribution = contribution,
+        })
+        global.research_sync.technologies[tech.name].progress = progress
+    end
+end
+
+sync.events[defines.events.on_research_started] = function(event)
+    local tech = event.last_research
+    if tech then
+        send_contribution(tech)
+    end
+end
+
+sync.events[defines.events.on_research_finished] = function(event)
+    if global.research_sync.ignore_research_finished then
+        return
+    end
+
+    local tech = event.research
+    global.research_sync.technologies[tech.name] = {
+        level = tech.level,
+        researched = tech.researched,
+    }
+
+    local level = tech.level
+    if not tech.researched then
+        level = level - 1
+    end
+
+    clusterio_api.send_json("research_sync:finished", {
+        name = tech.name,
+        level = level,
+    })
+end
+
+sync.on_nth_tick = {}
+sync.on_nth_tick[79] = function(event)
+    local tech = game.forces["player"].current_research
+    if tech then
+        send_contribution(tech)
+    end
+end
+
+research_sync = {}
+function research_sync.dump_technologies()
+    local force = game.forces["player"]
+
+    local techs = {}
+    for _, tech in pairs(force.technologies) do
+        if tech.enabled then
+            table.insert(techs, {
+                name = tech.name,
+                level = tech.level,
+                progress = get_technology_progress(tech),
+                researched = tech.researched,
+            })
+        end
+    end
+
+    if #techs == 0 then
+        rcon.print("[]")
+    else
+        rcon.print(game.table_to_json(techs))
+    end
+end
+
+function research_sync.sync_technologies(data)
+    local force = game.forces["player"]
+
+    local nameIndex = 1
+    local levelIndex = 2
+    local progressIndex = 3
+    local researchedIndex = 4
+
+    global.research_sync.ignore_research_finished = true
+    for _, tech_data in pairs(game.json_to_table(data)) do
+        local tech = force.technologies[tech_data[nameIndex]]
+        if tech and tech.enabled and tech.level <= tech_data[levelIndex] then
+            tech.level = tech_data[levelIndex]
+            local progress
+            if tech_data[researchedIndex] then
+                tech.researched = true
+                progress = nil
+            elseif tech_data[progressIndex] then
+                send_contribution(tech)
+                progress = tech_data[progressIndex]
+                set_technology_progress(tech, progress)
+            else
+                progress = get_technology_progress(tech)
+            end
+
+            global.research_sync.technologies[tech.name] = {
+                level = tech.level,
+                researched = tech.researched,
+                progress = progress,
+            }
+        end
+    end
+    global.research_sync.ignore_research_finished = false
+end
+
+function research_sync.update_progress(data)
+    local techs = game.json_to_table(data)
+    local force = game.forces["player"]
+
+    for _, masterTech in ipairs(techs) do
+        local tech = force.technologies[masterTech.name]
+        if tech and tech.enabled and tech.level == masterTech.level then
+            send_contribution(tech)
+            set_technology_progress(tech, masterTech.progress)
+            global.research_sync.technologies[tech.name] = {
+                level = tech.level,
+                progress = masterTech.progress
+            }
+        end
+    end
+end
+
+function research_sync.research_technology(name, level)
+    local force = game.forces["player"]
+    local tech = force.technologies[name]
+    if not tech or not tech.enabled or tech.level > level then
+        return
+    end
+
+    global.research_sync.ignore_research_finished = true
+    if tech == force.current_research and tech.level == level then
+        force.research_progress = 1
+
+    elseif tech.level < level or tech.level == level and not tech.researched then
+        tech.level = level
+        tech.researched = true
+
+        if tech.name:find("-%d+$") then
+            game.print {"", "Researched ", {"technology-name." .. tech.name:gsub("-%d+$", "")}, level}
+        else
+            game.print {"", "Researched ", {"technology-name." .. tech.name}}
+        end
+        game.play_sound { path = "utility/research_completed" }
+    end
+    global.research_sync.ignore_research_finished = false
+
+    global.research_sync.technologies[tech.name] = {
+        level = tech.level,
+        researched = tech.researched,
+    }
+end
+
+
+return sync

plugins/research_sync/test/missing.js

@@ -0,0 +1,5 @@
+// These are libraries that does not have any units test, they are
+// required here in order to track coverage for them.
+
+require('../instance');
+require('../master');

plugins/statistics_exporter/info.js

@@ -0,0 +1,9 @@
+let link = require("lib/link");
+
+module.exports = {
+	name: "statistics_exporter",
+	title: "Prometheus Statistics Export",
+	description: "Exports statistics to Prometheus",
+	version: "2.0.0-alpha",
+	instanceEntrypoint: "instance",
+}

plugins/statistics_exporter/instance.js

@@ -0,0 +1,67 @@
+const plugin = require('lib/plugin');
+const { Gauge } = require('lib/prometheus');
+
+
+const instancePlayerCount = new Gauge(
+	"clusterio_statistics_exporter_instance_player_count", "Amount of players connected to this cluster",
+	{ labels: ['instance_id'] }
+);
+const instanceGameTicksTotal = new Gauge(
+	"clusterio_statistics_exporter_instance_game_ticks_total", "Game tick an instance has progressed to",
+	{ labels: ['instance_id'] }
+);
+const instanceForceFlowStatistics = new Gauge(
+	"clusterio_statistics_exporter_instance_force_flow_statistics", "Items/fluids/enemies/buildings produced/built/killed by a force",
+	{ labels: ['instance_id', 'force', 'statistic', 'direction', 'name'] },
+);
+const instanceGameFlowStatistics = new Gauge(
+	"clusterio_statistics_exporter_instance_game_flow_statistics", "Pollution produced/consumed in the game",
+	{ labels: ['instance_id', 'statistic', 'direction', 'name'] },
+);
+
+
+class InstancePlugin extends plugin.BaseInstancePlugin {
+	async onMetrics() {
+		let string = await this.instance.server.sendRcon('/sc statistics_exporter.export()');
+		let stats;
+		try {
+			stats = JSON.parse(string);
+		} catch (err) {
+			throw new Error(`Error parsing statistics JSON: ${err.message}, content "${string}"`);
+		}
+
+		let instanceId = this.instance.config.get("instance.id");
+		instanceGameTicksTotal.labels(String(instanceId)).set(stats.game_tick);
+		instancePlayerCount.labels(String(instanceId)).set(stats.player_count);
+
+		for (let [forceName, flowStatistics] of Object.entries(stats.force_flow_statistics)) {
+			for (let [statisticName, statistic] of Object.entries(flowStatistics)) {
+				for (let [direction, counts] of Object.entries(statistic)) {
+					for (let [item, value] of Object.entries(counts)) {
+						instanceForceFlowStatistics.labels(
+							String(instanceId), forceName, statisticName, direction, item
+						).set(value);
+					}
+				}
+			}
+		}
+
+		for (let [direction, counts] of Object.entries(stats.game_flow_statistics.pollution_statistics)) {
+			for (let [item, value] of Object.entries(counts)) {
+				instanceGameFlowStatistics.labels(
+					String(instanceId), "pollution_statistics", direction, item
+				).set(value);
+			}
+		}
+	}
+}
+
+module.exports = {
+	InstancePlugin,
+
+	// For testing only
+	_instancePlayerCount: instancePlayerCount,
+	_instanceGameTicksTotal: instanceGameTicksTotal,
+	_instanceForceFlowStatistics: instanceForceFlowStatistics,
+	_instanceGameFlowStatistics: instanceGameFlowStatistics,
+}

plugins/statistics_exporter/module/export.lua

@@ -0,0 +1,34 @@
+local statistics = {
+    "item_production_statistics",
+    "fluid_production_statistics",
+    "kill_count_statistics",
+    "entity_build_count_statistics",
+}
+
+statistics_exporter = {}
+function statistics_exporter.export()
+    local stats = {
+        game_tick = game.tick,
+        player_count = #game.connected_players,
+        game_flow_statistics = {
+            pollution_statistics = {
+                input = game.pollution_statistics.input_counts,
+                output = game.pollution_statistics.output_counts,
+            },
+        },
+        force_flow_statistics = {}
+    }
+    for _, force in pairs(game.forces) do
+        local flow_statistics = {}
+        for _, statName in pairs(statistics) do
+            flow_statistics[statName] = {
+                input = force[statName].input_counts,
+                output = force[statName].output_counts,
+            }
+        end
+        stats.force_flow_statistics[force.name] = flow_statistics
+    end
+
+
+    rcon.print(game.table_to_json(stats))
+end

plugins/statistics_exporter/module/module.json

@@ -0,0 +1,4 @@
+{
+    "name": "statistics_exporter",
+    "require": ["export.lua"]
+}

plugins/statistics_exporter/test/plugin.js

@@ -0,0 +1,71 @@
+const assert = require('assert').strict;
+
+const mock = require('../../../test/mock');
+const instance = require('../instance');
+const info = require('../info');
+
+
+describe("statistics_exporter plugin", function() {
+	describe("class InstancePlugin", function() {
+		let instancePlugin;
+		before(async function() {
+			instancePlugin = new instance.InstancePlugin(info, new mock.MockInstance());
+			await instancePlugin.init();
+		});
+
+		describe(".onMetrics()", function() {
+			it("should record statistics", async function() {
+				instancePlugin.instance.server.rconCommandResults.set(
+					'/sc statistics_exporter.export()',
+					JSON.stringify({
+						game_tick: 100,
+						player_count: 3,
+						force_flow_statistics: {
+							player: {
+								item_production_statistics: {
+									input: { "inserter": 10, },
+									output: { "iron-plate": 24, }
+								},
+							}
+						},
+						game_flow_statistics: {
+							pollution_statistics: {
+								input: { "boiler": 2000 },
+								output: { "tree-proxy": 560 },
+							}
+						},
+					})
+				);
+
+				await instancePlugin.onMetrics();
+
+				assert.equal(instance._instancePlayerCount.labels('7357').get(), 3);
+				assert.equal(instance._instanceGameTicksTotal.labels('7357').get(), 100);
+				assert.equal(instance._instanceForceFlowStatistics.labels(
+					'7357', 'player', 'item_production_statistics', 'input', 'inserter').get(),
+				10);
+				assert.equal(instance._instanceGameFlowStatistics.labels(
+					'7357', 'pollution_statistics', 'input', 'boiler').get(),
+				2000);
+			});
+			it("should pass on JSON parse errors", async function() {
+				let string = 'An error occured\n';
+				instancePlugin.instance.server.rconCommandResults.set(
+					'/sc statistics_exporter.export()', string
+				);
+
+				let errorMessage;
+				try {
+					JSON.parse(string);
+				} catch (err) {
+					errorMessage = err.message;
+				}
+
+				await assert.rejects(
+					instancePlugin.onMetrics(),
+					new Error(`Error parsing statistics JSON: ${errorMessage}, content "${string}"`)
+				);
+			});
+		});
+	});
+});

plugins/subspace_storage/dole.js

@@ -0,0 +1,129 @@
+const doleNN = require("./dole_nn_base")
+
+const prometheus = require("lib/prometheus");
+
+
+const prometheusNNDoleGauge = new prometheus.Gauge(
+	"clusterio_subspace_storage_nn_dole_gauge",
+	"Current demand being supplied by Neural Network ; 1 means all demand covered, 0.5 means about half of each supply is covered, 0 means no items are given",
+	{ labels: ["resource"] }
+);
+const prometheusDoleFactorGauge = new prometheus.Gauge(
+	"clusterio_subspace_storage_dole_factor_gauge",
+	"The current dole division factor for this resource",
+	{ labels: ["resource"] }
+);
+
+class neuralDole {
+    
+    getRequestStats(itemname,samples)
+    {
+        var sum=0;
+        samples=Math.min(samples,(this.stats[itemname] || []).length-1);
+        if (samples<1) return 0.1;
+        for (var i = 1; i <= samples; i++) {
+            sum+=this.stats[itemname][i].req;
+        }
+        if (sum==0) return 0.1;
+        return sum/samples;
+    }
+    
+    constructor({
+		items
+    }){
+        // Set some storage variables for the dole divider
+		this.items = items
+		this.itemsLastTick = new Map(items._items)
+        this.dole = {}
+        this.carry = {}
+        this.lastRequest = {}
+        this.stats=[]
+		this.debt= {}
+	}
+	doMagic() {
+		for (let [name, count] of this.items._items) {
+			let magicData = doleNN.Tick(
+				count,
+				this.dole[name],
+				this.itemsLastTick.get(name),
+				this.getRequestStats(name,5)
+			)
+			this.stats[name]=this.stats[name] || []
+			this.stats[name].unshift({req:0,given:0});//stats[name][0] is the one we currently collect
+			if (this.stats[name].length>10) this.stats[name].pop();//remove if too many samples in stats
+
+			this.dole[name] = magicData[0]
+			// DONE handle magicData[1] for graphing for our users
+			prometheusNNDoleGauge.labels(name).set(magicData[1] || 0);
+		}
+		this.itemsLastTick = new Map(this.items._items)
+    }
+    divider(object) {
+        let magicData = doleNN.Dose(
+            object.count, // numReq
+			this.items.getItemCount(object.name),
+			this.itemsLastTick.get(object.name) || 0,
+            this.dole[object.name],
+            this.carry[object.name+" "+object.instanceID] || 0,
+            this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] || 0,
+            this.getRequestStats(object.name,5),
+			this.debt[object.name+" "+object.instanceID] || 0
+        )
+        if ((this.stats[object.name] || []).length>0)
+        {
+            this.stats[object.name][0].req+=Number(object.count);
+            this.stats[object.name][0].given+=Number(magicData[0]);
+        }
+        this.lastRequest[object.name+"_"+object.instanceID+"_"+object.instanceName] = object.count
+        //0.Number of items to give in that dose
+        //1.New dole for item X
+        //2.New carry for item X slave Y
+        this.dole[object.name] = magicData[1]
+        this.carry[object.name+" "+object.instanceID] = magicData[2]
+		this.debt[object.name+" "+object.instanceID] = magicData[3]
+
+		// Remove item from DB and send it
+		this.items.removeItem(object.name, magicData[0]);
+
+		return magicData[0];
+    }
+}
+
+// If the server regularly can't fulfill requests, this number grows until it can. Then it slowly shrinks back down.
+let _doleDivisionFactor = {};
+function doleDivider({
+    object,
+	items,
+}){
+	let itemCount = items.getItemCount(object.name);
+    const doleDivisionRetardation = 10; //lower rates will equal more dramatic swings
+    const maxDoleDivision = 250; //a higher cap will divide the store more ways, but will take longer to recover as supplies increase
+    
+    const originalCount = Number(object.count) || 0;
+    object.count /= ((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation;
+    object.count = Math.round(object.count);
+	if (itemCount > 40) {
+		console.info(`Serving ${object.count}/${originalCount} ${object.name} from ${itemCount} ${object.name} with dole division factor ${(_doleDivisionFactor[object.name]||0)} (real=${((_doleDivisionFactor[object.name]||0)+doleDivisionRetardation)/doleDivisionRetardation}), item is ${itemCount > object.count?'stocked':'short'}.`);
+	}
+
+	// Update existing items if item name already exists
+	if (itemCount > object.count) {
+        //If successful, increase dole
+        _doleDivisionFactor[object.name] = Math.max((_doleDivisionFactor[object.name]||0)||1, 1) - 1;
+		items.removeItem(object.name, object.count)
+
+        prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+        return object.count;
+    } else {
+        // if we didn't have enough, attempt giving out a smaller amount next time
+        _doleDivisionFactor[object.name] = Math.min(maxDoleDivision, Math.max((_doleDivisionFactor[object.name]||0)||1, 1) * 2);
+        prometheusDoleFactorGauge.labels(object.name).set(_doleDivisionFactor[object.name] || 0);
+        return 0;
+        //console.log('failure out of ' + object.name + " | " + object.count + " from " + object.instanceID + " ("+object.instanceName+")");
+    }
+}
+
+module.exports = {
+    doleDivider,
+    neuralDole
+}

plugins/subspace_storage/dole_nn_base.js

@@ -0,0 +1,119 @@
+//Sigmoids...
+
+function sigmoid0(x)
+{
+	return x/(1+Math.abs(x));
+}
+function sigmoid1(x)
+{
+	if (x>=0.0) {
+		return 1.0
+	} else {
+		return 0.0
+	}
+}
+function sigmoid2(x)
+{
+	if (x>=0.0){
+		return x
+	} else {
+		return 0.0
+	}
+}
+function sigmoid3(x)
+{
+	return x;
+}
+
+//DOSE - Doses items for item X slave Y
+//Numreq - number of items that are requested for item X for slave Y
+//Instore - number of items for item X in master gauge (currently)
+//store_last_tick - Number of items for item X in master gauge at time of last tick
+//dole - Current dole for item X
+//carry - carry for item X slave Y
+//prev_req - Numreq for previous request for item X slave Y
+//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+//debt - debt for item X slave Y
+
+//Returns: Array of 3 elements
+//0.Number of items to give in that dose
+//1.New dole for item X
+//2.New carry for item X slave Y
+//3.New debt
+function Dose(numreq,instore,store_last_tick,dole,carry,prev_req,numreq_total_adj,debt)
+{
+	numreq=Number(numreq);
+	instore=Number(instore);
+	var instore_adj=instore;
+	if (instore_adj==0) instore_adj=0.1;
+	
+	var b=Math.trunc(prev_req/instore_adj*numreq);
+	var inp1=[numreq,instore_adj,store_last_tick/numreq,dole,carry];//0..4
+	var inp2=[prev_req/numreq,numreq/numreq_total_adj,store_last_tick/numreq_total_adj,prev_req/instore_adj,b,numreq_total_adj,store_last_tick,debt];//5..12
+	var inputs=inp1.concat(inp2);
+	var internal=new Array(20).fill(0);
+	var outputs=[0,0,0,0];
+	//NN: Please don't touch :), code is generated
+	
+outputs[0]=inputs[7]*0.8058582544;
+outputs[0]=sigmoid2(outputs[0]);
+outputs[1]=sigmoid0(outputs[1]);
+outputs[2]=sigmoid0(outputs[2]);
+outputs[3]=inputs[6]*0.2294657379;
+outputs[3]=sigmoid0(outputs[3]);
+
+	debt=debt || 0;
+	if (outputs[0]>0.99) outputs[0]=1.0;
+	
+	var res1=Number(outputs[0])*numreq+outputs[3]+debt;
+	var res_capped=Math.round(res1);
+	
+	if (res_capped>numreq) res_capped=numreq;
+	if (res_capped>instore) res_capped=instore;
+	if (res_capped<0) res_capped=0;
+	
+	res1=res1 || 0;
+	res_capped=res_capped || 0;//Safeguard against wrong values
+	debt=res1-res_capped;
+	debt_cap=Math.round(numreq*1.5)+5;
+	if (debt>debt_cap) debt=debt_cap;
+	
+	carry=sigmoid0(outputs[2]*carry+outputs[0]) || 0;
+	return [res_capped,dole+outputs[1],carry,debt];
+}
+
+//TICK - Updates values for item X, should be called once for each item for each round of all slaves requesting that item
+//Instore - number of items for item X in master gauge (currently)
+//dole - Current dole for item X
+//store_last_tick - Number of items for item X in master gauge at time of last tick (not this one)
+//* doleinfo_last - Object with 2 values for last "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+//* doleinfo_new - Object with 2 values for new "tick period": numreq - total number or requests for item, numslave - number of slaves requesting that item
+
+
+//Returns: Array of 2 elements
+//0.New dole for item X
+//1.Guessed percentage average of items supplied in last 10 ticks
+function Tick(instore,dole,store_last_tick,numreq_total_adj)
+{
+	var instore_adj=instore;
+	if (instore_adj==0) instore_adj=0.1;
+	var inputs=[instore,dole,store_last_tick/instore_adj,store_last_tick/numreq_total_adj,store_last_tick,numreq_total_adj];
+	var internal=new Array(10).fill(0);
+	var outputs=[0,0];
+	//NN: Please don't touch :), code is generated
+	
+outputs[0]=inputs[3]*0.9364774823;
+outputs[0]=outputs[0]*1.0*0.4590721726;
+outputs[0]=sigmoid2(outputs[0]);
+outputs[1]=outputs[0]*1.828422785;
+outputs[1]=sigmoid3(outputs[1]);
+
+	avg=outputs[1];
+	if (numreq_total_adj<0.11) avg=1;//When no item's are requested, result is 0.1 ; In that case there is no deficit, so return 1 = 100% demand fulfilled
+	if (avg>1) avg=1;
+	if (avg<0) avg=0;
+	
+	return [sigmoid0(outputs[0]),avg];
+}
+
+module.exports = {Dose,Tick}

plugins/subspace_storage/info.js

@@ -0,0 +1,106 @@
+const link = require("lib/link");
+const config = require("lib/config");
+
+class MasterConfigGroup extends config.PluginConfigGroup {}
+MasterConfigGroup.groupName = "subspace_storage";
+MasterConfigGroup.define({
+	name: "autosave_interval",
+	title: "Autosave Interval",
+	description: "Interval the storage is autosaved at in seconds.",
+	type: "number",
+	initial_value: 60,
+});
+MasterConfigGroup.define({
+	name: "division_method",
+	title: "Division Method",
+	description: "Method for dividing resource requests between instances.",
+	type: "string",
+	enum: ["simple", "dole", "neural_dole"],
+	initial_value: "simple",
+});
+MasterConfigGroup.define({
+	name: "log_item_transfers",
+	title: "Log Item Trassfers",
+	description: "Spam master console with item transfers done.",
+	type: "boolean",
+	initial_value: false,
+});
+MasterConfigGroup.finalize();
+
+class InstanceConfigGroup extends config.PluginConfigGroup {}
+InstanceConfigGroup.groupName = "subspace_storage";
+InstanceConfigGroup.define({
+	name: "log_item_transfers",
+	title: "Log Item Trassfers",
+	description: "Spam slave console with item transfers done.",
+	type: "boolean",
+	initial_value: false,
+});
+InstanceConfigGroup.finalize();
+
+// JSON schema for subspace storage items
+const items = {
+	type: "array",
+	items: {
+		type: "array",
+		minItems: 2,
+		maxItems: 2,
+		items: [
+			{ type: "string" },
+			{ type: "integer" },
+		],
+	},
+}
+
+module.exports = {
+	name: "subspace_storage",
+	title: "Subspace Storage",
+	description: "Provides shared storage across instances for the Subspace Storage mod",
+	version: "2.0.0-alpha",
+	instanceEntrypoint: "instance",
+	InstanceConfigGroup,
+
+	masterEntrypoint: "master",
+	MasterConfigGroup,
+
+	messages: {
+		// XXX this should be a request to be reliable
+		place: new link.Event({
+			type: "subspace_storage:place",
+			links: ["instance-slave", "slave-master"],
+			forwardTo: "master",
+			eventProperties: {
+				"instance_id": { type: "integer" },
+				"items": items,
+			},
+		}),
+		remove: new link.Request({
+			type: "subspace_storage:remove",
+			links: ["instance-slave", "slave-master"],
+			forwardTo: "master",
+			requestProperties: {
+				"instance_id": { type: "integer" },
+				"items": items,
+			},
+			responseProperties: {
+				"items": items,
+			},
+		}),
+		getStorage: new link.Request({
+			type: "subspace_storage:get_storage",
+			links: ["instance-slave", "slave-master"],
+			forwardTo: "master",
+			responseProperties: {
+				"items": items,
+			},
+		}),
+		updateStorage: new link.Event({
+			type: "subspace_storage:update_storage",
+			links: ["master-slave", "slave-instance"],
+			broadcastTo: "instance",
+			eventProperties: {
+				"items": items,
+			},
+		}),
+	},
+}

plugins/subspace_storage/instance.js

@@ -0,0 +1,95 @@
+const fs = require("fs-extra");
+
+const plugin = require("lib/plugin");
+const luaTools = require("lib/luaTools");
+
+function unexpectedError(err) {
+	console.log("Unexpected error in subspace_storage")
+	console.log("------------------------------------")
+	console.log(err);
+}
+
+class InstancePlugin extends plugin.BaseInstancePlugin {
+	async init() {
+		this.instance.server.on("ipc-subspace_storage:output", (output) => {
+			this.provideItems(output).catch(unexpectedError);
+		});
+		this.instance.server.on("ipc-subspace_storage:orders", (orders) => {
+			this.requestItems(orders).catch(unexpectedError);
+		});
+	}
+
+	async onStart() {
+		this.pingId = setInterval(() => {
+			this.instance.server.sendRcon(
+				"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0", true
+			).catch(unexpectedError);
+		}, 5000);
+
+		let response = await this.info.messages.getStorage.send(this.instance);
+		// TODO Diff with dump of invdata produce minimal command to sync
+		let itemsJson = luaTools.escapeString(JSON.stringify(response.items));
+		await this.instance.server.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}", true)`, true);
+	}
+
+	onExit() {
+		clearInterval(this.pingId);
+	}
+
+	// provide items --------------------------------------------------------------
+	async provideItems(items) {
+		this.info.messages.place.send(this.instance, {
+			items,
+			instance_id: this.instance.config.get("instance.id"),
+		});
+
+		if (this.instance.config.get("subspace_storage.log_item_transfers")) {
+			console.log("Exported the following to master:")
+			console.log(items);
+		}
+	}
+
+	// request items --------------------------------------------------------------
+	async requestItems(items) {
+		// Request the items all at once
+		let response = await this.info.messages.remove.send(this.instance, {
+			instance_id: this.instance.config.get("instance.id"),
+			items,
+		});
+
+		if (!response.items.length) {
+			return;
+		}
+
+		if (this.instance.config.get("subspace_storage.log_item_transfers")) {
+			console.log("Imported following from master:")
+			console.log(response.items);
+		}
+
+		let itemsJson = luaTools.escapeString(JSON.stringify(response.items));
+		await this.instance.server.sendRcon(`/sc __subspace_storage__ Import("${itemsJson}")`, true);
+	}
+
+	// combinator signals ---------------------------------------------------------
+	async updateStorageEventHandler(message, event) {
+		let items = message.data.items;
+
+		// XXX this should be done on the lua side
+		// Ensure counts don't overflow
+		for (let item of items) {
+			if (item[1] > 0x7ffffff) {
+				item[1] = 0x7ffffff;
+			}
+		}
+
+		// XXX this should be moved to instance/clusterio api
+		items.push(["signal-unixtime", Math.floor(Date.now()/1000)]);
+
+		let itemsJson = luaTools.escapeString(JSON.stringify(items));
+		await this.instance.server.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`, true);
+	}
+}
+
+module.exports = {
+	InstancePlugin,
+};