v2.0.0-alpha.4

Remote master config management

.dockerignore

@@ -0,0 +1,23 @@
+# No need to invalidate cache just because we try speeding up the build
+.dockerignore
+Dockerfile
+docker-compose*
+
+# Install dependencies during the build
+node_modules
+packages/**/node_modules
+plugins/**/node_modules
+
+# Download factorio during the build
+factorio
+factorio.tar.gz
+
+# We build internal mods during build, mods from local test env should not be applied
+sharedMods
+sharedModsDisabled
+
+# These files should never be required during the build
+.git
+.github
+dist
+temp

.eslintignore

@@ -0,0 +1 @@
+dist/

.eslintrc.js

@@ -0,0 +1,283 @@
+"use strict";
+module.exports = {
+	"env": {
+		"node": true,
+		"commonjs": true,
+		"es6": true,
+	},
+	"parserOptions": {
+		"ecmaVersion": 11,
+	},
+
+	"overrides": [
+		{
+			"files": ["test/**/*.js", "plugins/*/test/**/*.js"],
+			"env": {
+				"mocha": "true",
+			},
+			"rules": {
+				"prefer-arrow-callback": "off",
+				"no-invalid-this": "off",
+			},
+		},
+		{
+			"files": ["packages/web_ui/src/**/*.jsx", "plugins/*/web/**/*.jsx"],
+			"env": {
+				"browser": true,
+			},
+			"parserOptions": {
+				"sourceType": "module",
+				"ecmaFeatures": {
+					"jsx": true,
+				},
+			},
+		},
+	],
+
+	"rules": {
+		"accessor-pairs": "error",
+		"array-bracket-newline": "off",
+		"array-bracket-spacing": ["error", "never"],
+		"array-callback-return": "error",
+		"array-element-newline": "off",
+		"arrow-body-style": "error",
+		"arrow-parens": "off",
+		"arrow-spacing": ["error", { "after": true, "before": true }],
+		"block-scoped-var": "error",
+		"block-spacing": "error",
+		"brace-style": ["error", "1tbs", { "allowSingleLine": true }],
+		"callback-return": "off",
+		"camelcase": "off",
+		"capitalized-comments": "off",
+		"class-methods-use-this": "off",
+		"comma-dangle": [
+			"error",
+			{
+				"arrays": "always-multiline",
+				"objects": "always-multiline",
+				"imports": "always-multiline",
+				"exports": "always-multiline",
+				"functions": "only-multiline",
+			},
+		],
+		"comma-spacing": "error",
+		"comma-style": ["error", "last"],
+		"complexity": "error",
+		"computed-property-spacing": ["error", "never"],
+		"consistent-return": "error",
+		"consistent-this": "error",
+		"curly": "error",
+		"default-case": "error",
+		"default-case-last": "error",
+		"default-param-last": "error",
+		"dot-location": ["error", "property"],
+		"dot-notation": ["error", { "allowPattern": "^[a-z]+(_[a-z]+)*$" }],
+		"eol-last": "error",
+		"eqeqeq": "error",
+		"func-call-spacing": "error",
+		"func-name-matching": "error",
+		"func-names": "off",
+		"func-style": ["error", "declaration"],
+		"function-call-argument-newline": "off",
+		"function-paren-newline": "off",
+		"generator-star-spacing": ["error", { "before": false, "after": true }],
+		"global-require": "error",
+		"grouped-accessor-pairs": "error",
+		"guard-for-in": "error",
+		"handle-callback-err": "error",
+		"id-blacklist": "error",
+		"id-length": "off",
+		"id-match": "error",
+		"implicit-arrow-linebreak": "error",
+		"indent": ["error", "tab", { "SwitchCase": 1 }],
+		"indent-legacy": "off",
+		"init-declarations": "off",
+		"jsx-quotes": "error",
+		"key-spacing": "error",
+		"keyword-spacing": "error",
+		"line-comment-position": "off",
+		"linebreak-style": ["error", "unix"],
+		"lines-around-comment": "off",
+		"lines-around-directive": "off",
+		"lines-between-class-members": ["error", "always"],
+		"max-classes-per-file": "off",
+		"max-depth": "error",
+		"max-len": ["error", { "code": 120 }],
+		"max-lines": "off",
+		"max-lines-per-function": "off",
+		"max-nested-callbacks": "error",
+		"max-params": "off",
+		"max-statements": "off",
+		"max-statements-per-line": "off",
+		"multiline-comment-style": ["error", "separate-lines"],
+		"multiline-ternary": "off",
+		"new-cap": "error",
+		"new-parens": "error",
+		"newline-after-var": "off",
+		"newline-before-return": "off",
+		"newline-per-chained-call": "off",
+		"no-alert": "error",
+		"no-array-constructor": "error",
+		"no-await-in-loop": "off",
+		"no-bitwise": "off",
+		"no-buffer-constructor": "error",
+		"no-caller": "error",
+		"no-catch-shadow": "error",
+		"no-confusing-arrow": "error",
+		"no-console": "error",
+		"no-constant-condition": ["error", { "checkLoops": false }],
+		"no-constructor-return": "error",
+		"no-continue": "off",
+		"no-div-regex": "error",
+		"no-duplicate-imports": "error",
+		"no-else-return": "error",
+		"no-empty-function": "off",
+		"no-eq-null": "error",
+		"no-eval": "error",
+		"no-extend-native": "error",
+		"no-extra-bind": "error",
+		"no-extra-label": "error",
+		"no-extra-parens": "off",
+		"no-floating-decimal": "error",
+		"no-implicit-coercion": "error",
+		"no-implicit-globals": "error",
+		"no-implied-eval": "error",
+		"no-inline-comments": "off",
+		"no-invalid-this": "error",
+		"no-iterator": "error",
+		"no-label-var": "error",
+		"no-lone-blocks": "error",
+		"no-lonely-if": "error",
+		"no-loop-func": "error",
+		"no-magic-numbers": "off",
+		"no-mixed-operators": [
+			"error",
+			{
+				"groups": [
+					["&", "|", "^", "~", "<<", ">>", ">>>"],
+					["==", "!=", "===", "!==", ">", ">=", "<", "<="],
+					["in", "instanceof"],
+				],
+			},
+		],
+		"no-mixed-requires": "error",
+		"no-multi-assign": "error",
+		"no-multi-spaces": "error",
+		"no-multi-str": "error",
+		"no-multiple-empty-lines": "error",
+		"no-native-reassign": "error",
+		"no-negated-condition": "off",
+		"no-negated-in-lhs": "error",
+		"no-nested-ternary": "error",
+		"no-new": "error",
+		"no-new-func": "error",
+		"no-new-object": "error",
+		"no-new-require": "error",
+		"no-new-wrappers": "error",
+		"no-octal-escape": "error",
+		"no-param-reassign": "off",
+		"no-path-concat": "error",
+		"no-plusplus": ["error", { "allowForLoopAfterthoughts": true }],
+		"no-process-env": "error",
+		"no-process-exit": "error",
+		"no-proto": "error",
+		"no-restricted-exports": "error",
+		"no-restricted-globals": "error",
+		"no-restricted-imports": "error",
+		"no-restricted-modules": "error",
+		"no-restricted-properties": "error",
+		"no-restricted-syntax": "error",
+		"no-return-assign": "error",
+		"no-return-await": "off",
+		"no-script-url": "error",
+		"no-self-compare": "error",
+		"no-sequences": "error",
+		"no-shadow": ["error", { "allow": ["yargs"] }],
+		"no-spaced-func": "error",
+		"no-sync": "error",
+		"no-tabs": ["error", { "allowIndentationTabs": true }],
+		"no-template-curly-in-string": "error",
+		"no-ternary": "off",
+		"no-throw-literal": "error",
+		"no-trailing-spaces": "error",
+		"no-undef-init": "error",
+		"no-undefined": "off",
+		"no-unmodified-loop-condition": "error",
+		"no-unneeded-ternary": "error",
+		"no-unused-expressions": "error",
+		"no-use-before-define": "error",
+		"no-useless-backreference": "error",
+		"no-useless-call": "error",
+		"no-useless-computed-key": "error",
+		"no-useless-concat": "error",
+		"no-useless-constructor": "error",
+		"no-useless-rename": "error",
+		"no-useless-return": "error",
+		"no-var": "error",
+		"no-void": ["error", { "allowAsStatement": true }],
+		"no-warning-comments": "off",
+		"no-whitespace-before-property": "error",
+		"nonblock-statement-body-position": "error",
+		"object-curly-newline": "error",
+		"object-curly-spacing": "off",
+		"object-shorthand": "off",
+		"one-var": "off",
+		"one-var-declaration-per-line": "error",
+		"operator-assignment": "off",
+		"operator-linebreak": ["error", "after", {
+			"overrides": {
+				"?": "before",
+				":": "before",
+				"||": "before",
+				"&&": "before",
+			},
+		}],
+		"padded-blocks": "off",
+		"padding-line-between-statements": "error",
+		"prefer-arrow-callback": "error",
+		"prefer-const": "off",
+		"prefer-destructuring": "off",
+		"prefer-exponentiation-operator": "error",
+		"prefer-named-capture-group": "off",
+		"prefer-numeric-literals": "error",
+		"prefer-object-spread": "error",
+		"prefer-promise-reject-errors": "error",
+		"prefer-reflect": "off",
+		"prefer-regex-literals": "error",
+		"prefer-rest-params": "error",
+		"prefer-spread": "error",
+		"prefer-template": "error",
+		"quote-props": "off",
+		"quotes": ["error", "double", { "avoidEscape": true }],
+		"radix": ["error", "always"],
+		"require-atomic-updates": "off", // XXX Not sure why this triggers on process.title = ...
+		"require-await": "off",
+		"require-jsdoc": "off",
+		"require-unicode-regexp": "off",
+		"rest-spread-spacing": ["error", "never"],
+		"semi": "error",
+		"semi-spacing": "error",
+		"semi-style": "off",
+		"sort-imports": "off",
+		"sort-keys": "off",
+		"sort-vars": "error",
+		"space-before-blocks": "error",
+		"space-before-function-paren": "off",
+		"space-in-parens": ["error", "never"],
+		"space-infix-ops": "off",
+		"space-unary-ops": "error",
+		"spaced-comment": ["error", "always"],
+		"strict": "error",
+		"switch-colon-spacing": "error",
+		"symbol-description": "error",
+		"template-curly-spacing": ["error", "never"],
+		"template-tag-spacing": "error",
+		"unicode-bom": ["error", "never"],
+		"valid-jsdoc": ["error", { "requireReturn": false }],
+		"vars-on-top": "error",
+		"wrap-iife": "error",
+		"wrap-regex": "off",
+		"yield-star-spacing": "error",
+		"yoda": ["error", "never", { "exceptRange": true }],
+	},
+};

.gitattributes

@@ -0,0 +1 @@
+package-lock.json -diff

.github/workflows/docker-publish.yml

@@ -0,0 +1,159 @@
+name: Docker
+
+on:
+  push:
+    # Publish `master` as Docker `latest` image.
+    branches:
+      - '**'
+
+    # Publish `v1.2.3` tags as releases.
+    tags:
+      - v*
+
+  # Run tests for any PRs.
+  pull_request:
+
+  release:
+    types: [released, prereleased]
+
+env:
+  # TODO: Change variable to your image's name.
+  IMAGE_NAME: clusterio
+
+jobs:
+  # Build dependencies and clusterio image
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build docker image
+        run: DOCKER_BUILDKIT=1 docker build . -t $IMAGE_NAME --target clusterio_final
+
+      - name: Export docker image
+        run: docker save -o clusterio_docker_image $IMAGE_NAME
+
+      - name: Upload docker image
+        uses: actions/upload-artifact@master
+        with:
+          name: clusterio_docker_image
+          path: clusterio_docker_image
+  # Build dependencies and clusterio image with test tooling
+  build_testing:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build docker image
+        run: DOCKER_BUILDKIT=1 docker build . -t $IMAGE_NAME --target clusterio_testing
+
+      - name: Export docker image
+        run: docker save -o clusterio_docker_image_testing $IMAGE_NAME
+
+      - name: Upload docker image
+        uses: actions/upload-artifact@master
+        with:
+          name: clusterio_docker_image_testing
+          path: clusterio_docker_image_testing
+  # Upload clusterio_lib and subspace_storage versions
+  upload_mods:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download docker image
+        uses: actions/download-artifact@master
+        with:
+          name: clusterio_docker_image
+          path: .
+
+      - name: Load docker image
+        run: docker load -i clusterio_docker_image
+      
+      - name: Create container
+        run: docker create --name clusterio_container $IMAGE_NAME
+
+      - name: Move mods out of container
+        run: docker cp clusterio_container:/clusterio/sharedMods ./mods
+
+      - name: Remove container
+        run: docker rm clusterio_container
+
+      - name: List extracted mods
+        run: ls mods
+
+      - name: Upload mods
+        uses: actions/upload-artifact@master
+        with:
+          name: mods
+          path: mods
+  # Run tests.
+  # See also https://docs.docker.com/docker-hub/builds/automated-testing/
+  test:
+    needs: build_testing
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Download docker image
+        uses: actions/download-artifact@master
+        with:
+          name: clusterio_docker_image_testing
+          path: .
+      - name: ls
+        run: ls && ls clusterio_docker_image_testing && ls /home/runner/work/factorioClusterio && ls /home/runner/work/factorioClusterio/factorioClusterio
+      - name: Load docker image
+        run: docker load -i clusterio_docker_image_testing
+
+      - name: Run tests
+        run: |
+          if [ -f docker-compose.test.yml ]; then
+            docker-compose --file docker-compose.test.yml run unit_test
+          else
+            docker build . --file Dockerfile
+          fi
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    # Ensure test job passes before pushing image.
+    needs: [test, build]
+
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Download artifact
+        uses: actions/download-artifact@master
+        with:
+          name: clusterio_docker_image
+          path: .
+
+      - name: Load docker image
+        run: docker load -i clusterio_docker_image
+
+      - name: Log into registry
+        run: echo "${{ secrets.CR_PAT }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Push image
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.actor }}/$IMAGE_NAME
+
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # Strip "v" prefix from tag name
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "master" ] && VERSION=latest
+
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION

.gitignore

@@ -28,35 +28,24 @@ dist/
 # Autogenerated docs output
 /docs/api
 
-node_modules/*
+node_modules
 
 Factorio*
+factorio*
 /factorio
 /instances
 /sharedMods
 /database
+/plugin-list.json
 /config-master.json
 /config-control.json
 /config-slave.json
 package-lock.json
-secret-api-token.txt
 mod_cache.json
-static/pictures/spritesheet.js
-static/pictures/spritesheet.png
-static/external
+static/export
 saves
 /temp/
 
-# weird pattern to ignore sharedPlugins except not ignore some core modules
-!sharedPlugins/
-sharedPlugins/*
-!sharedPlugins/inventoryImports/
-!sharedPlugins/researchSync/
-!sharedPlugins/clusterioMod/
-
-# File used for marking plugins as disabled
-sharedPlugins/*/DISABLED
-
 # IDE
 .idea/
 .vs/

.travis.yml

@@ -6,6 +6,10 @@ before_install:
   - tar -xf factorio.tar.gz
   - npm install -g codecov
 script:
+  - npx lerna bootstrap --no-ci
+  - node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/sharedMods
+  - curl -o temp/test/sharedMods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/factorioClusterioMod/releases/download/1.99.6/subspace_storage_1.99.8.zip
+  - npx lerna run build
   - npm run-script ci-cover
-  - cat temp/test/integration/log.txt
   - codecov -f coverage/lcov.info
+  - npm run-script lint

CHANGELOG.md

@@ -15,7 +15,7 @@ Version 2.0.0
 - Daemonized slaves.  Slaves now have the ability to run multiple Factorio
   instances, starting and stopping them individually.  To manage this the
   local command line interface has been replaced with a remote interface on
-  the master server that can be accessed through the clusterctl cli tool.
+  the master server that can be accessed through the clusterioctl cli tool.
 - Rewritten the communication between slaves and the master from scratch.  The
   new system is based on a WebSocket connection between the slaves and the
   master server and provides efficient validated bi-directonal communication.
@@ -25,6 +25,10 @@ Version 2.0.0
 - New configuration system with support for initializing configs when needed,
   modifying config entries using built-in commands, and updating instance
   configuration remotely.
+- Packaged and uploaded to npm again, greatly simplifying distribution and
+  installation of both Clusterio and plugins.
+- Added centralized logging for the cluster.  All logs from both Clusterio
+  and Factorio is stored in a shared log which can be inspected and queried.
 
 ### Features
 
@@ -37,11 +41,23 @@ Version 2.0.0
   allowing it to be proxied behind web-server.
 - Added WebSocket usage statistics.
 - Added commands sent statistic.
+- Added Factorio server CPU and memory usage statistic.
+- Added Factorio autosave size statistic.
+- Added support for extracting locale and item icons from Factorio and mods.
+- Added integrated user management for controlling access and storing per
+  player data.
+- Added synchronization of in-game adminlist, banlist and whitelist to
+  Clusterio core.  Previously this was handled by the playerManager plugin.
+- Added stripping of long paths in the Factorio server log.
+- Added option to configure maximum number of commands to send in parallel.
+- Added option to auto start instances on slave startup.
+- Added metric mapping of slave and instance ids to their names, allowing
+  them to be displayed by name in queries that join with the mapping.
+- Added option to configure timeout for exported Prometheus metrics.
+- Added option to set address to bind HTTP(S) port to.
 
 ### Changes
 
-- Fixed sslCert and sslPrivKey entries being ignored in the config.
-- Changed ssl key creation to be done at startup instead of on npm install.
 - Added error handling during master startup.
 - Factorio game and rcon port now defaults to a random port above 49151.
 - Removed unimplemented mods update command.
@@ -89,8 +105,7 @@ Version 2.0.0
 
 ### Breaking Changes
 
-- lib/authenticate no longer requires config to be passed to it.  Breaks
-  playerManager.
+- Removed lib/authenticate.  Breaks playerManager.
 - The masterIP and masterPort config entries has been merged into masterURL.
   Breaks discordChat, playerManager, serverSelect, trainTeleports and
   clusterioModel.
@@ -163,6 +178,8 @@ Version 2.0.0
 - Implemented a new config system that replaces the old.  Breaks all plugins.
 - Removed usage of socket.io entirely in favor of a plain WebSocket connection.
 - Renamed clusterioMod plugin to subspace_storage.
+- Master no longer creates secret-api-token.txt.
+- Removed automatic creation of self-signed TLS certificate.
 
 
 Version 1.2.4

Dockerfile

@@ -1,23 +1,64 @@
-# Warning: This is currently broken as a result of
-# changes done in the 2.0 refactoring.
-FROM node:12
-RUN apt-get update && apt install git curl tar -y
-RUN mkdir factorioClusterio
+FROM node:12 as subspace_storage_builder
+RUN apt update && apt install -y git
+WORKDIR /
+RUN git clone https://github.com/clusterio/factorioClusterioMod.git
+WORKDIR /factorioClusterioMod
+RUN git checkout clusterio-2.0 \
+&& npm install \
+&& node build
 
-RUN git clone -b master https://github.com/clusterio/factorioClusterio.git && cd factorioClusterio && npm install --only=production
-RUN cd factorioClusterio && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz
+FROM node:12 as clusterio_builder
+RUN apt update \
+&& apt install -y wget \
+&& mkdir /clusterio
+WORKDIR /clusterio
+RUN wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
+# Copy project files into the container
+COPY . .
 
-WORKDIR factorioClusterio
-RUN mkdir instances sharedMods
-RUN cp config.json.dist config.json
+RUN npm install \
+&& npx lerna bootstrap --hoist
 
-RUN node client manage shared mods add clusterio
+# Install plugins. This is intended as a reasonable default, enabling plugins to make for fun gameplay.
+# If you want a different set of plugins, consider using this as the base image for your own.
+#RUN npm install @clusterio/plugin-subspace_storage
+#RUN npx clusteriomaster plugin add @clusterio/plugin-subspace_storage
 
-LABEL maintainer "Sir3lit@gmail.com"
+COPY --from=subspace_storage_builder /factorioClusterioMod/dist/ /clusterio/sharedMods/
 
-EXPOSE 8080 34167
-VOLUME /factorioClusterio/instances
-VOLUME /factorioClusterio/sharedMods
-VOLUME /factorioClusterio/sharedPlugins
+# Build submodules (web UI, libraries, plugins etc)
+RUN npx lerna run build
+# Build Lua library mod
+RUN node packages/lib/build_mod --build --source-dir packages/slave/lua/clusterio_lib \
+&& mv dist/* sharedMods/ \
+&& mkdir temp \
+&& mkdir temp/test \
+&& cp sharedMods/ temp/test/ -r \
+&& ls sharedMods
 
-CMD MODE="$MODE" node $MODE start $INSTANCE
+# Remove node_modules
+RUN find . -name 'node_modules' -type d -prune -print -exec rm -rf '{}' \;
+
+FROM frolvlad/alpine-glibc AS clusterio_final
+
+RUN apk add --update bash nodejs npm
+
+COPY --from=clusterio_builder /clusterio /clusterio
+WORKDIR /clusterio
+
+# Install runtime dependencies
+RUN npm install --production
+RUN npx lerna bootstrap -- --production --no-optional
+LABEL maintainer "danielv@danielv.no"
+
+FROM frolvlad/alpine-glibc AS clusterio_testing
+
+RUN apk add --update bash nodejs npm
+
+COPY --from=clusterio_builder /clusterio /clusterio
+WORKDIR /clusterio
+
+# Install runtime dependencies
+RUN npm install
+RUN npx lerna bootstrap
+RUN npm install chalk semver

LICENSE.md

@@ -19,8 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
-
-The icons for angels modded items located in /static/pictures/ are licensed
-by https://mods.factorio.com/user/Arch666Angel under CC BY-NC-ND 4.0
-https://creativecommons.org/licenses/by-nc-nd/4.0/
-

README.md

@@ -10,8 +10,6 @@ Discord for development/support/play: https://discord.gg/5XuDkje
 
 ## Important notice
 
-**WARNING: The master branch is currently broken, do not use it!**
-
 This is the development branch for factorioClusterio 2.0 which is currently undergoing heavy
 restructuring and refactoring.  Expect plugins and existing installations to frequently break when
 using this branch.  If you don't want to be an alpha tester for 2.0 please use the stable
@@ -32,87 +30,93 @@ the [1.2.x branch][1.2.x] for instructions on how to install the stable version.
 
 ### Table of contents
 
-* [Introduction & methodology](#introduction)
+* [Introduction](#introduction)
+* [Features](#features)
+* [Plugins](#plugins)
 * [Ubuntu setup](#ubuntu-setup)
 * [Windows setup](#windows-setup)
+* [Installing Plugins](#installing-plugins)
+* [Configure Master Server](#configure-master-server)
+  * [Hosting Behind Proxy](#hosting-behind-proxy)
+  * [Setting up an admin account](#setting-up-an-admin-account)
 * [Running Clusterio](#running-clusterio)
   * [Master Server](#master-server)
   * [Slaves](#slaves)
   * [Instances](#instances)
-* [Optional plugins](#Plugins)
 * [Common problems](#Common-problems)
 
+
 ## Introduction
 
-Features:
+Clusterio is a clustered Factorio server manager that provides the
+tooling for implementing cross server interactions in Factorio.  It
+was previously best known for implementing cross server transfer and
+cloud storage of items via teleporter chests.  But this functionality
+has been pulled out of Clusterio into its own plugin for Clusterio named
+[Subspace Storage](https://github.com/clusterio/factorioClusterioMod).
 
-- Entities to send/recieve items
-
-- Cross dimensional storage
-
-- Sending of liquids
-
-- Sending of circuit network signals
-
-- Inventory combinator to display item levels in the cluster (and epoch time)
-
-- Reporting of graphs and UPS on master interface (Also has extensive Prometheus reporting)
-
-Optional extras (see [Plugins](#Plugins))
-
-- Have your inventory synchronize across servers
-
-- Teleport trans from the border of one world to te next
-
-- Show in-game chat in discord
+By itself Clusterio doesn't change the gameplay in any way, you could
+even use Clusterio to manage completely vanilla Factorio servers.
+Plugins do the work of modding in the visible changes into the game, see
+the [Plugins section](#plugins) for ready made plugins you can install
+into a Clusterio cluster.
 
 
-Connection diagram:
+## Features
 
-![http://i.imgur.com/7FdVfgB.png](http://i.imgur.com/7FdVfgB.png)
+- Clustered Factorio server management allowing you manage the running
+  of Factorio servers across a fleet of physical servers from both a web
+  interface and a command line interface.
 
-There can be any number of clients connected to each slave, and any number of slaves connected to a master but there can only be one master server in each cluster.
+- User list management for synchronizing in-game admins, whitelisted
+  users, and bans to all the servers in the cluster.
 
-**How does it work?**
+- Integrated support for exporting statistics for the whole cluster to
+  Prometheus via a single metrics endpoint.
 
-Traditional factorio mods have always been limited by the games deterministic design. This gives us a very bug free and predictable game, but doesn't allow us cool stuff such as internet communication.
-Clusterio exploits one of the games logging features, game.write_file and RCON to communicate between servers. Sending an item from one server to another takes this path:
+- Extensive plugin support for adding your own cross server features to
+  Factorio using Clusterio's communication backbone.
 
-1. server1: Chest has stuff in it, write the contents to a file and delete them from the game world
 
-2. client.js on server1: File has stuff in it, parse and send to the master for storage
+## Plugins
 
-3. master: server1 gave us stuff, store it in the storage and write some statistics
+The heart of Clusterio is its plugin system.  Plugins add functionality
+to Factorio servers, Clusterio itself or both.  These are the plugins
+supported and maintained by the Clusterio developers:
 
-4. server2: get-chest is empty, write a request to file
+- [Global Chat](/plugins/global_chat/README.md): share the in-game chat
+  between servers.
+- [Research Sync](/plugins/research_sync/README.md): synchronize
+  research progress and technologies unlocked between servers.
+- [Statistics Exporter](/plugins/statistics_exporter/README.md): collect
+  in-game statistics from all the servers and makes it available to the
+  Prometheus endpoint on the master server.
+- [Subspace Storage](https://github.com/clusterio/factorioClusterioMod):
+  Provide shared storage that can transport items between servers via
+  teleport chests.
 
-5. client.js on server2: Request file has stuff in it, parse and send a request to master for more items of that type
+Want to make your own plugin?  Check out the documentation on [Writing
+Plugins](/docs/writing-plugins.md) for where to start.
 
-6. master: server2 asked for stuff, check if we have enough and how much demand there is, then send however much is appropriate back
-
-7. client.js on server2: We were allowed to import x of item y, run command /c remote.call("clusterio", "importMany", "{'copper-plate':120}")
-
-This process works the same for both items and liquids, independent on what mods are used. Yes, modded items are fully supported.
-
-Clusterio can also do a few other neat things, such as giving you access to epoch time, syncing player inventories between servers, keeping track of playtime (playerManager plugin), teleporting trains between servers (trainTeleports) and exporting tons of factorio related statistics to Prometheus for graphing in grafana.
 
 ## Ubuntu setup
 
 **Warning**: These instructions are for the unstable master version and is not
 recommended for use, see [the 1.2.x branch][1.2.x] for how to install the stable version.
 
-Clusterio runs on Node.js v12 and up, v11.13.0+ and v10.16.0+.  Node.js itself is not
+Clusterio runs on Node.js v12 and up, and v10.16.0+.  Node.js itself is not
 supported on EOL Ubuntu releases so make sure you're on a recent release of Ubuntu.
 
 Master and all slaves:
 
     wget -qO - https://deb.nodesource.com/setup_12.x | sudo -E bash -
-    sudo apt install -y nodejs python-dev git build-essential
-    git clone -b master https://github.com/clusterio/factorioClusterio.git
-    cd factorioClusterio
+    sudo apt install -y nodejs
+    mkdir clusterio
+    cd clusterio
+    npm init -y
+    npm install @clusterio/master @clusterio/slave @clusterio/ctl
     wget -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64
     tar -xf factorio.tar.gz
-    npm install --only=production
 
 downloads and installs nodejs, git and clusterio. To specify a version, change "latest" in the link to a version number like 0.14.21.
 
@@ -138,30 +142,19 @@ The -v flag is used to specify the instance directory. Your instances (save file
 **Warning**: These instructions are for the unstable master version and is not
 recommended for use, see [the 1.2.x branch][1.2.x] for how to install the stable version.
 
-Clusterio is built up of multiple parts. Here is a quick guide:
-
-Master = master.js
-
-Server host (Slave) = slave.js + factorio server
-
-Game Client = The people connecting to the server
-
 **Requirements**
 
 download and install nodeJS 12 from http://nodejs.org.  Clusterio runs on Node.js v12 and
-up, v11.13.0+ and v10.16.0+.
-
-download and install git from https://git-scm.com/
-
-reboot when you are done, then proceed to the next steps. *reboots matter*
+up, and v10.16.0+.
 
 **Master**
 
 1. Open PowerShell or Command prompt in the directory you want to install to and run the following commands.
 
-        git clone -b master https://github.com/clusterio/factorioClusterio
-        cd factorioClusterio
-        npm install --only=production
+        mkdir clusterio
+        cd clusterio
+        npm init -y
+        npm install @clusterio/master @clusterio/slave @clusterio/ctl
 
 2. Obtain Factorio by either of these two methods:
 
@@ -173,7 +166,7 @@ reboot when you are done, then proceed to the next steps. *reboots matter*
         2.  Download the MS Windows (64-bit zip package) from
             https://www.factorio.com/download .
 
-        3.  Open the zip file and drag the folder called "Factorio_0.x.y" into
+        3.  Open the zip file and drag the folder called "Factorio_x.y.z" into
             the factorio folder created in step 1.
 
     - Via steam installation
@@ -188,10 +181,72 @@ reboot when you are done, then proceed to the next steps. *reboots matter*
             the Factorio folder into the factorio folder created in step 1.
 
 
+## Installing Plugins
+
+Installing plugins to make them work with Clusterio consists of two
+steps.  First install the package via npm, for example
+
+    npm install @clusterio/plugin-subspace_storage
+
+Then tell clusterio that this plugin exists by adding it as a plugin.
+
+    npx clusteriomaster plugin add @clusterio/plugin-subspace_storage
+
+This adds it to the default `plugin-list.json` file which in the shared
+folder setup is loaded by master, slave and ctl.  If you have slaves or
+ctl installed on separate computers (or directories) then you need to
+repeat the plugin install process for all of them.  The clusteriomaster,
+clusterioslave and clusterioctl commands has the plugin sub-command
+so you do not need to install clusteriomaster to add plugins.
+
+For development purposes the `plugin add` command supports adding
+plugins by the absolute path to them, or a relative path that must start
+with either . or .. (which will then be resolved to an absolute path).
+
+
+## Configure Master Server
+
+By default the master server will listen for HTTP on port 8080.  You can
+change the port used with the command
+
+    npx clusteriomaster config set master.http_port 1234
+
+If you plan to make your cluster available externally set the address
+that it will be accessible under with, for example
+
+    npx clusteriomaster config set master.external_address http://203.0.113.4:1234/
+
+Change the url to reflect the IP, protocol, and port the master server
+is accessible under, dns names are also supported.  If you're planning
+on making the master server accessible on the internet it's recommended
+to set up TLS, see the [Setting Up TLS](/docs/setting-up-tls.md)
+document for more details.
+
+You can list the config of the master server with the `npx
+clusteriomaster config list` command.  See the [readme for
+@clusterio/master](/packages/master/README.md) for more
+information.
+
+
+### Setting up an admin account
+
+Before you can manage the cluster you need to bootstrap an admin account
+for it.  Replace `<username>` with your Factorio username (do not make
+up a new username here).
+
+    npx clusteriomaster bootstrap create-admin <username>
+    npx clusteriomaster bootstrap create-ctl-config <username>
+
+The first command creates a user account with the given name and
+promotes it to a cluster admin.  The second one sets up a
+`config-control.json` config for clusterioctl to connect to the master
+server under the given user account.
+
+
 ## Running Clusterio
 
-After following the installation instructions you can use the following
-commands to run Clusterio.
+After following the installation and master configuration instructions
+you can use the following commands to run Clusterio.
 
 
 ### Master Server
@@ -200,69 +255,61 @@ It's necessary to run the master server in order for anything to work.
 Once you've completed the setup run the following command to start it
 up:
 
-    node master run
+    npx clusteriomaster run
 
 
 ### Slaves
 
 Slaves connect to the master server and are managed remotely from the
 master server.  In order for slaves to connect to the master server they
-need an authentication token from the master server.  This token is
-written to secret-api-token.txt on the master server when it is started
-up.
+need a valid authentication token, you can create a slave config with
+a valid token with the following command.
 
-To set up the configuration for a new local slave run the following.
-
-    node slave config set slave.name "Local"
-    node slave config set slave.master_token "<token>"
+    npx clusterioctl slave create-config --name Local --generate-token
 
 This will write a new `config-slave.json` file in the current directory
-(you can change the location with the `--config` option) with the name
-and token provided.  If you are connecting to a remote master server you
-will also need to set the `slave.master_url` option to that url.
+(you can change the location with the `--output` option) with the name,
+token and url to connect to the master server with.  If you are making
+the config for a remote slave you will need to have set the
+`master.external_address` option to the URL the master server can be
+reached on.
 
-You can list the config of a slave with the `node slave config list`
-command.  Use `node slave config --help` for more information.
+You can list the config of a slave on the slave itself with the `npx
+clusterioslave config list` command.  See the [readme for
+@clusterio/slave](/packages/slave/README.md) for more information.
 
 Once the config is set up run the slave with
 
-    node slave run
+    npx clusterioslave run
 
 
 ### Instances
 
 Instances are created, managed and started from the master server.  For
-now the only interface available is the `clusterctl` command line tool
+now the only interface available is the `clusterioctl` command line tool
 included in Clusterio.  You can run this tool from any slave, or the
 master server without having to set up a config, if you want to manage
 the cluster from somewhere else you will need to set the
 `control.master_url` and `control.master_token` options with the  `node
-clusterctl control-config set` command:
+clusterioctl control-config set` command:
 
 The basic operations to start a new instance is the following
 
-    node clusterctl create-instance --name "My instance"
-    node clusterctl assign-instance --instance "My instance" --slave "Local"
-    node clusterctl create-save --instance "My instance"
-    node clusterctl start-instance --instance "My instance"
+    npx clusterioctl instance create "My instance"
+    npx clusterioctl instance assign "My instance" "Local"
+    npx clusterioctl instance start "My instance"
 
 The first line creates the instance configuration on the master server.
 The second assigns the instance to a slave which creates the instance
 directory and files needed to run the instance on the given slave.  The
-third line creates a new savegame for the instance.  You could also
-upload your own save to the instance directory instead.  And finally the
-fourth line starts the instance.
+third line starts the instance, which creates a new save if there are no
+save games present.
 
-There are many more commands available with clusterctl.  See
-`node clusterctl --help` for a full list of them.
+There are many more commands available with clusterioctl.  See the
+[Managing a Cluster](/docs/managing-a-cluster.md) document or
+`npx clusterioctl --help` for a full list of them.
 
 
-## Plugins
-Here are the known Clusterio plugins in the wild:
-1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta)
-2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook.
-3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers.
-
 ## Common problems
 
 ### EACCESS [...] LISTEN 443
@@ -288,7 +335,3 @@ TLDR: the tested fix is:
 - select TCP and select specific local ports and type in the ports that you want to open (comma separated) and click next > 3 times
 
 - give the rule a name (like 'web server' or something), give it a description (optionally) and click finish
-
-### Other fixes for other potential problems:
-
-Sometimes the install fails. Try `node ./lib/npmPostinstall` to complete it.

build-mod.js

@@ -1,92 +0,0 @@
-"use strict";
-const events = require("events");
-const fs = require("fs-extra");
-const JSZip = require("jszip");
-const klaw = require("klaw");
-const path = require("path");
-const yargs = require("yargs");
-
-
-// Location of mod source files
-const sourceDir = "mod";
-
-// Paths to additional files to put into the mod's folder.
-const additionalFiles = [
-	path.join("modules", "clusterio", "serialize.lua"),
-];
-
-async function main() {
-	const args = yargs
-		.scriptName("build")
-		.options({
-			'clean': { describe: "Remove previous builds", type: 'boolean', default: false },
-			'build': { describe: "Build mod", type: 'boolean', default: true },
-			'pack': { describe: "Pack into zip file", type: 'boolean', default: true },
-			'output-dir': { describe: "Path to output built mod", nargs: 1, type: 'string', default: "dist" },
-		})
-		.argv
-	;
-
-	let info = JSON.parse(await fs.readFile(path.join(sourceDir, "info.json")));
-
-	if (args.clean) {
-		let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/
-		for (let entry of await fs.readdir(args.outputDir)) {
-			let match = splitter.exec(entry);
-			if (match) {
-				let [, name, version] = match;
-				if (name === info.name) {
-					let modPath = path.join(args.outputDir, entry);
-					console.log(`Removing ${modPath}`);
-					await fs.remove(modPath);
-				}
-			}
-		}
-	}
-
-	if (args.build) {
-		await fs.ensureDir(args.outputDir);
-		let modName = `${info.name}_${info.version}`;
-
-		if (args.pack) {
-			let zip = new JSZip();
-			let walker = klaw(sourceDir)
-				.on('data', item => {
-					if (item.stats.isFile()) {
-						// On Windows the path created uses backslashes as the directory sepparator
-						// but the zip file needs to use forward slashes.  We can't use the posix
-						// version of relative here as it doesn't work with Windows style paths.
-						let basePath = path.relative(sourceDir, item.path).replace(/\\/g, "/");
-						zip.file(path.posix.join(modName, basePath), fs.createReadStream(item.path));
-					}
-				})
-			;
-			await events.once(walker, 'end');
-
-			for (let filePath of additionalFiles) {
-				zip.file(path.posix.join(modName, path.basename(filePath)), fs.createReadStream(filePath));
-			}
-
-			let modPath = path.join(args.outputDir, `${modName}.zip`);
-			console.log(`Writing ${modPath}`);
-			let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath));
-			await events.once(writeStream, 'finish');
-
-		} else {
-			let modDir = path.join(args.outputDir, modName);
-			if (await fs.exists(modDir)) {
-				console.log(`Removing existing build ${modDir}`);
-				await fs.remove(modDir);
-			}
-			console.log(`Building ${modDir}`);
-			await fs.copy(sourceDir, modDir);
-			for (let filePath of additionalFiles) {
-				await fs.copy(filePath, path.join(modDir, path.basename(filePath)));
-			}
-		}
-	}
-}
-
-if (module === require.main) {
-	main().catch(err => { console.log(err) });
-}

clusterctl.js

@@ -1,566 +0,0 @@
-const jwt = require("jsonwebtoken");
-const fs = require("fs-extra");
-const yargs = require("yargs");
-const version = require("./package").version;
-const asTable = require("as-table").configure({delimiter: ' | '});
-const chalk = require("chalk");
-const events = require("events");
-
-const link = require("lib/link");
-const errors = require("lib/errors");
-const config = require("lib/config");
-
-
-/**
- * Format a parsed Factorio output message with colors
- *
- * Formats a parsed Factorio output from lib/factorio into a readable
- * colorized output using terminal escape codes that can be printed.
- */
-function formatOutputColored(output) {
-	let time = "";
-	if (output.format === 'seconds') {
-		time = chalk.yellow(output.time.padStart(8)) + ' ';
-	} else if (output.format === 'date') {
-		time = chalk.yellow(output.time) + ' ';
-	}
-
-	let info = "";
-	if (output.type === 'log') {
-		let level = output.level;
-		if (level === "Info") {
-			level = chalk.bold.blueBright(level);
-		} else if (output.level === "Warning") {
-			level = chalk.bold.yellowBright(level);
-		} else if (output.level === "Error") {
-			level = chalk.bold.redBright(level);
-		}
-
-		info = level + ' ' + chalk.gray(output.file) + ': ';
-
-	} else if (output.type === 'action') {
-		info = '[' + chalk.yellow(output.action) + '] ';
-	}
-
-	return time + info + output.message;
-}
-
-/**
- * Represents a command that can be runned by clusterctl
- */
-class Command {
-	constructor({ definition, handler }) {
-		this.name = definition[0];
-		this._definition = definition;
-		this._handler = handler;
-	}
-
-	register(yargs) {
-		yargs.command(...this._definition);
-	}
-
-	async run(args, control) {
-		await this._handler.call(this, args, control);
-	}
-}
-
-/**
- * Resolve a string to an instance ID
- *
- * Resolves a string with either an instance name or an id into an integer
- * with the instance ID.
- *
- * @param client - link to master server to query instance on.
- * @param instanceName - string with name or id of instance.
- * @returns {number} instance ID.
- */
-async function resolveInstance(client, instanceName) {
-	let instanceId;
-	if (/^-?\d+$/.test(instanceName)) {
-		instanceId = parseInt(instanceName, 10);
-	} else {
-		let response = await link.messages.listInstances.send(client);
-		for (let instance of response.list) {
-			if (instance.name === instanceName) {
-				instanceId = instance.id;
-				break;
-			}
-		}
-
-		if (instanceId === undefined) {
-			throw new errors.CommandError(`No instance named ${instanceName}`);
-		}
-	}
-
-	return instanceId;
-}
-
-/**
- * Resolve a string into a slave ID
- *
- * Resolves a string with either an slave name or an id into an integer with
- * the slave ID.
- *
- * @param client - link to master server to query slave on.
- * @param slaveName - string with name or id of slave.
- * @returns {number} slave ID.
- */
-async function resolveSlave(client, slaveName) {
-	let slaveId;
-	if (/^-?\d+$/.test(slaveName)) {
-		slaveId = parseInt(slaveName, 10);
-	} else {
-		let response = await link.messages.listSlaves.send(client);
-		for (let slave of response.list) {
-			if (slave.name === slaveName) {
-				slaveId = slave.id;
-				break;
-			}
-		}
-
-		if (slaveId === undefined) {
-			throw new errors.CommandError(`No slave named ${slaveName}`);
-		}
-	}
-
-	return slaveId;
-}
-
-let commands = [];
-commands.push(new Command({
-	definition: ["list-slaves", "List slaves connected to the master"],
-	handler: async function(args, control) {
-		let response = await link.messages.listSlaves.send(control);
-		console.log(asTable(response.list));
-	},
-}));
-
-commands.push(new Command({
-	definition: ["list-instances", "List instances known to the master"],
-	handler: async function(args, control) {
-		let response = await link.messages.listInstances.send(control);
-		console.log(asTable(response.list));
-	}
-}));
-
-commands.push(new Command({
-	definition: ['create-instance', "Create an instance", (yargs) => {
-		// XXX TODO: set any specific options?
-		yargs.option('name', {
-			type: "string",
-			nargs: 1,
-			describe: "Instance name",
-			group: "Instance Config:",
-		});
-		yargs.option('id', {
-			type: "number",
-			nargs: 1,
-			describe: "Instance ID",
-			group: "Instance Config:",
-		});
-		yargs.option('base', {
-			type: "boolean",
-			nargs: 0,
-			describe: "Specify this is a base config",
-			group: "Instance Config:"
-		});
-	}],
-	handler: async function(args, control) {
-		let instanceConfig = new config.InstanceConfig();
-		await instanceConfig.init();
-		if (args.id) {
-			instanceConfig.set("instance.id", args.id);
-		}
-		if (args.name) {
-			instanceConfig.set("instance.name", args.name);
-		}
-		let serialized_config = instanceConfig.serialize();
-		let response = await link.messages.createInstance.send(control, { serialized_config });
-	},
-}));
-
-commands.push(new Command({
-	definition: ["list-instance-config", "List configuration for an instance", (yargs) => {
-		yargs.option({
-			"instance": { describe: "Instance to list config for", nargs: 1, type: "string", demandOption: true },
-		});
-	}],
-	handler: async function(args, control) {
-		let instanceId = await resolveInstance(control, args.instance);
-		let response = await link.messages.getInstanceConfig.send(control, { instance_id: instanceId });
-
-		for (let group of response.serialized_config.groups) {
-			for (let [name, value] of Object.entries(group.fields)) {
-				console.log(`${group.name}.${name} ${JSON.stringify(value)}`);
-			}
-		}
-	},
-}));
-
-commands.push(new Command({
-	definition: ["set-instance-config", "Set field in instance config", (yargs) => {
-		yargs.option({
-			"instance": { describe: "Instance to set config on", nargs: 1, type: "string", demandOption: true },
-			"field": { describe: "Field to set", nargs: 1, type: "string", demandOption: true },
-			"value": { describe: "Value to set", nargs: 1, type: "string", demandOption: true },
-		});
-	}],
-	handler: async function(args, control) {
-		let instanceId = await resolveInstance(control, args.instance);
-		await link.messages.setInstanceConfigField.send(control, {
-			instance_id: instanceId,
-			field: args.field,
-			value: args.value,
-		});
-	},
-}));
-
-commands.push(new Command({
-	definition: ['assign-instance', "Assign instance to a slave", (yargs) => {
-		yargs.options({
-			'instance': { describe: "Instance to assign", nargs: 1, type: 'string', demandOption: true },
-			'slave': { describe: "Slave to assign to", nargs: 1, type: 'string', demandOption: true },
-		});
-	}],
-	handler: async function(args, control) {
-		let instanceId = await resolveInstance(control, args.instance);
-		let slaveId = await resolveSlave(control, args.slave);
-		await link.messages.assignInstanceCommand.send(control, {
-			instance_id: instanceId,
-			slave_id: slaveId,
-		});
-	},
-}));
-
-commands.push(new Command({
-	definition: ['create-save', "Create a new save on an instance", (yargs) => {
-		yargs.options({
-			'instance': { describe: "Instance to create on", nargs: 1, type: 'string', demandOption: true },
-		});
-	}],
-	handler: async function(args, control) {
-		let instanceId = await resolveInstance(control, args.instance);
-		await link.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
-		let response = await link.messages.createSave.send(control, {
-			instance_id: instanceId,
-		});
-	},
-}));
-
-commands.push(new Command({
-	definition: ['start-instance', "Start instance", (yargs) => {
-		yargs.options({
-			'instance': { describe: "Instance to start", nargs: 1, type: 'string', demandOption: true },
-		});
-	}],
-	handler: async function(args, control) {
-		let instanceId = await resolveInstance(control, args.instance);
-		await link.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
-		let response = await link.messages.startInstance.send(control, {
-			instance_id: instanceId,
-		});
-	},
-}));
-
-commands.push(new Command({
-	definition: ['stop-instance', "Stop instance", (yargs) => {
-		yargs.options({
-			'instance': { describe: "Instance to stop", nargs: 1, type: 'string', demandOption: true },
-		});
-	}],
-	handler: async function(args, control) {
-		let instanceId = await resolveInstance(control, args.instance);
-		await link.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
-		let response = await link.messages.stopInstance.send(control, {
-			instance_id: instanceId,
-		});
-	},
-}));
-
-commands.push(new Command({
-	definition: ['delete-instance', "Delete instance", (yargs) => {
-		yargs.options({
-			'instance': { describe: "Instance to delete", nargs: 1, type: 'string', demandOption: true },
-		});
-	}],
-	handler: async function(args, control) {
-		let response = await link.messages.deleteInstance.send(control, {
-			instance_id: await resolveInstance(control, args.instance),
-		});
-	},
-}));
-
-commands.push(new Command({
-	definition: ['send-rcon', "Send RCON command", (yargs) => {
-		yargs.options({
-			'instance': { describe: "Instance to sent to", nargs: 1, type: 'string', demandOption: true },
-			'command': { describe: "command to send", nargs: 1, type: 'string', demandOption: true },
-		});
-	}],
-	handler: async function(args, control) {
-		let response = await link.messages.sendRcon.send(control, {
-			instance_id: await resolveInstance(control, args.instance),
-			command: args.command
-		})
-
-		// Factorio includes a newline in it's response output.
-		process.stdout.write(response.result);
-	},
-}));
-
-// Convert to mapping from name to command instance
-commands = new Map([...commands.map(command => [command.name, command])]);
-
-
-/**
- * Connector for control connection to master server
- */
-class ControlConnector extends link.WebSocketClientConnector {
-	constructor(url, reconnectDelay, token) {
-		super(url, reconnectDelay);
-		this._token = token;
-	}
-
-	register() {
-		console.log("SOCKET | registering control");
-		this.sendHandshake("register_control", {
-			token: this._token,
-			agent: "clusterctl",
-			version: version,
-		});
-	}
-}
-
-/**
- * Handles running the control
- *
- * Connects to the master server over WebSocket and sends commands to it.
- */
-class Control extends link.Link {
-
-	// I don't like God classes, but the alternative of putting all this state
-	// into global variables is not much better.
-	constructor(connector) {
-		super('control', 'master', connector);
-		link.attachAllMessages(this);
-	}
-
-	async instanceOutputEventHandler(message) {
-		let { instance_id, output } = message.data;
-		console.log(formatOutputColored(output));
-	}
-
-	async shutdown() {
-		await link.messages.shutdownConnection.send(this);
-		this.connector.close(1001, "Control Quit")
-		await events.once(this.connector, "close");
-	}
-}
-
-async function findCredentials(controlConfig) {
-	// Try looking in the default master config location
-	try {
-		let masterConfig = new config.MasterConfig();
-		await masterConfig.load(JSON.parse(await fs.readFile("config-master.json")));
-
-		let token = jwt.sign({ id: "api" }, masterConfig.get("master.auth_secret"), { expiresIn: 600 });
-		controlConfig.set("control.master_token", token);
-
-		let url;
-		if (masterConfig.get("master.https_port")) {
-			url = `https://localhost:${masterConfig.get("master.https_port")}`;
-		} else {
-			url = `http://localhost:${masterConfig.get("master.http_port")}`;
-		}
-		controlConfig.set("control.master_url", url);
-
-		return;
-
-	} catch (err) {
-		if (err.code !== "ENOENT") {
-			throw err;
-		}
-	}
-
-	// Try looking in the default slave config location
-	// In the future it's likely that tokens will be assigned more granular
-	// authorizations, meaning that you cannot use a slave token to control
-	// the cluster.
-	try {
-		let slaveConfig = new config.SlaveConfig();
-		await slaveConfig.load(JSON.parse(await fs.readFile("config-slave.json")));
-
-		controlConfig.set("control.master_token", slaveConfig.get("slave.master_token"));
-		controlConfig.set("control.master_url", slaveConfig.get("slave.master_url"));
-		return;
-
-	} catch (err) {
-		if (err.code !== "ENOENT") {
-			throw err;
-		}
-	}
-
-	throw new errors.StartupError("Couldn't find credentials to connect to the master server with");
-}
-
-
-class ControlGroup extends config.ConfigGroup {}
-ControlGroup.groupName = "control";
-ControlGroup.define({
-	name: "master_url",
-	description: "URL to connect to the master server at",
-	type: "string",
-	optional: true,
-});
-ControlGroup.define({
-	name: "master_token",
-	description: "Token to authenticate to master server with.",
-	type: "string",
-	optional: true,
-});
-ControlGroup.define({
-	name: "reconnect_delay",
-	title: "Reconnect Delay",
-	description: "Maximum delay to wait before attempting to reconnect WebSocket",
-	type: "number",
-	initial_value: 2,
-});
-ControlGroup.finalize();
-
-class ControlConfig extends config.Config { }
-ControlConfig.registerGroup(ControlGroup);
-ControlConfig.finalize();
-
-
-async function startControl() {
-	yargs
-		.scriptName("clusterctl")
-		.usage("$0 <command> [options]")
-		.option('config', {
-			nargs: 1,
-			describe: "config file to get credentails from",
-			default: "config-control.json",
-			defaultDescription: "auto",
-			type: 'string',
-		})
-		.command("control-config", "Manage Control config", config.configCommand)
-		.wrap(yargs.terminalWidth())
-	;
-
-	for (let command of commands.values()) {
-		command.register(yargs);
-	}
-
-	const args = yargs.demandCommand(1, "You need to specify a command to run")
-		.strict()
-		.argv
-	;
-
-	config.finalizeConfigs();
-
-	console.log(`Loading config from ${args.config}`);
-	let controlConfig = new ControlConfig();
-	try {
-		await controlConfig.load(JSON.parse(await fs.readFile(args.config)));
-
-	} catch (err) {
-		if (err.code === 'ENOENT') {
-			console.log("Config not found, initializing new config");
-			await controlConfig.init();
-
-		} else {
-			throw err;
-		}
-	}
-
-	let commandName = args._[0];
-	if (commandName === "control-config") {
-		await config.handleConfigCommand(args, controlConfig, args.config);
-		return;
-	}
-
-	// The remaining commands require connecting to the master server.
-	if (!controlConfig.get("control.master_url") || !controlConfig.get("control.master_token")) {
-		await findCredentials(controlConfig);
-	}
-
-	let controlConnector = new ControlConnector(
-		controlConfig.get("control.master_url"),
-		controlConfig.get("control.reconnect_delay"),
-		controlConfig.get("control.master_token")
-	);
-	let control = new Control(controlConnector);
-	try {
-		await controlConnector.connect();
-	} catch(err) {
-		if (err instanceof errors.AuthenticationFailed) {
-			throw new errors.StartupError(err.message);
-		}
-		throw err;
-	}
-
-	if (commands.has(commandName)) {
-		command = commands.get(commandName);
-
-		try {
-			await command.run(args, control)
-
-		} catch (err) {
-			if (err instanceof errors.CommandError) {
-				console.error(`Error running command: ${err.message}`);
-				process.exitCode = 1;
-
-			} else if (err instanceof errors.RequestError) {
-				console.error(`Error sending request: ${err.message}`);
-				process.exitCode = 1;
-
-			} else {
-				throw err;
-			}
-
-		} finally {
-			await control.shutdown();
-		}
-	}
-
-	//XXX control.close("done");
-}
-
-module.exports = {
-	// for testing only
-	_formatOutputColored: formatOutputColored,
-	_resolveInstance: resolveInstance,
-	_Control: Control,
-
-	_commands: commands,
-}
-
-
-if (module === require.main) {
-	console.warn(`
-+==========================================================+
-I WARNING:  This is the development branch for the 2.0     I
-I           version of clusterio.  Expect things to break. I
-+==========================================================+
-`
-	);
-	startControl().catch(err => {
-		if (!(err instanceof errors.StartupError)) {
-			console.error(`
-+----------------------------------------------------------------+
-| Unexpected error occured while starting control, please report |
-| it to https://github.com/clusterio/factorioClusterio/issues    |
-+----------------------------------------------------------------+`
-			);
-		} else {
-			console.error(`
-+-------------------------------+
-| Unable to to start clusterctl |
-+-------------------------------+`
-			);
-		}
-
-		console.error(err);
-		process.exitCode = 1;
-	});
-}

docker-compose.test.yml

@@ -0,0 +1,7 @@
+version: '3'
+
+services:
+  unit_test:
+    image: clusterio
+    build: .
+    command: npm run test

docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3'
+
+services:
+  master:
+    build: .
+    command: npx clusteriomaster run
+  slave:
+    build: .
+    command: npx clusterioslave run

docs/communication.md

@@ -49,9 +49,10 @@ Message
 
 There's two types of messages currently implemented apart from the
 Link builtin close message and that's Request and Event.  The builtin
-messages are defined in [lib/link/messages.js](../lib/link/messages.js)
-and are attached to links via the attachAllMessages function, which is
-usally called in the Link subclass constructor.
+messages are defined in
+[packages/lib/link/messages.js](/packages/lib/link/messages.js) and are
+attached to links via the attachAllMessages function, which is usally
+called in the Link subclass constructor.
 
 The messages define which links they are valid on and only attaches
 handlers for those links.  There's also a forwarding mechanism for when

docs/config-system.md

@@ -6,16 +6,16 @@ configuration system in Clusterio, and unless you're developing
 Clusterio it's probably not going to be very useful for you.</sub>
 
 Clusterio uses a group based configuration system to manage settings for
-the master server, instances, slaves, and the clusterctl utility.
+the master server, instances, slaves, and the clusterioctl utility.
 Plugins get their own config group for the master server and instances
 which they can add their own fields to.  The fields are defined early in
 the startup, and the groups and their fields for disabled plugins will
 still be created.
 
 The built-in config groups and their fields are defined in
-[/lib/config/definitions.js](lib/config/definitions.js) while plugins
-can add their own config groups to the `MasterConfigGroup` and
-`InstanceConfigGroup` properties of the `info.js` export using
+[packages/lib/config/definitions.js](/packages/lib/config/definitions.js)
+while plugins can add their own config groups to the `MasterConfigGroup`
+and `InstanceConfigGroup` properties of the `info.js` export using
 subclasses of `PluginConfigGroup`.  The plugin group has the enabled
 field pre-defined on it, and it's used to enable/disable plugins.
 
@@ -28,6 +28,16 @@ after the groupName have been set and before the class has been
 finalized.  The `define` method takes an object as argument with the
 following properties:
 
+**access**:
+    The locations this config field can be read and modified from.  This
+    is an array of strings which can contain the values `"master"`,
+    `"slave"` and `"control"`.  If a party is missing in this array then
+    that party will not receive the value of this field when the config
+    is shared, and will be unable to modify the field.  Takes the value
+    of `defaultAccess` set on the config group class if not passed.
+    Note that instance config groups should have both `"master"` and
+    `"slave"` for all of its fields or unexpected behaviour may occur.
+
 **type**:
     The type of config value this field will support.  The supported
     values are boolean, string, number, and object, and if an attept is

docs/configuration.md

@@ -0,0 +1,475 @@
+Configuration
+=============
+
+
+Master Configuration
+--------------------
+
+### master.name
+
+Name of the cluster.  Used to distinguish it from other clusters and is
+prepended to the name of servers in the server list.
+
+Defaluts to "Your Cluster".
+
+
+### master.database_directory
+
+Directory used to store cluster data to.  Needs to be writeable by the
+master server.  This is also used by plugins to store its data.
+
+Defaults to "database".
+
+
+### master.http_port
+
+Port to host HTTP server on.  If set to null no HTTP server will be
+exposed.  At least one of this and master.https_port needs to be set to
+a port.
+
+Defaults to 8080.
+
+
+### master.https_port
+
+Port to host HTTPS server on.  Uses master.tls_certificate and
+master.tls_private_key as the certificate and private key for the HTTPS
+server.  If set to null no HTTPS server will be exposed.  At least one
+of this and master.http_port needs to be set to a port.
+
+See the [Setting up TLS](/docs/setting-up-tls.md) document for a guide
+to setting up HTTPS with Clusterio.
+
+Defaults to null.
+
+
+### master.bind_address
+
+IP address to bind the HTTP and HTTPS ports on.  Useful to limit which
+interface to accept connections from.  If set to null the unspecified
+IPv6 address will be used, which on most systems means it will also
+listen on the unspecified IPv4 address and accept connections from all
+interfaces.
+
+Defaults to null.
+
+
+### master.external_address
+
+External address the master server is accessible on.  Currently only
+used for `clusteriomaster bootstrap create-ctl-config` in order to give
+the right url to connect to.  This should be a full URL ending with a /.
+
+Defaults to null meaning assume localhost.
+
+
+### master.tls_certificate
+
+Path to TLS certificate to use for the HTTPS server when
+master.https_port is configured.
+
+Defaults to "database/certificates/cert.crt".
+
+
+### master.tls_private_key
+
+Path to TLS private key to use for the HTTPS server when
+master.https_port is configured.
+
+Defaults to "database/certificates/cert.key".
+
+
+### master.auth_secret
+
+Base64 encoded authentication secret used to verify tokens issued to
+slaves, users, and sessions.  This should contain 256 bytes of random
+data, and if changed will cause all tokens become invalid.  Keep this
+secret if leaked an attacker could easily compromise the cluster.
+
+Defaults to null which means generate a secure secret on startup.
+
+
+### master.heartbeat_interval
+
+Interval in seconds heartbeats are sent out at for WebSocket
+connections.  If a connection hasn't received a heartbeat in 2 times the
+heartbeat interval it will be considered stale and closed.  A lower
+value means less time between connections going stale and them being
+closed.
+
+Defaults to 45
+
+
+### master.connector_shutdown_timeout
+
+Timeout in seconds to use while shutting down the master server for link
+sessions that currently don't have an active connection and haven't been
+closed down properly.  This happens when the network between a slave and
+the master server goes down unexpectedly, or the slave goes down without
+closing the connection properly either do to crashing or the machine
+it's hosted on going down.  If the timeout expires the session is
+terminated and data loss may occur, if the slave went down this is
+unavoidable.
+
+Defaults to 30.
+
+
+### master.metrics_timeout
+
+Timeout in seconds before a call to gather metrics from a slave times
+out.  This should be less than both the timeout and collection interval
+configured for the collection job in Prometheus.
+
+Defaults to 8.
+
+
+### master.default_role_id
+
+Role to automatically grant to new users.  If null no role is granted.
+
+Defaults to 1 which correspond to the default Player role.
+
+
+### <plugin_name>.enabled
+
+Whether to enable the given plugin on the master server.  Plugins
+disabled on the master server will not be loaded on instances.
+
+Defaults to true.
+
+
+Slave Configuration
+-------------------
+
+### slave.name
+
+Name of the slave.  Shows up in slave lists and is used to reference
+this slave in the clusterioctl command line interface.
+
+Defaults to "New Slave".
+
+
+### slave.id
+
+Immutable numeric id of the slave which uniquely identifies this slave in
+the cluster.
+
+Defaults to a random 31 bit number.
+
+
+### slave.factorio_directory
+
+Directory to look for the Factorio server(s) in.  This can either point
+to a Factorio server directory, or to a directory containing multiple
+versions of the Factorio server.
+
+Defaults to "factorio".
+
+
+### slave.instances_directory
+
+Directory to store instances in.  One sub-directory will be created in
+it for each instance assigned to this slave and these sub-directories
+will contain logs, saves, and various auto generated configuration files
+for the instances.
+
+Defaults to "instances".
+
+
+### slave.master_url
+
+URL to connect to the master server to.
+
+Defaults to "http://localhost:8080/".
+
+
+### slave.master_token
+
+Access token used for authenticating with the master server.  You can
+generate an access token with `clusterioctl slave generate-token
+--id <slave-id>`, or use the `clusterioctl slave create-config` to
+create a new slave config with the correct url and token in it.
+
+Defaults to "enter token here".
+
+
+### slave.tls_ca
+
+Path to certificate in PEM format to use for validating a TLS connection
+to the master server.  If you have a self signed certificate on the
+master server you will need to copy the certificate to your slaves and
+set it as the certificate authority with this option.
+
+Defaults to null meaning use Node.js's default set of trusted
+certificate authorities.
+
+
+### slave.public_address
+
+External address instances hosted on this slave can be accessed on.
+This is used by plugins like Server Select to give the correct address
+to switch between instances in-game.
+
+Defaults to "localhost".
+
+
+### slave.reconnect_delay
+
+Delay in seconds to wait after connection to the master server is
+dropped before attempting to reconnect to it.  The actual delay on each
+reconnect will be a random number between 0 and this configured value to
+avoid all slaves trying to reconnect at the same time.
+
+Defaults to 5.
+
+
+Instance Configuration
+----------------------
+
+### instance.name
+
+Name of the instance, shown in instance lists and used to reference to this
+instance in the clusterioctl command line interface.
+
+Defaults to "New Instance"
+
+
+### instance.id
+
+Imutable numeric id of the instance which uniquely identifies this
+instance in the cluster.
+
+Defaluts to a random 31 bit integer.
+
+
+### instance.assigned_slave
+
+Slave this instance is assigned to.  To change this you need to use the
+separate assign instance interface.  After being assigned to a slave the
+directory for the instance is created on the slave.
+
+Defaults to null meaning not assigned.
+
+
+### instance.auto_start
+
+If enabled start this instance when the slave it is assigned to is
+started up.  Does not affect start or stop of the instance while the
+slave is running.
+
+Defaults to false
+
+
+### factorio.version
+
+Version of Factorio to use for this instance.  Can be a version like
+"1.0.0" or special string "latest" meaning the latest version of
+Factorio found on the slave.
+
+Defaults to "latest"
+
+
+### factorio.game_port
+
+UDP port to run the Factorio server on.  If null a random port in the
+dynamyc range assigned each time the instance starts.
+
+Defaults to null
+
+
+### factorio.rcon_port
+
+TCP port to start the RCON interface on the Factorio server on.  If null
+a random port in the dynamic range is assigned each time the instance
+starts.
+
+Defaults to null
+
+
+### factorio.rcon_password
+
+Password to use for the RCON interafec.  If null a random secure
+password is generated and used each time the instance starts.
+
+Defaults to null
+
+
+### factorio.enable_save_patching
+
+Whether to use save patching or not.  When enabled lua code is patched
+into the game save before starting it.  Most plugins require the use of
+save patching to function.  Turning it off makes it possible to use
+Clusterio to run and manage regular vanilla games and scenarios not
+compatible with Clusterio.
+
+Defaults to true
+
+
+### factorio.enable_whitelist
+
+Turn on the whitelist on the server.
+
+Defaults to false
+
+
+### factorio.settings
+
+Object with the settings to put into `server-settings.json` for the
+Factorio server.  The settings in `data/server-settings.example.json` of
+the Factorio installation is used as the base and properties specified
+here overrides the properties there.
+
+Changes to the following properties will be applied live if the instance
+is running while it is changed: `afk_autokick_interval`,
+`allow_commands`, `autosave_interval`, `autosave_only_on_server`,
+`description`, `ignore_player_limit_for_returning_players`,
+`max_players`, `max_upload_slots`, `max_upload_in_kilobytes_per_second`,
+`name`, `only_admins_can_pause_the_game`, `game_password`,
+`require_user_verification`, `tags`, `visibility`.
+
+Defaults to {"tags":["clusterio"],"auto_pause":false}.
+
+
+### factorio.verbose_logging
+
+Pass the `--verbose` flag to Factorio when starting the server.  This
+prints more messages to the log.
+
+Defaults to false
+
+
+### factorio.strip_paths
+
+Strip down absolute paths in the server log going to files inside the
+instance directory such that they are relative to the instance
+directory.  This improves the signal to noise ratio of the log.
+
+Sample output when disabled:
+
+    0.487 Loading map /srv/clusterio/instances/test/saves/world.zip: 2615888 bytes.
+    0.614 Checksum for script /srv/clusterio/instances/test/temp/currently-playing/control.lua: 2390553941
+    1.277 Script @/srv/clusterio/instances/test/temp/currently-playing/modules/example/test.lua:7: Example log line.
+
+Sample output when enabled:
+
+    0.487 Loading map saves/world.zip: 2615888 bytes.
+    0.614 Checksum for script temp/currently-playing/control.lua: 2390553941
+    1.277 Script @modules/example/test.lua:7: Example log line.
+
+Defaults to true
+
+### factorio.sync_adminlist
+
+Synchronize in-game admin list with admin status of the users in
+cluster.  If enabled Clusterio will generate a new
+`server-adminlist.json` on instance startup based on the users in the
+cluster with admin status set to true, and keep the admins in sync with
+the users that have admin status while the instance is running.
+
+Defaults to true
+
+
+### factorio.sync_whitelist
+
+Synchronize in-game whitelist with the whitelisted status of the users
+in the cluster.  If enabled Clusterio will generate a new
+`server-whitelist.json` on instance startup based on the users in the
+cluster with whitelisted status set to true, and keep the whitelisted
+users in sync with the users that have whitelisted status while the
+instance in running.
+
+Defaults to true
+
+
+### factorio.sync_banlist
+
+Synchronize in-game banlist with the banned status of users in the
+cluster.  If enabled Clusterio will generate a new `server-banlist.json`
+on instance startup based on users in the cluster with banned status set
+to true, and keep the banned users in sync with the users that have the
+banned status while the instance is running.
+
+Defaults to true
+
+
+### factorio.max_concurrent_commands
+
+Maximum number of commands being transmitted into the game in parallel.
+Since the rate at which long commands are streamed into the game is by
+default limeted to 100 bytes/tick and scales down to 25 bytes/tick when
+there's 20 players connected, long commands can hold up the command
+interface and cause large latencies in getting data pushed into the
+game. (The rates are controlled by the `segment_size` options in the
+server settings.)  To counteract this Clusterio sends up to this
+configured value number of commands in parallel over the RCON interface.
+This causes the game updates sent to clients to have this many command
+streams in parallel contained in them, drastically increasing the
+maxiumum size they can become.  Since commands are only processed so
+ofter the maximum rate of commands that can be sustained is roughly 30
+times this configured value per second, though note that large commands
+will lower this value.
+
+The game updates sent by the server is split into roughly 500 byte
+packets, and if any one of those parts are lost the entire update is
+resent.  This means a high number of concurrent commands can amplify
+resends due to packet loss and degrade the player experience up to the
+point of becomming completely unplayable.
+
+Prior to 0.18.7 the maximum practical size of game update sent where
+arround 8 kB due to the UDP receive buffer on windows defaulting to this
+value, larger updates would not get through the receive buffer and cause
+the connection to drop.  To stay below the 8 kB limit the maximum safe
+value for this option would be around 7000 / (3 * game_speed *
+maximum_segment_size), which is around 20 in normal circumstances.
+
+Defaults to 5
+
+
+### <plugin_name>.enabled
+
+Whether to enable the given plugin on the instance.  Note that plugins
+need to be enabled on the master server in order for them to be loaded
+on instances.
+
+Defaults to true
+
+
+Control Configuration
+---------------------
+
+### control.master_url
+
+URL to connect to the master server to.
+
+Defaults to null meaning complain about it not being set and exit.
+
+
+### control.master_token
+
+Access token used for authenticating with the master server.  You can
+generate an access token with `clusteriomaster bootstrap
+generate-user-token <username>`, or use the `clusteriomaster bootstrap
+create-ctl-config <username>` to create a new ctl config with the
+correct url and token in it.
+
+Defaults to null meaning complain about it not being set and exit.
+
+
+### control.tls_ca
+
+Path to certificate in PEM format to use for validating a TLS connection
+to the master server.  If you have a self signed certificate on the
+master server you will need to copy the certificate to the computer you
+run clusterctl from and set it as the certificate authority with this
+option.
+
+Defaults to null meaning use Node.js's default set of trusted
+certificate authorities.
+
+
+### control.reconnect_delay
+
+Duration in seconds to wait before attempting to reconnect with the
+master server after the connection is dropped.
+
+Defaults to 2.

docs/contributing.md

@@ -11,6 +11,8 @@ Contents
 
 - [Workflow](#workflow)
     - [Starting a Feature Branch](#starting-a-feature-branch)
+    - [Setting up the Development Environment](#setting-up-the-development-environment)
+    - [Checking your code for errors](#checking-your-code-for-errors)
     - [Submitting a Pull Request](#submitting-a-pull-request)
     - [Editing your Feature Branch](#editing-your-feature-branch)
     - [Resolving Merge Conflicts](#resolving-merge-conflicts)
@@ -18,6 +20,7 @@ Contents
 - [Changelog](#changelog)
 - [Supported Node.js and Factorio Version](#supported-node.js-and-factorio-version)
 - [Code Style](#code-style)
+    - [Naming Style](#naming-style)
     - [Indenting](#indenting)
     - [Line Length](#line-length)
 
@@ -55,6 +58,19 @@ from the main repository.  Do not run `git pull upstream` it will mess
 thing up!
 
 
+### Setting up the Development Environment
+
+The project is organized into several packages that are all managed
+through Lerna.  In order to get the development environment up and
+running you will need to run the following commands:
+
+    npm install
+    npx lerna bootstrap
+
+This installs dependencies needed by tests and links the packages up so
+they work from the git work tree.
+
+
 ### Starting a Feature Branch
 
 Development is done exclusively in feature branches.  This means the
@@ -76,6 +92,18 @@ you've used before.  But it's nice to have its name be somewhat
 descriptive of what it implements.
 
 
+### Checking your code for errors
+
+There's automated tests and linting setup for the project.  Please do
+add tests for any code you add and run both:
+
+    npm run test
+    npm run lint
+
+To check that your changes pass the integration tests as well as the
+ESLint rules set up for the project.
+
+
 ### Submitting a Pull Request
 
 Once you've commited changes to your feature branch that you want to
@@ -159,7 +187,7 @@ of this log should be self explanatory, please follow it carefully.
 Supported Node.js and Factorio Versions
 ---------------------------------------
 
-Clusterio run on Node.js v12 and up, v11.13.0+ and v10.16.0+, make sure
+Clusterio run on Node.js v12 and up, and v10 from v10.16.0, make sure
 to check that the Node builtin API's and npm packages you use are
 supported on these versions.  The Travis CI tests are runned on whatever
 version of Node.js v10 is the default, which at the time of writing is
@@ -179,25 +207,40 @@ The style of the code is a bit of a mixed bag at the moment.  But there
 are at least a few things that have been agreed upon.
 
 
+### Naming style
+
+JavaScript variables in general uses camelCase.  The exception is for
+classes/constructors which use PascalCase and variables destructured
+from over the wire objects.
+
+Fields in configs and messages sent over links uses
+lowercase_underscore.
+
+JavaScript files are named using lowercase_underscore for files that
+export multiple items.  Files containing and exporting a single class
+should be named the same as the class in PascalCase.
+
+Imported modules from the lib package are prefixed with lib to make them
+easier to distinguish, as the names tend to be very generic.  E.g.:
+
+    const libLink = require("@clusterio/lib/link");
+    const libLuaTools = require("@clusterio/lib/lua_tools");
+
+
+For lua code lowercase_underscore is used for everything.
+
+
 ### Strings
 
 JavaScript and Lua code should use double quoted strings (`"string"`)
 for all strings with the only exception that strings that contain double
-quotes may use single quotes (i.e., `'String with "quotes"'`.)  For
+quotes may use single quotes (i.e., `'String with "quotes"'`).
 JavaScript template literals may also be used where appropriate.
 
-There are places in the codebase that are using `'single'` quoted
-strings.  Should you be modifying such code use double quotes on new and
-modified lines and leave the rest untouched.
-
 
 ### Indenting
 
-JavaScript code should be indented with tabs.  There are places in the
-codebase that are mixing tabs and spaces or uses only spaces for
-indents.  Should you be modifiying such code that is mixing spaces and
-tabs as indending you should update the indenting to use tabs _on the
-code you change_ while leaving the rest of the code untouched.
+JavaScript code should be indented with tabs.
 
 Lua code should be indented with 4 spaces.
 

docs/developing-for-clusterio.md

@@ -22,7 +22,7 @@ Factorio Mods
 -------------
 
 Factorio mods that do not interact with Clusterio should not have any
-conciderations that need to be taken into account for them to work with
+considerations that need to be taken into account for them to work with
 Clusterio.
 
 
@@ -36,12 +36,12 @@ Clusterio to work.  The main limitation is that the
 lib must be used, and control.lua cannot contain any code other than
 calls to event_handler loading the relevant libs of the scenario.
 This is because the control.lua file will be overwritten by the save
-patcher.  Scenarios also cannot use the event registring functions
+patcher.  Scenarios also cannot use the event registering functions
 exposed by `script` such as `script.on_event` or `script.on_nth_tick` as
 using these will overwrite the handlers registered by the
 event_handler lib.
 
-A breif description of the usage of the event_handler library is
+A brief description of the usage of the event_handler library is
 provided in the [event_handler interface](#event_handler-interface)
 section.
 
@@ -82,10 +82,10 @@ a part of Factorio since at least 0.17.4, and was moved from base to
 core in 0.17.63 but is still not documented anywhere.  A brief
 description is provided here for this reason.</sub>
 
-The event_handler lib provides a simple interface for registring
+The event_handler lib provides a simple interface for registering
 multiple callbacks to the same events without having to be concerned
 with callback chaining or overwriting callbacks defined elsewhere.  It
-works by taking over the task of registring the actual callbacks with
+works by taking over the task of registering the actual callbacks with
 Factorio, providing its own interface for the rest of the code to use.
 
 The sole function of interest exported by event_handler is `add_lib`
@@ -111,11 +111,11 @@ of which are optional:
 - `add_remote_interface`:
     Callback called before on_init/on_load callbacks<sup>[1]</sup>.  It
     has no special meaning and receives no arguments but should be used
-    for registring remote interfaces.
+    for registering remote interfaces.
 - `add_commands`:
     Callback called before on_init/on_load callbacks<sup>[1]</sup>.  It
     has no special meaning and receives no arguments but should be used
-    for registring commands.
+    for registering commands.
 
 <sub>1: Before 0.17.69 these callbacks were called after
 on_init/on_load.</sub>
@@ -161,11 +161,13 @@ which accepts a table and calls `add_lib` for each value in the table.
 Communicating with Clusterio
 ----------------------------
 
-There's a poorly to virtuarly undocumented HTTP interface that can be
-used to communicate with Clusterio to do things like get the status of
-instances, grab items from the store, run commands on instances, etc,
-etc.  For now you can read the source code, and maybe ask on the Discord
-for some pointers if you get stuck.
+Clusterio uses a homebrew protocol based on sending JSON payloads over
+a WebSocket connection.  See [socket.md](socket.md) for the
+implementation details of it.
+
+It's also possible to write a plugin for Clusterio that exposes a custom
+interface over HTTP, WebSocket or any other technology supported by
+Node.js.
 
 
 Clusterio Modules
@@ -177,7 +179,9 @@ modules that are loaded into the game if you don't need the capabilites
 of the plugin system.  The save patcher puts modules into the `modules`
 folder of the Factorio save and adds code to `control.lua` to load the
 module according to the `load` and `require` options to the module.json
-file.
+file.  Like with scenarios for Clusterio the [event_handler
+interface](#event_handler-interface) has to be used for any event
+subscriptions in modules.
 
 Stand alone modules are placed into the modules folder of Clusterio,
 plugin modules are located in the module folder of the plugin.  In
@@ -298,7 +302,7 @@ Raised after the name and id of an instance has been updated.  This may
 occur even if the id and name didn't change.
 
 Event data:
-- `instance_id`: The id of the instance.
+- `instance_id`: the id of the instance.
 - `instance_name`: the name of the instance.
 
 ##### on_server_startup
@@ -309,14 +313,14 @@ the `on_init` and `on_configuration_changed` event for modules.  It is
 invoked on the first tick the server runs after the save has been
 patched, before most other events.
 
-Use this event to initialize and and/or migrate the data structures
+Use this event to initialize and/or migrate the data structures
 you need.  Keep in mind that both mods and Clusterio modules can switch
 from any version to any version so it should be able to handle both
 forwards and backwards migrations gracefully.
 
 #### clusterio_api.send_json(channel, data)
 
-Send json data to Clusterio over the given channel.  The `data` argument
+Send JSON data to Clusterio over the given channel.  The `data` argument
 should be table that can be serialized with `game.table_to_json`.
 Clusterio plugins can listen to channels and will receive an event with
 the data sent here.  See the [Communicating with Factorio
@@ -376,14 +380,12 @@ passed.
 #### serialize.deserialize_inventory(LuaInventory, serialized_inventory)
 
 Deserialize a previously serialized inventory into the target
-LuaInventory.  Overwrites slots that has content in the serialized
+LuaInventory.  Overwrites slots that have content in the serialized
 inventory.
 
 
 Clusterio Plugins
 -----------------
 
-The plugin interface is undergoing a major rewrite, and will be
-documented once that is done.  See
-[issue #220](https://github.com/clusterio/factorioClusterio/issues/220)
-for details.
+See the [Writing Plugins](writing-plugins.md) document for details on
+the plugin interface and how to write plugins for Clusterio.

docs/how-it-works.md

@@ -9,7 +9,22 @@ based on the built in
 library to Factorio and requires scenarios to be coded specifically for
 it.  Mods on the other hand work without modifications.
 
+This can be disabled with the `factorio.enable_save_patching` config on
+a per instance basis, however disabling it means that modules are not
+loaded into the game and most plugins will not function.
+
 <sub>1: Before version 2.0 of Clusterio this was done at runtime
 with a patcher called Hotpatch, but due to the difficulty in supporting
 run time patching and the lack of documentation it was replace it with
 save patching instead.</sub>
+
+
+User Accounts
+-------------
+
+The master server creates and stores accounts for all Factorio users
+that log into instances under its control.  These accounts are used to
+store per player data and authorize management of the cluster.  By
+default users accounts are assigned a Player role that gives a limited
+read access to the cluster resources, and admins of the cluster can be
+assigned a Cluster Admin role to get access to manage it.

docs/jsdoc.json

@@ -11,13 +11,13 @@
     },
     "source": {
         "include": [
-            "master.js",
-            "slave.js",
-            "clusterctl.js",
-            "lib/",
-            "plugins/",
-            "routes/",
-            "routes.js"
-        ]
+            "packages/master/src/",
+            "packages/master/master.js",
+            "packages/slave/slave.js",
+            "packages/ctl/ctl.js",
+            "packages/lib/",
+            "plugins/"
+        ],
+        "excludePattern": "node_modules"
     }
 }

docs/managing-a-cluster.md

@@ -0,0 +1,132 @@
+Managing a Cluster
+==================
+
+Clusterio clusters are managed through the master server by using the
+`clusterioctl` command line interface which is invoked by running
+`npx clusterioctl <command>` in the clusterio directory.  This document
+uses the shorthand `ctl> foo` to indicate `npx clusterioctl foo` should
+be executed.  Mandatory parameters are shown in `<angles bracket>` and
+optional pameters are in `[square brackets]`.
+
+Before `clusterioctl` can be used it needs to be configured for the
+cluster it will connect to.  The easiest way to do this is to run
+`npx clusteriomaster bootstrap create-ctl-config <username>` on the
+master server, which creates the necessary `config-control.json` for
+managing the cluster as the given user.
+
+
+Slaves
+------
+
+To be written.
+
+
+Instances
+---------
+
+To be written.
+
+
+Users
+-----
+
+Clusterio automatically creates user accounts for all players that join
+an instance when save patching is enabled.  These accounts are used to
+store per player data shared between instances, like if the player
+should be whitelisted, admin or banned.
+
+### List users
+
+    ctl> user list
+
+Lists all user accounts in the cluster along with some data for each.
+
+
+### Create User
+
+    ctl> user create <name>
+
+Creates a new empty user account for the given Factorio player.  Note
+that case matters here.  This is usually not required as user accounts
+are created automatically for players that join instances with save
+patching enabled.
+
+
+### Promote User to Server Admin
+
+    ctl> user set-admin <name> [--revoke] [--create]
+
+Promotes the user given to in-game admin on instances with the
+`sync_adminlist` option enabled.  If the `--revoke` switch is used the
+user is removed from the adminlist.
+
+Since admin status is a part of the account data the account must exist
+for this to succeed, passing `--create` will create the account if it
+does not exist.
+
+**Note:** Being a server admin does not grant any access to manage the
+cluster.  See [set-roles](#set-cluster-roles) for adding roles which
+grant access to managing the clusetr.
+
+
+### Whitelist User
+
+    ctl> user set-whitelisted <name> [--remove] [--create]
+
+Add the user given to the whitelist on instances with the
+`sync_whitelist` option enabled.  If the `--remove` switch is used the
+user is removed from the whitelist.
+
+Since whitelisted status is a part of the account data the account must
+exist for this to succeed, passing `--create` will create the account if
+it does not exist.
+
+
+### Ban User
+
+    ctl> user set-banned <name> [--reason <message>] [--pardon] [--create]
+
+Ban user in-game from instances with the `sync_banlist` option enabled.
+Reason is a message that will be shown to the user when attemption to
+log in.  If the `--pardon` switch is used it removes the ban.
+
+Since ban status is a part of the account data the account must exist
+for this to succeed, passing `--create` will create the account if it
+does not exist.
+
+Note: This bans the user from logging in to Factorio servers in the
+cluster, it does not revoke access to any cluster management they might
+have, see next section on setting roles for revoking tha.
+
+
+### Set Cluster Roles
+
+    ctl> user set-roles <name> [roles...]
+
+Replaces the roles the user has in the cluster with a new list of roles.
+Calling with an empty roles argument will remove all roles from the
+user.
+
+By default there's a roled named Cluster Admin which grants access to
+everything and a role named Player which grants a limited read access to
+the cluster, see [the section on roles](#roles) for more information
+about setting up roles and permissions.
+
+
+### Delete user
+
+    ctl> user delete <name>
+
+Deletes everything stored on the master server for this user.
+
+Note: If the player is banned from the cluster this will effectively
+unban them, as the ban status is stored with the user account.
+
+Note: If the player joins the cluster again a new account will be made
+for them automatically.
+
+
+Roles
+-----
+
+To be written.

docs/setting-up-tls.md

@@ -0,0 +1,112 @@
+Setting up TLS
+==============
+
+By default Clusterio will listen for connections over HTTP which is fine
+for local networks but over the Internet it should idealy be over HTTPS
+as there are some credentials that could be exposed to eavesdroppers.
+The recommended setup is to have an ordinary HTTPS server software like
+Apache or Nginx to act as a reverse proxy in front of the Clusterio
+master server.  The reverse proxy would then be set up with a TLS
+certificate from a Certificate Authoritiy like [Let's
+Encrypt](https://letsencrypt.org).  Alternatively you can have Clusterio
+listen for connections over HTTPS directly by providing it with a TLS
+certificate and private key.
+
+Note that a TLS certificate signed by a recognized Certificate Authority
+in general requires that you own a domain.  See the last section for how
+to create and use a self-signed certificate for an IP address if you
+don't have a domain.
+
+
+Using a reverse proxy
+---------------------
+
+Configure the proxy to forward requests over HTTP to the port the master
+server is listening to so that for example a request to
+`https://example.com/instances/` is forwarded to
+`http://localhost:8080/instances/`.  It is possible to host the
+interface under a sub-path, in that case a request under the sub-path
+should be mapped to the master server with the sub-path stripped out.
+For example `https://example.com/sub/path/instances/` would be mapped to
+`http://localhost:8080/instances`.
+
+Additianlly WebSocket connections to `/api/socket` also needs to be
+forwarded.  Typically this requires explicit configuration and support
+for forwarding WebSocket connections.  If the interface hosted under a
+sub-path the `/api/socket` address is relative to the sub-path and the
+sub-path also needs to be stripped out when forwarded.
+
+You can also optionally set the master server to listen only on the
+loopback interface to prevent it being directly connected to from the
+outside.
+
+    npx clusteriomaster config set master.bind_address 127.0.0.1
+
+
+### Apache Config
+
+To proxy with Apache make sure that `proxy_module`, `proxy_http_module`
+and `proxy_wstunnel_module` is loaded in the config, then add the
+following directives to wherever is the approriate place for your
+flavour of Apache config organization.
+
+```apache
+<Location /sub/path/>
+    ProxyPass "http://localhost:8080/"
+</location>
+<Location /sub/path/api/socket>
+    ProxyPass "ws://localhost:8080/api/socket"
+</Location>
+```
+
+Replace `/sub/path/` with the location on the proxy you want to host the
+Clusterio interface under.  If the whole domain is to be dedicated to
+the Clusterio interface then `/sub/path/` should be just `/`.  Pay
+attention to the use of trailing slashes, as they are important.
+
+
+Hosting HTTPS directly from Node.js
+-----------------------------------
+
+Listening on HTTPS is supported by the Clusterio master server via the
+`master.https_port` option provided paths to a certificate file and a
+private key file is configured for the `master.tls_certificate` and
+`master.tls_private_key` options.  Both of these are expected to be
+unencrypted PEM files.
+
+
+Creating and using a self-signed certificate
+--------------------------------------------
+
+Create a self-signed certificate with OpenSSL by running
+
+    openssl req -x509 -newkey rsa:2048 -nodes -keyout key.pem -out cert.pem -days 3650 -subj /CN=localhost -addext "subjectAltName=IP:<ip>"
+
+substituting `<ip>` with the IP address the server will be accessed
+under, for example `IP:203.0.113.4`.  Multiple addresses can be given by
+supplying a comma sepparated list of `IP:address` pairs and DNS names
+can also be specified using the `DNS:example.com` format.  You need to
+supply every name you want to connect to the server under the
+subjectAltName field, for example to connect to the HTTPS port locally
+via an address like `https://localhost:4443/` you need to add
+`DNS:localhost`.
+
+This creates a private key file named `key.pem` and a public certificate
+file named `cert.pem` and these are used for the
+`master.tls_certificate` and `master.tls_private_key` config options
+respectively on the master server:
+
+    npx clusteriomaster config set master.tls_certificate cert.pem
+    npx clusteriomaster config set master.tls_private_key key.pem
+
+Additionally you will need copy the `cert.pem` file to all of the
+computers you want to set up as slaves and configure it as the
+`slave.tls_ca` option:
+
+    npx clusterioslave config set slave.tls_ca cert.pem
+
+You will also need to copy the `cert.pem` file to all of the computers
+you want to remotely manage the cluster via clusterctl and configure it
+as `control.tls_ca` option:
+
+    npx clusterioctl control-config set control.tls_ca cert.pem

docs/socket.md

@@ -114,6 +114,9 @@ Handshake messages
 Sent by the master server when the WebSocket connection has been opened.
 
 - version - string - The version of the master, e.g. "2.0.0".
+- plugins - Object<string, string> -
+    Object mapping plugin names to plugin version for plugins that are
+    loaded on the master server.
 
 ### `register_slave`
 
@@ -124,6 +127,9 @@ Slave handshake for establishing a new connection session.
 - version - string - The protocol version of the slave, e.g. "2.0.0".
 - name - string - Name of the slave.
 - id - integer - ID of the slave.
+- plugins - Object<string, string> -
+    Object mapping plugin names to plugin version for plugins that are
+    available on the slave.
 
 ### `register_control`
 
@@ -224,8 +230,8 @@ there's [Requests](#requests).  By convention message types ending in
 `*_event` are events.  The data payload of an event consists solely of
 event specific properties.
 
-See [lib/link/messages.js](/lib/link/messages.js) for the recognized
-events and their contents.
+See [packages/lib/link/messages.js](/packages/lib/link/messages.js) for
+the recognized events and their contents.
 
 
 Requests
@@ -239,8 +245,8 @@ replied to with a corresponding `*_response` message.  If an error
 occured while processing the request an error response containing an
 error property in the data payload is sent instead.
 
-See [lib/link/messages.js](/lib/link/messages.js) for the recognized
-requests and their contents.
+See [packages/lib/link/messages.js](/packages/lib/link/messages.js) for
+the recognized requests and their contents.
 
 ### `*_response`
 
@@ -253,28 +259,29 @@ requests and their contents.
 - error - string - Human readable text message describing the error.
 
 
-Graceful shutdown
------------------
+Graceful Disconnect
+-------------------
 
 To ensure no data is lost when closing the link it's important to
-send connection shutdown request and wait for it to complete before
+send a prepare disconnect request and wait for it to complete before
 closing the WebSocket.  Either party of the link can send this request
 and the party sending the request should be the one that initates the
 close of the WebSocket.
 
-### `connection_shutdown_request`
+### `prepare_disconnect_request`
 
-Request graceful shutdown of the connection.  The receiving party should
+Request graceful disconnect of the connection.  The receiving party should
 stop tasks that send events and requests over the link and wait for
 pending requests it has over the link to complete, but it should not
 stop responding to requests sent to it.  Only after this process is done
 should the response be sent.
 
-### `connection_shutdown_response`
+### `prepare_disconnect_response`
 
-Signals the shutdown is complete and that the connection can be closed.
-The party that initiatied the shutdown request may still send other
-requests/events over the link to finish its part of the shutdown.
+Signals the preparation for disconnect is complete and that the
+connection can be closed.  The party that initiatied the prepare
+disconnect request may still send other requests/events over the link to
+finish its part of the disconnection proceedure.
 
 If the error response variant is received it should be logged if
 possible and treated as a success.

docs/writing-plugins.md

@@ -11,13 +11,16 @@ Contents
 
 - [Plugin Structure](#plugin-structure)
 - [Defining the plugin class](#defining-the-plugin-class)
+- [Logging Messages](#logging-messages)
 - [Plugin Configuration](#plugin-configuration)
 - [Communicating with Factorio](#communicating-with-factorio)
 - [Defining Link Messages](#defining-link-messages)
     - [Defining Events](#defining-events)
     - [Defining Requests](#defining-requests)
 - [Sending Link Messages](#sending-link-messages)
+    - [Handling connection events](#handling-connection-events)
 - [Collecting Statistics](#collecting-statistics)
+- [Adding Custom Commands to clusterioctl](#adding-custom-commands-to-clusterioctl)
 
 
 Plugin Structure
@@ -30,22 +33,35 @@ The basic file structure of a plugin is the following.
       +- package.json
       +- master.js
       +- instance.js
+      +- control.js
       +- test/
       |  +- plugin.js
       +- module/
          +- module.json
          +- plugin.lua
 
-The most important file is the `info.js` file.  Without it the plugin
-will not recognized by Clusterio.  Here's an example of it:
+Clusterio plugins are Node.js packages that can be published on npm for
+ease of installation and distribution.  The usual guides for creating
+such packages apply.  At minimum the `package.json` file must contain a
+version entry.
 
-    const link = require('lib/link'); // For messages
+A possible workflow for developing plugins is to place the plugin in a
+sub-directory of where clusterio has been installed, and rely on Node.js
+searching up the folder heirarchy for it to find `@clusterio/lib`.  To
+add it to `plugin-list.json` so that it gets loaded use the `plugin add
+<path>` sub-command to either clusteriomaster, clusterioslave or
+clusterioctl.  Note that it's important that the path starts with ./ or
+../ (use .\ or ..\ on Windows).
+
+For a plugin the most important file is the `info.js` file.  Without it
+the plugin will not recognized by Clusterio.  Here's an example of it:
+
+    const libLink = require("@clusterio/lib/link"); // For messages
 
     module.exports = {
         name: "foo_frobber",
         title: "Foo Frobber",
-        description: "Does advance frobnication",
-        version: "0.8.1",
+        description: "Does advanced frobnication",
         instanceEntrypoint: "instance",
         masterEntrypoint: "master",
         messages: {
@@ -65,20 +81,15 @@ The following properties are recognized:
 **description**:
     Brief description of what the plugin does.  Currently not used.
 
-**version**:
-    Semver compatible version string of the plugin.  Currently only
-    servers as the default for the Clusterio module embedded in the
-    plugin, if any.
-
 **instanceEntrypoint**:
     Path to a Node.js module relative to the plugin directory which
     contains the InstancePlugin class definition for this plugin.  This
-    is an optional paramater.  A plugin can be for instances only but
-    for it to be able to send any messages to other instances it will
-    still have to be loaded on the master server.
+    is an optional paramater.  A plugin may have code only for instances
+    but it must still be loaded on the master in order for it to be
+    possible to load it on an instance.
 
 **InstanceConfigGroup**:
-    Subclass of `PluginConfigGroup` to defining the per instance
+    Subclass of `PluginConfigGroup` for defining the per instance
     configuration fields for this plugin.  See [Plugin
     Configuration](#plugin-configuration)
 
@@ -89,10 +100,16 @@ The following properties are recognized:
     master server.
 
 **MasterConfigGroup**:
-    Subclass of `PluginConfigGroup` to defining the master server
+    Subclass of `PluginConfigGroup` for defining the master server
     configuration fields for this plugin.  See [Plugin
     Configuration](#plugin-configuration)
 
+**controlEntrypoint**:
+    Path to a Node.js module relative to the plugin directory which
+    contains the ControlPlugin class definition for this plugin.  This
+    is an optional paramater.  A plugin can be made that only runs on
+    the clusterioctl side.
+
 **messages**:
     Object with link messages definitions for this plugin.  See guide
     for [defining link messages](#defining-link-messages) below.
@@ -106,10 +123,10 @@ into plugins is that they must be named the same as the plugin.
 While there is no standard for how to organize a plugin it's recommended
 to put the MasterPlugin class definition into master.js and the
 InstancePlugin class definition into instance.js.  You can put them into
-whatever file you want (even the same one for both)
+whatever file you want (even the same one for both).
 
 For both instanceEntrypoint and masterEntrypoint the path should not end
-with .js and it should use forward slashes for directory sepparators if
+with .js and it should use forward slashes for directory separators if
 any.
 
 
@@ -117,12 +134,12 @@ Defining the plugin class
 -------------------------
 
 The plugin class should derive from its respective base class defined in
-`lib/plugin`.  For example to define a MasterPlugin class the following
+`lib/plugin`.  For example, to define a MasterPlugin class the following
 code can be used:
 
-    const plugin = require('lib/plugin');
+    const libPlugin = require("@clusterio/lib/plugin");
 
-    class MasterPlugin extends plugin.BaseMasterPlugin {
+    class MasterPlugin extends libPlugin.BaseMasterPlugin {
         async init() {
             this.foo = 42;
             await this.startFrobnication();
@@ -136,11 +153,12 @@ code can be used:
     }
 
 For the instance plugin it's exactly the same except "Master" is
-replaced with "Instance".  The available hooks that you can override is
-documented on the base class [in lib/plugin.js](../lib/plugin.js).
+replaced with "Instance", and for the clusterioctl plugin "Control" is
+used.  The available hooks that you can override are documented in the
+base class [in lib/plugin.js](/packages/lib/plugin.js).
 
 It's best to avoid defining a constructor, but if you insist on defining
-one forward all arguments to be the base class.  E.g.:
+one, forward all arguments to the base class.  E.g.:
 
         constructor(...args) {
             super(...args);
@@ -149,23 +167,78 @@ one forward all arguments to be the base class.  E.g.:
         }
 
 The arguments passed may change, and attempting to modify them will
-result in unpredicatable behaviour.  The async init method always called
-immediatly after the constructor so there's little reason to do this.
+result in unpredicatable behaviour.  The async init method is always called
+immediatly after the constructor, so there's little reason to do this.
 
 
-Defining Configuration
-----------------------
+Logging Messages
+----------------
+
+The base plugin classes provide a winston logger for logging messages to
+the shared cluster log.  For instances a copy of the log is also stored
+on the slave the instance is on.  To use it, pass a string to one of the
+log levels functions, for example:
+
+    async init() {
+        this.logger.info("Initializing frobbing");
+    }
+
+Metadata covering which plugin and instance (for instance plugin
+classes) the log event happened on will automatically be attached to the
+log message.
+
+The available levels are fatal, error, warn, audit, info, and verbose.
+These levels have roughly the following meanings:
+
+- fatal: Unrecoverable error that prevents continuing execution of
+  the Node.js process.  Note that Plugins shouldn't have fatal errors.
+- error: An unexpected error condition occured, usually an exception
+  thrown when it wasn't expected.  Should be investigated and fixed.
+- warn: An potential error was detected, but operation continued
+  gracefully.  These may indicate that something is running
+  sub-optimally, or is not working as intedend.
+- audit: Record of actions performed by users of the cluster.  This
+  is usually emitted in response to management requests by a control
+  link.
+- info: A general log message.  May indicae a status or operation
+  completed.  This should not be emitted by periodic tasks that run
+  often.
+- verbose: A verbose log message which might be useful when
+  investigating issues, does not show by default.  Note that logging
+  messages is not a cheap operation even when the messages do not show
+  up and spammy logging should be avoided for the verbose level.
+  If the logs are still valuable in certain cases consider putting them
+  behind a config option.
+
+For guarding unexpected errors the best option is to log a short
+description along with the`stack` property of the Error:
+
+    try {
+        // Operation that should not throw but may end up throwing
+    } catch (err) {
+        this.logger.error(`Operation failed:\n${err.stack}`);
+    )
+
+For plugin hooks expections thrown are automatically catched and logged,
+but for event handlers registered on EventEmitters it's critical that
+exceptions are catched and handled appropriately.
+
+
+Plugin Configuration
+--------------------
 
 Clusterio provides a configuration system that handles storing,
 distributing, editing and validating config fields for you.  You can
-take advantage of it by subclassing `PluginConfigGroup`, setting the
+take advantage of it by subclassing `PluginConfigGroup`, setting
+`defaultAccess` to where config entries can be accessed from, setting the
 `groupName` to your plugin name, defining fields on it, finalizing it,
 and passing it as either `MasterConfigGroup` or `InstanceConfigGroup` in
 the `info.js` export.  For example in info.js:
 
-    const config = require("lib/config");
+    const libConfig = require("@clusterio/lib/config");
 
-    class MasterConfigGroup extends config.PluginConfigGroup { }
+    class MasterConfigGroup extends libConfig.PluginConfigGroup { }
+    MasterConfigGroup.defaultAccess = ["master", "slave", "control"];
     MasterConfigGroup.groupName = "foo_frobber";
     MasterConfigGroup.define({
         name: "level",
@@ -186,24 +259,38 @@ for example in the MasterPluginClass:
 
     async init() {
         let level = this.master.config.get("foo_frobber.level");
-        console.log("I got a frobnication level of", level);
+        this.logger.info(`I got a frobnication level of ${level}`);
     }
 
 The same applies for instance configs, replace "master" with "instance"
-where appropriate.  See [Configuration System](configuration-system.md)
+where appropriate.  See [Configuration System](config-system.md)
 for more details on how this system works.
 
 
+### Handling Invalid Configuration
+
+If the plugin requires a certain feature to be enabled to function it
+should throw an error during init if this is not the case.  The most
+common such feature is the save patching, which can be disabled to run
+vanilla or scenarios not compatible with Clusterio.  For example:
+
+    async init() {
+        if (!this.instance.config.get("factorio.enable_save_patching")) {
+            throw new Error("foo_frobber plugin requires save patching.");
+        }
+    }
+
+
 Communicating with Factorio
 ---------------------------
 
-For pushing data into Factorio there's RCON which lets you send
-arbitrary Lua command to invoke whatever code you want in the game.
-This is done by calling sendRcon method on the instance's server object.
+For pushing data into Factorio there's RCON, which lets you send
+arbitrary Lua commands to invoke whatever code you want in the game.
+This is done by calling the `sendRcon` method on the plugin object.
 For example:
 
     async onStart() {
-        let response = await this.instance.server.sendRcon(
+        let response = await this.sendRcon(
             "/sc rcon.print('data')"
         );
 
@@ -211,7 +298,7 @@ For example:
     }
 
 
-Because data into Factorio is streamed at a rate of 3-6 kB/s by default
+Because data into Factorio is streamed at a rate of 3-6 kB/s by default,
 it is recommended to avoid sending large commands as much as possible,
 and to strip down the data on the ones you send to only what's strictly
 necessary.  You can have lua code injected into the game via the module
@@ -222,13 +309,13 @@ For getting data out from Factorio there's both RCON and the `send_json`
 API of the Clusterio module.  Returning data via RCON is prefered if the
 action is initiated from the Node.js side.  The `send_json` API allows
 sending JSON payloads on channels that plugins can listen to.  From a
-plugin you listen for an event named ipc-channel_name in order to get
+plugin you listen for an event named `ipc-channel_name` in order to get
 data sent by `send_json`.  For example in the plugin code:
 
     async init() {
-        this.instance.server.on('ipc-my_plugin_foo', content =>
-            this.handleFoo(content).catch(err => console.log(
-                "Error handling foo:", foo
+        this.instance.server.on("ipc-my_plugin_foo", content =>
+            this.handleFoo(content).catch(err => this.logger.error(
+                `Error handling foo:\n${err.stack}`
             ))
         );
     }
@@ -239,10 +326,10 @@ data sent by `send_json`.  For example in the plugin code:
 
 And then in the module for the plugin:
 
-    local clusterio_api = require('modules/clusterio/api')
+    local clusterio_api = require("modules/clusterio/api")
 
     -- inside some event handler
-    clusterio_api.send_json('my_plugin_foo', { data = 123 })
+    clusterio_api.send_json("my_plugin_foo", { data = 123 })
 
 It's recommended to either use the plugin name as the channel name or to
 prefix the channel name with the name of the plugin if you need multiple
@@ -257,7 +344,7 @@ server has a fast enough storage system.
 
 **Note:** both `send_json` and RCON can operate out of order.  For
 `send_json` it's possible that payloads greater than 4kB are received
-after payloads that was sent at a later point in time.  For RCON,
+after payloads that were sent at a later point in time.  For RCON,
 commands longer than 50 characters may end up being executed after
 shorter commands sent after it.
 
@@ -269,18 +356,18 @@ You will most likely have to communicate with the master or other
 instances in your plugin for it to do anything useful.  For this there's
 a WebSocket communication channel established between the slaves and the
 master server that plugins can define their own messages to send over
-it.  This channel is bi-directional and all messages send over it are
+it.  This channel is bi-directional and all messages sent over it are
 validated with a JSON schema (see [this guide][guide] for an
-introduction to writing JSON schema.)
+introduction to writing JSON schema).
 
 [guide]: https://json-schema.org/learn/getting-started-step-by-step.html
 
-There are currently two kinds of messages that can be defined.  Events
-and requests.  Events are simple one way notifications that invoke a
-handler on target it's sent to.  Requests are pairs of request and
+There are currently two kinds of messages that can be defined: events
+and requests.  Events are simple one-way notifications that invoke a
+handler on the target it's sent to.  Requests are pairs of request and
 response messages where the request is sent to the target and the
 response is the reply back from the target.  The requests are similar to
-HTTP requests only that both parties of a link may innitiate one.
+HTTP requests, except both parties of a link may initiate one.
 
 
 ### Defining Events
@@ -292,84 +379,108 @@ class.  The Event constructor takes an object of properties that define
 the event, for example the following could be defined in `info.js`:
 
     messages: {
-        startFrobnication: new link.Event({
-            type: 'foo_frobber:start_frobnication',
-            links: ['master-slave', 'slave-instance'],
-            forwardTo: 'instance',
+        startFrobnication: new libLink.Event({
+            type: "foo_frobber:start_frobnication",
+            links: ["master-slave", "slave-instance"],
+            forwardTo: "instance",
+            eventRequired: ["frobnication_type"],
             eventProperties: {
                 "frobnication_type": { type: "string" },
+                "urgent": { type: "boolean" },
             },
         }),
     },
 
 This specifies an event that can be sent from the master to a slave,
-and from a slave to an instance.  And that the event must contain the
-property `frobnication_type` with a string value in the data payload.
-It will also be forwarded by slaves to a specific instance.
+and from a slave to an instance.  It also specifies that the event must
+contain the property `frobnication_type`, with a string value in the
+data payload and that it may optionally contain a boolean `urgent`
+property.  It will also be forwarded by slaves to a specific instance.
 
 The following properties are recognized by the Event constructor:
 
-**type**:
-    The message type sent over the wire.  This can be any string but
-    it must be unique across all plugins and it's recommended that it is
-    of format `plugin_name:message_name`.  The suffix `_event` will be
-    appened to the type.
+#### type
 
-**links**:
-    An array of strings describing which links this event can be sent
-    over.  Direction matters, `'master-slave'` means the event can be
-    sent from the master to slave, but can't be sent back the other way
-    unless `'slave-master'` is also present in the links array.
+The message type sent over the wire.  This must start with the name of
+the plugin followed by colon and and be unique for the plugin.  The type
+of the message sent over the socket will have the suffix `_event`
+appended to it.
 
-**forwardTo**:
-    Target to forward event to.  Can be either `'master'` to indicate a
-    slave should forward it to the master server or `'instance'` to
-    indicate it should be forwarded to the instances specified by the
-    `instance_id` event property.  This works by using a default handler
-    for the event at the links that forward it.
+#### links
 
-**broadcastTo**:
-    Target to broadcast this message towards.  Currently only
-    `'instance'` is supported and means the event will be broadcast to
-    all instances downstream of the target it's sent to, but not back
-    from where it came from.  This means that sending an event to a
-    slave from an instance will cause it to be broadcast to all
-    instances of that slave except for the instance it came from.
+An array of strings describing which links this event can be sent over.
+Direction matters, `"master-slave"` means the event can be sent from the
+master to the slave, but can't be sent back the other way, unless
+`"slave-master"` is also present in the links array.
 
-**eventProperties**:
-    Object with properties mapping to a JSON schema of that property
-    that specifies what's valid to send in the event.  This is
-    equivalent to using the `properties` keyword in JSON schema except
-    that the properties specified are implicitly required and additional
-    properties are not allowed.  See [this guide][guide] for an
-    introduction to writing JSON schemas
+The available endpoints are `master`, `slave`, `instance`, and
+`control`.  Master talks with slave and control, and slave talks to
+instance.  The full chain must be specified as the individual links in
+order for a message to travers multiple hops, (i.e., for a message to go
+from master to instance it must have both `"master-slave"` and
+`"slave-instance"` in the links array).  See `forwardTo` and
+`broadcastTo` for ways to forward an event to the next link in a chain.
+
+#### forwardTo
+
+Target to forward an event to.  Can either be `"master"`, to indicate a
+slave should forward it to the master server, or `"instance"`, to
+indicate it should be forwarded to the instances specified by the
+`instance_id` event property.  This works by using a default handler for
+the event at the links that forward it.
+
+#### broadcastTo
+
+Target to broadcast this message towards.  A value of "instance" means
+the event will be broadcast to all instances downstream of the target
+it's sent to, but not back from where it came from. Currently, only
+"instance" is supported. This means that sending an event to a slave
+that slave except for the instance it came from.  from an instance will
+cause it to be broadcast to all instances of
+
+#### eventRequired
+
+By default all properties defined in `eventProperties` are required to
+be present in the payload for the event.  This can be overriden by
+specifying an array of properties which are required here.  This is
+equivalent to using the `required` keyword in JSON schema.
+
+#### eventProperties
+
+Object with properties mapping to a JSON schema of that property that
+specifies what's valid to send in the event.  This is equivalent to
+using the `properties` keyword in JSON schema, except that the
+properties specified are by default required and additional properties
+are not allowed.  See [this guide][guide] for an introduction to writing
+JSON schemas.
 
 The forwardTo and broadcastTo can be combined such that specifying
-`'master'` as the forwardTo value and `'instance'` as the broadcastTo
+`"master"` as the forwardTo value and `"instance"` as the broadcastTo
 value will cause the event to be broadcast to all instances in the
-cluster.  For this to work you will need to specify instance-slave,
-slave-master and master-slave, slave-instance as the links.
+cluster.  For this to work, you will need to specify `instance-slave`,
+`slave-master`, `master-slave`, and `slave-instance` as the links.
 
 Keep in mind when forwarding events that if the target an event is being
-forwarded to is not online the event will be dropped.  If you need a
-confirmation that the message was received use a request.
-
+forwarded to is not online, the event will be dropped.  Use a request if
+you need a confirmation that the message was received.
 
 ### Definining Requests
 
 Requests are defined as properties of the messages object exported by
 `info.js` that map to instances of the `Request` class from `lib/link`.
-The name of the property correspond to the handler invoked on the plugin
+The name of the property corresponds to the handler invoked on the plugin
 class.  The Request constructor takes an object of properties that define
-the event, for example the following could be defined in `info.js`:
+the event. For example, the following could be defined in `info.js`:
 
     messages: {
-        reportFrobnication: new link.Request({
-            type: 'foo_frobber:report_frobnication',
-            links: ['master-slave', 'slave-instance'],
-            forwardTo: 'instance',
+        reportFrobnication: new libLink.Request({
+            type: "foo_frobber:report_frobnication",
+            links: ["master-slave", "slave-instance"],
+            forwardTo: "instance",
+            requestRequired: ["verbosity"],
             requestProperties: {
                 "verbosity": { type: "integer" },
+                "special": { type: "boolean" },
             },
             responseProperties: {
                 "report": {
@@ -382,47 +493,72 @@ the event, for example the following could be defined in `info.js`:
 
 This specifies a request that can be sent from the master to a slave,
 and from a slave to an instance.  The request data must contain the
-property `verbosity` with an integer number as the value as well as the
-`instance_id` property (implied by `forwardTo: 'instance'`), and the
+property `verbosity` with an integer number as the value, as well as the
+`instance_id` property (implied by `forwardTo: "instance"`) and it may
+also contain a boolean `special` property.  It also defines that the
 response sent must contain a `report` property mapping to an array of
-strings.  When received by a slave it will also be forwarded to the
+strings.  When received by a slave, it will also be forwarded to the
 instance specified by `instance_id`.
 
 The following properties are recognized by the Request constructor:
 
-**type**:
-    The message type sent over the wire.  This can be any string but
-    it must be unique across all plugins and it's recommended that it is
-    of format `plugin_name:message_name`.  The suffix `_request` will be
-    appened to the type for the request message sent, and the suffix
-    `_response` will be appended to the type for the response.
+#### type
 
-**links**:
-    An array of strings describing which links this request can be sent
-    over.  Direction matters, `'master-slave'` means the request can be
-    sent from the master to slave and the slave can reply to the master,
-    but the slave can't sent a request to the master unless
-    `'slave-master'` is also present in the links array.
+The message type sent over the wire.  This must start with the name of
+the plugin followed by colon and and be unique for the plugin.  The type
+of the message sent over the socket will have the suffix `_request`
+appended to it for the request and `_response` appended to it for the
+response.
 
-**forwardTo**:
-    Target to forward request to.  Can be either `'master'` to indicate
-    a slave should forward it to the master server when receiving it
-    from an instance or `'instance'` to indicate it should be forwarded
-    to the instances specified by the `instance_id` request property.
-    This works by using a default handler for the request at the links
-    that forward it.
+#### links
 
-**requestProperties**:
-    Object with properties mapping to a JSON schema of that property
-    that specifies what's valid to send in the request.  This is
-    equivalent to using the `properties` keyword in JSON schema except
-    that the properties specified are implicitly required and additional
-    properties are not allowed.  See [this guide][guide] for an
-    introduction to writing JSON schemas
+An array of strings describing which links this request can be sent
+over.  Direction matters; `"master-slave"` means the request can be sent
+from the master to the slave and the slave can reply to the master, but
+the slave can't send a request to the master unless `"slave-master"` is
+also present in the links array.
 
-**responseProperties**:
-    Same as the requestProperties only for the response sent back by the
-    target.
+The available endpoints are `master`, `slave`, `instance`, and
+`control`.  Master talks with slave and control, and slave talks to
+instance.  The full chain must be specified as the individual links in
+order for a message to travers multiple hops, (i.e., for a message to go
+from master to instance it must have both `"master-slave"` and
+`"slave-instance"` in the links array).  See `forwardTo` for ways to
+forward a request to the next link in a chain.
+
+#### forwardTo
+
+Target to forward the request to.  Can either be `"master"` to indicate
+a slave should forward it to the master server when receiving it from an
+instance, or `"instance"` to indicate it should be forwarded to the
+instances specified by the `instance_id` request property.  This works
+by using a default handler for the request by the links that forward it.
+
+#### requestRequired
+
+By default all properties defined in `requestProperties` are required to
+be present in the payload for the request.  This can be overriden by
+specifying an array of properties which are required here.  This is
+equivalent to using the `required` keyword in JSON schema.
+
+#### requestProperties
+
+Object with properties mapping to a JSON schema of that property that
+specifies what's valid to send in the request.  This is equivalent to
+using the `properties` keyword in JSON schema, except that the
+properties specified are by default required and additional properties
+are not allowed.  See [this guide][guide] for an introduction to writing
+JSON schemas
+
+#### responseRequired
+
+Same as the requestRequired only for the response sent back by the
+target.
+
+#### responseProperties
+
+Same as the requestProperties only for the response sent back by the
+target.
 
 
 Sending Link Messages
@@ -430,10 +566,10 @@ Sending Link Messages
 
 Link messages are sent by calling the `.send()` method on the
 Event/Request instance with the link you want to send it over and the
-data to send.  For instance plugins the link is the instance itself
-which is accessible through the `.instance` property on the plugin.  The
-`.info` property of the plugin class exposes the data exported from the
-plugin's `info.js` module.  In other words:
+data to send. For `InstancePlugin` code the link to the slave is the
+`instance` itself, which is accessible through the `.instance` property
+of the `InstancePlugin`. The `.info` property of the plugin class exposes
+the data exported from the plugin's `info.js` module.  In other words:
 
     // In an InstancePlugin class
     async frobnicate() {
@@ -445,21 +581,66 @@ data received from the target it was sent to, or throws an error if the
 request failed.
 
 
+### Handling connection events
+
+There are a few connection related events that plugins neeed to repsond
+to in order to avoid data loss and connection problems.  The most
+important is the prepare disconnect for the link between master and
+slave.  This is signaled to `MasterPlugin` classes via the
+`onPrepareSlaveDisconnect` hook and to `InstancePlugin` classes via the
+`onPrepareMasterDisconnect` hook.
+
+After the prepare disconnect the connection will be closed, which will
+result in pending requests and events being dropped.  Plugins must
+respond to the prepare disconnect by stopping any processess it does
+that send events or requests over the link in question.  This can be
+accomplished either through listening for the prepare disconnect hook,
+or by checking the `connected` or `closing` property of the connector
+for the master/slave connection.  For example the sending of an event
+from an `InstancePlugin` class can be stopped while the connection is
+closing by using the following code:
+
+    if (!this.slave.connector.closing) {
+        this.info.messages.frobnicate.send(this.instance, { foo: "bar" });
+    }
+
+If the event or request needs to be sent to the master it can be put
+into a queue stored on the plugin instance and sent out when the
+connection is established again.  The re-establishement of the
+connection is  notified to plugins via the `connect` event to the
+`onMasterConnectionEvent` and `onSlaveConnectionEvent` hooks.
+
+The second connection event which is of lesser importance to respond to
+is the `drop` connection event served through `onMasterConnectionEvent`
+for `InstancePlugin` classes and through `onSlaveConnectionEvent` for
+`MasterPlugin` classes.  This is raised when the connection between the
+master and slave in question is lost, most likely due to networking
+issues.  When in the dropped state the slave will keep trying to
+reconnect to the master server in order to re-establish it, and if
+successful no events or requests will be lost.  However while in the
+dropped state any requests and events sent gets queued up in memory
+until the connection is re-established.  This means that if your plugin
+sends a lot of events or requests, they can end up being queued up in a
+buffer for a long time and sent out all at once.  To avoid this you
+should be throtteling and/or stopping your requests/events after `drop`
+has been raised, and continue back as normal when `resume` is raised.
+
+
 Collecting Statistics
 ---------------------
 
-Clusterio comes with it's own Prometheus client implementation, one part
+Clusterio comes with its own Prometheus client implementation, one part
 due to Not Invented Here and another part due to collectors in
 prom-client being difficult to get to work nicely with collecting data
 from plugins optionally loaded at runtime on different computers.
 
-In its simplest form collecting data from a plugins consists of defining
+In its simplest form collecting data from plugins consists of defining
 the metric and updating it somewhere in the plugin code.  For example:
 
-    const { Counter } = require('lib/prometheus');
+    const { Counter } = require("@clusterio/lib/prometheus");
 
     const fooMetric = new Counter(
-        'clusterio_foo_frobber_foo_metric', "Measures the level of foo",
+        "clusterio_foo_frobber_foo_metric", "Measures the level of foo",
     );
 
     // Somewhere in the master plugin code
@@ -470,28 +651,91 @@ available through the /metric HTTP endpoint.  It's recommended that
 plugin metrics follow `clusterio_<plugin_name>_<metric_name>` as the
 naming scheme.
 
-For metrics that are per-instance you must define an `instance_id` label and
+For metrics that are per-instance, you must define an `instance_id` label and
 set it accordingly, for example:
 
-    const { Counter } = require('lib/prometheus');
+    const { Counter } = require("@clusterio/lib/prometheus");
 
     const barMetric = new Gauge(
-        'clusterio_foo_frobber_bar_metric', "Bar instance level",
-        { labels: ['instance_id'] }
+        "clusterio_foo_frobber_bar_metric", "Bar instance level",
+        { labels: ["instance_id"] }
     );
 
     // Somewhere in the instance plugin code
-    barMetric.labels(String(this.instance.config.get("instance.id"))).set(someValue);
+    barMetric.labels(String(this.instance.id)).set(someValue);
 
 Metrics are automatically registered to the default registry, and this
 default registry is automatically polled by the master server on slaves.
 This means that it's important that you place the definition of the
 metric at module level so that it's not created more than once over the
 lifetime of a slave.  Since the metrics remember their values and would
-continue to be exported after an instance is shutdown there's code at
+continue to be exported after an instance is shutdown, there's code at
 instance shutdown that removes all the values where the `instance_id`
 label matches the id of the instance shut down.
 
 For statistics you need to update on collection there's an `onMetrics`
-hook on both master and instance plugins that is runned before the
-metrics in the default registry is collected.
+hook on both master and instance plugins that is run before the
+metrics in the default registry are collected.
+
+
+Adding Custom Commands to clusterioctl
+--------------------------------------
+
+The control entrypoint for plugins allows you to extend clustectl with
+your own commands.  The creation of custom commands typically starts
+with defining a command tree for the plugin:
+
+    const { Command, CommandTree } = require("@clusterio/lib/command");
+    const fooFrobberCommands = new CommandTree({
+        name: "foo-frobber", description: "Foo Frobber Plugin commands"
+    });
+
+Then commands are added to the the plugin's command tree:
+
+    const info = require("./info");
+
+    fooFrobberCommands.add(new Command({
+        definition: ["frobnicate <type>", "Do frobnications", (yargs) => {
+            yargs.positional("level", {
+                describe: "type of frobnication", type: "string"
+            });
+        }],
+        handler: async function(args, control) {
+            await info.messages.frobnicate.send(control, {
+                instance_name: "Console",
+                content: args.message,
+            });
+        },
+    }));
+
+For a command the `definition` is the arguments to pass to
+[yargs.command](http://yargs.js.org/docs/#api-reference-commandcmd-desc-builder-handler)
+(see also
+[yargs.positional](http://yargs.js.org/docs/#api-reference-positionalkey-opt)
+and
+[yargs.options](http://yargs.js.org/docs/#api-reference-optionskey-opt)
+for setting up positional and optional arguments to commands).  The
+`handler` is an async function that's invoked when the command is
+executed and it's passed the parsed command line arguments and a
+reference to the `Control` class of clusterioctl.  It's possible to optain
+a reference to the plugin class with `control.plugins.get(info.name)`.
+
+Note that messages sent from clusterioctl needs to have
+`"control-master"` as a part of the links array for it to be accepted by
+the master server, see [Defining Link Messages](#defining-link-messages)
+for how to define the messages that can be sent to the master.
+
+To have the command tree become part of clusterioctl it needs to be added
+to the rootCommand tree in `addCommands` callback of the Control plugin:
+
+    const libPlugin = require("@clusterio/lib/plugin");
+
+    class ControlPlugin extends libPlugin.BaseControlPlugin {
+        async addCommands(rootCommand) {
+            rootCommand.add(fooFrobberCommands);
+        }
+    }
+
+    module.exports = {
+        ControlPlugin,
+    }

lerna.json

@@ -0,0 +1,7 @@
+{
+  "packages": [
+    "packages/*",
+    "plugins/*"
+  ],
+  "version": "2.0.0-alpha.4"
+}

lib/authenticate.js

@@ -1,48 +0,0 @@
-const jwt = require('jsonwebtoken');
-
-let authSecret = null;
-function setAuthSecret(secret) {
-	authSecret = secret;
-}
-
-function middleware(req, res, next) {
-	// This should not happen
-	if (!authSecret) {
-		return res.status(500).send({ auth: false, message: 'Secret not set', endpoint: req.route.path });
-	}
-
-	var token = req.headers['x-access-token'];
-	if (!token) {
-		return res.status(401).send({ auth: false, message: 'No token provided.', endpoint: req.route.path });
-	}
-
-	jwt.verify(token, authSecret, function(err, decoded) {
-		if (err) {
-			return res.status(401).send({ auth: false, message: 'Failed to authenticate token.', endpoint: req.route.path });
-		}
-		next();
-	});
-}
-
-async function check(token) {
-	return new Promise((resolve, reject) => {
-		// This should not happen
-		if (!authSecret) {
-			resolve({ ok: false, msg: 'Secret not set' });
-		}
-
-		jwt.verify(token, authSecret, function(err, decoded) {
-			if (err) {
-				resolve({ok:false, msg:err})
-			} else {
-				resolve({ok:true, msg:"Successfully authenticated"});
-			}
-		});
-	});
-}
-
-module.exports = {
-	setAuthSecret,
-	middleware,
-	check,
-};

lib/factorio/index.js

@@ -1,5 +0,0 @@
-module.exports = {
-	...require('lib/factorio/locale'),
-	...require('lib/factorio/patch'),
-	...require('lib/factorio/server'),
-};

lib/factorio/locale.js

@@ -1,28 +0,0 @@
-/**
- * Function for parsing factorio locale from a local factorio install
- */
-
-"use strict";
-const fs = require("fs-extra");
-const path = require("path");
-const ini = require("ini");
-
-/**
- * Gets factorios locale as an object, does not respect mods.
- *
- * @param {string} factorioDirectory
- * @param {string} languageCode
- * @param {function} callback
- */
-async function getLocale(dataDir, languageCode) {
-	if (typeof dataDir !== "string") throw new TypeError("dataDir must be a string");
-	if (typeof languageCode !== "string") throw new TypeError("languageCode must be a string");
-
-	let localeFilePath = path.join(dataDir, "base", "locale", languageCode, "base.cfg");
-	let content = await fs.readFile(localeFilePath, "utf8");
-	return ini.parse(content);
-}
-
-module.exports = {
-	getLocale,
-};

lib/fileOps/index.js

@@ -1,2 +0,0 @@
-const fileOps = require("./fileOps");
-module.exports = fileOps;

lib/generateSSLcert.js

@@ -1,44 +0,0 @@
-const forge = require("node-forge");
-const fs = require("fs-extra");
-const util = require("util");
-
-const hrtime = process.hrtime.bigint;
-
-
-module.exports = async function({
-	bits = 2048,
-	sslCertPath,
-	sslPrivKeyPath,
-	doLogging = false,
-} = {}){
-	function log(t) {
-		if(doLogging) console.log("lib/generateSSLcert.js | "+t);
-	}
-	log("Generating SSL certificate with "+bits+" bits at "+sslCertPath);
-
-	let startNs = hrtime();
-	let keypair = await util.promisify(forge.pki.rsa.generateKeyPair)({bits: bits, e: 0x10001});
-	log("Generated 2048bit keypair in: "+(Number(hrtime()-startNs) / 1e6)+" ms");
-
-	startNs = hrtime();
-	let cert = forge.pki.createCertificate();
-	cert.publicKey = keypair.publicKey;
-	cert.serialNumber = "01";
-
-	cert.validity.notBefore = new Date();
-	cert.validity.notAfter = new Date(Date.now()+(1000*60*60*24*365)); // valid for one year
-
-	// XXX This should probably be limited to the actual hostname
-	let attrs = [{ name: "commonName", value: "*" }];
-	cert.setSubject(attrs);
-	cert.setIssuer(attrs);
-
-	// self-sign certificate
-	cert.sign(keypair.privateKey);
-
-	// Write certificate and private key to pem files
-	await fs.outputFile(sslPrivKeyPath, forge.pki.privateKeyToPem(keypair.privateKey));
-	await fs.outputFile(sslCertPath, forge.pki.certificateToPem(cert));
-
-	log("Generated certificate in: "+(Number(hrtime()-startNs) / 1e6)+" ms");
-}

lib/helpers.js

@@ -1,23 +0,0 @@
-/**
- * Collection of small utilites that are useful in multiple places.
- * @module
- */
-
-/**
- * Return a string describing the type of the value passed
- *
- * Works the same as typeof, excpet that null and array types get their
- * own string.
- *
- * @param value - value to return the type of.
- * @returns {string} basic type of the value passed.
- */
-function basicType(value) {
-	if (value === null) { return "null"; }
-	if (value instanceof Array) { return "array"; }
-	return typeof value;
-}
-
-module.exports = {
-	basicType,
-}

lib/link/index.js

@@ -1,5 +0,0 @@
-module.exports = {
-	...require('lib/link/link'),
-	...require('lib/link/messages'),
-	...require('lib/link/connectors'),
-};

lib/link/messages.js

@@ -1,558 +0,0 @@
-"use strict";
-const schema = require("lib/schema");
-const errors = require("lib/errors");
-
-/**
- * Represents a request sent over the link
- *
- * A request-response message that can be innitated by either party of the
- * link.
- */
-class Request {
-
-	/**
-	 * Creates a request
-	 *
-	 * @param {string} type -
-	 *     message type used.  Will have suffix '_request' for the request
-	 *     message and suffix '_response' for the response message.
-	 * @param {Array<string>} links -
-	 *     Links this request is valid on.  Takes an array of strings
-	 *     containing "<source>-<target>" specifications.
-	 * @param {?string} forwardTo -
-	 *     Optional target to forward this request to.  'instance' add
-	 *     instance_id into the requestProperties and forward to the given
-	 *     instance.  'master' forwards it to the master server.
-	 * @param {Object<string, Object>} requestProperties -
-	 *     Mapping of property values to JSON schema specifications for the
-	 *     properties that are valid in the data payload of this requst.
-	 * @param {Object<string, Object>} responseProperties -
-	 *     Mapping of property values to JSON schema specifications for the
-	 *     properties that are valid in the data payload of the response to
-	 *     this requst.
-	 */
-	constructor({
-		type, links, forwardTo = null, requestProperties = {}, responseProperties = {}
-	}) {
-		this.type = type;
-		this.links = links;
-		this.forwardTo = forwardTo;
-
-		this.requestType = type + '_request';
-		this.responseType = type + '_response';
-
-		if (forwardTo === 'instance') {
-			requestProperties = Object.assign(
-				{"instance_id": { type: "integer" }}, requestProperties
-			);
-
-		} else if (forwardTo !== "master" && forwardTo !== null) {
-			throw new Error(`Invalid forwardTo value ${forwardTo}`);
-		}
-
-		this._requestValidator = schema.compile({
-			$schema: "http://json-schema.org/draft-07/schema#",
-			properties: {
-				"type": { const: this.requestType },
-				"data": {
-					additionalProperties: false,
-					required: Object.keys(requestProperties),
-					properties: requestProperties,
-				},
-			},
-		});
-
-		this._responseValidator = schema.compile({
-			$schema: "http://json-schema.org/draft-07/schema#",
-			properties: {
-				"type": { const: this.responseType },
-				"data": {
-					anyOf: [
-						{
-							additionalProperties: false,
-							required: ["seq", ...Object.keys(responseProperties)],
-							properties: {
-								"seq": { type: "integer" },
-								...responseProperties,
-							},
-						},
-						{
-							additionalProperties: false,
-							required: ["seq", "error"],
-							properties: {
-								"seq": { type: "integer" },
-								"error": { type: "string" },
-							},
-						},
-					],
-				},
-			}
-		});
-	}
-
-	/**
-	 * Attach to a link
-	 *
-	 * Set a the validator for the response on the given link if it's a
-	 * source and sets the handler for the request if it is a target.
-	 *
-	 * If forwardTo was set to 'instance' the `forwardRequestToInstance`
-	 * method on the link is used as the default handler.
-	 *
-	 * If forwardTo was set to 'master' the `forwardRequesToMaster` method
-	 * on the link is used as the default handler.
-	 *
-	 * Does nothing if the link is neither a source nor a target for this
-	 * request.
-	 *
-	 * @param {Link} link - The link to attach to.
-	 * @param {Function} handler -
-	 *    Async function to invoke with link set as this to handle the
-	 *    request request.  Only used on the target side.
-	 * @throws {Error} if the handler is needed and not defined.
-	 */
-	attach(link, handler) {
-		// Source side.
-		if (this.links.includes(`${link.source}-${link.target}`)) {
-			link.setValidator(this.responseType, this._responseValidator);
-		}
-
-		// Target side.  Note: Source and target is reversed here.
-		if (this.links.includes(`${link.target}-${link.source}`)) {
-
-			// Use forwarder if handler is not present.
-			if (!handler) {
-				if (this.forwardTo === 'instance') {
-					handler = link.forwardRequestToInstance;
-				} else if (this.forwardTo === "master") {
-					handler = link.forwardRequestToMaster;
-				}
-			}
-
-			if (!handler) {
-				throw new Error(`Missing handler for ${this.requestType} on ${link.source}-${link.target} link`);
-			}
-
-			link.setHandler(this.requestType, message => {
-				handler.call(link, message, this).then(response => {
-					if (response === undefined) {
-						// XXX Should we allow implicit responses like this?
-						response = {};
-					}
-
-					if (!this._responseValidator({ seq: 0, type: this.responseType, response})) {
-						console.error(this._responseValidator.errors);
-						throw new Error(`Validation failed responding to ${this.requestType}`);
-					}
-
-					link.connector.send(this.responseType, { ...response, seq: message.seq });
-
-				}).catch(err => {
-					if (!(err instanceof errors.RequestError)) {
-						console.error(`Unexpected error while responding to ${this.requestType}`, err);
-					}
-					link.connector.send(this.responseType, { seq: message.seq, error: err.message });
-				});
-			}, this._requestValidator);
-		}
-	}
-
-	/**
-	 * Send request over the given link
-	 *
-	 * Sends the given data over the link as the request payload and waits
-	 * for the response.
-	 *
-	 * @returns {object} response data
-	 */
-	async send(link, data = {}) {
-		// XXX validate link target/source?
-		if (!this._requestValidator({ seq: 0, type: this.requestType, data})) {
-			console.error(this._requestValidator.errors);
-			throw new Error(`Validation failed sending ${this.requestType}`);
-		}
-		let seq = link.connector.send(this.requestType, data);
-		let responseMessage = await link.waitFor(this.responseType, { seq });
-		if (responseMessage.data.error) {
-			throw new errors.RequestError(responseMessage.data.error);
-		}
-		return responseMessage.data;
-	}
-}
-
-
-let messages = {}
-
-// Connection requests
-let wsLinks = ["master-control", "control-master", "master-slave", "slave-master"];
-messages.shutdownConnection = new Request({
-	type: "shutdown_connection",
-	links: wsLinks,
-});
-
-messages.ping = new Request({
-	type: "ping",
-	links: wsLinks,
-});
-
-// Management requests
-messages.listSlaves = new Request({
-	type: 'list_slaves',
-	links: ['control-master'],
-	responseProperties: {
-		"list": {
-			type: "array",
-			items: {
-				additionalProperties: false,
-				required: ["agent", "version", "name", "id", "connected"],
-				properties: {
-					"agent": { type: "string" },
-					"version": { type: "string" },
-					"name": { type: "string" },
-					"id": { type: "integer" },
-					"connected": { type: "boolean" },
-				},
-			},
-		},
-	},
-});
-
-messages.listInstances = new Request({
-	type: 'list_instances',
-	links: ['control-master'],
-	responseProperties: {
-		"list": {
-			type: "array",
-			items: {
-				additionalProperties: false,
-				required: ["name", "id", "assigned_slave"],
-				properties: {
-					"name": { type: "string" },
-					"id": { type: "integer" },
-					"assigned_slave": { type: ["null", "integer"] },
-				},
-			},
-		},
-	},
-});
-
-messages.setInstanceOutputSubscriptions = new Request({
-	type: 'set_instance_output_subscriptions',
-	links: ['control-master'],
-	requestProperties: {
-		"instance_ids": {
-			type: "array",
-			items: { type: "integer" },
-		},
-	},
-});
-
-messages.createInstance = new Request({
-	type: 'create_instance',
-	links: ['control-master'],
-	requestProperties: {
-		"serialized_config": { type: "object" },
-	},
-});
-
-messages.getInstanceConfig = new Request({
-	type: "get_instance_config",
-	links: ["control-master"],
-	requestProperties: {
-		"instance_id": { type: "integer" },
-	},
-	responseProperties: {
-		"serialized_config": { type: "object" },
-	},
-});
-
-messages.setInstanceConfigField = new Request({
-	type: "set_instance_config_field",
-	links: ["control-master"],
-	requestProperties: {
-		"instance_id": { type: "integer" },
-		"field": { type: "string" },
-		"value": { type: "string" },
-	},
-});
-
-messages.assignInstanceCommand = new Request({
-	type: 'assign_instance_command',
-	links: ['control-master'],
-	requestProperties: {
-		"instance_id": { type: "integer" },
-		"slave_id": { type: "integer" },
-	},
-});
-
-messages.assignInstance = new Request({
-	type: 'assign_instance',
-	links: ['master-slave'],
-	requestProperties: {
-		"serialized_config": { type: "object" },
-	},
-	forwardTo: 'instance',
-})
-
-messages.startInstance = new Request({
-	type: 'start_instance',
-	links: ['control-master', 'master-slave', 'slave-instance'],
-	requestProperties: {
-		"instance_id": { type: "integer" },
-	},
-	forwardTo: 'instance',
-});
-
-messages.createSave = new Request({
-	type: 'create_save',
-	links: ['control-master', 'master-slave', 'slave-instance'],
-	forwardTo: 'instance',
-});
-
-messages.stopInstance = new Request({
-	type: 'stop_instance',
-	links: ['control-master', 'master-slave', 'slave-instance'],
-	forwardTo: 'instance',
-});
-
-messages.deleteInstance = new Request({
-	type: 'delete_instance',
-	links: ['control-master', 'master-slave'],
-	forwardTo: 'instance',
-});
-
-messages.sendRcon = new Request({
-	type: 'send_rcon',
-	links: ['control-master', 'master-slave', 'slave-instance'],
-	forwardTo: 'instance',
-	requestProperties: {
-		"command": { type: "string" },
-	},
-	responseProperties: {
-		"result": { type: "string" },
-	}
-});
-
-messages.getMetrics = new Request({
-	type: 'statistics_exporter:get_metrics',
-	links: ['master-slave', 'slave-instance'],
-	responseProperties: {
-		"results": { type: "array" },
-	},
-});
-
-
-/**
- * Represents an event sent over the link
- *
- * A one way message without any response or recipt confirmation that
- * can be innitiated by either party of the link.
- */
-class Event {
-
-	/**
-	 * Creates an event
-	 *
-	 * @param {string} type -
-	 *     message type used.  Will have suffix '_event' appended to it for
-	 *     the messages sent through this event.
-	 * @param {Array<string>} links -
-	 *     Links this event is valid on.  Takes an array of strings
-	 *     containing "<source>-<target>" specifications.
-	 * @param {?string} forwardTo -
-	 *     Optional target to forward this request to.  Supported values are
-	 *     'master' and 'instance'.  If set to 'instance' an instance_id
-	 *     property is implicity added to eventProperties.
-	 * @param {?string} broadcastTo -
-	 *     Optional target to broadcast this event to.  Only 'instance' is
-	 *     currently supported.  An event will only be broadcast towards
-	 *     links that are downstream, so sending an event set to instance
-	 *     broadcast that's sent to a slave will only be broadcast to that
-	 *     slave's instances.
-	 * @param {Object<string, Object>} eventProperties -
-	 *     Mapping of property values to JSON schema specifications for the
-	 *     properties that are valid in the data payload of this event.
-	 */
-	constructor({ type, links, forwardTo = null, broadcastTo = null, eventProperties = {} }) {
-		this.type = type;
-		this.links = links;
-		this.forwardTo = forwardTo;
-		this.broadcastTo = broadcastTo;
-
-		if (forwardTo === 'instance') {
-			eventProperties = Object.assign(
-				{"instance_id": { type: "integer" }}, eventProperties
-			);
-
-		} else if (forwardTo !== 'master' && forwardTo !== null) {
-			throw new Error(`Invalid forwardTo value ${forwardTo}`);
-		}
-
-		if (broadcastTo !== 'instance' && broadcastTo !== null) {
-			throw new Error(`Invalid broadcastTo value ${broadcastTo}`);
-		}
-
-		this.eventType = type + '_event';
-		this._eventValidator = schema.compile({
-			$schema: "http://json-schema.org/draft-07/schema#",
-			properties: {
-				"type": { const: this.eventType },
-				"data": {
-					additionalProperties: false,
-					required: Object.keys(eventProperties),
-					properties: eventProperties,
-				},
-			},
-		});
-	}
-
-	/**
-	 * Attach to a link
-	 *
-	 * Set a the handler for the event on the given link if it's a target
-	 * for the event.
-	 *
-	 * If forwardTo was set to 'master' the `forwardEventToMaster` method on
-	 * the link is used as the default handler.  If it was set to 'instance'
-	 * the `forwardEventToInstance` method on the link is used by default.
-	 *
-	 * If broadcastTo was set to 'instance' the `broadcastEventToInstance`
-	 * method on the link is invoked before the handler.
-	 *
-	 * Does nothing if the link is not a target for this event.
-	 *
-	 * @param {Link} link - The link to attach to.
-	 * @param {Function} handler -
-	 *    Async function to invoke with link set as this to handle the
-	 *    event.  Only used on the target side.
-	 * @throws {Error} if the handler is needed and not defined.
-	 */
-	attach(link, handler) {
-		// Target side.  Note: Source and target is reversed here.
-		if (this.links.includes(`${link.target}-${link.source}`)) {
-			// Use forwarder if handler is not present.
-			if (!handler) {
-				if (this.forwardTo === 'instance') {
-					handler = link.forwardEventToInstance;
-				} else if (this.forwardTo === 'master') {
-					handler = link.forwardEventToMaster;
-				}
-			}
-
-			if (
-				this.broadcastTo === 'instance'
-				&& [
-					'instance-slave', 'slave-master', 'control-master', 'master-slave'
-				].includes(`${link.target}-${link.source}`)
-			) {
-				if (!handler) {
-					handler = link.broadcastEventToInstance;
-				} else {
-					let originalHandler = handler;
-					handler = async (message, event) => {
-						await link.broadcastEventToInstance(message, event);
-						await originalHandler.call(link, message, event);
-					}
-				}
-			}
-
-			if (!handler) {
-				throw new Error(`Missing handler for ${this.eventType} on ${link.source}-${link.target} link`);
-			}
-
-			link.setHandler(this.eventType, message => {
-				// XXX Should event handlers even be allowed to be async?
-				handler.call(link, message, this).catch(err => {
-					console.error(`Unexpected error while handling ${this.eventType}`, err);
-				});
-			}, this._eventValidator);
-		}
-	}
-
-	/**
-	 * Send event over the given link
-	 *
-	 * Sends the given event data over the link.
-	 */
-	send(link, data = {}) {
-		if (!this._eventValidator({ seq: 0, type: this.eventType, data})) {
-			console.error(this._eventValidator.errors);
-			throw new Error(`Validation failed sending ${this.eventType}`);
-		}
-
-		link.connector.send(this.eventType, data);
-	}
-}
-
-messages.instanceStopped = new Event({
-	type: 'instance_stopped',
-	links: ['instance-slave'],
-});
-
-messages.updateInstances = new Event({
-	type: 'update_instances',
-	links: ['slave-master'],
-	eventProperties: {
-		"instances": {
-			type: "array",
-			items: {
-				type: "object",
-				additionalProperties: false,
-				required: ["serialized_config"],
-				properties: {
-					"serialized_config": { type: "object" },
-				},
-			},
-		},
-	},
-});
-
-messages.instanceOutput = new Event({
-	type: 'instance_output',
-	links: ['instance-slave', 'slave-master', 'master-control'],
-	forwardTo: 'master',
-	eventProperties: {
-		"instance_id": { type: "integer" },
-		"output": {
-			type: "object",
-			additionalProperties: false,
-			required: ["source", "received", "format", "type", "message"],
-			properties: {
-				"source": { type: "string" },
-				"received": { type: "number" },
-				"format": { type: "string" },
-				"time": { type: "string" },
-				"level": { type: "string" },
-				"file": { type: "string" },
-				"type": { type: "string" },
-				"action": { type: "string" },
-				"message": { type: "string" },
-			},
-		},
-	},
-});
-
-
-/**
- * Attaches all requests and events to the given link
- *
- * Loops through all builtin messages defined and attaches all of them
- * to the link.  The handler used for the messageis looked up on the
- * link instance as the concatenation of the name of the message, the
- * name of message class, and 'Handler'.
- */
-function attachAllMessages(link) {
-	for (let [name, message] of Object.entries(messages)) {
-		message.attach(link, link[name + message.constructor.name + 'Handler']);
-	}
-}
-
-
-module.exports = {
-	Request,
-	Event,
-
-	attachAllMessages,
-
-	messages,
-}

lib/manager/modManager.js

@@ -1,294 +0,0 @@
-/**
-Provides tools to manage mods in a clusterio install
-@module
-*/
-
-const asTable = require("as-table").configure({delimiter: ' | '});
-const moment = require("moment");
-const syncRequest = require("sync-request");
-const fs = require("fs-extra");
-const https = require('follow-redirects').https;
-const factorioAPI = require("node-factorio-api");
-const sLog = require("single-line-log").stdout;
-const query = require('cli-interact').getYesNo;
-const path = require("path");
-
-const fileOps = require("lib/fileOps");
-
-
-function ellipse(str, max){
-   return str.length > (max - 3) ? str.substring(0,max-3) + '...' : str;
-}
-
-// https://stackoverflow.com/questions/15900485/correct-way-to-convert-size-in-bytes-to-kb-mb-gb-in-javascript
-function formatBytes(size, decimals) {
-	if (bytes === 0) return '0 Bytes';
-
-	const k = 1024;
-	const dm = decimals < 0 ? 0 : decimals;
-	const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
-
-	const i = Math.floor(Math.log(bytes) / Math.log(k));
-
-	return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
-}
-
-
-/**
-Lists mods for an instance as a table. Warning, handles printing internally.
-@param {Instance} instance - Instance object, see slave.js
-
-@example
-var modManager = require("modManager");
-modManager.listMods(config, instance);
-// console.logs table of mods installed on instance
-// if instance.name is shared:
-// console.logs mods located in /sharedMods as a nice table
-*/
-async function listMods(config, instance) {
-	return new Promise((resolve, reject) => {
-		if (true) {
-			factorioAPI.init(false, instance.path("instanceMods"), instance.path("saves"), instance.server.version);
-			factorioAPI.authenticate({
-				username: config.username,
-				token: config.token,
-				require_ownership: false
-			}).then(token => {
-				console.log("* Currently installed:");
-				factorioAPI.readModZips().then(list => {
-					console.log(asTable(list.map(mod => {
-						return {
-							name:(mod.title || mod.name),
-							version:mod.version,
-							date:mod.date,
-							factorio_ver:mod.factorio_version,
-							author:mod.author,
-						}
-					})));
-				});
-				// Had issues earlier due to bug in node-factorio-api, see https://github.com/Danacus/node-factorio-api/issues/2
-				// bug is resolved, thanks to @Danacus and @brotherpot
-				factorioAPI.getModsFromSaves().then(list => {
-					console.log("* Mods used in save:");
-					console.log(asTable(list[0].mods))
-				});
-			}).catch(reason => {
-				console.log("Failed to authenticate to factorio.com! Please check your connection and the username+token used for authentication in config.json");
-				resolve();
-			});
-		} else {
-			// list shared mods in /sharedMods/
-			factorioAPI.init(false, 'sharedMods/', '', instance.server.version);
-			factorioAPI.authenticate({
-				username: config.username,
-				token: config.token,
-				require_ownership: false
-			}).then(() => {
-				console.log("\r\n")
-				console.log("* Mods shared by all instances:")
-				factorioAPI.getInstalledMods().then(mods => {
-					console.log(asTable(mods));
-					resolve();
-				});
-			}).catch(e => {
-				reject(new Error(e));
-			});
-		}
-
-		// get JSON data about GH releases
-		/*let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"factorioClusterio"}});
-		let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
-		let name = JSON.parse(res.getBody())[0].assets[0].name;
-		console.log(JSON.parse(res.getBody()))
-		*/
-	});
-}
-
-/**
-Searches mods.factorio.com for mods matching a search term and prints
-the results to console as a neat looking table
-
-@param {string} searchTerm What to search for
-*/
-async function findMods(config, searchTerm) {
-	return new Promise((resolve, reject) => {
-		factorioAPI.init(false);
-		factorioAPI.authenticate({
-			username: config.username,
-			token: config.token,
-			require_ownership: false
-		}).then(token => {
-			factorioAPI.searchMods(searchTerm).then(mods => {
-				let result = asTable(mods.reverse().map(mod => {
-					return {
-						title:ellipse(mod.title, 50),
-						name:ellipse(mod.name, 30),
-						downloads:mod.downloads_count,
-						updated: moment(mod.updated_at).format("DD.MM.YY"),
-						latest_version: mod.latest_release.version,
-						owner:mod.owner,
-						//license: mod.license_name,
-					}
-				}));
-				resolve(result);
-				// console.log(result);
-			});
-		}).catch(e => {
-			if (e.contains("Insufficient information")) console.log("We are missing some parameters required for logging in! Make sure your factorio username and token are specified in config.json.");
-			reject(new Error(e));
-		});
-	});
-}
-
-async function addMod(config, term, instance) {
-	return new Promise((resolve, reject) => {
-		if (instance.name == "shared") {
-			var modDirectory = "sharedMods";
-		} else {
-			var modDirectory = instance.path("instanceMods");
-		}
-		if (term && term.length > 0) {
-			if (term.includes("mods.factorio.com/mods/")) { // https://mods.factorio.com/mods/Danielv123/ChooChoo
-				// treat it as a link download
-				factorioAPI.init(false, modDirectory, "", instance.server.version);
-				factorioAPI.authenticate({
-					username: config.username,
-					token: config.token,
-					require_ownership: false
-				}).then(() => {
-					factorioAPI.searchMods(term.split("/")[term.split("/").length-1]).then(body => {
-						console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version);
-						factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{
-							console.log("Mod downloaded");
-							resolve();
-						});
-					});
-				});
-			} else if (term.includes("github.com")) { // https://github.com/Danielv123/factorioClusterioMod
-				// treat it as a github releases download
-				let apiURL = "https://api.github.com/repos/" + term.split("/")[3] + "/" + term.split("/")[4] + "/releases"; // https://api.github.com/repos/Danielv123/factorioClusterioMod/releases
-
-				let res = syncRequest('GET', apiURL, {"headers":{"User-Agent":"factorioClusterio"}});
-				let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
-				let name = JSON.parse(res.getBody())[0].assets[0].name;
-				if (url) {
-					console.log("Downloading " + url);
-					let file = fs.createWriteStream(path.join(modDirectory, name));
-					let bytesDownloaded = 0;
-					let downloadStartTime = Date.now();
-					https.get(url, function(response) {
-						let bytesToDownload = response.headers['content-length'];
-						response.pipe(file);
-						response.on("data", chunk => {
-							bytesDownloaded += chunk.length;
-							let downloadTimeElapsed = Date.now() - downloadStartTime;
-							let bytesPerSecond = bytesDownloaded / (downloadTimeElapsed/1000);
-							let timeToComplete = Math.floor((bytesToDownload - bytesDownloaded)/bytesPerSecond);
-							sLog("Downloading "+name+" "+(bytesDownloaded/bytesToDownload*100).toPrecision(3)+"% ("+formatBytes(bytesDownloaded)+" / "+formatBytes(bytesToDownload)+")\r\n"+
-							"about "+timeToComplete+" seconds remaining ("+formatBytes(bytesPerSecond)+"/s)");
-						}).on("end", x => {
-							sLog.clear();
-							console.log(" Downloaded in "+moment.utc(moment.duration(Date.now()-downloadStartTime).as('milliseconds')).format('mm:ss')); // because sLog.clear() doesn't seem to do the job of ending the line properly
-							resolve();
-						});
-					});
-				}
-			} else {
-				// treat it as a search and download the top result
-				factorioAPI.init(false, modDirectory, "", instance.server.version);
-				factorioAPI.authenticate({
-					username: config.username,
-					token: config.token,
-					require_ownership: false
-				}).then(() => {
-					factorioAPI.searchMods(term).then(body => {
-						console.log(body)
-						console.log("Im gonna download: "+body[0].name + " in version: " + body[0].latest_release.version);
-						factorioAPI.downloadMod( {name: body[0].name, version: body[0].latest_release.version}).then(()=>{
-							console.log("Mod downloaded");
-							resolve();
-						});
-					});
-				});
-			}
-		} else {
-			reject(new Error("Invalid search term"));
-		}
-	});
-}
-
-async function removeMod(config, term, instance, force = false) {
-	return new Promise((resolve, reject) => {
-		if (instance.name == "shared") {
-			var modDirectory = "sharedMods";
-		} else {
-			var modDirectory = instance.path("instanceMods");
-		}
-		if (term && typeof term == "string" && term.length > 0) {
-			let mods = fs.readdirSync(modDirectory);
-			let modsMatchingFilter = mods.filter(mod => mod.toLowerCase().includes(term.toLowerCase()));
-			if (modsMatchingFilter.length < 1) {
-				console.log("No mods found matching search");
-			} else if (modsMatchingFilter.length == 1) {
-				console.log("Deleting "+modsMatchingFilter[0]);
-				fs.unlinkSync(path.join(modDirectory, modsMatchingFilter[0]));
-				console.log("Deleted.");
-			} else if (modsMatchingFilter.length > 1) {
-				console.log("Found multiple mods");
-				modsMatchingFilter.forEach(mod => console.log(mod));
-				if (query("Delete all mods matching filter?")) {
-					let modsToDelete = modsMatchingFilter.length;
-					for (let i = 0; i < modsToDelete; i++) {
-						fs.unlinkSync(path.join(modDirectory, modsMatchingFilter[i]));
-						console.log("Deleted "+modsMatchingFilter[i]);
-					}
-				} else console.log("Did not delete mods.")
-			}
-			resolve();
-		} else {
-			reject(new Error("Invalid search term"));
-		}
-	});
-}
-
-// 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases'
-/**
-Download mod from github repo by looking for the first binary download in
-the latest release, saving it to /sharedMods
-
-@param {string} url a github repo  URL, for example https://api.github.com/repos/Danielv123/factorioClusterioMod/releases. Must link to /releases
-*/
-function downloadFromGitURL(url) {
-	console.log("Downloading mod from "+url);
-	// get JSON data about releases
-	let res = syncRequest('GET', url, {"headers":{"User-Agent":"factorioClusterio"}});
-	let downloadUrl = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
-	let name = JSON.parse(res.getBody())[0].assets[0].name;
-	console.log(JSON.parse(res.getBody()))
-	if (downloadUrl) {
-		console.log(downloadUrl);
-		let file = fs.createWriteStream("sharedMods/"+name);
-		https.get(downloadUrl, function(response) {
-			response.pipe(file);
-			console.log("Downloaded "+name);
-		});
-	}
-}
-
-/*
-function findUpdates() {
-	console.log("Downloading mods...");
-	// get JSON data about releases
-	let res = syncRequest('GET', 'https://api.github.com/repos/Danielv123/factorioClusterioMod/releases', {"headers":{"User-Agent":"factorioClusterio"}});
-	let url = JSON.parse(res.getBody())[0].assets[0].browser_download_url;
-	let name = JSON.parse(res.getBody())[0].assets[0].name;
-	console.log(JSON.parse(res.getBody()))
-}*/
-
-module.exports = {
-	listMods,
-	findMods,
-	addMod,
-	removeMod,
-	downloadFromGitURL,
-}

lib/npmPostinstall.js

@@ -1,116 +0,0 @@
-var http = require('http');
-var https = require('https');
-var fs = require('fs-extra');
-
-
-function makeDirectory(path){
-	return new Promise((resolve, reject) => {
-		fs.mkdirp(path, function(err){
-			if(err){
-				console.error("Unable to mkdirp "+path);
-				console.error(err);
-				resolve(false);
-			} else {
-				console.log("mkdirp "+path);
-				resolve(true);
-			}
-		});
-	});
-}
-function download(url, file, callback){
-	return new Promise((resolve, reject) => {
-		let pathArray = file.split("/");
-		pathArray.pop(); // remove filename from the end
-		if(pathArray.length >= 1){
-			// this might be an invalid path!
-			fs.stat(pathArray.join("/"), (err, stat) => {
-				if(err){
-					makeDirectory(pathArray.join("/")).then(()=>{
-						doDownload(url, file, callback);
-					});
-				} else {
-					doDownload(url, file, callback);
-				}
-			});
-		} else {
-			doDownload(url, file, callback);
-		}
-		function doDownload(url, file, callback){
-			var writeStream = fs.createWriteStream(file);
-			let protocol = http;
-			if(url.split(":")[0] == "https") protocol = https;
-			var request = protocol.get(url, function(response) {
-				response.pipe(writeStream);
-				response.on("end", () => {
-					if(callback && typeof callback == "function") callback(file);
-					console.log("Downloaded "+file+" from "+url);
-					resolve();
-				});
-			});
-		}
-	});
-}
-
-var directories = [
-	"sharedMods",
-	"instances",
-];
-let downloadList = [
-	// {
-		// url: "http://fbpviewer.trakos.pl/images/spritesheet.json",
-		// file: "static/pictures/spritesheet.js",
-		// callback: function(file){
-			// fs.writeFileSync("static/pictures/spritesheet.js", "export default\n"+fs.readFileSync("static/pictures/spritesheet.js"));
-		// },
-	// },
-	// {
-		// url: "http://fbpviewer.trakos.pl/images/spritesheet.png",
-		// file: "static/pictures/spritesheet.png",
-	// },
-	{
-		url: "https://code.jquery.com/jquery.js",
-		file: "static/external/jquery.js",
-	},
-	{
-		url: "https://code.jquery.com/ui/1.12.1/jquery-ui.js",
-		file: "static/external/jquery-ui.js",
-	},
-	{
-		url: "https://momentjs.com/downloads/moment.min.js",
-		file: "static/external/moment.min.js",
-	},
-	{
-		url: "https://craig.global.ssl.fastly.net/js/mousetrap/mousetrap.min.js",
-		file: "static/external/mousetrap.min.js",
-	},
-	{
-		url: "https://fonts.googleapis.com/css?family=Roboto",
-		file: "static/external/css/Roboto.css",
-	},
-]
-
-// initiate folder creations and downloads
-function processMkdirRecursive(callback){
-	let item = directories.shift();
-	makeDirectory(item).then(()=>{
-		if(directories.length == 0){
-			if(callback) callback();
-		} else {
-			processMkdirRecursive(callback);
-		}
-	});
-}
-function processDownloadRecursive(callback){
-	let item = downloadList.shift();
-	download(item.url, item.file, item.callback).then(()=>{
-		if(downloadList.length == 0){
-			if(callback) callback();
-		} else {
-			processDownloadRecursive(callback);
-		}
-	});
-}
-
-if (module === require.main) {
-	processMkdirRecursive(processDownloadRecursive);
-}

lib/objectOps.js

@@ -1,44 +0,0 @@
-/**
-Helper functions related to objects
-@module
-*/
-
-/**
-Deep clones an object, is actually just JSON.parse(JSON.stringify()) but looks cleaner.
-@param {object} obj to clone
-@returns {object}
-
-@example
-a = {"thingy":"with a value"};
-b = a;
-a == b; // true
-c = objectOps.deepclone(a);
-a == c; // false
-*/
-module.exports.deepclone = function(obj){
-	return JSON.parse(JSON.stringify(obj));
-}
-/**
-Checks if a string is valid JSON using try/catch. If passed an object it will always return true.
-@param {string} string to verify
-@returns {boolean}
-
-@example
-objectOps.isJSON("99 bottles of beer"); // false
-objectOps.isJSON("99"); // true
-objectOps.isJSON({"thingy":"that is an object"}); // true
-objectOps.isJSON(JSON.stringify({"thingy":"that is an object"})); // true
-*/
-module.exports.isJSON = function(string){
-	let stringIsJson = false;
-	let x
-	try {
-		x = JSON.parse(string);
-	} catch (e){
-		return false;
-	}
-	if(typeof x == "object" || typeof string == "object"){
-		stringIsJson = true;
-	}
-	return stringIsJson;
-}

lib/package.json

@@ -1,5 +0,0 @@
-{
-    "name": "lib",
-    "description": "Internal library for factorioClusterio"
-}
-

lib/plugin.js

@@ -1,192 +0,0 @@
-/**
- * Plugin interfaces and utilities.
- * @module
- */
-"use strict";
-const fs = require("fs-extra");
-const path = require("path");
-
-const errors = require("lib/errors");
-
-
-/**
- * Base class for instance plugins
- */
-class BaseInstancePlugin {
-	constructor(info, instance, slave) {
-		/**
-		 * The plugin's own info module
-		 */
-		this.info = info;
-
-		/**
-		 * Instance the plugin started for
-		 * @type {Instance}
-		 */
-		this.instance = instance;
-
-		/**
-		 * Slave running the instance
-		 *
-		 * With the exepction of accessing the slave's config you should
-		 * avoid ineracting with the slave object directly.
-		 *
-		 * @type {Slave}
-		 */
-		this.slave = slave;
-	}
-
-	/**
-	 * Called immediately after the class is instantiated
-	 */
-	async init() { }
-
-	/**
-	 * Called before collecting Prometheus metrics
-	 *
-	 * Invoked before the default metrics of prometheus is collected.  Note
-	 * that since this is done while replying to /metrics this should not
-	 * take a long time, otherwise the requst will time out.
-	 *
-	 * Although you can return the results of collecting your prometheus
-	 * collectors directly, all gauges in the default registry is already
-	 * automatically collected.  It's therefore recommended to only update
-	 * the Gauges/Counters/etc in onMetric and let Clusterio deal will
-	 * collecting the values.
-	 *
-	 * @returns an async iterator of prometheus metric results or undefined.
-	 */
-	async onMetrics() { }
-
-	/**
-	 * Called after the Factorio server is started
-	 */
-	async onStart() { }
-
-	/**
-	 * Called before the Factorio server is stopped
-	 *
-	 * This will not be called if for example the Factorio server crashes or
-	 * is killed.
-	 */
-	async onStop() { }
-
-	/**
-	 * Called when the Factorio server exited
-	 */
-	onExit() { }
-
-	/**
-	 * Called when the Factorio outputs a line
-	 *
-	 * Called when the Factorio server outputs a line on either stdout or
-	 * stderr.  The output is a parsed form of the line and contains the
-	 * following properties (those marked with ? are optional):
-	 *
-	 * - source: Where the message came from, one of "stdout" and "stderr".
-	 * - format: Timestamp format, one of "date", "seconds" and "none".
-	 * - time?: Timestamp of the message.  Not present if format is "none".
-	 * - type: Type of message, one of "log", "action" and "generic".
-	 * - level?: Log level for "log" type.  i.e "Info" normally.
-	 * - file?: File reported for "log" type.
-	 * - action?: Kind of action for "action" type. i.e "CHAT" for chat.
-	 * - message: Main content of the line.
-	 */
-	async onOutput(output) { }
-}
-
-class BaseMasterPlugin {
-	constructor(info, master, metrics) {
-		this.info = info;
-		this.master = master;
-		this.metrics = metrics;
-	}
-
-	/**
-	 * Called immediately after the class is instantiated
-	 */
-	async init() { }
-
-	/**
-	 * Called before collecting Prometheus metrics
-	 *
-	 * Invoked before the default metrics of prometheus is collected.  Note
-	 * that since this is done while replying to /metrics this should not
-	 * take a long time, otherwise the requst will time out.
-	 *
-	 * Although you can return the results of collecting your prometheus
-	 * collectors directly, all gauges in the default registry is already
-	 * automatically collected.  It's therefore recommended to only update
-	 * the Gauges/Counters/etc in onMetric and let Clusterio deal will
-	 * collecting the values.
-	 *
-	 * @returns an async iterator of prometheus metric results or undefined.
-	 */
-	async onMetrics() { }
-
-	/**
-	 * Called when the master server is shutting down
-	 */
-	async onExit() { }
-}
-
-/**
- * Load plugin information
- *
- * Searches through the plugin directory and loads each plugin's info
- * module.  Once loaded the info modules will not be reloaded should this
- * function be called again.
- *
- * @returns {Map<string, Object>} mapping of plugin name to info module.
- */
-async function loadPluginInfos(baseDir) {
-	let plugins = [];
-	for (let pluginDir of await fs.readdir(baseDir)) {
-		let pluginInfo;
-
-		try {
-			// Note: Require path is relative to this module
-			pluginInfo = require(path.posix.join('..', baseDir, pluginDir, 'info'));
-
-		} catch (err) {
-			if (err.code === 'MODULE_NOT_FOUND') {
-				continue;
-			} else {
-				throw new errors.PluginError(pluginDir, err);
-			}
-		}
-
-		if (pluginInfo.name !== pluginDir) {
-			throw new errors.EnvironmentError(
-				`Plugin dir ${baseDir}/${pluginDir} does not match the name of the plugin (${pluginInfo.name})`
-			);
-		}
-
-		plugins.push(pluginInfo);
-	}
-	return plugins;
-}
-
-function attachPluginMessages(link, pluginInfo, plugin) {
-	let messageDefinitions = pluginInfo.messages || [];
-	for (let [name, messageFormat] of Object.entries(messageDefinitions)) {
-		let handler = name + messageFormat.constructor.name + 'Handler';
-		if (plugin === null || !plugin[handler]) {
-			messageFormat.attach(link);
-
-		} else {
-			messageFormat.attach(link, async function(message, format) {
-				return await plugin[handler](message, format, this);
-			});
-		}
-	}
-}
-
-
-module.exports = {
-	BaseInstancePlugin,
-	BaseMasterPlugin,
-
-	loadPluginInfos,
-	attachPluginMessages,
-};

lib/prometheus.js

@@ -1,432 +0,0 @@
-class Metric {
-	constructor(type, name, help, labels = []) {
-		if (typeof name !== 'string') {
-			throw new Error("Expected name to be a string");
-		}
-
-		if (typeof help !== 'string') {
-			throw new Error("Expected help to be a string");
-		}
-
-		if (!/^[a-zA-Z_:][a-zA-Z0-9_:]*$/.test(name)) {
-			throw new Error(`Invalid name '${name}'`);
-		}
-
-		for (let name of labels) {
-			if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(name)) {
-				throw new Error(`Invalid label '${name}'`);
-			}
-		}
-
-		this.name = name;
-		this.help = help;
-		this.type = type;
-		this.labels = labels;
-	}
-}
-
-let defaultRegistry;
-
-class Collector {
-	constructor(register = true) {
-		if (register) {
-			defaultRegistry.register(this);
-		}
-	}
-
-	async* collect() { }
-}
-
-function escapeLabelValue(value) {
-	return value
-		.replace(/\\/g, "\\\\")
-		.replace(/\n/g, "\\n")
-		.replace(/"/g, "\\\"")
-	;
-}
-
-function labelsToKey(labels, metricLabels) {
-	let items = []
-	if (labels.length === 1 && typeof labels[0] === "object") {
-		labels = labels[0];
-		for (let name of metricLabels) {
-			if (!Object.hasOwnProperty.call(labels, name)) {
-				throw new Error(`Missing label '${name}'`);
-			}
-			if (typeof labels[name] !== "string") {
-				throw new Error(
-					`Expected value for label '${name}' to be a string`
-				);
-			}
-			items.push(`${name}="${escapeLabelValue(labels[name])}"`);
-		}
-
-		for (let name of Object.keys(labels)) {
-			if (!metricLabels.includes(name)) {
-				throw new Error(`Extra label '${name}'`);
-			}
-		}
-
-	} else {
-		if (metricLabels.length > labels.length) {
-			throw new Error(`Missing label '${metricLabels[labels.length]}'`);
-		}
-
-		if (labels.length > metricLabels.length) {
-			throw new Error(`Extra positional label`);
-		}
-
-		for (let i=0; i < metricLabels.length; ++i) {
-			if (typeof labels[i] !== "string") {
-				throw new Error(
-					`Expected value for label '${metricLabels[i]}' to be a string`
-				);
-			}
-			items.push(`${metricLabels[i]}="${escapeLabelValue(labels[i])}"`);
-		}
-	}
-
-	return items.join(',');
-}
-
-function keyToLabels(key) {
-	let labels = new Map();
-
-	if (key !== '') for (let pair of key.split(',')) {
-		let [name, value] = pair.split('=', 2);
-		labels.set(name, value
-			.slice(1, -1)
-			.replace(/\\"/g, "\"")
-			.replace(/\\n/g, "\n")
-			.replace(/\\\\/g, "\\")
-		);
-	}
-
-	return labels;
-}
-
-class ValueCollector extends Collector {
-	constructor(type, name, help, options, childClass) {
-		let labels = [];
-		let register = true;
-		let callback = null;
-		for (let [name, value] of Object.entries(options)) {
-			if (name === 'labels') { labels = value; }
-			else if (name === 'register') { register = value; }
-			else if (name === 'callback') { callback = value; }
-			else { throw new Error(`Unrecognized option '${name}'`); }
-		}
-
-		// Make sure we don't register to the default registry if metric throws.
-		let metric = new Metric(type, name, help, labels);
-
-		super(register);
-
-		this.callback = callback;
-		this.metric = metric;
-		this._values = new Map();
-		this._children = new Map();
-		this._childClass = childClass;
-		this._defaultChild = labels.length ? null : this.labels({});
-	}
-
-	async* collect() {
-		if (this.callback) { await this.callback(this); }
-		yield { metric: this.metric, samples: this._values };
-	}
-
-	get() {
-		return this._defaultChild.get();
-	}
-
-	labels(...labels) {
-		let key = labelsToKey(labels, this.metric.labels);
-		let child = this._children.get(key);
-		if (child === undefined) {
-			child = new this._childClass(this._values, key);
-			this._children.set(key, child);
-		}
-		return child;
-	}
-
-	remove(...labels) {
-		let key = labelsToKey(labels, this.metric.labels);
-		if (key === '') {
-			throw new Error("labels cannot be empty");
-		}
-		this._children.delete(key);
-		this._values.delete(key);
-	}
-
-	removeAll(labels) {
-		if (!Object.keys(labels).length) {
-			throw new Error("labels cannot be empty");
-		}
-
-		function removeMatchingLabels(mapping, labels) {
-			for (let key of mapping.keys()) {
-				let candidate = keyToLabels(key);
-				let hasLabels = Object.entries(labels).every(([name, value]) => {
-					return candidate.get(name) === value;
-				});
-				if (hasLabels) {
-					mapping.delete(key)
-				}
-			}
-		}
-
-		removeMatchingLabels(this._values, labels);
-		removeMatchingLabels(this._children, labels);
-	}
-
-	clear() {
-		if (!this.metric.labels.length) {
-			throw new Error("Cannot clear unlabeled metric");
-		}
-		this._children = new Map();
-		this._values = new Map();
-	}
-}
-
-class ValueCollectorChild {
-	constructor(values, key) {
-		this._values = values;
-		this._key = key;
-
-		this._values.set(key, 0);
-	}
-
-	get() {
-		return this._values.get(this._key);
-	}
-}
-
-class CounterChild extends ValueCollectorChild {
-	inc(value = 1) {
-		// Note: Inverted to also catch NaN
-		if (!(value >= 0)) {
-			throw new Error("Expected value to be a positive number");
-		}
-
-		this._values.set(this._key, this._values.get(this._key) + value)
-	}
-}
-
-class Counter extends ValueCollector {
-	constructor(name, help, options = {}) {
-		super("counter", name, help, options, CounterChild);
-	}
-
-	inc(value = 1) {
-		this._defaultChild.inc(value);
-	}
-}
-
-class GaugeChild extends ValueCollectorChild {
-	inc(value = 1) {
-		this._values.set(this._key, this._values.get(this._key) + value)
-	}
-
-	dec(value = 1) {
-		this._values.set(this._key, this._values.get(this._key) - value)
-	}
-
-	set(value) {
-		this._values.set(this._key, value)
-	}
-
-	setToCurrentTime() {
-		this._values.set(this._key, Date.now() / 1000)
-	}
-}
-
-class Gauge extends ValueCollector {
-	constructor(name, help, options = {}) {
-		super("gauge", name, help, options, GaugeChild);
-	}
-
-	inc(value = 1) {
-		this._defaultChild.inc(value);
-	}
-
-	dec(value = 1) {
-		this._defaultChild.dec(value);
-	}
-
-	set(value) {
-		this._defaultChild.set(value);
-	}
-
-	setToCurrentTime() {
-		this._defaultChild.setToCurrentTime();
-	}
-}
-
-class CollectorRegistry {
-	constructor() {
-		this.collectors = [];
-	}
-
-	async* collect() {
-		for (let collector of this.collectors) {
-			for await (let result of collector.collect()) {
-				yield result;
-			}
-		}
-	}
-
-	register(metric) {
-		let index = this.collectors.lastIndexOf(metric);
-		if (index !== -1) {
-			throw new Error(
-				"Collector is already registered in this registry."
-			);
-		}
-
-		this.collectors.push(metric);
-	}
-
-	unregister(metric) {
-		let index = this.collectors.lastIndexOf(metric);
-		if (index === -1) {
-			throw new Error(
-				"Collector is not registered in this registry."
-			);
-		}
-
-		this.collectors.splice(index, 1);
-	}
-}
-
-defaultRegistry = new CollectorRegistry();
-
-function escapeHelp(help) {
-	return help
-		.replace(/\\/g, "\\\\")
-		.replace(/\n/g, "\\n")
-	;
-}
-
-function formatValue(value) {
-	if (value === Infinity) {
-		return '+Inf';
-	} else if (value === -Infinity) {
-		return '-Inf';
-	} else {
-		return value.toString();
-	}
-}
-
-async function* expositionLines(resultsIterator) {
-	let first = true;
-	for await (let result of resultsIterator) {
-		if (first) {
-			first = false;
-		} else {
-			yield `\n`
-		}
-
-		yield `# HELP ${result.metric.name} ${escapeHelp(result.metric.help)}\n`;
-		yield `# TYPE ${result.metric.name} ${result.metric.type}\n`;
-
-		for (let [key, value] of result.samples) {
-			if (key === '') {
-				yield `${result.metric.name} ${formatValue(value)}\n`;
-			} else {
-				yield `${result.metric.name}{${key}} ${formatValue(value)}\n`;
-			}
-		}
-	}
-}
-
-async function exposition(resultsIterator = defaultRegistry.collect()) {
-	let exposition = '';
-	for await (let line of expositionLines(resultsIterator)) {
-		exposition += line;
-	}
-	return exposition;
-}
-
-// HTTP Content-Type for the exposition format that's implemented
-exposition.contentType = 'text/plain; version=0.0.4';
-
-function serializeResult(result, options = {}) {
-	let addLabels = null;
-	let metricName = result.metric.name;
-	let metricHelp = result.metric.help;
-	for (let [name, value] of Object.entries(options)) {
-		if (name === 'addLabels') { addLabels = value; }
-		else if (name === 'metricName') { metricName = value; }
-		else if (name === 'metricHelp') { metricHelp = value; }
-		else { throw new Error(`Unrecognized option '${name}'`); }
-	}
-
-	let samples;
-	if (addLabels === null) {
-		samples = result.samples;
-		addLabels = {};
-
-	} else {
-		samples = new Map();
-		let key = labelsToKey([addLabels], [...Object.keys(addLabels)]);
-		for (let [labels, value] of result.samples.entries()) {
-			if (labels === '') { samples.set(key, value); }
-			else { samples.set(`${labels},${key}`, value); }
-		}
-	}
-
-	return {
-		metric: {
-			type: result.metric.type,
-			name: metricName,
-			help: metricHelp,
-			labels: [...result.metric.labels, ...Object.keys(addLabels)],
-		},
-		samples: [...samples.entries()],
-	};
-}
-
-function deserializeResult(serializedResult) {
-	return {
-		metric: new Metric(
-			serializedResult.metric.type,
-			serializedResult.metric.name,
-			serializedResult.metric.help,
-			serializedResult.metric.labels,
-		),
-		samples: new Map(serializedResult.samples),
-	}
-}
-
-let defaultCollectors = {};
-defaultCollectors.processCpuSecondsTotal = new Gauge(
-	'process_cpu_seconds', "Total user and system CPU time spent in seconds.",
-	{
-		callback: async function() {
-			let usage = process.cpuUsage();
-			this.set((usage.user + usage.system) / 1e6);
-		},
-	},
-);
-
-defaultCollectors.processStartTimeSeconds = new Gauge(
-	'process_start_time_seconds',
-	"Start time of the process since unix epoch in seconds."
-);
-defaultCollectors.processStartTimeSeconds.setToCurrentTime();
-
-
-module.exports = {
-	Counter,
-	Gauge,
-	CollectorRegistry,
-	Collector,
-	ValueCollector,
-	Metric,
-
-	exposition,
-	defaultRegistry,
-	defaultCollectors,
-	serializeResult,
-	deserializeResult,
-};

mod/changelog.txt

@@ -1,5 +0,0 @@
----------------------------------------------------------------------------------------------------
-Version: 0.1.0
-  Major Features:
-    - API for mods to interact with Clusterio
-    - Serialization library for converting items to JSON compatible tables.

mod/info.json

@@ -1,9 +0,0 @@
-{
-    "name": "clusterio_lib",
-    "version": "0.1.0",
-    "title": "Clusterio Library (Alpha)",
-    "author": "Clusterio Team",
-    "homepage": "https://github.com/clusterio/factorioClusterio",
-    "description": "Library for interfacing with Clusterio.  Warning: work in progress alpha.",
-    "factorio_version": "0.17"
-}

package.json

@@ -1,17 +1,14 @@
 {
-  "name": "clusterio",
-  "version": "2.0.0-alpha",
-  "description": "Interserver communication for factorio",
-  "repository": "https://github.com/clusterio/factorioClusterio",
-  "main": "master.js",
+  "name": "root",
+  "private": true,
   "scripts": {
-    "test": "mocha --exit --check-leaks -R spec --recursive test plugins/*/test",
+    "test": "mocha --exit --check-leaks -R spec --recursive test plugins/global_chat/test plugins/research_sync/test plugins/statistics_exporter/test plugins/subspace_storage/test",
     "fast-test": "FAST_TEST=y npm test",
     "cover": "nyc npm test",
     "fast-cover": "FAST_TEST=y nyc npm test",
     "ci-cover": "nyc --reporter=lcovonly npm run-script test",
+    "lint": "eslint \"*.js\" packages plugins test",
     "docs": "jsdoc --configure docs/jsdoc.json",
-    "install": "node lib/npmPostinstall",
     "bundle-dependencies": "bundle-dependencies"
   },
   "nyc": {
@@ -20,61 +17,25 @@
       "plugins/*/test/**"
     ]
   },
-  "keywords": [
-    "factorio"
-  ],
-  "author": "Danielv123",
-  "license": "MIT",
   "engines": {
-    "node": ">=10.16 <11 || >= 11.13"
-  },
-  "dependencies": {
-    "as-table": "^1.0.31",
-    "base64url": "^3.0.0",
-    "bcrypt": "=3.0.6",
-    "bcrypt-promise": "^2.0.0",
-    "body-parser": "^1.18.2",
-    "chalk": "^3.0.0",
-    "cli-interact": "^0.1.9",
-    "compression": "^1.7.2",
-    "cookie-parser": "^1.4.3",
-    "deepmerge": "^4.0.0",
-    "ejs": "^2.5.7",
-    "express": "^4.16.2",
-    "express-fileupload": "1.1.5",
-    "follow-redirects": "^1.5.0",
-    "fs-extra": "^8.1.0",
-    "ini": "^1.3.5",
-    "jsonwebtoken": "^8.2.1",
-    "jszip": "^3.2.2",
-    "lib": "file:./lib",
-    "luamin": "^1.0.4",
-    "minimist": "^1.2.0",
-    "moment": "^2.22.2",
-    "needle": "^2.4.0",
-    "node-factorio-api": "^0.3.8",
-    "node-forge": "^0.7.1",
-    "rcon-client": "^4.2.0",
-    "request": "^2.83.0",
-    "sanitizer": "^0.1.3",
-    "semver": "^7.1.1",
-    "set-blocking": "^2.0.0",
-    "simple-git": "^1.96.0",
-    "single-line-log": "^1.1.2",
-    "sync-request": "^4.0.1",
-    "typescript-collections": "^1.3.2",
-    "ws": "^7.2.3",
-    "yargs": "^14.2.0"
+    "node": ">=10.16 <11 || >= 12"
   },
   "devDependencies": {
-    "html5-validator": "^1.2.1",
+    "@babel/core": "^7.12.3",
+    "@babel/preset-react": "^7.12.1",
+    "@clusterio/ctl": "file:./packages/ctl",
+    "@clusterio/lib": "file:./packages/lib",
+    "@clusterio/master": "file:packages/master",
+    "@clusterio/slave": "file:./packages/slave",
+    "eslint": "^7.2.0",
+    "fs-extra": "^8.1.0",
     "jsdoc": "^3.6.3",
-    "klaw": "^3.0.0",
+    "jsonwebtoken": "^8.2.1",
+    "jszip": "^3.2.2",
+    "lerna": "^3.22.1",
     "mocha": "^6.2.0",
-    "mocha-sinon": "^2.1.0",
-    "mocha.parallel": "^0.15.5",
-    "nock": "^11.3.0",
     "nyc": "^14.1.1",
-    "sinon": "^7.4.1"
+    "phin": "^3.4.1",
+    "webpack": "^5.0.0"
   }
 }

packages/ctl/README.md

@@ -0,0 +1,108 @@
+Clusterio Command Line Interface
+================================
+
+Provides a command line interface for managing a Clusterio cluster.
+
+Note that this document only describes the options and commands for
+managing clusterioctl itself, see [Managing a
+cluster](/docs/managing-a-cluster.md) for the available commands used
+for managing a cluster.
+
+
+Usage
+-----
+
+    npx clusterioctl <command>
+
+Common options:
+
+ * `--plugin-list <file>` JSON file to use for storing the list of
+   plugins that are available to ctl.  Defaults to `plugin-list.json`
+   and will be created if it does not exist. See the `plugin` command
+   for managing this list.
+
+ * `--config <file>` JSON file to use for storing configuration for
+   ctl.  Defaults to `config-control.json` and will be created if it
+   does not exist.  See the `config` command for inspecting and
+   modifying the configuration.
+
+
+### `plugin <command>`
+
+Configure plugins available to be loaded by ctl.  The available plugins
+will be automatically loaded and the commands they provide will be added
+to the ctl utility.
+
+
+#### `plugin add <path>`
+
+Add plugin either by require path or relative/absolute path to the
+plugin directory.  A relative path must start with ./ or ../ (or .\ and
+..\ on Windows) otherwise it will be assumed to be a require path for an
+installed package in node_modules.
+
+For example, installing the Subspace Storage plugin:
+
+    npm install @clusterio/plugin-subspace_storage
+    npx clusterioctl plugin add @clusterio/plugin-subspace_storage
+
+Since the `plugin-list.json` is shared between master, slave and ctl you
+usually only need to do this once per machine.
+
+
+#### `plugin remove <name>`
+
+Remove a plugin by its name.  This should be done before uninstalling
+the plugin, otherwise there will be an error when Clusterio tries to
+load the info from the plugin.  Removing and unistalling a plugin is
+usually not neccessary as the functions provided by the plugin can be
+disabled in the config.
+
+For example, uninstalling the Subspace Storage plugin:
+
+    npx clusterioctl plugin remove subspace_storage
+    npm uninstall @clusterio/plugin-subspace_storage
+
+Since the `plugin-list.json` is shared between master, slave and ctl you
+usually only need to do this once per machine.
+
+
+#### `plugin list`
+
+Lists the plugins set up to be available by name followed by path.
+
+
+### `config-control`
+
+Manage the ctl configuration.  This allows setting the url and token to
+the master server to manage.
+
+
+#### `config-control set <config-entry> [value]`
+
+Set a config entry to the given value.  If value is not provided the
+entry is set to null.  If the config entry is of type object the value
+must be a valid JSON serialization of an object.
+
+See docs/configuration.md in the main repositiory for the available
+configuration.
+
+
+#### `config-control show <config-entry>`
+
+Shows the value for a single config entry.
+
+
+#### `config-control list`
+
+Lists up all configuration entries with their currently configured values.
+
+
+## See Also
+
+[Managing a cluster](/docs/managing-a-cluster.md) for the available
+commands used for managing a cluster.
+
+[The Clusterio
+repository](https://github.com/clusterio/factorioClusterio)
+for instructions on how to set up a cluster.

packages/ctl/ctl.js

@@ -0,0 +1,998 @@
+#!/usr/bin/env node
+
+/**
+ * Command line interface for controlling a Clusterio cluster
+ * @module ctl/ctl
+ */
+"use strict";
+const fs = require("fs-extra");
+const yargs = require("yargs");
+const version = require("./package").version;
+const asTable = require("as-table").configure({ delimiter: " | " });
+const events = require("events");
+const path = require("path");
+const winston = require("winston");
+const setBlocking = require("set-blocking");
+
+const libLink = require("@clusterio/lib/link");
+const libErrors = require("@clusterio/lib/errors");
+const libConfig = require("@clusterio/lib/config");
+const libPlugin = require("@clusterio/lib/plugin");
+const libPluginLoader = require("@clusterio/lib/plugin_loader");
+const libCommand = require("@clusterio/lib/command");
+const libSharedCommands = require("@clusterio/lib/shared_commands");
+const { ConsoleTransport, levels, logger } = require("@clusterio/lib/logging");
+const libLoggingUtils = require("@clusterio/lib/logging_utils");
+const libHelpers = require("@clusterio/lib/helpers");
+
+
+function print(...content) {
+	// eslint-disable-next-line no-console
+	console.log(...content);
+}
+
+
+const masterCommands = new libCommand.CommandTree({ name: "master", description: "Master management" });
+const masterConfigCommands = new libCommand.CommandTree({
+	name: "config", alias: ["c"], description: "master config management",
+});
+masterConfigCommands.add(new libCommand.Command({
+	definition: ["list", "List master configuration"],
+	handler: async function(args, control) {
+		let response = await libLink.messages.getMasterConfig.send(control);
+
+		for (let group of response.serialized_config.groups) {
+			for (let [name, value] of Object.entries(group.fields)) {
+				print(`${group.name}.${name} ${JSON.stringify(value)}`);
+			}
+		}
+	},
+}));
+
+masterConfigCommands.add(new libCommand.Command({
+	definition: ["set <field> [value]", "Set field in master config", (yargs) => {
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("value", { describe: "Value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args, control) {
+		if (args.stdin) {
+			args.value = (await libHelpers.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		} else if (args.value === undefined) {
+			args.value = "";
+		}
+		await libLink.messages.setMasterConfigField.send(control, {
+			field: args.field,
+			value: args.value,
+		});
+	},
+}));
+
+masterConfigCommands.add(new libCommand.Command({
+	definition: ["set-prop <field> <prop> [value]", "Set property of field in master config", (yargs) => {
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("prop", { describe: "Property to set", type: "string" });
+		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args, control) {
+		if (args.stdin) {
+			args.value = (await libHelpers.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		}
+		let request = {
+			field: args.field,
+			prop: args.prop,
+		};
+		try {
+			if (args.value !== undefined) {
+				request.value = JSON.parse(args.value);
+			}
+		} catch (err) {
+			// See note for the instance version of set-prop
+			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value)) {
+				throw new libErrors.CommandError(`In parsing value '${args.value}': ${err.message}`);
+			}
+			request.value = args.value;
+		}
+		await libLink.messages.setMasterConfigProp.send(control, request);
+	},
+}));
+masterCommands.add(masterConfigCommands);
+
+
+const slaveCommands = new libCommand.CommandTree({ name: "slave", description: "Slave management" });
+slaveCommands.add(new libCommand.Command({
+	definition: [["list", "l"], "List slaves connected to the master"],
+	handler: async function(args, control) {
+		let response = await libLink.messages.listSlaves.send(control);
+		print(asTable(response.list));
+	},
+}));
+
+slaveCommands.add(new libCommand.Command({
+	definition: ["generate-token", "Generate token for a slave", (yargs) => {
+		yargs.option("id", { type: "number", nargs: 1, describe: "Slave id", demandOption: true });
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.generateSlaveToken.send(control, { slave_id: args.id });
+		print(response.token);
+	},
+}));
+
+slaveCommands.add(new libCommand.Command({
+	definition: ["create-config", "Create slave config", (yargs) => {
+		yargs.option("id", { type: "number", nargs: 1, describe: "Slave id", default: null });
+		yargs.option("name", { type: "string", nargs: 1, describe: "Slave name", default: null });
+		yargs.option("generate-token", {
+			type: "boolean", nargs: 0, describe: "Generate authentication token", default: false,
+		});
+		yargs.option("output", {
+			type: "string", nargs: 1, describe: "Path to output config (- for stdout)", default: "config-slave.json",
+		});
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.createSlaveConfig.send(control, {
+			id: args.id, name: args.name, generate_token: args.generateToken,
+		});
+
+		let content = JSON.stringify(response.serialized_config, null, 4);
+		if (args.output === "-") {
+			print(content);
+		} else {
+			logger.info(`Writing ${args.output}`);
+			try {
+				await fs.outputFile(args.output, content, { flag: "wx" });
+			} catch (err) {
+				if (err.code === "EEXIST") {
+					throw new libErrors.CommandError(`File ${args.output} already exists`);
+				}
+				throw err;
+			}
+		}
+	},
+}));
+
+
+const instanceCommands = new libCommand.CommandTree({
+	name: "instance", alias: ["i"], description: "Instance management",
+});
+instanceCommands.add(new libCommand.Command({
+	definition: [["list", "l"], "List instances known to the master"],
+	handler: async function(args, control) {
+		let response = await libLink.messages.listInstances.send(control);
+		print(asTable(response.list));
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["create <name>", "Create an instance", (yargs) => {
+		// XXX TODO: set any specific options?
+		yargs.positional("name", { describe: "Instance name", type: "string" });
+		yargs.options({
+			"id": { type: "number", nargs: 1, describe: "Instance id" },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceConfig = new libConfig.InstanceConfig("control");
+		await instanceConfig.init();
+		if (args.id) {
+			instanceConfig.set("instance.id", args.id);
+		}
+		instanceConfig.set("instance.name", args.name);
+		let serialized_config = instanceConfig.serialize("master");
+		let response = await libLink.messages.createInstance.send(control, { serialized_config });
+	},
+}));
+
+const instanceConfigCommands = new libCommand.CommandTree({
+	name: "config", alias: ["c"], description: "Instance config management",
+});
+instanceConfigCommands.add(new libCommand.Command({
+	definition: ["list <instance>", "List configuration for an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to list config for", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		let response = await libLink.messages.getInstanceConfig.send(control, { instance_id: instanceId });
+
+		for (let group of response.serialized_config.groups) {
+			for (let [name, value] of Object.entries(group.fields)) {
+				print(`${group.name}.${name} ${JSON.stringify(value)}`);
+			}
+		}
+	},
+}));
+
+instanceConfigCommands.add(new libCommand.Command({
+	definition: ["set <instance> <field> [value]", "Set field in instance config", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("value", { describe: "Value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		if (args.stdin) {
+			args.value = (await libHelpers.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		} else if (args.value === undefined) {
+			args.value = "";
+		}
+		await libLink.messages.setInstanceConfigField.send(control, {
+			instance_id: instanceId,
+			field: args.field,
+			value: args.value,
+		});
+	},
+}));
+
+instanceConfigCommands.add(new libCommand.Command({
+	definition: ["set-prop <instance> <field> <prop> [value]", "Set property of field in instance config", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("prop", { describe: "Property to set", type: "string" });
+		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		if (args.stdin) {
+			args.value = (await libHelpers.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		}
+		let request = {
+			instance_id: instanceId,
+			field: args.field,
+			prop: args.prop,
+		};
+		try {
+			if (args.value !== undefined) {
+				request.value = JSON.parse(args.value);
+			}
+		} catch (err) {
+			// If this is from stdin or looks like an array, object or string
+			// literal throw the parse error, otherwise assume this is a string.
+			// The resoning behind this is that correctly quoting the string
+			// with the all the layers of quote removal at play is difficult.
+			// See the following table for how to pass "That's a \" quote" in
+			// different environments:
+			// cmd              : """""That's a \\"" quote"""""
+			// cmd + npx        : """""""""""That's a \\\\"""" quote"""""""""""
+			// PowerShell       : '"""""That''s a \\"" quote"""""'
+			// PowerShell + npx : '"""""""""""That''s a \\\\"""" quote"""""""""""'
+			// bash             : '""That'\''s a \" quote""'
+			// bash + npx       : '""That'\''s a \" quote""'
+			// bash + npx -s sh : "'\"\"That'\\''s a \\\" quote\"\"'"
+			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value)) {
+				throw new libErrors.CommandError(`In parsing value '${args.value}': ${err.message}`);
+			}
+			request.value = args.value;
+		}
+		await libLink.messages.setInstanceConfigProp.send(control, request);
+	},
+}));
+instanceCommands.add(instanceConfigCommands);
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["assign <instance> [slave]", "Assign instance to a slave", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to assign", type: "string" });
+		yargs.positional("slave", { describe: "Slave to assign to or unassign if none", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		let slaveId = args.slave ? await libCommand.resolveSlave(control, args.slave) : null;
+		await libLink.messages.assignInstanceCommand.send(control, {
+			instance_id: instanceId,
+			slave_id: slaveId,
+		});
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["create-save <instance>", "Create a new save on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to create on", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instance_ids: [instanceId] });
+		let response = await libLink.messages.createSave.send(control, {
+			instance_id: instanceId,
+		});
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["export-data <instance>", "Export item icons and locale from instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to export from", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instance_ids: [instanceId] });
+		let response = await libLink.messages.exportData.send(control, {
+			instance_id: instanceId,
+		});
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["start <instance>", "Start instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to start", type: "string" });
+		yargs.options({
+			"save": { describe: "Save load, defaults to latest", nargs: 1, type: "string" },
+			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instance_ids: [instanceId] });
+		let response = await libLink.messages.startInstance.send(control, {
+			instance_id: instanceId,
+			save: args.save || null,
+		});
+		control.keepOpen = args.keepOpen;
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["load-scenario <instance> <scenario>", "Start instance by loading a scenario", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to start", type: "string" });
+		yargs.positional("scenario", { describe: "Scenario to load", type: "string" });
+		yargs.options({
+			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instance_ids: [instanceId] });
+		let response = await libLink.messages.loadScenario.send(control, {
+			instance_id: instanceId,
+			scenario: args.scenario || null,
+		});
+		control.keepOpen = args.keepOpen;
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["stop <instance>", "Stop instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to stop", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		await control.setLogSubscriptions({ instance_ids: [instanceId] });
+		let response = await libLink.messages.stopInstance.send(control, {
+			instance_id: instanceId,
+		});
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["delete <instance>", "Delete instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to delete", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.deleteInstance.send(control, {
+			instance_id: await libCommand.resolveInstance(control, args.instance),
+		});
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["send-rcon <instance> <command>", "Send RCON command", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to send to", type: "string" });
+		yargs.positional("command", { describe: "command to send", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.sendRcon.send(control, {
+			instance_id: await libCommand.resolveInstance(control, args.instance),
+			command: args.command,
+		});
+
+		// Factorio includes a newline in it's response output.
+		process.stdout.write(response.result);
+	},
+}));
+
+const permissionCommands = new libCommand.CommandTree({ name: "permission", description: "Permission inspection" });
+permissionCommands.add(new libCommand.Command({
+	definition: [["list", "l"], "List permissions in the cluster"],
+	handler: async function(args, control) {
+		let response = await libLink.messages.listPermissions.send(control);
+		print(asTable(response.list));
+	},
+}));
+
+
+const roleCommands = new libCommand.CommandTree({ name: "role", description: "Role management" });
+roleCommands.add(new libCommand.Command({
+	definition: [["list", "l"], "List roles in the cluster"],
+	handler: async function(args, control) {
+		let response = await libLink.messages.listRoles.send(control);
+		print(asTable(response.list));
+	},
+}));
+
+roleCommands.add(new libCommand.Command({
+	definition: ["create <name>", "Create a new role", (yargs) => {
+		yargs.positional("name", { describe: "Name of role to create", type: "string" });
+		yargs.options({
+			"description": { describe: "Description for role", nargs: 1, type: "string", default: "" },
+			"permissions": { describe: "Permissions role grants", nargs: 1, array: true, type: "string", default: [] },
+		});
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.createRole.send(control, {
+			name: args.name,
+			description: args.description,
+			permissions: args.permissions,
+		});
+		logger.info(`Created role ID ${response.id}`);
+	},
+}));
+
+roleCommands.add(new libCommand.Command({
+	definition: ["edit <role>", "Edit existing role", (yargs) => {
+		yargs.positional("role", { describe: "Role to edit", type: "string" });
+		yargs.options({
+			"name": { describe: "New name for role", nargs: 1, type: "string" },
+			"description": { describe: "New description for role", nargs: 1, type: "string" },
+			"set-perms": { describe: "Set permissions for role", array: true, type: "string" },
+			"add-perms": { describe: "Add permissions to role", array: true, type: "string", conflicts: "set-perms" },
+			"remove-perms": {
+				describe: "Remove permissions from role", array: true, type: "string", conflicts: "set-perms",
+			},
+			"grant-default": { describe: "Add default permissions to role", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args, control) {
+		let role = await libCommand.retrieveRole(control, args.role);
+
+		if (args.name !== undefined) {
+			role.name = args.name;
+		}
+		if (args.description !== undefined) {
+			role.description = args.description;
+		}
+		if (args.addPerms) {
+			role.permissions = role.permissions.concat(args.addPerms);
+		}
+		if (args.removePerms) {
+			let perms = new Set(role.permissions);
+			for (let perm of args.removePerms) {
+				perms.delete(perm);
+			}
+			role.permissions = [...perms];
+		}
+		if (args.setPerms !== undefined) {
+			role.permissions = args.setPerms;
+		}
+		await libLink.messages.updateRole.send(control, role);
+
+		if (args.grantDefault) {
+			await libLink.messages.grantDefaultRolePermissions.send(control, { id: role.id });
+		}
+	},
+}));
+
+roleCommands.add(new libCommand.Command({
+	definition: ["delete <role>", "Delete role", (yargs) => {
+		yargs.positional("role", { describe: "Role to delete", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let role = await libCommand.retrieveRole(control, args.role);
+		await libLink.messages.deleteRole.send(control, { id: role.id });
+	},
+}));
+
+
+const userCommands = new libCommand.CommandTree({ name: "user", alias: ["u"], description: "User management" });
+userCommands.add(new libCommand.Command({
+	definition: [["list", "l"], "List user in the cluster"],
+	handler: async function(args, control) {
+		let response = await libLink.messages.listUsers.send(control);
+		print(asTable(response.list));
+	},
+}));
+
+userCommands.add(new libCommand.Command({
+	definition: ["create <name>", "Create a user", (yargs) => {
+		yargs.positional("name", { describe: "Name of user to create", type: "string" });
+	}],
+	handler: async function(args, control) {
+		await libLink.messages.createUser.send(control, { name: args.name });
+	},
+}));
+
+userCommands.add(new libCommand.Command({
+	definition: ["set-admin <user>", "Promote or demote a user to admin", (yargs) => {
+		yargs.positional("user", { describe: "Name of user set admin status for", type: "string" });
+		yargs.options({
+			"revoke": { describe: "Revoke admin status", nargs: 0, type: "boolean", default: false },
+			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args, control) {
+		await libLink.messages.setUserAdmin.send(control, {
+			name: args.user, create: args.create, admin: !args.revoke,
+		});
+	},
+}));
+
+userCommands.add(new libCommand.Command({
+	definition: ["set-whitelisted <user>", "Add or remove user from the whitelist", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to set whitelist status for", type: "string" });
+		yargs.options({
+			"remove": { describe: "Remove from whitelist", nargs: 0, type: "boolean", default: false },
+			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args, control) {
+		await libLink.messages.setUserWhitelisted.send(control, {
+			name: args.user, create: args.create, whitelisted: !args.remove,
+		});
+	},
+}));
+
+userCommands.add(new libCommand.Command({
+	definition: ["set-banned <user>", "Ban or pardon user from banlist", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to set ban status for", type: "string" });
+		yargs.options({
+			"pardon": { describe: "Remove from banlist", nargs: 0, type: "boolean", default: false },
+			"reason": { describe: "Ban reason", nargs: 1, type: "string", default: "" },
+			"create": { describe: "Create user if it does not exist", nargs: 0, type: "boolean", default: false },
+		});
+	}],
+	handler: async function(args, control) {
+		await libLink.messages.setUserBanned.send(control, {
+			name: args.user, create: args.create, banned: !args.pardon, reason: args.reason,
+		});
+	},
+}));
+
+userCommands.add(new libCommand.Command({
+	definition: ["set-roles <user> [roles...]", "Replace user roles", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to change roles for", type: "string" });
+		yargs.positional("roles", { describe: "roles to assign", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.listRoles.send(control);
+
+		let resolvedRoles = [];
+		for (let roleName of args.roles) {
+			if (/^-?\d+$/.test(roleName)) {
+				let roleId = parseInt(roleName, 10);
+				resolvedRoles.push(roleId);
+
+			} else {
+				let found = false;
+				for (let role of response.list) {
+					if (role.name === roleName) {
+						resolvedRoles.push(role.id);
+						found = true;
+						break;
+					}
+				}
+
+				if (!found) {
+					throw new libErrors.CommandError(`No role named ${roleName}`);
+				}
+			}
+		}
+
+		await libLink.messages.updateUserRoles.send(control, { name: args.user, roles: resolvedRoles });
+	},
+}));
+
+userCommands.add(new libCommand.Command({
+	definition: ["delete <user>", "Delete user", (yargs) => {
+		yargs.positional("user", { describe: "Name of user to delete", type: "string" });
+	}],
+	handler: async function(args, control) {
+		await libLink.messages.deleteUser.send(control, { name: args.user });
+	},
+}));
+
+const logCommands = new libCommand.CommandTree({ name: "log", description: "Log inspection" });
+logCommands.add(new libCommand.Command({
+	definition: ["follow", "follow cluster log", (yargs) => {
+		yargs.options({
+			"all": { describe: "Follow the whole cluster log", nargs: 0, type: "boolean", default: false },
+			"master": { describe: "Follow log of the master server", nargs: 0, type: "boolean", default: false },
+			"slave": { describe: "Follow log of given slave", nargs: 1, type: "string", default: null },
+			"instance": { describe: "Follow log of given instance", nargs: 1, type: "string", default: null },
+		});
+	}],
+	handler: async function(args, control) {
+		if (!args.all && !args.master && !args.slave && !args.instance) {
+			logger.error("At least one of --all, --master, --slave and --instance must be passed");
+			process.exitCode = 1;
+			return;
+		}
+		let instance_ids = args.instance ? [await libCommand.resolveInstance(control, args.instance)] : [];
+		let slave_ids = args.slave ? [await libCommand.resolveSlave(control, args.slave)] : [];
+		await control.setLogSubscriptions({ all: args.all, master: args.master, slave_ids, instance_ids });
+		control.keepOpen = true;
+	},
+}));
+
+logCommands.add(new libCommand.Command({
+	definition: ["query", "Query cluster log", (yargs) => {
+		yargs.options({
+			"all": { describe: "Query the whole cluster log", nargs: 0, type: "boolean", default: false },
+			"master": { describe: "Query log of the master server", nargs: 0, type: "boolean", default: false },
+			"slave": { describe: "Query log of given slave", nargs: 1, type: "string", default: null },
+			"instance": { describe: "Query log of given instance", nargs: 1, type: "string", default: null },
+			"max-level": { describe: "Maximum log level to return", nargs: 1, type: "string", default: null },
+		});
+	}],
+	handler: async function(args, control) {
+		if (!args.all && !args.master && !args.slave && !args.instance) {
+			logger.error("At least one of --all, --master, --slave and --instance must be passed");
+			process.exitCode = 1;
+			return;
+		}
+		let instance_ids = args.instance ? [await libCommand.resolveInstance(control, args.instance)] : [];
+		let slave_ids = args.slave ? [await libCommand.resolveSlave(control, args.slave)] : [];
+		let result = await libLink.messages.queryLog.send(control, {
+			all: args.all,
+			master: args.master,
+			slave_ids,
+			instance_ids,
+			max_level: args.maxLevel,
+		});
+
+		let stdoutLogger = winston.createLogger({
+			level: "verbose",
+			levels,
+			format: new libLoggingUtils.TerminalFormat({ showTimestamp: true }),
+			transports: [
+				new ConsoleTransport({ errorLevels: [], warnLevels: [] }),
+			],
+		});
+		for (let info of result.log) {
+			stdoutLogger.log(info);
+		}
+	},
+}));
+
+const debugCommands = new libCommand.CommandTree({ name: "debug", description: "Debugging utilities" });
+debugCommands.add(new libCommand.Command({
+	definition: ["dump-ws", "Dump WebSocket messages sent and received by master", (yargs) => { }],
+	handler: async function(args, control) {
+		await libLink.messages.debugDumpWs.send(control);
+		control.keepOpen = true;
+	},
+}));
+
+
+/**
+ * Connector for control connection to master server
+ * @private
+ */
+class ControlConnector extends libLink.WebSocketClientConnector {
+	constructor(url, reconnectDelay, tlsCa, token) {
+		super(url, reconnectDelay, tlsCa);
+		this._token = token;
+	}
+
+	register() {
+		logger.verbose("SOCKET | registering control");
+		this.sendHandshake("register_control", {
+			token: this._token,
+			agent: "clusterioctl",
+			version: version,
+		});
+	}
+}
+
+/**
+ * Handles running the control
+ *
+ * Connects to the master server over WebSocket and sends commands to it.
+ * @static
+ */
+class Control extends libLink.Link {
+
+	constructor(connector, controlPlugins) {
+		super("control", "master", connector);
+		libLink.attachAllMessages(this);
+
+		/**
+		 * Mapping of plugin names to their instance for loaded plugins.
+		 * @type {Map<string, module:lib/plugin.BaseControlPlugin>}
+		 */
+		this.plugins = controlPlugins;
+		for (let controlPlugin of controlPlugins.values()) {
+			libPlugin.attachPluginMessages(this, controlPlugin);
+		}
+
+		/**
+		 * Keep the control connection alive after the command completes.
+		 * @type {boolean}
+		 */
+		this.keepOpen = false;
+	}
+
+	async setLogSubscriptions({
+		all = false,
+		master = false,
+		slave_ids = [],
+		instance_ids = [],
+		max_level = null,
+	}) {
+		await libLink.messages.setLogSubscriptions.send(this, {
+			all, master, slave_ids, instance_ids, max_level,
+		});
+	}
+
+	async logMessageEventHandler(message) {
+		logger.log(message.data.info);
+	}
+
+	async debugWsMessageEventHandler(message) {
+		print("WS", message.data.direction, message.data.content);
+	}
+
+	async shutdown() {
+		this.connector.setTimeout(30);
+
+		try {
+			await libLink.messages.prepareDisconnect.send(this);
+		} catch (err) {
+			if (!(err instanceof libErrors.SessionLost)) {
+				throw err;
+			}
+		}
+
+		await this.connector.close(1001, "Control Quit");
+	}
+}
+
+async function loadPlugins(pluginList) {
+	let pluginInfos = await libPluginLoader.loadPluginInfos(pluginList);
+	libConfig.registerPluginConfigGroups(pluginInfos);
+	libConfig.finalizeConfigs();
+
+	let controlPlugins = new Map();
+	for (let pluginInfo of pluginInfos) {
+		if (!pluginInfo.controlEntrypoint) {
+			continue;
+		}
+
+		let ControlPluginClass = await libPluginLoader.loadControlPluginClass(pluginInfo);
+		let controlPlugin = new ControlPluginClass(pluginInfo, logger);
+		controlPlugins.set(pluginInfo.name, controlPlugin);
+		await controlPlugin.init();
+	}
+	return controlPlugins;
+}
+
+async function registerCommands(controlPlugins, yargs) {
+	const rootCommands = new libCommand.CommandTree({ name: "clusterioctl", description: "Manage cluster" });
+	rootCommands.add(masterCommands);
+	rootCommands.add(slaveCommands);
+	rootCommands.add(instanceCommands);
+	rootCommands.add(permissionCommands);
+	rootCommands.add(roleCommands);
+	rootCommands.add(userCommands);
+	rootCommands.add(logCommands);
+	rootCommands.add(debugCommands);
+
+	for (let controlPlugin of controlPlugins.values()) {
+		await controlPlugin.addCommands(rootCommands);
+	}
+
+	for (let [name, command] of rootCommands.subCommands) {
+		if (name === command.name) {
+			command.register(yargs);
+		}
+	}
+
+	return rootCommands;
+}
+
+async function startControl() {
+	yargs
+		.scriptName("clusterioctl")
+		.usage("$0 <command> [options]")
+		.option("log-level", {
+			nargs: 1,
+			describe: "Log level to print to stderr",
+			default: "server",
+			choices: Object.keys(levels),
+			type: "string",
+		})
+		.option("config", {
+			nargs: 1,
+			describe: "config file to get credentails from",
+			default: "config-control.json",
+			defaultDescription: "auto",
+			type: "string",
+		})
+		.option("plugin-list", {
+			nargs: 1,
+			describe: "File containing list of plugins available with their install path",
+			default: "plugin-list.json",
+			type: "string",
+		})
+		.command("plugin", "Manage available plugins", libSharedCommands.pluginCommand)
+		.command("control-config", "Manage Control config", libSharedCommands.configCommand)
+		.wrap(yargs.terminalWidth())
+		.help(false) // Disable help to avoid triggering it on the first parse.
+	;
+
+	// Parse the args first to get the configured plugin list.
+	let args = yargs.parse();
+
+	// Log stream for the ctl session.
+	logger.add(new ConsoleTransport({
+		errorLevels: Object.keys(levels),
+		level: args.logLevel,
+		format: new libLoggingUtils.TerminalFormat(),
+	}));
+
+	logger.verbose(`Loading available plugins from ${args.pluginList}`);
+	let pluginList = new Map();
+	try {
+		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
+	} catch (err) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+	}
+
+	// If the command is plugin management we don't try to load plugins
+	if (args._[0] === "plugin") {
+		await libSharedCommands.handlePluginCommand(args, pluginList, args.pluginList);
+		return;
+	}
+
+	logger.verbose("Loading Plugins");
+	let controlPlugins = await loadPlugins(pluginList);
+
+	// Add all cluster management commands including ones from plugins
+	let rootCommands = await registerCommands(controlPlugins, yargs);
+
+	// Reparse after commands have been added with help and strict checking.
+	args = yargs
+		.help()
+		.strict()
+		.parse()
+	;
+
+	logger.verbose(`Loading config from ${args.config}`);
+	let controlConfig = new libConfig.ControlConfig("control");
+	try {
+		await controlConfig.load(JSON.parse(await fs.readFile(args.config)));
+
+	} catch (err) {
+		if (err.code === "ENOENT") {
+			logger.verbose("Config not found, initializing new config");
+			await controlConfig.init();
+
+		} else {
+			throw err;
+		}
+	}
+
+	if (args._.length === 0) {
+		yargs.showHelp();
+		yargs.exit();
+	}
+
+	// Handle the control-config command before trying to connect.
+	if (args._[0] === "control-config") {
+		await libSharedCommands.handleConfigCommand(args, controlConfig, args.config);
+		return;
+	}
+
+	// Determine which command is being executed.
+	let commandPath = [...args._];
+	let targetCommand = rootCommands;
+	while (commandPath.length && targetCommand instanceof libCommand.CommandTree) {
+		targetCommand = targetCommand.get(commandPath.shift());
+	}
+
+	// The remaining commands require connecting to the master server.
+	if (!controlConfig.get("control.master_url") || !controlConfig.get("control.master_token")) {
+		logger.error("Missing URL and/or token to connect with.  See README.md for setting up access.");
+		process.exitCode = 1;
+		return;
+	}
+
+	let tlsCa = null;
+	let tlsCaPath = controlConfig.get("control.tls_ca");
+	if (tlsCaPath) {
+		tlsCa = await fs.readFile(tlsCaPath);
+	}
+
+	let controlConnector = new ControlConnector(
+		controlConfig.get("control.master_url"),
+		controlConfig.get("control.reconnect_delay"),
+		tlsCa,
+		controlConfig.get("control.master_token")
+	);
+	let control = new Control(controlConnector, controlPlugins);
+	try {
+		await controlConnector.connect();
+	} catch (err) {
+		if (err instanceof libErrors.AuthenticationFailed) {
+			throw new libErrors.StartupError(err.message);
+		}
+		throw err;
+	}
+
+	process.on("SIGINT", () => {
+		logger.info("Caught interrupt signal, closing connection");
+		control.shutdown().catch(err => {
+			setBlocking(true);
+			logger.error(err.stack);
+			// eslint-disable-next-line no-process-exit
+			process.exit(1);
+		});
+	});
+
+	try {
+		await targetCommand.run(args, control);
+
+	} catch (err) {
+		control.keepOpen = false;
+		if (err instanceof libErrors.CommandError) {
+			logger.error(`Error running command: ${err.message}`);
+			process.exitCode = 1;
+
+		} else if (err instanceof libErrors.RequestError) {
+			logger.error(`Error sending request: ${err.message}`);
+			process.exitCode = 1;
+
+		} else {
+			throw err;
+		}
+
+	} finally {
+		if (!control.keepOpen) {
+			await control.shutdown();
+		}
+	}
+}
+
+module.exports = {
+	Control,
+};
+
+
+if (module === require.main) {
+	// eslint-disable-next-line no-console
+	console.warn(`
++==========================================================+
+I WARNING:  This is the development branch for the 2.0     I
+I           version of clusterio.  Expect things to break. I
++==========================================================+
+`
+	);
+	startControl().catch(err => {
+		if (!(err instanceof libErrors.StartupError)) {
+			logger.fatal(`
++----------------------------------------------------------------+
+| Unexpected error occured while starting control, please report |
+| it to https://github.com/clusterio/factorioClusterio/issues    |
++----------------------------------------------------------------+
+${err.stack}`
+			);
+		} else {
+			logger.error(`
++---------------------------------+
+| Unable to to start clusterioctl |
++---------------------------------+
+${err.stack}`
+			);
+		}
+
+		process.exitCode = 1;
+	});
+}

packages/ctl/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@clusterio/ctl",
+  "description": "Clusterio command line management tool",
+  "author": "Hornwitser",
+  "license": "MIT",
+  "version": "2.0.0-alpha.4",
+  "main": "ctl.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "bin": {
+    "clusterioctl": "./ctl.js"
+  },
+  "keywords": [
+    "factorio"
+  ],
+  "engines": {
+    "node": ">=10.16 <11 || >= 12"
+  },
+  "dependencies": {
+    "@clusterio/lib": "^2.0.0-alpha.4",
+    "as-table": "^1.0.31",
+    "chalk": "^3.0.0",
+    "fs-extra": "^8.1.0",
+    "winston": "^3.3.3",
+    "yargs": "^14.2.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/lib/README.md

@@ -0,0 +1,8 @@
+Common Library for Clusterio
+============================
+
+This package contains functionality shared between master, slave, ctl
+and/or plugins of Clusterio.
+
+See [the Clusterio repository](https://github.com/clusterio/factorioClusterio)
+for more information on Clusterio.

packages/lib/RateLimiter.js

@@ -0,0 +1,104 @@
+"use strict";
+
+
+/**
+ * Helper class for rate limiting actions
+ *
+ * Provides a simple interface for implementing actions that are rate
+ * limited on invocations.
+ *
+ * @example
+ * const rateLimiter = RateLimiter({
+ *     action: () => { console.log("action"); },
+ *     maxRate: 1,
+ * });
+ *
+ * for (let i = 0; i < 100; i++) {
+ *     rateLimiter.activate();
+ *     // Prints action twice to the console, once for the first activation
+ *     // and one more time 1 second later for 99 other activations that were
+ *     // exceeding the rate limit.
+ * }
+ * @alias module:lib/RateLimiter
+ */
+class RateLimiter {
+	/**
+	 * Construct RateLimiter helper
+	 *
+	 * The options passed may also be modified on the instance at runtime.
+	 *
+	 * @param {object=} options - Options for this rate limiter
+	 * @param {number=} options.maxRate -
+	 *     Maximum rate of activations per second.
+	 * @param {function()=} options.action -
+	 *     Callback invoked at maste `maxRate` per second on activation.
+	 */
+	constructor(options = {}) {
+		/**
+		 * Maximum rate of activations per second.
+		 * @type {number}
+		 */
+		this.maxRate = options.maxRate || 1;
+
+		/**
+		 * Callback invoked at most `maxRate` per second on activation.
+		 * @type {function()}
+		 */
+		this.action = options.action || null;
+
+		this._lastRun = 0;
+		this._runTimeout = null;
+		this.lastProgressBroadcast = Date.now();
+	}
+
+	/**
+	 * Activate the rate limited action
+	 *
+	 * Checks if the invocation of the rate limited action now would fall
+	 * within the rate limit.  If it is within the limit then the action is
+	 * invoked immediatly, otherwise a timeout is registered to invoke the
+	 * action at next point in time where it will be within the rate limit.
+	 *
+	 * @returns {boolean} true if activation was within the rate limit.
+	 */
+	activate() {
+		let now = Date.now();
+		if (now < this._lastRun + 1000 / this.maxRate) {
+			if (!this._runTimeout && this.action !== null) {
+				this._runTimeout = setTimeout(() => {
+					this._runTimeout = null;
+					this._lastRun = Date.now();
+					this.action();
+				}, this._lastRun + 1000 / this.maxRate - now);
+			}
+			return false;
+		}
+
+		// We may end up activating before the timeout does.
+		if (this._runTimeout) {
+			clearTimeout(this._runTimeout);
+			this._runTimeout = null;
+		}
+
+		this._lastRun = now;
+		if (this.action !== null) {
+			this.action();
+		}
+		return true;
+	}
+
+	/**
+	 * Cancel pending action
+	 *
+	 * Cancel pending action invocation.  This is necessary when shutting
+	 * down the use of the rate limit as a timeout is registered to invoke
+	 * the next action when activated faster than the rate limit allows.
+	 */
+	cancel() {
+		if (this._runTimeout) {
+			clearTimeout(this._runTimeout);
+		}
+	}
+}
+
+module.exports = RateLimiter;

packages/lib/build_mod.js

@@ -0,0 +1,131 @@
+/* eslint-disable no-console */
+"use strict";
+const events = require("events");
+const fs = require("fs-extra");
+const JSZip = require("jszip");
+const klaw = require("klaw");
+const path = require("path");
+const yargs = require("yargs");
+
+
+async function buildMod(args, info) {
+	if (args.build) {
+		await fs.ensureDir(args.outputDir);
+		let modName = `${info.name}_${info.version}`;
+
+		if (args.pack) {
+			let zip = new JSZip();
+			let walker = klaw(args.sourceDir)
+				.on("data", item => {
+					if (item.stats.isFile()) {
+						// On Windows the path created uses backslashes as the directory sepparator
+						// but the zip file needs to use forward slashes.  We can't use the posix
+						// version of relative here as it doesn't work with Windows style paths.
+						let basePath = path.relative(args.sourceDir, item.path).replace(/\\/g, "/");
+						zip.file(path.posix.join(modName, basePath), fs.createReadStream(item.path));
+					}
+				})
+			;
+			await events.once(walker, "end");
+
+			for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) {
+				let filePath = path.join(args.sourceDir, ...pathParts);
+				if (!await fs.pathExists(filePath)) {
+					throw new Error(`Additional file ${filePath} does not exist`);
+				}
+				zip.file(path.posix.join(modName, fileName), fs.createReadStream(filePath));
+			}
+			delete info.additional_files;
+
+			zip.file(path.posix.join(modName, "info.json"), JSON.stringify(info, null, 4));
+
+			let modPath = path.join(args.outputDir, `${modName}.zip`);
+			console.log(`Writing ${modPath}`);
+			let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath));
+			await events.once(writeStream, "finish");
+
+		} else {
+			let modDir = path.join(args.outputDir, modName);
+			if (await fs.exists(modDir)) {
+				console.log(`Removing existing build ${modDir}`);
+				await fs.remove(modDir);
+			}
+			console.log(`Building ${modDir}`);
+			await fs.copy(args.sourceDir, modDir);
+			for (let [fileName, pathParts] of Object.entries(info.additional_files) || []) {
+				let filePath = path.join(...pathParts);
+				await fs.copy(filePath, path.join(modDir, fileName));
+			}
+			delete info.additional_files;
+
+			await fs.writeFile(path.join(modDir, "info.json"), JSON.stringify(info, null, 4));
+		}
+	}
+}
+
+async function build(args) {
+	let info = JSON.parse(await fs.readFile(path.join(args.sourceDir, "info.json")));
+
+	if (args.factorioVersion) {
+		info.factorio_version = args.factorioVersion;
+	}
+
+	if (args.dependencies) {
+		info.dependencies = args.dependencies;
+	}
+
+	if (args.clean) {
+		let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/;
+		for (let entry of await fs.readdir(args.outputDir)) {
+			let match = splitter.exec(entry);
+			if (match) {
+				let [, name, version] = match;
+				if (name === info.name) {
+					let modPath = path.join(args.outputDir, entry);
+					console.log(`Removing ${modPath}`);
+					await fs.remove(modPath);
+				}
+			}
+		}
+	}
+
+	if (info.variants) {
+		for (let [variant, variantOverrides] of Object.entries(info.variants)) {
+			let variantInfo = {
+				...info,
+				...variantOverrides,
+			};
+			delete variantInfo.variants;
+			await buildMod(args, variantInfo);
+		}
+	} else {
+		await buildMod(args, info);
+	}
+}
+
+
+async function main() {
+	const args = yargs
+		.scriptName("build")
+		.options({
+			"clean": { describe: "Remove previous builds", type: "boolean", default: false },
+			"build": { describe: "Build mod(s)", type: "boolean", default: true },
+			"pack": { describe: "Pack into zip file", type: "boolean", default: true },
+			"source-dir": { describe: "Path to mod source files", nargs: 1, type: "string", demandOption: true },
+			"output-dir": { describe: "Path to output built mod(s)", nargs: 1, type: "string", default: "dist" },
+			"factorio-version": { describe: "Override factorio_version", type: "string" },
+		})
+		.strict()
+		.argv
+	;
+
+	await build(args);
+}
+
+if (module === require.main) {
+	main().catch(err => { console.log(err); });
+}
+
+module.exports = {
+	build,
+};

packages/lib/command.js

@@ -0,0 +1,248 @@
+/**
+ * Command library
+ *
+ * Data types and utilites for commands to clusterioctl's CLI.
+ *
+ * @author Hornwitser
+ * @module lib/command
+ */
+"use strict";
+const libLink = require("@clusterio/lib/link");
+const libErrors = require("@clusterio/lib/errors");
+
+/**
+ * Represents a command that can be runned by clusterioctl
+ * @static
+ */
+class Command {
+	/**
+	 * Define an executable command.
+	 *
+	 * @param {Object} cmd - Command definiton.
+	 * @param {Array} cmd.definiton -
+	 *     Arguments to pass to yargs .command method to define this
+	 *     command.
+	 * @param {function(args, control)} handler -
+	 *     Async function invoked when the command is executed.  Is given
+	 *     the parsed args and a reference to the {@link
+	 *     module:ctl/ctl.Control} instance.
+	 */
+	constructor({ definition, handler }) {
+		if (definition[0] instanceof Array) {
+			this.name = definition[0][0].split(" ")[0];
+			this.alias = definition[0].slice(1);
+		} else {
+			this.name = definition[0].split(" ")[0];
+			this.alias = [];
+		}
+		this._definition = definition;
+		this._handler = handler;
+	}
+
+	register(yargs) {
+		yargs.command(...this._definition);
+	}
+
+	async run(args, control) {
+		await this._handler(args, control);
+	}
+}
+
+/**
+ * A node in the command tree that can hold commands and sub trees
+ *
+ * Container which can contain {@link module:lib/command.Command}s and other
+ * CommandTrees.  This is used by clusterioctl to hold the full tree of
+ * available commands, you may extend this tree by using a control plugin,
+ * see {@link module:lib/plugin.BaseControlPlugin#addCommands}
+ *
+ * @static
+ */
+class CommandTree {
+	/**
+	 * Define a command containing sub commands.
+	 *
+	 * @param {Object} cmd - Command definition
+	 * @param {string} cmd.name - Name of the command.
+	 * @param {Array<string>} cmd.alias - Aliases for this tree.
+	 * @param {description} cmd.description -
+	 *     Descripton to provide for this command tree node.
+	 */
+	constructor({ name, alias, description }) {
+		if (typeof name !== "string") {
+			throw new Error("name must be a string");
+		}
+
+		this.name = name;
+		this.alias = alias || [];
+		this.description = description;
+		this.subCommands = new Map();
+	}
+
+	/**
+	 * Add a command or command tree to this tree
+	 *
+	 * @param {module:lib/command.Command|module:lib/command.CommandTree} command -
+	 *    The command to add to this command tree.
+	 */
+	add(command) {
+		if (this.subCommands.has(command.name)) {
+			throw new Error(`Command ${command.name} already exists`);
+		}
+		this.subCommands.set(command.name, command);
+		for (let alias of command.alias) {
+			if (this.subCommands.has(alias)) {
+				throw new Error(`Alias ${alias} for command ${command.name} already exists`);
+			}
+			this.subCommands.set(alias, command);
+		}
+	}
+
+	/**
+	 * Get a command or command tree from this tree
+	 *
+	 * @param {string} name -
+	 *     The name of the command or command tree to retrieve.
+	 * @return {?module:lib/command.Command|module:lib/command.CommandTree}
+	 *    The command to add to this command tree.
+	 */
+	get(name) {
+		return this.subCommands.get(name) || null;
+	}
+
+	/**
+	 * Insert the registered commands into the given yargs parser
+	 *
+	 * Traverses the the tree of sub-commands recursively and adds all of
+	 * them to the given yargs parser.
+	 *
+	 * @param {Object} yargs - yargs parser to register with.
+	 */
+	register(yargs) {
+		yargs.command([this.name].concat(this.alias), this.description, (yargs) => {
+			for (let [name, command] of this.subCommands) {
+				// Check if the entry is not an alias.
+				if (name === command.name) {
+					command.register(yargs);
+				}
+			}
+		}, () => {
+			yargs.showHelp();
+			yargs.exit();
+		});
+	}
+}
+
+
+/**
+ * Resolve a string into a slave ID
+ *
+ * Resolves a string with either an slave name or an id into an integer with
+ * the slave ID.
+ *
+ * @param {module:lib/link.Link} client - link to master server to query slave on.
+ * @param {string} slaveName - string with name or id of slave.
+ * @returns {number} slave ID.
+ * @static
+ */
+async function resolveSlave(client, slaveName) {
+	let slaveId;
+	if (/^-?\d+$/.test(slaveName)) {
+		slaveId = parseInt(slaveName, 10);
+	} else {
+		let response = await libLink.messages.listSlaves.send(client);
+		for (let slave of response.list) {
+			if (slave.name === slaveName) {
+				slaveId = slave.id;
+				break;
+			}
+		}
+
+		if (slaveId === undefined) {
+			throw new libErrors.CommandError(`No slave named ${slaveName}`);
+		}
+	}
+
+	return slaveId;
+}
+
+/**
+ * Resolve a string to an instance ID
+ *
+ * Resolves a string with either an instance name or an id into an integer
+ * with the instance ID.
+ *
+ * @param {module:lib/link.Link} client - link to master server to query instance on.
+ * @param {string} instanceName - string with name or id of instance.
+ * @returns {number} instance ID.
+ * @static
+ */
+async function resolveInstance(client, instanceName) {
+	let instanceId;
+	if (/^-?\d+$/.test(instanceName)) {
+		instanceId = parseInt(instanceName, 10);
+	} else {
+		let response = await libLink.messages.listInstances.send(client);
+		for (let instance of response.list) {
+			if (instance.name === instanceName) {
+				instanceId = instance.id;
+				break;
+			}
+		}
+
+		if (instanceId === undefined) {
+			throw new libErrors.CommandError(`No instance named ${instanceName}`);
+		}
+	}
+
+	return instanceId;
+}
+
+/**
+ * Retrieve role object from string
+ *
+ * Resolves a string with either a role name or an id into an object
+ * representing the role.
+ *
+ * @param {module:lib/link.Link} client - link to master server to query role on.
+ * @param {string} roleName - string with name or id of role.
+ * @returns {Object} Role info.
+ * @static
+ */
+async function retrieveRole(client, roleName) {
+	let response = await libLink.messages.listRoles.send(client);
+
+	let resolvedRole;
+	if (/^-?\d+$/.test(roleName)) {
+		let roleId = parseInt(roleName, 10);
+		for (let role of response.list) {
+			if (role.id === roleId) {
+				resolvedRole = role;
+				break;
+			}
+		}
+
+	} else {
+		for (let role of response.list) {
+			if (role.name === roleName) {
+				resolvedRole = role;
+				break;
+			}
+		}
+	}
+
+	if (!resolvedRole) {
+		throw new libErrors.CommandError(`No role named ${roleName}`);
+	}
+
+	return resolvedRole;
+}
+
+module.exports = {
+	Command,
+	CommandTree,
+
+	resolveSlave,
+	resolveInstance,
+	retrieveRole,
+};

packages/lib/config/classes.js

@@ -1,17 +1,31 @@
-/**
- * @file Configuration classes
- * @author Hornwitser
- */
+// Configuration classes
 "use strict";
 
-const { basicType } = require("lib/helpers");
+const events = require("events");
 
+const isDeepStrictEqual = require("../is_deep_strict_equal");
+const { basicType } = require("@clusterio/lib/helpers");
+
+
+/**
+ * Invalid Access exception
+ *
+ * Exception class for when an access attempt is made to a field that
+ * does not permitt this access is done.
+ *
+ * @extends Error
+ * @memberof module:lib/config
+ */
+class InvalidAccess extends Error { };
 
 /**
  * Invalid Value exception
  *
  * Exception class for when invalid values are attempted to be set on a
  * field.
+ *
+ * @extends Error
+ * @memberof module:lib/config
  */
 class InvalidValue extends Error { };
 
@@ -20,23 +34,31 @@ class InvalidValue extends Error { };
  *
  * Exception class for when a group or field that does not exist is
  * attempted to be accessed.
+ *
+ * @extends Error
+ * @memberof module:lib/config
  */
 class InvalidField extends Error { };
 
 /**
  * Split the given string on the first instance of separator
  *
- * Splits @p string on the first instance of @p separator and returns an
+ * Splits `string` on the first instance of `separator` and returns an
  * array consisting of the string up to the separator and the string
  * after the separator.  Returns an array with the string and an empty
  * string if the separator is not present.
  *
+ * @param {string} separator - Separator to split string by.
+ * @param {string} string - String to split
  * @returns {Array} string split on separator.
+ * @memberof module:lib/config
+ * @private
+ * @inner
  */
 function splitOn(separator, string) {
 	let index = string.indexOf(separator);
 	if (index === -1) {
-		return [string, ''];
+		return [string, ""];
 	}
 	return [string.slice(0, index), string.slice(index + separator.length)];
 }
@@ -46,13 +68,36 @@ function splitOn(separator, string) {
  *
  * Represents a collection of ConfigGroup instances that can be
  * managed as one.
+ *
+ * This is an EventEmitter with the following events:
+ * - fieldChanged -
+ *     invoked after a field has been updated in one of the config groups
+ *     belonging to this config.  Takes three parameters: group, field,
+ *     prev.
+ *
+ * @extends events.EventEmitter
+ *
+ * @memberof module:lib/config
  */
-class Config {
-	constructor() {
+class Config extends events.EventEmitter {
+	/**
+	 * Create a new instance of the given config
+	 *
+	 * @param {string} location -
+	 *     Location to evaluate access for this instance from
+	 */
+	constructor(location) {
+		super();
+
 		if (!this.constructor._finalized) {
-			throw new Error(`Cannot instantiate incomplete Config class ${this.constructor.name}`)
+			throw new Error(`Cannot instantiate incomplete Config class ${this.constructor.name}`);
 		}
 
+		if (typeof location !== "string") {
+			throw new Error("location must be a string");
+		}
+
+		this.location = location;
 		this._groups = new Map();
 		this._unknownGroups = new Map();
 	}
@@ -60,7 +105,7 @@ class Config {
 	async init() {
 		for (let [name, GroupClass] of this.constructor._groups) {
 			if (!this._groups.has(name)) {
-				let group = new GroupClass();
+				let group = new GroupClass(this);
 				await group.init();
 				this._groups.set(name, group);
 			}
@@ -78,7 +123,13 @@ class Config {
 		}
 	}
 
-	async load(serializedConfig) {
+	/**
+	 * Load config from a serialized object
+	 *
+	 * @param {object} serializedConfig - Serialized config to load.
+	 * @param {string=} location - Location used for access control.
+	 */
+	async load(serializedConfig, location = this.location) {
 		this._validate(serializedConfig);
 		for (let serializedGroup of serializedConfig.groups) {
 			let GroupClass = this.constructor._groups.get(serializedGroup.name);
@@ -87,9 +138,9 @@ class Config {
 				continue;
 			}
 
-			let group = new GroupClass();
-			await group.load(serializedGroup);
-			this._groups.set(GroupClass.groupName, group);
+			let group = new GroupClass(this);
+			await group.load(serializedGroup, location);
+			this._groups.set(group.name, group);
 		}
 
 		await this.init();
@@ -97,11 +148,14 @@ class Config {
 
 	/**
 	 * Serialize the config to a plain JavaScript object
+	 *
+	 * @param {string=} location - Location used for access control.
+	 * @returns {Object} JSON serializable representation of the config.
 	 */
-	serialize() {
+	serialize(location = this.location) {
 		let groups = [...this._unknownGroups.values()];
 		for (let group of this._groups.values()) {
-			groups.push(group.serialize());
+			groups.push(group.serialize(location));
 		}
 
 		return { groups };
@@ -112,13 +166,17 @@ class Config {
 	 *
 	 * Updates all groups in the config with groups in the serialized
 	 * config passed.
+	 *
+	 * @param {Object} serializedConfig - Output from .serialize().
+	 * @param {boolean} notify - Invoke fieldChanged events if true.
+	 * @param {string=} location - Location used for access control.
 	 */
-	update(serializedConfig) {
+	update(serializedConfig, notify, location = this.location) {
 		this._validate(serializedConfig);
 		for (let serializedGroup of serializedConfig.groups) {
 			let group = this._groups.get(serializedGroup.name);
 			if (group) {
-				group.update(serializedGroup);
+				group.update(serializedGroup, notify, location);
 			} else {
 				this._unknownGroups.set(serializedGroup.name, serializedGroup);
 			}
@@ -126,27 +184,63 @@ class Config {
 	}
 
 	/**
-	 * Get the value for a config field
+	 * Check if field can be accessed from the given location
+	 *
+	 * @param {string} name - Name of the field to check.
+	 * @param {string=} location - Location used for access control.
+	 * @returns {boolean} true if field is accessible
 	 */
-	get(name) {
-		let [group, field] = splitOn('.', name);
-		return this.group(group).get(field);
+	canAccess(name, location = this.location) {
+		let [group, field] = splitOn(".", name);
+		return this.group(group).canAccess(field, location);
+	}
+
+	/**
+	 * Get the value for a config field
+	 *
+	 * @param {string} name - Name of field to get.
+	 * @param {string=} location - Location used for access control.
+	 * @returns {*} the value for the field.
+	 */
+	get(name, location = this.location) {
+		let [group, field] = splitOn(".", name);
+		return this.group(group).get(field, location);
 	}
 
 	/**
 	 * Set the value for a config field
+	 *
+	 * @param {string} name - Name of field to set.
+	 * @param {*} value - the value to set on field
+	 * @param {string=} location - Location used for access control.
 	 */
-	set(name, value) {
-		let [group, field] = splitOn('.', name);
-		return this.group(group).set(field, value);
+	set(name, value, location = this.location) {
+		let [group, field] = splitOn(".", name);
+		this.group(group).set(field, value, location);
+	}
+
+	/**
+	 * Set property of an object value for a config field
+	 *
+	 * @param {string} name - Name of field to set property on.
+	 * @param {string} prop - Name of property to set on field.
+	 * @param {*} value - the value to set the property to.
+	 * @param {string=} location - Location used for access control.
+	 */
+	setProp(name, prop, value, location = this.location) {
+		let [group, field] = splitOn(".", name);
+		this.group(group).setProp(field, prop, value, location);
 	}
 
 	/**
 	 * Get the config group instance with the given name
+	 *
+	 * @param {string} name - Name of group to get.
+	 * @returns {module:lib/config.ConfigGroup} config group.
 	 */
 	group(name) {
 		if (!this._initialized) {
-			throw new Error(`${this.constructor.name} instance is uninitialized`)
+			throw new Error(`${this.constructor.name} instance is uninitialized`);
 		}
 		let group = this._groups.get(name);
 		if (!group) {
@@ -192,6 +286,8 @@ class Config {
 	 *
 	 * Adds the ConfigGroup to this config as a group that will get
 	 * initialized, loaded and serialized with this config class.
+	 *
+	 * @param {module:lib/config.ConfigGroup} group - class to register.
 	 */
 	static registerGroup(group) {
 		this._initSubclass();
@@ -215,6 +311,13 @@ class Config {
 
 /**
  * Collection of related config entries
+ *
+ * Type checks and stores a collection of related config entries.  Attempt
+ * to set invalid values for fields will result in an error thrown while
+ * loading a config with invalid values will result in those being replaced
+ * by the innitial value for those fields.
+ *
+ * @memberof module:lib/config
  */
 class ConfigGroup {
 	/**
@@ -222,14 +325,42 @@ class ConfigGroup {
 	 *
 	 * After the creation of the new config group you have to call
 	 * either .init() or .load() on it in order to fully initialize it.
+	 *
+	 * @param {module:lib/config.Config} config -
+	 *     Parent config for this group instance.
 	 */
-	constructor(serializedGroup) {
+	constructor(config) {
 		if (!this.constructor._finalized) {
-			throw new Error(`Cannot instantiate incomplete ConfigGroup class ${this.constructor.name}`)
+			throw new Error(`Cannot instantiate incomplete ConfigGroup class ${this.constructor.name}`);
 		}
 
 		this._fields = new Map();
 		this._unknownFields = new Map();
+		this.config = config;
+	}
+
+	/**
+	 * Check access for a field
+	 *
+	 * @param {object} def - definition for for the field to check.
+	 * @param {string} remote - location of remote access to check for.
+	 * @param {boolean} error - throw on failure to pass check.
+	 * @throws {module:lib/config.InvalidAccess}
+	 *     if access is not granted and error is true.
+	 * @returns {boolean} true if access is granted
+	 * @private
+	 */
+	_checkAccess(def, remote, error) {
+		for (let loc of [remote, this.config.location]) {
+			if (!def.access.includes(loc)) {
+				if (error) {
+					throw new InvalidAccess(`Field '${def.name}' is not accessible from ${loc}`);
+				} else {
+					return false;
+				}
+			}
+		}
+		return true;
 	}
 
 	/**
@@ -243,10 +374,13 @@ class ConfigGroup {
 			if (this._fields.has(name)) {
 				continue;
 			}
+			if (!def.access.includes(this.config.location)) {
+				continue;
+			}
 
 			let value = null;
 			if (typeof def.initial_value === "function") {
-				value = await def.initial_value()
+				value = await def.initial_value();
 
 			} else if (def.initial_value !== undefined) {
 				value = def.initial_value;
@@ -262,19 +396,53 @@ class ConfigGroup {
 	 * Loads all the values from the passed serialized version of the
 	 * group and then initializes any possible missing fields from it to
 	 * their default values.
+	 *
+	 * @param {Object} serializedGroup -
+	 *     Result from a previous call to .serialize().
+	 * @param {string=} location - Location used for access control.
 	 */
-	async load(serializedGroup) {
-		this.update(serializedGroup);
+	async load(serializedGroup, location = this.config.location) {
+		this.update(serializedGroup, false, location);
 		await this.init();
 	}
 
 	/**
-	 * Get value for field
+	 * Name of the group
+	 *
+	 * Shortcut for `group.constructor.groupName`
 	 */
-	get(name) {
-		if (!this.constructor._definitions.has(name)) {
+	get name() {
+		return this.constructor.groupName;
+	}
+
+	/**
+	 * Check if field can be accessed from the given location
+	 *
+	 * @param {string} name - Name of field to check.
+	 * @param {string=} location - Location used for access control.
+	 * @returns {boolean} true if field is accessible
+	 */
+	canAccess(name, location = this.config.location) {
+		let def = this.constructor._definitions.get(name);
+		if (!def) {
 			throw new InvalidField(`No field named '${name}'`);
 		}
+		return this._checkAccess(def, location, false);
+	}
+
+	/**
+	 * Get value for field
+	 *
+	 * @param {string} name - Name of field to get.
+	 * @param {string=} location - Location used for access control.
+	 * @returns {*} Value stored for the field.
+	 */
+	get(name, location = this.config.location) {
+		let def = this.constructor._definitions.get(name);
+		if (!def) {
+			throw new InvalidField(`No field named '${name}'`);
+		}
+		this._checkAccess(def, location, true);
 
 		return this._fields.get(name);
 	}
@@ -285,13 +453,18 @@ class ConfigGroup {
 	 * The field must be defined for the config group and the value is
 	 * type checked against the field definition.
 	 *
-	 * @throws {InvalidValue} if value is not allowed for the field.
+	 * @param {string} name - Name of field to set.
+	 * @param {*} value - Value to set for field.
+	 * @param {string=} location - Location used for access control.
+	 * @throws {module:lib/config.InvalidField} if field is not defined.
+	 * @throws {module:lib/config.InvalidValue} if value is not allowed for the field.
 	 */
-	set(name, value) {
+	set(name, value, location = this.config.location) {
 		let def = this.constructor._definitions.get(name);
 		if (!def) {
-			throw new Error(`No field named '${name}'`);
+			throw new InvalidField(`No field named '${name}'`);
 		}
+		this._checkAccess(def, location, true);
 
 		// Empty strings are treates as null
 		if (value === "") {
@@ -299,13 +472,16 @@ class ConfigGroup {
 		}
 
 		if (def.enum && !def.enum.includes(value)) {
-			throw new InvalidValue(`Expected one of [${def.enum.join(', ')}], not ${value}`);
+			throw new InvalidValue(`Expected one of [${def.enum.join(", ")}], not ${value}`);
 		}
 
 		if (basicType(value) === "string") {
 			if (def.type === "boolean") {
-				if (value === "true") { value = true; }
-				else if (value === "false") { value = false; }
+				if (value === "true") {
+					value = true;
+				} else if (value === "false") {
+					value = false;
+				}
 
 			} else if (def.type === "number") {
 				let numberRegExp = /^[+-]?(Infinity|\d+\.(\d+)?([eE][+-]?\d+)?|\.?\d+([eE][+-]?\d+)?)$/;
@@ -313,27 +489,67 @@ class ConfigGroup {
 					value = Number.parseFloat(value);
 				}
 
-			} else if (def.type === "object" ) {
+			} else if (def.type === "object") {
 				try {
 					value = JSON.parse(value);
 				} catch (err) {
-					throw new InvalidValue(`Error parsing value for ${name}: ${err.message}`)
+					throw new InvalidValue(`Error parsing value for ${name}: ${err.message}`);
 				}
 			}
 		}
 
 		if (value === null) {
 			if (!def.optional) {
-				throw new InvalidValue(`Field ${name} cannot be null`)
+				throw new InvalidValue(`Field ${name} cannot be null`);
 			}
 
-		} else {
-			if (basicType(value) !== def.type) {
-				throw new InvalidValue(`Expected type of ${name} to be ${def.type}, not ${basicType(value)}`);
-			}
+		} else if (basicType(value) !== def.type) {
+			throw new InvalidValue(`Expected type of ${name} to be ${def.type}, not ${basicType(value)}`);
 		}
 
+		let prev = this._fields.get(name);
 		this._fields.set(name, value);
+		if (this.config && !isDeepStrictEqual(value, prev)) {
+			this.config.emit("fieldChanged", this, name, prev);
+		}
+	}
+
+	/**
+	 * Set property of object field
+	 *
+	 * Update value of stored object field by setting the specified property
+	 * on it.  The field must be defined as an object for the config group.
+	 *
+	 * @param {string} name - Name of field to set property on.
+	 * @param {string} prop - Name of property to set on field.
+	 * @param {*} value - the value to set the property to.
+	 * @param {string=} location - Location used for access control.
+	 * @throws {module:lib/config.InvalidField} if field is not defined.
+	 * @throws {module:lib/config.InvalidValue} if field is not an object.
+	 */
+	setProp(name, prop, value, location = this.config.location) {
+		let def = this.constructor._definitions.get(name);
+		if (!def) {
+			throw new InvalidField(`No field named '${name}'`);
+		}
+		this._checkAccess(def, location, true);
+
+		if (def.type !== "object") {
+			throw new InvalidValue(`Cannot set property on non-object field '${name}'`);
+		}
+
+		let prev = this._fields.get(name);
+		let updated = {...prev || {}};
+
+		if (value !== undefined) {
+			updated[prop] = value;
+		} else {
+			delete updated[prop];
+		}
+		this._fields.set(name, updated);
+		if (this.config && !isDeepStrictEqual(updated, prev)) {
+			this.config.emit("fieldChanged", this, name, prev);
+		}
 	}
 
 	/**
@@ -341,14 +557,19 @@ class ConfigGroup {
 	 *
 	 * Overwrites the values of all the fields in this config group that
 	 * has a valid value in the passed serialized group with that value.
+	 *
+	 * @param {Object} serializedGroup -
+	 *     Result from a previous call to .serialize().
+	 * @param {boolean} notify - Invoke fieldChanged events if true.
+	 * @param {string=} location - Location used for access control.
 	 */
-	update(serializedGroup) {
+	update(serializedGroup, notify, location = this.config.location) {
 		if (basicType(serializedGroup) !== "object") {
 			throw new Error(`Expected object, not ${basicType(serializedGroup)} for ConfigGroup`);
 		}
 
-		if (serializedGroup.name !== this.constructor.groupName) {
-			throw new Error(`Expected group name ${this.constructor.groupName}, not ${serializedGroup.name}`);
+		if (serializedGroup.name !== this.name) {
+			throw new Error(`Expected group name ${this.name}, not ${serializedGroup.name}`);
 		}
 
 		if (basicType(serializedGroup.fields) !== "object") {
@@ -362,30 +583,45 @@ class ConfigGroup {
 				continue;
 			}
 
+			if (!this._checkAccess(def, location, false)) {
+				continue;
+			}
+
 			if (def.optional) {
 				if (value !== null && basicType(value) !== def.type) {
 					continue;
 				}
-			} else {
-				if (basicType(value) !== def.type) {
-					continue;
-				}
+
+			} else if (basicType(value) !== def.type) {
+				continue;
 			}
 
 			if (def.enum && !def.enum.includes(value)) {
 				continue;
 			}
 
+			let prev = this._fields.get(name);
 			this._fields.set(name, value);
+			if (notify && this.config && !isDeepStrictEqual(value, prev)) {
+				this.config.emit("fieldChanged", this, name, prev);
+			}
 		}
 	}
 
 	/**
 	 * Serialize the config to a plain JavaScript object
+	 *
+	 * @param {string=} location - Location used for access control.
+	 * @returns {Object} JSON serializable representation of the group.
 	 */
-	serialize() {
+	serialize(location = this.config.location) {
 		let fields = {};
 		for (let [name, value] of this._fields) {
+			let def = this.constructor._definitions.get(name);
+			if (!this._checkAccess(def, location, false)) {
+				continue;
+			}
+
 			fields[name] = value;
 		}
 
@@ -394,7 +630,7 @@ class ConfigGroup {
 		}
 
 		return {
-			name: this.constructor.groupName,
+			name: this.name,
 			fields,
 		};
 	}
@@ -424,34 +660,37 @@ class ConfigGroup {
 	/**
 	 * Define a new configuration field
 	 *
-	 * The following properties are supported:
-	 * - type:
+	 * @param {Object} item - Field definition.
+	 * @param {Array<string>=} item.access -
+	 *     Array of strings declaring where this config entry can be read
+	 *     and modified from.
+	 * @param {string} item.type -
 	 *     string declaring the type of the config value, supports
 	 *     boolean, string, number and object.
-	 * - name:
+	 * @param {string} item.name -
 	 *     Name of the configuration field.  Should follow the
 	 *     lower_case_underscore style.
-	 * - title:
+	 * @param {string} item.title -
 	 *     Text used to identify the config field in user interfaces.
-	 * - description:
+	 * @param {string} item.description -
 	 *     Text used to describe the config field in user interfaces.
-	 * - enum (optional):
+	 * @param {Array=} item.enum -
 	 *     Array of all values that are acceptible.  Useful for making
 	 *     enumerated values.
-	 * - optional (optional):
+	 * @param {boolean=} item.optional -
 	 *     True if this field can be set to null.  Defaults to false.
-	 * - initial_value (optional):
+	 * @param {*=} item.initial_value -
 	 *     Value this config field should take in a newly initialized
 	 *     config.  This can also be an async function returning the
 	 *     value to use.
 	 */
 	static define(item) {
 		if (this._finalized) {
-			throw new Error(`Cannot define field for ConfigGroup class ${this.name} after it has been finalized`)
+			throw new Error(`Cannot define field for ConfigGroup class ${this.name} after it has been finalized`);
 		}
 
 		if (basicType(this.groupName) !== "string") {
-			throw new Error(`Expected ConfigGroup class ${this.name} to have the groupName property set to a string`)
+			throw new Error(`Expected ConfigGroup class ${this.name} to have the groupName property set to a string`);
 		}
 
 		this._initSubclass();
@@ -463,7 +702,7 @@ class ConfigGroup {
 			if (!itemKeys.includes(prop)) {
 				throw new Error(`${prop} is required when defining an field`);
 			}
-		})
+		});
 
 		// Validate properties
 		let validTypes = ["boolean", "string", "number", "object"];
@@ -477,6 +716,12 @@ class ConfigGroup {
 					}
 					break;
 
+				case "access":
+					if (valueType !== "array") {
+						throw new Error("access must be an array");
+					}
+					break;
+
 				case "enum":
 					if (valueType !== "array") {
 						throw new Error("enum must be an array");
@@ -522,10 +767,16 @@ class ConfigGroup {
 			throw new Error(`Config field ${item.name} has already been defined`);
 		}
 
-		let computed = Object.assign({
+		let computed = {
+			access: this.defaultAccess,
 			optional: false,
 			fullName: `${this.groupName}.${item.name}`,
-		}, item)
+			...item,
+		};
+
+		if (!computed.access) {
+			throw new Error(`access not set and no defaultAccess defined for ${this.name}`);
+		}
 
 		if (!computed.optional && computed.initial_value === undefined) {
 			throw new Error(`Non-optional field ${item.name} needs an initial_value`);
@@ -539,7 +790,7 @@ class ConfigGroup {
 	 */
 	static finalize() {
 		if (basicType(this.groupName) !== "string") {
-			throw new Error(`Expected ConfigGroup class ${this.name} to have the groupName property set to a string`)
+			throw new Error(`Expected ConfigGroup class ${this.name} to have the groupName property set to a string`);
 		}
 
 		this._initSubclass();
@@ -553,6 +804,9 @@ class ConfigGroup {
  * Grouping that includes per plugin fields clusterio uses to manage
  * plugins.  This is currently only the enabled field.  It is otherwise
  * identical to the ConfigGroup class.
+ *
+ * @extends module:lib/config.ConfigGroup
+ * @memberof module:lib/config
  */
 class PluginConfigGroup extends ConfigGroup {
 	static _initSubclass() {
@@ -570,10 +824,11 @@ class PluginConfigGroup extends ConfigGroup {
 
 
 module.exports = {
+	InvalidAccess,
 	InvalidField,
 	InvalidValue,
 
 	Config,
 	ConfigGroup,
 	PluginConfigGroup,
-}
+};

packages/lib/config/definitions.js

@@ -1,19 +1,24 @@
-/**
- * @file Core definitions for the configuration system
- * @author Hornwitser
- */
+// Core definitions for the configuration system
 "use strict";
 
-const util = require("util");
-const crypto = require("crypto");
-const path = require("path");
-const fs = require("fs-extra");
-
 const classes = require("./classes");
 
 
+/**
+ * Master config group for {@link module:lib/config.MasterConfig}
+ * @extends module:lib/config.ConfigGroup
+ * @memberof module:lib/config
+ */
 class MasterGroup extends classes.ConfigGroup { }
-MasterGroup.groupName = "master"
+MasterGroup.defaultAccess = ["master", "slave", "control"];
+MasterGroup.groupName = "master";
+MasterGroup.define({
+	name: "name",
+	title: "Name",
+	description: "Name of the cluster",
+	type: "string",
+	initial_value: "Your Cluster",
+});
 MasterGroup.define({
 	name: "database_directory",
 	title: "Database directory",
@@ -21,18 +26,13 @@ MasterGroup.define({
 	type: "string",
 	initial_value: "database",
 });
-MasterGroup.define({
-	name: "factorio_directory",
-	description: "Path to directory to look for factorio installs",
-	type: "string",
-	initial_value: "factorio"
-});
 MasterGroup.define({
 	name: "http_port",
 	title: "HTTP Port",
 	description: "Port to listen for HTTP connections on, set to null to not listen for HTTP connections.",
 	type: "number",
-	optional: true
+	optional: true,
+	initial_value: 8080,
 });
 MasterGroup.define({
 	name: "https_port",
@@ -40,14 +40,20 @@ MasterGroup.define({
 	description: "Port to listen for HTTPS connection on, set to null to not listen for HTTPS connections.",
 	type: "number",
 	optional: true,
-	initial_value: 8443,
 });
 MasterGroup.define({
-	name: "web_root",
-	title: "Web Root",
-	description: "Sub-path to where the web interface is hosted.  Needed when proxying the web interface.",
+	name: "bind_address",
+	title: "Bind Address",
+	description: "IP address to bind the HTTP and HTTPS ports on.",
 	type: "string",
-	initial_value: "/",
+	optional: true,
+});
+MasterGroup.define({
+	name: "external_address",
+	title: "External Address",
+	description: "Public facing address the master server is hosted on.",
+	type: "string",
+	optional: true,
 });
 MasterGroup.define({
 	name: "tls_certificate",
@@ -55,7 +61,6 @@ MasterGroup.define({
 	description: "Path to the certificate to use for HTTPS.",
 	type: "string",
 	optional: true,
-	initial_value: "database/certificates/cert.crt",
 });
 MasterGroup.define({
 	name: "tls_private_key",
@@ -63,37 +68,65 @@ MasterGroup.define({
 	description: "Path to the private key to use for HTTPS.",
 	type: "string",
 	optional: true,
-	initial_value: "database/certificates/cert.key",
 });
 MasterGroup.define({
+	access: ["master"],
 	name: "auth_secret",
 	title: "Master Authentication Secret",
 	description:
-		"Secret used to generate and verify authentication tokens."
-		+"  Should be a long string of random letters and numbers."
-		+"  Do not share this.",
+		"Secret used to generate and verify authentication tokens.  " +
+		"Should be a long string of random letters and numbers.  " +
+		"Do not share this.",
 	type: "string",
-	initial_value: async function() {
-		console.log("Generating new master authentication secret");
-		let asyncRandomBytes = util.promisify(crypto.randomBytes);
-		let bytes = await asyncRandomBytes(256);
-		return bytes.toString("base64");
-	},
+	optional: true,
 });
 MasterGroup.define({
 	name: "heartbeat_interval",
 	title: "Heartbeat Interval",
 	description: "Interval heartbeats are sent out on WebSocket connections",
 	type: "number",
-	initial_value: 60,
+	initial_value: 45,
+});
+MasterGroup.define({
+	name: "connector_shutdown_timeout",
+	title: "Connector Shutdown Timeout",
+	description: "Timeout in seconds for connectors to properly disconnect on shutdown",
+	type: "number",
+	initial_value: 30,
+});
+MasterGroup.define({
+	name: "metrics_timeout",
+	title: "Metrics Timeout",
+	description: "Timeout in seconds for metrics gathering from slaves",
+	type: "number",
+	initial_value: 8,
+});
+MasterGroup.define({
+	name: "default_role_id",
+	title: "Default role",
+	description: "ID of role assigned by default to new users",
+	type: "number",
+	optional: true,
+	initial_value: 1,
 });
 MasterGroup.finalize();
 
+/**
+ * Master Config class
+ * @extends module:lib/config.Config
+ * @memberof module:lib/config
+ */
 class MasterConfig extends classes.Config { }
 MasterConfig.registerGroup(MasterGroup);
 
 
+/**
+ * Slave config group for {@link module:lib/config.SlaveConfig}
+ * @extends module:lib/config.ConfigGroup
+ * @memberof module:lib/config
+ */
 class SlaveGroup extends classes.ConfigGroup {}
+SlaveGroup.defaultAccess = ["master", "slave", "control"];
 SlaveGroup.groupName = "slave";
 SlaveGroup.define({
 	name: "name",
@@ -111,7 +144,7 @@ SlaveGroup.define({
 	name: "factorio_directory",
 	description: "Path to directory to look for factorio installs",
 	type: "string",
-	initial_value: "factorio"
+	initial_value: "factorio",
 });
 SlaveGroup.define({
 	name: "instances_directory",
@@ -123,7 +156,7 @@ SlaveGroup.define({
 	name: "master_url",
 	description: "URL to connect to the master server at",
 	type: "string",
-	initial_value: "https://localhost:8443/",
+	initial_value: "http://localhost:8080/",
 });
 SlaveGroup.define({
 	name: "master_token",
@@ -131,6 +164,12 @@ SlaveGroup.define({
 	type: "string",
 	initial_value: "enter token here",
 });
+SlaveGroup.define({
+	name: "tls_ca",
+	description: "Path to Certificate Authority to validate TLS connection to master against.",
+	type: "string",
+	optional: true,
+});
 SlaveGroup.define({
 	name: "public_address",
 	description: "Public facing address players should connect to in order to join instances on this slave",
@@ -146,12 +185,23 @@ SlaveGroup.define({
 });
 SlaveGroup.finalize();
 
+/**
+ * Slave Config class
+ * @extends module:lib/config.Config
+ * @memberof module:lib/config
+ */
 class SlaveConfig extends classes.Config { }
 SlaveConfig.registerGroup(SlaveGroup);
 
 
+/**
+ * Instance config group for {@link module:lib/config.InstanceConfig}
+ * @extends module:lib/config.ConfigGroup
+ * @memberof module:lib/config
+ */
 class InstanceGroup extends classes.ConfigGroup { }
-InstanceGroup.groupName = "instance"
+InstanceGroup.defaultAccess = ["master", "slave", "control"];
+InstanceGroup.groupName = "instance";
 InstanceGroup.define({
 	name: "name",
 	type: "string",
@@ -168,9 +218,21 @@ InstanceGroup.define({
 	type: "number",
 	optional: true,
 });
+InstanceGroup.define({
+	name: "auto_start",
+	description: "Automatically start this instance when the slave hosting it is started up",
+	type: "boolean",
+	initial_value: false,
+});
 InstanceGroup.finalize();
 
+/**
+ * Factorio config group for {@link module:lib/config.InstanceConfig}
+ * @extends module:lib/config.ConfigGroup
+ * @memberof module:lib/config
+ */
 class FactorioGroup extends classes.ConfigGroup { }
+FactorioGroup.defaultAccess = ["master", "slave", "control"];
 FactorioGroup.groupName = "factorio";
 FactorioGroup.define({
 	name: "version",
@@ -196,22 +258,116 @@ FactorioGroup.define({
 	type: "string",
 	optional: true,
 });
+FactorioGroup.define({
+	name: "enable_save_patching",
+	description: "Patch saves with Lua code. Required for Clusterio integrations, lua modules, and most plugins.",
+	type: "boolean",
+	initial_value: true,
+});
+FactorioGroup.define({
+	name: "enable_whitelist",
+	description: "Turn on whitelist for joining the server.",
+	type: "boolean",
+	initial_value: false,
+});
 FactorioGroup.define({
 	name: "settings",
 	description: "Settings overridden in server-settings.json",
 	type: "object",
-	initial_value: {
-		"tags": ["clusterio"],
-		"auto_pause": false,
-	},
+	initial_value: {}, // See create instance handler in master.
+});
+FactorioGroup.define({
+	name: "verbose_logging",
+	description: "Enable verbose logging on the Factorio server",
+	type: "boolean",
+	initial_value: false,
+});
+FactorioGroup.define({
+	name: "strip_paths",
+	description: "Strip down instance paths in the log",
+	type: "boolean",
+	initial_value: true,
+});
+FactorioGroup.define({
+	name: "sync_adminlist",
+	description: "Synchronize adminlist with master server",
+	type: "boolean",
+	initial_value: true,
+});
+FactorioGroup.define({
+	name: "sync_whitelist",
+	description: "Synchronize whitelist with master server",
+	type: "boolean",
+	initial_value: true,
+});
+FactorioGroup.define({
+	name: "sync_banlist",
+	description: "Synchronize banlist with master server",
+	type: "boolean",
+	initial_value: true,
+});
+FactorioGroup.define({
+	name: "max_concurrent_commands",
+	description: "Maximum number of RCON commands trasmitted in parallel",
+	type: "number",
+	initial_value: 5,
 });
 FactorioGroup.finalize();
 
+/**
+ * Instance config class
+ * @extends module:lib/config.Config
+ * @memberof module:lib/config
+ */
 class InstanceConfig extends classes.Config { }
 InstanceConfig.registerGroup(InstanceGroup);
 InstanceConfig.registerGroup(FactorioGroup);
 
 
+/**
+ * Control config group for {@link module:lib/config.ControlConfig}
+ * @extends module:lib/config.ConfigGroup
+ * @memberof module:lib/config
+ */
+class ControlGroup extends classes.ConfigGroup {}
+ControlGroup.defaultAccess = ["control"];
+ControlGroup.groupName = "control";
+ControlGroup.define({
+	name: "master_url",
+	description: "URL to connect to the master server at",
+	type: "string",
+	optional: true,
+});
+ControlGroup.define({
+	name: "master_token",
+	description: "Token to authenticate to master server with.",
+	type: "string",
+	optional: true,
+});
+ControlGroup.define({
+	name: "tls_ca",
+	description: "Path to Certificate Authority to validate TLS connection to master against.",
+	type: "string",
+	optional: true,
+});
+ControlGroup.define({
+	name: "reconnect_delay",
+	title: "Reconnect Delay",
+	description: "Maximum delay to wait before attempting to reconnect WebSocket",
+	type: "number",
+	initial_value: 2,
+});
+ControlGroup.finalize();
+
+/**
+ * Control config class
+ * @extends module:lib/config.Config
+ * @memberof module:lib/config
+ */
+class ControlConfig extends classes.Config { }
+ControlConfig.registerGroup(ControlGroup);
+
+
 function validateGroup(pluginInfo, groupName) {
 	if (!pluginInfo[groupName] instanceof classes.PluginConfigGroup) {
 		throw new Error(
@@ -228,6 +384,9 @@ function validateGroup(pluginInfo, groupName) {
 
 /**
  * Registers the config groups for the provided plugin infos
+ *
+ * @param {Array<Object>} pluginInfos - Array of plugin info objects.
+ * @memberof module:lib/config
  */
 function registerPluginConfigGroups(pluginInfos) {
 	for (let pluginInfo of pluginInfos) {
@@ -237,6 +396,7 @@ function registerPluginConfigGroups(pluginInfos) {
 
 		} else {
 			class MasterConfigGroup extends classes.PluginConfigGroup { }
+			MasterConfigGroup.defaultAccess = ["master", "slave", "control"];
 			MasterConfigGroup.groupName = pluginInfo.name;
 			MasterConfigGroup.finalize();
 			MasterConfig.registerGroup(MasterConfigGroup);
@@ -249,6 +409,7 @@ function registerPluginConfigGroups(pluginInfos) {
 
 			} else {
 				class InstanceConfigGroup extends classes.PluginConfigGroup { }
+				InstanceConfigGroup.defaultAccess = ["master", "slave", "control"];
 				InstanceConfigGroup.groupName = pluginInfo.name;
 				InstanceConfigGroup.finalize();
 				InstanceConfig.registerGroup(InstanceConfigGroup);
@@ -259,87 +420,28 @@ function registerPluginConfigGroups(pluginInfos) {
 
 /**
  * Lock configs from adding more groups and make them usable
+ *
+ * @memberof module:lib/config
  */
 function finalizeConfigs() {
 	MasterConfig.finalize();
 	SlaveConfig.finalize();
 	InstanceConfig.finalize();
+	ControlConfig.finalize();
 }
 
-/**
- * Yargs config command
- *
- * Can be passed to yargs.command to implement a config command.  Use
- * handleConfigCommand to do the requested action.
- */
-function configCommand(yargs) {
-	yargs
-		.command("set <field> [value]", "Set config field")
-		.command("show <field>", "Show value of the given config field")
-		.command("list", "List all configuration fields and their values")
-		.demandCommand(1, "You need to specify a command to run")
-		.strict()
-	;
-}
-
-/**
- * Handle yargs command
- *
- * Handle the actions that are made available by configCommand.
- */
-async function handleConfigCommand(args, instance, configPath) {
-	let command = args._[1];
-
-	if (command === "list") {
-		for (let GroupClass of instance.constructor.groups.values()) {
-			for (let def of GroupClass.definitions.values()) {
-				let value = instance.get(def.fullName);
-				console.log(`${def.fullName} ${JSON.stringify(value)}`);
-			}
-		}
-
-	} else if (command === "show") {
-		try {
-			console.log(instance.get(args.field));
-		} catch (err) {
-			if (err instanceof classes.InvalidField) {
-				console.error(err.message);
-			} else {
-				throw err;
-			}
-		}
-
-	} else if (command === "set") {
-		if (args.value === undefined) {
-			args.value = null;
-		}
-
-		try {
-			instance.set(args.field, args.value);
-			await fs.outputFile(configPath, JSON.stringify(instance.serialize(), null, 4));
-		} catch (err) {
-			if (err instanceof classes.InvalidField || err instanceof classes.InvalidValue) {
-				console.error(err.message);
-			} else {
-				throw err;
-			}
-		}
-	}
-}
-
-
 module.exports = {
 	MasterGroup,
 	SlaveGroup,
 	InstanceGroup,
 	FactorioGroup,
+	ControlGroup,
 
 	MasterConfig,
 	SlaveConfig,
 	InstanceConfig,
+	ControlConfig,
 
 	registerPluginConfigGroups,
 	finalizeConfigs,
-	configCommand,
-	handleConfigCommand,
-}
+};

packages/lib/config/index.js

@@ -1,9 +1,11 @@
+/* eslint-disable global-require */
 /**
- * @file Configuration System
+ * Configuration System
  * @author Hornwitser
+ * @module lib/config
  */
-
+"use strict";
 module.exports = {
 	...require("./classes"),
 	...require("./definitions"),
-}
+};

packages/lib/database.js

@@ -1,14 +1,14 @@
 /**
  * Implementation of master databases
  *
- * @module
+ * @module lib/database
  * @author Hornwitser
  */
 
 "use strict";
 const fs = require("fs-extra");
 
-const { basicType } = require("lib/helpers");
+const { basicType } = require("@clusterio/lib/helpers");
 
 
 /**
@@ -21,7 +21,7 @@ const { basicType } = require("lib/helpers");
 function mapToObject(map) {
 	let obj = {};
 	for (let [key, value] of map) {
-		if (typeof key !== 'string') {
+		if (typeof key !== "string") {
 			throw new Error(
 				`Expected all keys to be string but got ${typeof key}`
 			);
@@ -55,13 +55,12 @@ async function loadJsonAsMap(filePath) {
 		return new Map(Object.entries(parsed));
 
 	} catch (err) {
-		if (err.code == "ENOENT") {
+		if (err.code === "ENOENT") {
 			// File does not exist, return empty map
 			return new Map();
 
-		} else {
-			throw err;
 		}
+		throw err;
 	}
 }
 
@@ -105,11 +104,11 @@ async function loadJsonArrayAsMap(filePath) {
 		let map = new Map();
 		for (let element of parsed) {
 			if (basicType(element) !== "object") {
-				throw new Error(`Expected all elements to be objects`);
+				throw new Error("Expected all elements to be objects");
 			}
 
 			if (element.id === undefined) {
-				throw new Error(`Expected all elements to have an id property`);
+				throw new Error("Expected all elements to have an id property");
 			}
 
 			map.set(element.id, element);
@@ -117,13 +116,12 @@ async function loadJsonArrayAsMap(filePath) {
 		return map;
 
 	} catch (err) {
-		if (err.code == "ENOENT") {
+		if (err.code === "ENOENT") {
 			// File does not exist, return empty map
 			return new Map();
 
-		} else {
-			throw err;
 		}
+		throw err;
 	}
 }
 
@@ -144,7 +142,6 @@ async function saveMapAsJsonArray(filePath, map) {
 }
 
 
-
 function checkName(name) {
 	if (typeof name !== "string") {
 		throw new Error("name must be a string");
@@ -232,7 +229,7 @@ class ItemDatabase {
 		checkName(name);
 
 		if (!this._items.has(name)) {
-			return 0
+			return 0;
 		}
 
 		return this._items.get(name);
@@ -287,4 +284,4 @@ module.exports = {
 	loadJsonArrayAsMap,
 	saveMapAsJsonArray,
 	ItemDatabase,
-}
+};

packages/lib/errors.js

@@ -1,12 +1,25 @@
 /**
  * Errors thrown in Clusterio
- * @module
+ * @module lib/errors
  */
+"use strict";
 
+/**
+ * Thrown by commands when they fail
+ * @static
+ */
 class CommandError extends Error { }
+
+/**
+ * Thrown from requests sent when an error occured handling it
+ * @static
+ */
 class RequestError extends Error { }
 
-// Signal for messages that fail validation
+/**
+ * Signal for messages that fail validation
+ * @static
+ */
 class InvalidMessage extends Error {
 	constructor(msg, errors) {
 		super(msg);
@@ -14,16 +27,34 @@ class InvalidMessage extends Error {
 	}
 }
 
-// Thrown when WebSocket authentication failed
+/**
+ * Thrown from requests when the session it was sent on was lost
+ * @static
+ */
+class SessionLost extends Error { }
+
+/**
+ * Thrown when WebSocket authentication failed
+ * @static
+ */
 class AuthenticationFailed extends Error { }
 
-// Errror class for known errors occuring during startup
+/**
+ * Errror class for known errors occuring during startup
+ * @static
+ */
 class StartupError extends Error { }
 
-// Errors outside of our control.
+/**
+ * Errors outside of our control
+ * @static
+ */
 class EnvironmentError extends Error { }
 
-// Errors caused by plugins
+/**
+ * Errors caused by plugins
+ * @static
+ */
 class PluginError extends Error {
 	constructor(pluginName, original) {
 		super(`PluginError: ${original.message}`);
@@ -36,8 +67,9 @@ module.exports = {
 	CommandError,
 	RequestError,
 	InvalidMessage,
+	SessionLost,
 	AuthenticationFailed,
 	StartupError,
 	EnvironmentError,
 	PluginError,
-}
+};

packages/lib/factorio/export.js

@@ -0,0 +1,379 @@
+// Export of item icons and locale
+"use strict";
+const fs = require("fs-extra");
+const path = require("path");
+const ini = require("ini");
+const Jimp = require("jimp");
+const JSZip = require("jszip");
+
+const libBuildMod = require("@clusterio/lib/build_mod");
+
+/**
+ * Generate the export mod needed for exportData
+ *
+ * Packs the lua export mod located in lua/export into the mods folder for
+ * the server given, with dependencies generated for all the mods present in
+ * the server's mods folder.
+ *
+ * @param {module:lib/factorio.FactorioServer} server -
+ *     The server to generate the export mod for.
+ * @memberof module:lib/factorio
+ */
+async function generateExportMod(server) {
+	let dependencies = [];
+	let splitter = /^(.*)_(\d+\.\d+\.\d+)(\.zip)?$/;
+	for (let entry of await fs.readdir(server.writePath("mods"))) {
+		let match = splitter.exec(entry);
+		if (match && match[1] !== "export") {
+			dependencies.push(`? ${match[1]}`);
+		}
+	}
+
+	await libBuildMod.build({
+		clean: false,
+		build: true,
+		pack: true,
+		// XXX fix path when moving lib/factorio to slave
+		sourceDir: path.join(__dirname, "..", "..", "slave", "lua", "export"),
+		outputDir: server.writePath("mods"),
+		bumpPatch: false,
+		factorioVersion: server.version.replace(/\.\d+$/, ""),
+		dependencies,
+	});
+}
+
+let zipCache = new Map();
+async function loadZip(server, modVersions, mod) {
+	let modVersion = modVersions.get(mod);
+	if (!modVersion) {
+		throw new Error(`Got path for unknown mod ${mod}`);
+	}
+
+	let zipPath = server.writePath("mods", `${mod}_${modVersion}.zip`);
+	let zip = zipCache.get(zipPath);
+	if (!zip) {
+		zip = await JSZip.loadAsync(await fs.readFile(zipPath));
+		zipCache.set(zipPath, zip);
+	}
+
+	return zip.folder(`${mod}_${modVersion}`);
+}
+
+/**
+ * Load the given Factorio file path into a Buffer
+ *
+ * @param {module:lib/factorio.FactorioServer} server -
+ *     The server to load the file from.
+ * @param {Map<string, string>} modVersions - Mapping of mod to version used.
+ * @param {string} modPath - Factorio style path to the file to load.
+ * @returns {?Buffer} The content of the file or null if not found.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
+ */
+async function loadFile(server, modVersions, modPath) {
+	let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
+	if (!match) {
+		throw new Error(`Bad mod path ${match}`);
+	}
+
+	let [, mod, filePath] = match;
+
+	if (["core", "base"].includes(mod)) {
+		try {
+			return await fs.readFile(server.dataPath(mod, filePath));
+		} catch (err) {
+			if (err.code === "ENOENT") {
+				return null;
+			}
+			throw err;
+		}
+	}
+
+	let zip = await loadZip(server, modVersions, mod);
+	let file = zip.file(filePath);
+	if (!file) {
+		return null;
+	}
+
+	return await file.async("nodebuffer");
+}
+
+async function loadIcon(server, modVersions, iconPath, iconSize, iconMipmaps, iconCache) {
+	let icon = iconCache.get(iconPath);
+	if (icon === undefined) {
+		let fileContent = await loadFile(server, modVersions, iconPath);
+		if (fileContent) {
+			icon = await Jimp.read(fileContent);
+			icon.crop(0, 0, iconSize, iconSize);
+			iconCache.set(iconPath, icon);
+		} else {
+			icon = null;
+			server._logger.warn(`${iconPath} not found`);
+		}
+		iconCache.set(iconPath, icon);
+	}
+	return icon;
+}
+
+async function loadSimpleIcon(server, modVersions, item, size, iconCache) {
+	let icon = await loadIcon(server, modVersions, item.icon, item.icon_size, 0, iconCache);
+	if (icon) {
+		let iconScale = size / item.icon_size;
+		if (iconScale !== 1) {
+			icon = icon.clone();
+			icon.scale(iconScale);
+		}
+	}
+	return icon;
+}
+
+async function loadLayeredIcon(server, modVersions, item, size, iconCache) {
+	let baseLayerSize = item.icons[0].icon_size || item.icon_size;
+	let icon = await Jimp.create(size, size);
+
+	// The scaling factor of the base layer
+	let baseLayerScale = item.icons[0].scale || 32 / baseLayerSize;
+
+	// The size in pixels of one unit
+	let baseUnit = size / (baseLayerSize * baseLayerScale);
+
+	for (let layer of item.icons) {
+		let layerSize = layer.icon_size || item.icon_size;
+		let iconLayer = await loadIcon(server, modVersions, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache);
+
+		if (!iconLayer) {
+			continue;
+		}
+
+		iconLayer = iconLayer.clone();
+
+		let tint;
+		if (layer.tint) {
+			let divisor = (layer.tint.r > 1 || layer.tint.g > 1 || layer.tint.b > 1) ? 255 : 1;
+			tint = {
+				r: (layer.tint.r || 0) / divisor,
+				g: (layer.tint.g || 0) / divisor,
+				b: (layer.tint.b || 0) / divisor,
+				a: layer.tint.a || 1,
+			};
+		} else {
+			tint = { r: 1, b: 1, g: 1, a: 1 };
+		}
+
+		let layerScale = layer.scale || 32 / layerSize;
+		let [xs, ys] = layer.shift || [0, 0];
+		let realScale = layerScale * baseUnit;
+		let sizeShift = (size - layerSize * realScale) / 2;
+
+		xs = xs * baseUnit + sizeShift;
+		ys = ys * baseUnit + sizeShift;
+
+		if (realScale !== 1) {
+			iconLayer.scale(realScale);
+		}
+
+		// eslint-disable-next-line no-loop-func
+		iconLayer.scan(0, 0, iconLayer.bitmap.width, iconLayer.bitmap.height, (x, y, sidx) => {
+			x += xs;
+			y += ys;
+			if (x < 0 || x >= size || y < 0 || y >= size) {
+				return;
+			}
+			let sdata = iconLayer.bitmap.data;
+			let ddata = icon.bitmap.data;
+			let didx = icon.getPixelIndex(x, y);
+
+			let sa = sdata[sidx + 3] / 255;
+			let da = ddata[didx + 3] / 255;
+			let db = da * (1 - sa * tint.a);
+			let cb = sa + da * (1 - sa);
+			ddata[didx + 0] = Math.min(255, (sdata[sidx + 0] * sa * tint.r + ddata[didx + 0] * db) / cb);
+			ddata[didx + 1] = Math.min(255, (sdata[sidx + 1] * sa * tint.g + ddata[didx + 1] * db) / cb);
+			ddata[didx + 2] = Math.min(255, (sdata[sidx + 2] * sa * tint.b + ddata[didx + 2] * db) / cb);
+			ddata[didx + 3] = Math.min(255, sdata[sidx + 3] + ddata[didx + 3] * (1 - sa));
+		});
+	}
+
+	return icon;
+}
+
+/**
+ * Export item icons and data
+ *
+ * Assembles and packs the icons for the item prototypes given into a single
+ * spritesheet and json file with meta data.
+ *
+ * @param {module:lib/factorio.FactorioServer} server -
+ *     The server to generate the export mod for.
+ * @param {Map<string, string>} modVersions -
+ *     Mapping of mod name to versions to get icons from.
+ * @param {Array<Object>} items - Array of item prototypes.
+ * @return {{itemSheet: Jimp, itemData: Map<string,Object>}}
+ *     Item spritesheet and metadata.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
+ */
+async function exportItems(server, modVersions, items) {
+
+	// Size to render icons at
+	let size = 32;
+
+	// Width of spritesheet
+	let width = 1024;
+
+	let rows = Math.ceil(items.length / (width / size));
+	let iconSheet = await Jimp.create(width, rows * size);
+	let itemData = new Map();
+	let pos = 0;
+
+	let iconCache = new Map();
+	let simpleIcons = new Map();
+	for (let item of items) {
+		let icon;
+		let iconPos;
+		if (item.icons) {
+			icon = await loadLayeredIcon(server, modVersions, item, size, iconCache);
+			iconPos = pos;
+
+		} else {
+			iconPos = simpleIcons.get(item.icon);
+			if (iconPos === undefined) {
+				icon = await loadSimpleIcon(server, modVersions, item, size, iconCache);
+				if (icon) {
+					iconPos = pos;
+					simpleIcons.set(item.icon, pos);
+				}
+			}
+		}
+
+		if (iconPos !== undefined) {
+			itemData.set(item.name, {
+				x: iconPos * size % width,
+				y: Math.floor(iconPos / (width / size)) * size,
+				size,
+				localised_name: item.localised_name,
+				localised_description: item.localised_description,
+			});
+		}
+
+		if (icon) {
+			iconSheet.composite(icon, pos * size % width, Math.floor(pos / (width / size)) * size);
+			pos += 1;
+		}
+	}
+
+	iconSheet.crop(0, 0, width, Math.ceil(pos / (width / size)) * size);
+
+	return { iconSheet, itemData };
+}
+
+/**
+ * Exports the locale files for the base game and the given mods
+ *
+ * Parses and merges all the locales for the all the mods given through
+ * `modVersions` and `modOrder`.
+ *
+ * @param {module:lib/factorio.FactorioServer} server -
+ *     The server to export the locale from.
+ * @param {Map<string, string>} modVersions -
+ *     Mapping of mod name to version to export locale from.
+ * @param {Array<string>} modOrder - Load order of the mods.
+ * @param {string} languageCode - Language to export locale for.
+ * @returns {Map<string, string>} merged locale information
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
+ */
+async function exportLocale(server, modVersions, modOrder, languageCode) {
+	let mergedLocales = new Map();
+
+	function mergeLocale(locale) {
+		for (let [category, entries] of Object.entries(locale)) {
+			if (typeof entries === "string") {
+				mergedLocales.set(category, entries);
+
+			} else {
+				for (let [key, value] of Object.entries(entries)) {
+					mergedLocales.set(`${category}.${key}`, value);
+				}
+			}
+		}
+	}
+
+	let baseLocaleFilePath = server.dataPath("base", "locale", languageCode, "base.cfg");
+	mergeLocale(ini.parse(await fs.readFile(baseLocaleFilePath, "utf8")));
+
+	for (let mod of modOrder) {
+		if (["core", "base", "export"].includes(mod)) {
+			continue;
+		}
+
+		let zip = await loadZip(server, modVersions, mod);
+		for (let file of zip.file(new RegExp(`locale\\/${languageCode}\\/.*\\.cfg`))) {
+			let content = await file.async("nodebuffer");
+			mergeLocale(ini.parse(content.toString("utf8")));
+		}
+	}
+
+	return mergedLocales;
+}
+
+/**
+ * Export the locale and item icons for the given factorio server
+ *
+ * @param {module:lib/factorio.FactorioServer} server -
+ *     The server to export the data from.
+ * @returns {JSZip} zip file with exported data.
+ * @memberof module:lib/factorio
+ */
+async function exportData(server) {
+	await generateExportMod(server);
+
+	let items = [];
+	let modVersions = new Map();
+	let modOrder = [];
+
+	server.on("ipc-item_export", data => items.push(data));
+	server.on("ipc-mod_list", data => { modVersions = new Map(Object.entries(data)); });
+	server.on("output", parsed => {
+		if (parsed.format === "seconds" && parsed.type === "generic") {
+			let match = /^Checksum of (.*): \d+$/.exec(parsed.message);
+			if (match) {
+				modOrder.push(match[1]);
+			}
+		}
+	});
+
+	try {
+		await server.startScenario("base/freeplay");
+		await server.kill();
+	} finally {
+		await fs.unlink(server.writePath("mods", "export_0.0.0.zip"));
+	}
+
+	if (!items.length) {
+		throw new Error("No items got exported");
+	}
+
+	let { iconSheet, itemData } = await exportItems(server, modVersions, items);
+	let locale = await exportLocale(server, modVersions, modOrder, "en");
+
+	// Free up the memory used by zip files loaded during the export.
+	zipCache.clear();
+
+	let zip = new JSZip();
+	zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
+	zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
+	zip.file("export/locale.json", JSON.stringify([...locale.entries()]));
+	return zip;
+}
+
+
+module.exports = {
+	exportData,
+
+	// For testing only
+	_exportLocale: exportLocale,
+};

packages/lib/factorio/index.js

@@ -0,0 +1,12 @@
+/* eslint-disable global-require */
+/**
+ * Library for interfacing with Factorio Servers and saves.
+ * @module lib/factorio
+ * @author Hornwitser
+ */
+"use strict";
+module.exports = {
+	...require("./export"),
+	...require("./patch"),
+	...require("./server"),
+};

packages/lib/factorio/patch.js

@@ -1,16 +1,13 @@
-/**
- * Library for patching Factorio saves with scenario code.
- * @author Hornwitser
- */
+// Library for patching Factorio saves with scenario code.
 
 "use strict";
 const events = require("events");
 const fs = require("fs-extra");
-const jszip = require("jszip");
+const JSZip = require("jszip");
 const path = require("path");
 const semver = require("semver");
 
-const hash = require("lib/hash");
+const libHash = require("@clusterio/lib/hash");
 
 
 const knownScenarios = {
@@ -30,7 +27,11 @@ const knownScenarios = {
  * Returns the name of the folder that all files in the zip file is
  * contained in.  Throws an error if there are multiple such folders.
  *
+ * @param {JSZip} zip - Zip to search through.
  * @returns {string} name of the root folder.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
  */
 function findRoot(zip) {
 	let root = null;
@@ -44,12 +45,12 @@ function findRoot(zip) {
 		if (root === null) {
 			root = pathRoot;
 		} else if (root !== pathRoot) {
-			throw new Error(`Zip contains multiple root folders`);
+			throw new Error("Zip contains multiple root folders");
 		}
 	});
 
 	if (root === null) {
-		throw new Error(`Empty zip file`);
+		throw new Error("Empty zip file");
 	}
 
 	return root;
@@ -60,43 +61,47 @@ function findRoot(zip) {
  * Generates control.lua code for loading the Clusterio modules
  *
  * @param {Object} patchInfo - The patch info files's json content
+ * @returns {string} Generated control.lua code.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
  */
 function generateLoader(patchInfo) {
 	let lines = [
-		'-- Auto generated scenario module loader created by Clusterio',
-		'-- Modifications to this file will be lost when loaded in Clusterio',
+		"-- Auto generated scenario module loader created by Clusterio",
+		"-- Modifications to this file will be lost when loaded in Clusterio",
 		`clusterio_patch_number = ${patchInfo.patch_number}`,
-		'',
+		"",
 		'local event_handler = require("event_handler")',
-		'',
-		'-- Scenario modules',
+		"",
+		"-- Scenario modules",
 	];
 
 	for (let moduleName of patchInfo["scenario"]["modules"]) {
-		lines.push('event_handler.add_lib(require("'+moduleName+'"))');
+		lines.push(`event_handler.add_lib(require("${moduleName}"))`);
 	}
 
 	lines.push(...[
-		'',
-		'-- Clusterio modules',
+		"",
+		"-- Clusterio modules",
 	]);
 
 	for (let module of patchInfo["modules"]) {
 		for (let file of module["files"]) {
 			let requirePath = `modules/${module.name}/${file["path"].slice(0, -4)}`;
 			if (file["load"]) {
-				lines.push('event_handler.add_lib(require("'+requirePath+'"))');
+				lines.push(`event_handler.add_lib(require("${requirePath}"))`);
 			}
 			if (file["require"]) {
-				lines.push('require("'+requirePath+'")');
+				lines.push(`require("${requirePath}")`);
 			}
 		}
 	}
 
 	// End last line with a newline
-	lines.push('');
+	lines.push("");
 
-	return lines.join('\n');
+	return lines.join("\n");
 }
 
 /**
@@ -107,6 +112,9 @@ function generateLoader(patchInfo) {
  * their dependents.  Throws an error if this is not possible.
  *
  * @param {Array<Object>} modules - Array of modules to reorder
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
  */
 function reorderDependencies(modules) {
 	let index = 0;
@@ -144,7 +152,7 @@ function reorderDependencies(modules) {
 
 		// No unmet dependencies, record and continue
 		present.set(module.name, module.version);
-		index++;
+		index += 1;
 
 		if (hold.has(module.name)) {
 			modules.splice(index, 0, ...hold.get(module.name));
@@ -161,8 +169,8 @@ function reorderDependencies(modules) {
 	// the module depends on a module that satisfy any of these conditions.
 
 	let remaining = new Map();
-	for (let modules of hold.values()) {
-		for (let module of modules) {
+	for (let heldModules of hold.values()) {
+		for (let module of heldModules) {
 			remaining.set(module.name, module);
 		}
 	}
@@ -182,7 +190,7 @@ function reorderDependencies(modules) {
 
 			if (cycle.includes(module.name)) {
 				cycle.push(module.name);
-				cycle.splice(0, cycle.indexOf(module.name))
+				cycle.splice(0, cycle.indexOf(module.name));
 				throw new Error(`Module dependency loop detected: ${cycle.join(" -> ")}`);
 			}
 
@@ -202,9 +210,10 @@ function reorderDependencies(modules) {
  *
  * @param {string} savePath - Path to the Factorio save to patch.
  * @param {Array<Object>} modules - Description of the modules to patch.
+ * @memberof module:lib/factorio
  */
 async function patch(savePath, modules) {
-	let zip = await jszip.loadAsync(await fs.readFile(savePath));
+	let zip = await JSZip.loadAsync(await fs.readFile(savePath));
 	let root = zip.folder(findRoot(zip));
 
 	let patchInfoFile = root.file("clusterio.json");
@@ -216,7 +225,7 @@ async function patch(savePath, modules) {
 	// No info file present, try to detect if it's a known compatible scenario.
 	} else {
 		let controlStream = root.file("control.lua").nodeStream("nodebuffer");
-		let controlHash = await hash.hashStream(controlStream);
+		let controlHash = await libHash.hashStream(controlStream);
 
 		if (controlHash in knownScenarios) {
 			patchInfo = {
@@ -244,9 +253,9 @@ async function patch(savePath, modules) {
 		let moduleInfo = {
 			"name": module.name,
 			"files": [],
-		}
+		};
 
-		let dirs = [[module.path, '']];
+		let dirs = [[module.path, ""]];
 		while (dirs.length) {
 			let [dir, relativeDir] = dirs.pop();
 			for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
@@ -275,7 +284,7 @@ async function patch(savePath, modules) {
 	root.file("clusterio.json", JSON.stringify(patchInfo, null, 4));
 
 	// Write back the save
-	let stream = zip.generateNodeStream({compression: "DEFLATE"});
+	let stream = zip.generateNodeStream({ compression: "DEFLATE" });
 	let pipe = stream.pipe(fs.createWriteStream(savePath));
 	await events.once(pipe, "finish");
 }
@@ -287,4 +296,4 @@ module.exports = {
 	_findRoot: findRoot,
 	_generateLoader: generateLoader,
 	_reorderDependencies: reorderDependencies,
-}
+};

packages/lib/factorio/server.js

@@ -1,9 +1,4 @@
-/**
- * Factorio interfaces
- *
- * @module
- */
-
+// Factorio Server interface
 "use strict";
 const fs = require("fs-extra");
 const child_process = require("child_process");
@@ -14,12 +9,19 @@ const crypto = require("crypto");
 
 const ini = require("ini");
 const rconClient = require("rcon-client");
-const errors = require("lib/errors");
+const libErrors = require("@clusterio/lib/errors");
+const { logger } = require("@clusterio/lib/logging");
+const { escapeRegExp } = require("@clusterio/lib/helpers");
 
 
 /**
  * Determines the version of Factorio the datadir is pointing to by
  * reading the changelog.txt in it.
+ *
+ * @param {string} changelogPath - Path to changelog.txt.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
  */
 async function getVersion(changelogPath) {
 	let changelog;
@@ -47,6 +49,13 @@ async function getVersion(changelogPath) {
 
 /**
  * Comparison function for sorting Factorio version strings
+ *
+ * @param {string} a - Version to compare.
+ * @param {string} b - Version to compare.
+ * @returns {number} 1 if a < b, 0 if a = b and -1 if a > b.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
  */
 function versionOrder(a, b) {
 	let aParts = a.split(".").map(s => Number.parseInt(s, 10));
@@ -70,6 +79,15 @@ function versionOrder(a, b) {
  *
  * Searches the given factorio dir for an installation of Factorio of
  * the target version.
+ *
+ * @param {string} factorioDir - Path to Factorio installation dir(s).
+ * @param {string} targetVersion -
+ *     Version to look for, supports the special value "latest" for the
+ *     latest version available.
+ * @returns {Array} Array with path to data dir and version found.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
  */
 async function findVersion(factorioDir, targetVersion) {
 
@@ -125,12 +143,15 @@ async function findVersion(factorioDir, targetVersion) {
  * RFC 6335.
  *
  * @return {number} a number in the range 49152 to 65535.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
  */
 function randomDynamicPort() {
 	const start = 49152;
 	const end = 65535 + 1;
 
-	return Math.floor(Math.random() * (end - start) + start)
+	return Math.floor(Math.random() * (end - start) + start);
 }
 
 /**
@@ -141,13 +162,16 @@ function randomDynamicPort() {
  *
  * @param {number} length - the length of the password to generate.
  * @return {string} password of the given length
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
  */
 async function generatePassword(length) {
 	function validChar(byte) {
-		const ranges = ['az', 'AZ', '09'];
-		return ranges.some(range =>
+		const ranges = ["az", "AZ", "09"];
+		return ranges.some(range => (
 			range.codePointAt(0) <= byte && byte <= range.codePointAt(1)
-		);
+		));
 	}
 	let randomBytesAsync = util.promisify(crypto.randomBytes);
 
@@ -161,7 +185,7 @@ async function generatePassword(length) {
 
 			if (validChar(byte)) {
 				password += String.fromCharCode(byte);
-				if (password.length == length) {
+				if (password.length === length) {
 					return password;
 				}
 			}
@@ -179,6 +203,10 @@ async function generatePassword(length) {
  * Data is streamed via the data method, and once the stream ends the
  * end method should be called.  This ensures an unterminated line at
  * the end of the stream is passed to the callback.
+ *
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
  */
 class LineSplitter {
 
@@ -189,7 +217,7 @@ class LineSplitter {
 	 */
 	constructor(callback) {
 		this._callback = callback;
-		this._partial = null
+		this._partial = null;
 	}
 
 	/**
@@ -253,17 +281,19 @@ class LineSplitter {
  * @param {string} source - Passed into the output structure as source
  *
  * @returns {object} - An object with interpeted data.
+ * @memberof module:lib/factorio
+ * @private
+ * @inner
  */
 function parseOutput(line, source) {
 	let output = {
 		source,
-		received: Date.now(),
-	}
+	};
 
 	// There are three broad categories of output from Factorio, the first kind
 	// starts with the seconds since the server started and has a format of
 	// "   0.704 message"
-	const secRegex = /^ {0,3}(\d+\.\d+) (.*)$/
+	const secRegex = /^ {0,3}(\d+\.\d+) (.*)$/;
 	let secMatch = secRegex.exec(line);
 	if (secMatch) {
 		output.format = "seconds";
@@ -273,13 +303,15 @@ function parseOutput(line, source) {
 		// A seconds output has two different kinds of formats: The first is a
 		// log level and source location and has a format of
 		// "Level File.cpp:123: message"
-		const secLogRegex = /^(\w+) (\w+\.cpp:\d+): (.*)$/
+		// "Script @/path/to/file.lua:123: message"
+		// "Script =(command):123: message"
+		const secLogRegex = /^(\w+) ((\w+\.cpp:\d+)|([@=].*?:\d+)): (.*)$/;
 		let secLogMatch = secLogRegex.exec(secContent);
 		if (secLogMatch) {
 			output.type = "log";
 			output.level = secLogMatch[1];
 			output.file = secLogMatch[2];
-			output.message = secLogMatch[3];
+			output.message = secLogMatch[5];
 
 		// The other possibility is that the content is a generic message
 		} else {
@@ -290,7 +322,7 @@ function parseOutput(line, source) {
 	// The second category of output starts with a date stamp of the format
 	// "yyyy-mm-dd hh:mm:ss message"
 	} else {
-		const dateRegex = /^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d) (.*)$/
+		const dateRegex = /^(\d{4}-\d\d-\d\d \d\d:\d\d:\d\d) (.*)$/;
 		let dateMatch = dateRegex.exec(line);
 		if (dateMatch) {
 			output.format = "date";
@@ -299,7 +331,7 @@ function parseOutput(line, source) {
 
 			// A date output has two general formats.  The first is an action
 			// followed by a message and has a format of "[ACTION] message"
-			const dateActionRegex = /^\[(\w+)\] (.*)$/
+			const dateActionRegex = /^\[(\w+)\] (.*)$/;
 			let dateActionMatch = dateActionRegex.exec(dateContent);
 			if (dateActionMatch) {
 				output.type = "action";
@@ -335,24 +367,47 @@ function parseOutput(line, source) {
 const outputHeuristics = [
 	// Message indicating the RCON interface has started
 	{
-		filter: { type: 'log', message: /^Starting RCON interface/ },
-		action: function(output) {
-			this._startRcon().catch((err) => { this.emit('error', err); });
+		filter: { type: "log", message: /^Starting RCON interface/ },
+		action: function() {
+			this._startRcon().catch((err) => { this.emit("error", err); });
 		},
 	},
 
 	// Message indicating the server is done starting up
 	{
-		filter: { type: 'log', message: /^updateTick\(\d+\) changing state from\(CreatingGame\) to\(InGame\)$/ },
-		action: function(output) {
-			this._notifyGameReady().catch((err) => { this.emit('error', err); });
-		}
+		filter: { type: "log", message: /^updateTick\(\d+\) changing state from\(CreatingGame\) to\(InGame\)$/ },
+		action: function() {
+			this._notifyGameReady().catch((err) => { this.emit("error", err); });
+		},
+	},
+
+	// Message indicating the server has started an autosave
+	{
+		filter: {
+			type: "log",
+			message: /^Saving to _autosave\d+ \((non-)?blocking\)\.$/,
+		},
+		action: function(parsed) {
+			let name = /^Saving to (_autosave\d+) /.exec(parsed.message)[1];
+			this.emit("_autosave", name);
+		},
+	},
+
+	// Message indicating the server has started saving
+	{
+		filter: {
+			type: "log",
+			message: /^Saving game as /,
+		},
+		action: function() {
+			this.emit("_saving");
+		},
 	},
 
 	// Message indicating the server has finished saving
 	{
 		filter: {
-			type: 'log',
+			type: "log",
 			message: [
 				"Saving finished",
 
@@ -360,47 +415,54 @@ const outputHeuristics = [
 				"Can't save to default location: Default location not known",
 			],
 		},
-		action: function(output) {
-			this.emit('_saved');
-		}
+		action: function() {
+			this.emit("_saved");
+		},
 	},
 
 	// Message indicating the server is shutting down
 	{
 		filter: {
-			type: 'generic',
-			message: /^Quitting: /
+			type: "generic",
+			message: /^Quitting: /,
+		},
+		action: function(parsed) {
+			this.emit("_quitting");
+
+			// On windows the Factorio server will wait for input on the console input buffer
+			// when an error occurs.  Since we can't send input there we terminate the server
+			// process when this happens.
+			if (process.platform === "win32" && parsed.message === "Quitting: multiplayer error.") {
+				this._server.kill();
+			}
 		},
-		action: function(output) {
-			this.emit('_quitting');
-		}
 	},
 
 	// Messages that might be tha cause of an unexpected shutdown.
 	{
 		filter: {
-			type: 'log',
-			level: 'Error',
+			type: "log",
+			level: "Error",
 			message: [
 				"MultiplayerManager failed: \"Binding IPv4 socket failed: Permission denied\"",
 				"MultiplayerManager failed: Host address is already in use.",
 			],
 		},
-		action: function(output) {
-			this._unexpected.push(`Factorio failed to bind to game port: ${output.message}`);
+		action: function(parsed) {
+			this._unexpected.push(`Factorio failed to bind to game port: ${parsed.message}`);
 		},
 	},
 	{
 		filter: {
-			type: 'log',
-			level: 'Error',
+			type: "log",
+			level: "Error",
 			message: [
 				"Can't bind socket: Address already in use",
 				"Can't bind socket: Permission denied",
 			],
 		},
-		action: function(output) {
-			this._unexpected.push(`Factorio failed to bind to RCON port: ${output.message}`);
+		action: function(parsed) {
+			this._unexpected.push(`Factorio failed to bind to RCON port: ${parsed.message}`);
 		},
 	},
 ];
@@ -413,13 +475,17 @@ const outputHeuristics = [
  * stopping and sending commands to the server.  It does not deal with
  * creating or managing servers, or downloading Factorio.
  *
- * This is an events emitter with the following events:
- * stdout - invoked when Factorio outputs a line to stdout
- * stderr - invoked when Factorio outputs a line to stderr
- * output - invoked with parsed output from Factorio
- * rcon-ready - invoked when the RCON client has connected
- * game-ready - invoked when the server is finished starting up
- * exit - invoked when the sterver has exited
+ * This is an EventEmitter with the following events:
+ * - stdout - invoked when Factorio outputs a line to stdout
+ * - stderr - invoked when Factorio outputs a line to stderr
+ * - output - invoked with parsed output from Factorio
+ * - rcon-ready - invoked when the RCON client has connected
+ * - game-ready - invoked when the server is finished starting up
+ * - autosave-start - invoked when the server starts an autosave
+ * - autosave-fnished - invoked when the autosave finished
+ * - exit - invoked when the sterver has exited
+ * @extends events.EventEmitter
+ * @memberof module:lib/factorio
  */
 class FactorioServer extends events.EventEmitter {
 	/**
@@ -428,6 +494,21 @@ class FactorioServer extends events.EventEmitter {
 	 * @param {string} factorioDir - Directory of the Factorio install(s).
 	 * @param {string} writeDir - Directory to write runtime data to.
 	 * @param {object} options - Optional parameters.
+	 * @param {Logger} options.logger - Logger to use for reporting errors.
+	 * @param {string} options.version -
+	 *     Version of Factorio to use.  Can also be the string "latest" to
+	 *     use the latest version found in `factorioDir`.
+	 * @param {number} options.gamePort - UDP port to host game on.
+	 * @param {number} options.rconPort - TCP port to use for RCON.
+	 * @param {string} options.rconPassword - Password use for RCON.
+	 * @param {boolean} options.enableWhitelist - Turn on whitelisting.
+	 * @param {boolean} options.verboseLogging - Enable verbose logging.
+	 * @param {boolean} options.stripPaths - Strip paths in the console.
+	 * @param {number} options.maxConcurrentCommands -
+	 *     Maximum number of RCON commands transmitted in parallel on the
+	 *     RCON connection.
+	 * @param {number} options.hangTimeout -
+	 *     Timeout in ms to wait for a response to a shutdown command.
 	 */
 	constructor(factorioDir, writeDir, options) {
 		super();
@@ -439,21 +520,60 @@ class FactorioServer extends events.EventEmitter {
 		this._version = null;
 		this._dataDir = null;
 
+		this._logger = options.logger || logger;
 		this._targetVersion = options.version || "latest";
+		/** UDP port used for hosting the Factorio game server on */
 		this.gamePort = options.gamePort || randomDynamicPort();
+		/** TCP port used for RCON on the Factorio game server */
 		this.rconPort = options.rconPort || randomDynamicPort();
+		/** Password used for RCON on the Factorio game server */
 		this.rconPassword = options.rconPassword;
+		/** Enable player whitelist */
+		this.enableWhitelist = options.enableWhitelist;
+		/** Enable verbose logging */
+		this.verboseLogging = options.verboseLogging;
+		/** Maximum number of RCON commands transmitted in parallel on the RCON connection  */
+		this.maxConcurrentCommands = options.maxConcurrentCommands || 5;
+		/** Timeout in ms to wait for a response to a shutdown command */
+		this.hangTimeout = options.hangTimeout || 5000;
 		this._state = "new";
 		this._server = null;
 		this._rconClient = null;
 		this._rconReady = false;
 		this._gameReady = false;
 
+		if (options.stripPaths) {
+			let chars = new Set(this.writePath("temp"));
+			chars.delete(":"); // Having a colon could lead to matching the line number
+			chars = [...chars].join("");
+
+			this._stripRegExp = new RegExp(
+				`(${escapeRegExp(this.writePath("temp", "currently-playing", path.sep))})|` +
+				`(${escapeRegExp(this.writePath(path.sep))})|` +
+				`(\\.\\.\\.[${escapeRegExp(chars)}]*?currently-playing\\${path.sep})`,
+				"g"
+			);
+		}
+
 		// Array of possible causes for an unexpected shutdown
 		this._unexpected = [];
 
-		this._stdout = new LineSplitter((line) => { this._handleOutput(line, 'stdout'); });
-		this._stderr = new LineSplitter((line) => { this._handleOutput(line, 'stderr'); });
+		this._stdout = new LineSplitter((line) => { this._handleOutput(line, "stdout"); });
+		this._stderr = new LineSplitter((line) => { this._handleOutput(line, "stderr"); });
+
+		// Track autosaving
+		this._runningAutosave = null;
+		this.on("_autosave", name => {
+			this.emit("autosave-start", name);
+			this._runningAutosave = name;
+		});
+
+		this.on("_saved", () => {
+			if (this._runningAutosave) {
+				this.emit("autosave-finished", this._runningAutosave);
+				this._runningAutosave = null;
+			}
+		});
 	}
 
 	_check(expectedStates) {
@@ -465,40 +585,40 @@ class FactorioServer extends events.EventEmitter {
 	}
 
 	async _handleIpc(line) {
-		let channelEnd = line.indexOf('?');
+		let channelEnd = line.indexOf("?");
 		if (channelEnd === -1) {
 			throw new Error(`Malformed IPC line "${line.toString()}"`);
 		}
 
 		let channel = line
 			.slice(6, channelEnd)
-			.toString('utf-8')
-			.replace(/\\x([0-9a-f]{2})/g, (match, p1) =>
+			.toString("utf-8")
+			.replace(/\\x([0-9a-f]{2})/g, (match, p1) => (
 				String.fromCharCode(parseInt(p1, 16))
-			)
+			))
 		;
 
-		let type = line.slice(channelEnd + 1, channelEnd + 2).toString('utf-8');
+		let type = line.slice(channelEnd + 1, channelEnd + 2).toString("utf-8");
 		let content;
-		if (type === 'j') {
-			content = JSON.parse(line.slice(channelEnd + 2).toString('utf-8'));
+		if (type === "j") {
+			content = JSON.parse(line.slice(channelEnd + 2).toString("utf-8"));
 
-		} else if (type === 'f') {
-			let fileName = line.slice(channelEnd + 2).toString('utf-8');
-			let filePath = this.writePath('script-output', fileName);
+		} else if (type === "f") {
+			let fileName = line.slice(channelEnd + 2).toString("utf-8");
+			let filePath = this.writePath("script-output", fileName);
 
 			// Prevent malicious names
 			if (/[\\/\0]/.test(fileName)) {
-				throw new Error(`Invalid IPC file name '${fileName}'`)
+				throw new Error(`Invalid IPC file name '${fileName}'`);
 			}
 
-			if (fileName.endsWith('.json')) {
-				content = JSON.parse(await fs.readFile(filePath, 'utf-8'));
+			if (fileName.endsWith(".json")) {
+				content = JSON.parse(await fs.readFile(filePath, "utf-8"));
 				await fs.unlink(filePath);
 
 			} else {
-				let format = fileName.slice(fileName.lastIndexOf('.') + 1);
-				throw new Error(`Unknown IPC file format '${format}'`)
+				let format = fileName.slice(fileName.lastIndexOf(".") + 1);
+				throw new Error(`Unknown IPC file format '${format}'`);
 			}
 
 		} else {
@@ -506,45 +626,49 @@ class FactorioServer extends events.EventEmitter {
 		}
 
 		if (!this.emit(`ipc-${channel}`, content)) {
-			console.warn(`Warning: Unhandled ipc-${channel}`, content);
+			this._logger.warn(`Warning: Unhandled ipc-${channel}`, { content });
 		}
 	}
 
 	_handleOutput(line, source) {
 		if (line.slice(0, 6).equals(Buffer.from("\f$ipc:"))) {
-			this._handleIpc(line).catch(err => this.emit('error', err));
+			this._handleIpc(line).catch(err => this.emit("error", err));
 			return;
 		}
 
 		this.emit(source, line);
 
-		let output = parseOutput(line.toString('utf-8'), source);
+		line = line.toString("utf-8");
+		if (this._stripRegExp) {
+			line = line.replace(this._stripRegExp, "");
+		}
+
+
+		let parsed = parseOutput(line, source);
 		heuristicLoop: for (let heuristic of outputHeuristics) {
 			for (let [name, expected] of Object.entries(heuristic.filter)) {
-				if (!Object.hasOwnProperty.call(output, name)) {
+				if (!Object.hasOwnProperty.call(parsed, name)) {
 					continue heuristicLoop;
 				}
 
 				if (expected instanceof RegExp) {
-					if (!expected.test(output[name])) {
+					if (!expected.test(parsed[name])) {
 						continue heuristicLoop;
 					}
 				} else if (expected instanceof Array) {
-					if (!expected.includes(output[name])) {
-						continue heuristicLoop;
-					}
-				} else {
-					if (expected !== output[name]) {
+					if (!expected.includes(parsed[name])) {
 						continue heuristicLoop;
 					}
+				} else if (expected !== parsed[name]) {
+					continue heuristicLoop;
 				}
 			}
 
 			// If we get here the filter matched the output
-			heuristic.action.call(this, output);
+			heuristic.action.call(this, parsed);
 		}
 
-		this.emit('output', output);
+		this.emit("output", parsed, line);
 	}
 
 	async _startRcon() {
@@ -561,20 +685,45 @@ class FactorioServer extends events.EventEmitter {
 			port: this.rconPort,
 			password: this.rconPassword,
 			timeout: 200000, // 200s, should allow for commands up to 1250kB in length
-			maxPending: 5,
+			maxPending: this.maxConcurrentCommands,
 		};
 
 		this._rconClient = new rconClient.Rcon(config);
-		this._rconClient.on('error', (err) => this.emit('error', err));
-		this._rconClient.on('authenticated', () => { this._rconReady = true; this.emit('rcon-ready'); });
-		this._rconClient.on('end', () => {/* XXX TODO */});
+		this._rconClient.on("error", () => { /* Ignore */ });
+		this._rconClient.on("authenticated", () => { this._rconReady = true; this.emit("rcon-ready"); });
+		this._rconClient.on("end", () => {
+			this._rconClient = null;
+		});
+		// XXX: Workaround to suppress bogus event listener warning
+		if (this._rconClient.emitter instanceof events.EventEmitter) {
+			this._rconClient.emitter.setMaxListeners(this.maxConcurrentCommands + 5);
+		}
 		await this._rconClient.connect();
 	}
 
+	get maxConcurrentCommands() {
+		return this._maxConcurrentCommands;
+	}
+
+	set maxConcurrentCommands(value) {
+		this._maxConcurrentCommands = value;
+		this.setMaxListeners(value + 5);
+		if (this._rconClient) {
+			if (this._rconClient.sendQueue) {
+				this._rconClient.sendQueue.maxConcurrent = value;
+			}
+			// XXX: Workaround to suppress bogus event listener warning
+			if (this._rconClient.emitter instanceof events.EventEmitter) {
+				this._rconClient.emitter.setMaxListeners(value + 5);
+			}
+		}
+	}
+
 	/**
 	 * Initialize class instance
 	 *
 	 * Must be called before instances of this class can be used.
+	 * @throws {Error} if the requested version of Factorio was not found.
 	 */
 	async init() {
 		this._check(["new"]);
@@ -584,50 +733,66 @@ class FactorioServer extends events.EventEmitter {
 	}
 
 	/**
-	 * The version of Factorio that was detected
+	 * The version of Factorio in use.  This will be the actual version if
+	 * "latest" (the default) was specified as `factorioVersion` to the
+	 * constructor.
 	 */
 	get version() {
 		return this._version;
 	}
 
+	/**
+	 * The pid of the server process, or null if not running
+	 */
+	get pid() {
+		if (!this._server) {
+			return null;
+		}
+		return this._server.pid;
+	}
+
 	_attachStdio() {
-		this._server.stdout.on('data', chunk => { this._stdout.data(chunk); });
-		this._server.stdout.on('close', () => { this._stdout.end(); });
-		this._server.stderr.on('data', chunk => { this._stderr.data(chunk); });
-		this._server.stderr.on('close', () => { this._stderr.end(); });
+		this._server.stdout.on("data", chunk => { this._stdout.data(chunk); });
+		this._server.stdout.on("close", () => { this._stdout.end(); });
+		this._server.stderr.on("data", chunk => { this._stderr.data(chunk); });
+		this._server.stderr.on("close", () => { this._stderr.end(); });
 	}
 
 	_watchExit() {
-		this._server.on('exit', (code, signal) => {
+		this._server.on("exit", (code, signal) => {
 			if (this._state !== "stopping") {
 				if (this._unexpected.length === 0) {
-					if (signal === 'SIGKILL') {
-						this.emit('error', new errors.EnvironmentError(
+					if (signal === "SIGKILL") {
+						this.emit("error", new libErrors.EnvironmentError(
 							"Factorio server was unexpectedly killed, is the system low on memory?"
 						));
 
 					} else {
 						let msg;
 						if (code !== null) {
-							msg = `Factorio server unexpectedly shut down with code ${code}`
+							msg = `Factorio server unexpectedly shut down with code ${code}`;
 						} else {
 							msg = `Factorio server was unexpectedly shut by down by signal ${signal}`;
 						}
 
-						this.emit('error', new Error(msg));
+						this.emit("error", new Error(msg));
 					}
 
 				} else if (this._unexpected.length === 1) {
-					this.emit('error', new Error(this._unexpected[0]));
+					this.emit("error", new Error(this._unexpected[0]));
 
 				} else {
-					this.emit('error', new Error(
-						"Factorio server unexpectedly shut down. Possible causes:\n- "+
-						this._unexpected.join("\n- ")
+					this.emit("error", new Error(
+						"Factorio server unexpectedly shut down. Possible causes:"+
+						`\n- ${this._unexpected.join("\n- ")}`
 					));
 				}
 			}
 
+			if (this._rconClient) {
+				this._rconClient.end().catch(() => {});
+			}
+
 			// Reset server state
 			this._state = "init";
 			this._server = null;
@@ -635,6 +800,7 @@ class FactorioServer extends events.EventEmitter {
 			this._rconReady = false;
 			this._gameReady = false;
 			this._unexpected = [];
+			this._runningAutosave = null;
 
 			this.emit("exit");
 		});
@@ -642,12 +808,12 @@ class FactorioServer extends events.EventEmitter {
 
 	async _notifyGameReady() {
 		this._gameReady = true;
-		this.emit('game-ready');
+		this.emit("game-ready");
 	}
 
 	async _waitForReady() {
 		if (!this._gameReady) {
-			await events.once(this, 'game-ready');
+			await events.once(this, "game-ready");
 		}
 	}
 
@@ -666,14 +832,15 @@ class FactorioServer extends events.EventEmitter {
 
 		await this._writeConfigIni();
 		this._server = child_process.spawn(
-			this.dataPath('..', 'bin', 'x64', 'factorio'),
+			this.dataPath("..", "bin", "x64", "factorio"),
 			[
-				'--config', this.writePath('config.ini'),
-				'--create', this.writePath('saves', name),
+				"--config", this.writePath("config.ini"),
+				"--create", this.writePath("saves", name),
+				...(this.verboseLogging ? ["--verbose"] : []),
 			],
 			{
 				detached: true,
-				stdio: 'pipe',
+				stdio: "pipe",
 			}
 		);
 
@@ -681,7 +848,7 @@ class FactorioServer extends events.EventEmitter {
 		this._server.once("exit", () => this.emit("exit"));
 
 		try {
-			let [code, signal] = await events.once(this._server, 'exit');
+			let [code, signal] = await events.once(this._server, "exit");
 			if (code !== 0) {
 				throw new Error(`Factorio exited with status ${code}`);
 			}
@@ -704,17 +871,19 @@ class FactorioServer extends events.EventEmitter {
 
 		await this._writeConfigIni();
 		this._server = child_process.spawn(
-			this.dataPath('..', 'bin', 'x64', 'factorio'),
+			this.dataPath("..", "bin", "x64", "factorio"),
 			[
-				'--config', this.writePath('config.ini'),
-				'--start-server', this.writePath('saves', save),
-				'--port', this.gamePort,
-				'--rcon-port', this.rconPort,
-				'--rcon-password', this.rconPassword,
+				"--config", this.writePath("config.ini"),
+				"--start-server", this.writePath("saves", save),
+				"--port", this.gamePort,
+				"--rcon-port", this.rconPort,
+				"--rcon-password", this.rconPassword,
+				...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
+				...(this.verboseLogging ? ["--verbose"] : []),
 			],
 			{
 				detached: true,
-				stdio: 'pipe',
+				stdio: "pipe",
 			}
 		);
 
@@ -737,17 +906,19 @@ class FactorioServer extends events.EventEmitter {
 
 		await this._writeConfigIni();
 		this._server = child_process.spawn(
-			this.dataPath('..', 'bin', 'x64', 'factorio'),
+			this.dataPath("..", "bin", "x64", "factorio"),
 			[
-				'--config', this.writePath('config.ini'),
-				'--start-server-load-scenario', scenario,
-				'--port', this.gamePort,
-				'--rcon-port', this.rconPort,
-				'--rcon-password', this.rconPassword,
+				"--config", this.writePath("config.ini"),
+				"--start-server-load-scenario", scenario,
+				"--port", this.gamePort,
+				"--rcon-port", this.rconPort,
+				"--rcon-password", this.rconPassword,
+				...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
+				...(this.verboseLogging ? ["--verbose"] : []),
 			],
 			{
 				detached: true,
-				stdio: 'pipe',
+				stdio: "pipe",
 			}
 		);
 
@@ -770,13 +941,16 @@ class FactorioServer extends events.EventEmitter {
 	 * @returns {string} response from server.
 	 */
 	async sendRcon(message, expectEmpty) {
-		this._check(["running"])
+		this._check(["running", "stopping"]);
 		if (!this._rconReady) {
-			await events.once(this, 'rcon-ready');
+			await events.once(this, "rcon-ready");
+		}
+		if (!this._rconClient) {
+			throw new Error("RCON connection lost");
 		}
 
 		let response = await this._rconClient.send(message);
-		if (expectEmpty && response !== '') {
+		if (expectEmpty && response !== "") {
 			throw new Error(`Expected empty response but got "${response}"`);
 		}
 		return response;
@@ -791,25 +965,63 @@ class FactorioServer extends events.EventEmitter {
 	async stop() {
 		this._check(["running"]);
 
-		// On Windows we need to save the map as there's no graceful shutdown.
-		if (this._rconClient && process.platform === "win32") {
-			let saved = events.once(this, '_saved');
-			await this.sendRcon("/server-save");
-			await saved;
-		}
+		let hanged = true;
+		let stopped = false;
+		function setAlive() { hanged = false; }
+
+		let timeoutId = setTimeout(() => {
+			if (hanged) {
+				this._logger.error("Factorio appears to have hanged, sending SIGKILL");
+				if (this._rconClient) {
+					this._rconClient.end().catch(() => {});
+				}
+				this._server.kill("SIGKILL");
+			}
+		}, this.hangTimeout);
+
+		// It's possible the server decides to exit on its own, in which
+		// case the stop operation has to be cancelled.
+		this._server.once("exit", () => {
+			stopped = true;
+			clearTimeout(timeoutId);
+		});
 
 		this._state = "stopping";
+		// On Windows we need to save the map as there's no graceful shutdown.
+		if (this._rconClient && process.platform === "win32") {
+			// Not using events.once here as that will reject on error.
+			let saved = new Promise(resolve => this.once("_saved", resolve));
+			this.on("_saving", setAlive);
+
+			try {
+				await this.sendRcon("/server-save");
+				await saved;
+
+			} catch (err) {
+				// Ignore
+			}
+
+			clearTimeout(timeoutId);
+			this.off("_saving", setAlive);
+		}
+
 		if (this._rconClient) {
 
 			// If RCON is not yet fully connected that operation needs to
 			// complete before the RCON connection can be closed cleanly.
 			if (!this._rconReady) {
-				await events.once(this, 'rcon-ready');
+				await events.once(this, "rcon-ready");
 			}
 
 			await this._rconClient.end();
 		}
 
+		// The Factorio server may have decided to get ahead of us and
+		// stop by itself before we had the chance to signal it.
+		if (stopped) {
+			return;
+		}
+
 		// On linux this sends SIGTERM to the process.  On windows this
 		// will terminate the process with no cleanup.  You can send your
 		// complaint for the lack of graceful shutdown to the Factorio
@@ -823,29 +1035,31 @@ class FactorioServer extends events.EventEmitter {
 		// stop if it's stuck with an infinite lua code loop.  In either
 		// case there's no recovering from it.
 		if (process.platform !== "win32") {
-			let hanged = true;
-			function setAlive(output) { hanged = false; }
-			this.on('_quitting', setAlive);
-
-			let timeoutId = setTimeout(() => {
-				if (hanged) {
-					console.error("Factorio appears to have hanged, sending SIGKILL");
-					this._server.kill('SIGKILL');
-				}
-			}, 5000);
-
-			await events.once(this._server, 'exit');
+			this.on("_quitting", setAlive);
+			await events.once(this._server, "exit");
 
 			clearTimeout(timeoutId);
-			this.off('_quitting', setAlive);
+			this.off("_quitting", setAlive);
 
 		// On windows the process is terminated immediately, but to keep
 		// ordering wait until after the exit event here.
 		} else {
-			await events.once(this._server, 'exit');
+			await events.once(this._server, "exit");
 		}
 	}
 
+	/**
+	 * Kill the server
+	 *
+	 * Terminates the server without any cleanup or saving.
+	 */
+	async kill() {
+		this._check(["running", "stopping"]);
+		this._state = "stopping";
+		this._server.kill("SIGKILL");
+		await events.once(this._server, "exit");
+	}
+
 	/**
 	 * Disable achievements on the running server
 	 *
@@ -860,14 +1074,14 @@ class FactorioServer extends events.EventEmitter {
 		this._check(["running"]);
 
 		let check = await this.sendRcon("/sc rcon.print('disabled')");
-		if (check === 'disabled\n') {
+		if (check === "disabled\n") {
 			return false;
 		}
 
 		// Factorio will print a warning to the console and ask for the command
 		// to be runned again before disabling achievements and allowing commands.
 		check = await this.sendRcon("/sc rcon.print('disabled')");
-		if (check === 'disabled\n') {
+		if (check === "disabled\n") {
 			return true;
 		}
 
@@ -880,6 +1094,9 @@ class FactorioServer extends events.EventEmitter {
 	 * Creates a path using path.join with the given parts that's relative to
 	 * the data directory of the Factorio server.  Not valid before init has
 	 * been called.
+	 *
+	 * @param {...string} parts - Extra parts to add to the data path.
+	 * @returns {string} Data directory path.
 	 */
 	dataPath(...parts) {
 		return path.join(this._dataDir, ...parts);
@@ -890,6 +1107,9 @@ class FactorioServer extends events.EventEmitter {
 	 *
 	 * Creates a path using path.join with the given parts that's relative to
 	 * the write directory of the Factorio server.
+	 *
+	 * @param {...string} parts - Extra parts to add to the write path.
+	 * @returns {string} Write directory path.
 	 */
 	writePath(...parts) {
 		return path.join(this._writeDir, ...parts);
@@ -911,7 +1131,7 @@ class FactorioServer extends events.EventEmitter {
 			path: {
 				"read-data": this.dataPath(),
 				"write-data": this.writePath(),
-			}
+			},
 		});
 		await fs.outputFile(this.writePath("config.ini"), content);
 	}

packages/lib/file_ops.js

@@ -1,3 +1,7 @@
+/**
+ * @module lib/file_ops
+ */
+"use strict";
 const fs = require("fs-extra");
 const path = require("path");
 
@@ -30,4 +34,4 @@ async function getNewestFile(directory, filter = (name) => true) {
 
 module.exports = {
 	getNewestFile,
-}
+};

packages/lib/hash.js

@@ -1,21 +1,27 @@
-const crypto = require('crypto');
-const fs = require('fs');
+/**
+ * Hashing functions
+ * @module lib/hash
+ */
+"use strict";
+const crypto = require("crypto");
+const fs = require("fs");
 
 
 /**
  * Returns a promise that resolves to the SHA1 hash of the stream given
  *
- * @param stream - Node stream of the content to hash.
+ * @param {Stream} stream - Node stream of the content to hash.
+ * @returns {Promise<string>} hash of the stream.
  */
 function hashStream(stream) {
-	return new Promise(function(resolve, reject) {
-		let hasher = crypto.createHash('sha1')
-		hasher.setEncoding('hex')
-		hasher.on('finish', function() {
+	return new Promise((resolve, reject) => {
+		let hasher = crypto.createHash("sha1");
+		hasher.setEncoding("hex");
+		hasher.on("finish", () => {
 			resolve(hasher.read());
 		});
 
-		stream.on('error', function(error) {
+		stream.on("error", (error) => {
 			// the docs doesn't say anything about what's passed on error
 			reject(error);
 		});
@@ -28,6 +34,7 @@ function hashStream(stream) {
  * Returns a promise that resolves to the SHA1 hash of the file given by path
  *
  * @param {string} path - Path to the file to hash.
+ * @returns {Promise<string>} hash of the file given.
  */
 function hashFile(path) {
 	return hashStream(fs.createReadStream(path));
@@ -36,4 +43,4 @@ function hashFile(path) {
 module.exports = {
 	hashStream,
 	hashFile,
-}
+};

packages/lib/helpers.js

@@ -0,0 +1,100 @@
+/**
+ * Collection of small utilites that are useful in multiple places.
+ * @module lib/helpers
+ */
+"use strict";
+
+/**
+ * Return a string describing the type of the value passed
+ *
+ * Works the same as typeof, excpet that null and array types get their
+ * own string.
+ *
+ * @param {*} value - value to return the type of.
+ * @returns {string} basic type of the value passed.
+ */
+function basicType(value) {
+	if (value === null) { return "null"; }
+	if (value instanceof Array) { return "array"; }
+	return typeof value;
+}
+
+
+/**
+ * Asynchronously wait for the given duration
+ *
+ * @param {number} duration - Time to wait for in milliseconds.
+ */
+async function wait(duration) {
+	await new Promise(resolve => { setTimeout(resolve, duration); });
+}
+
+
+/**
+ * Resolve a promise with a timeout.
+ *
+ * @param {Promise} promise - Promise to wait for.
+ * @param {number} time - Maximum time im milliseconds to wait for.
+ * @param {*=} timeoutResult - Value to return if the operation timed out.
+ */
+async function timeout(promise, time, timeoutResult) {
+	let timer;
+	try {
+		return await Promise.race([
+			promise,
+			new Promise(resolve => {
+				timer = setTimeout(() => resolve(timeoutResult), time);
+			}),
+		]);
+
+	} finally {
+		clearTimeout(timer);
+	}
+}
+
+
+/**
+ * Read stream to the end and return its content
+ *
+ * Reads the stream given asynchronously until the end is reached and
+ * returns all the data which was read from the stream.
+ *
+ * @param {Readable} stream - byte stream to read to the end.
+ * @returns {Buffer} content of the stream.
+ */
+async function readStream(stream) {
+	let chunks = [];
+	for await (let chunk of stream) {
+		// Support using ^Z to end input on Windows
+		if (process.platform === "win32" && stream.isTTY && chunk.toString() === "\x1a\r\n") {
+			break;
+		}
+		chunks.push(chunk);
+	}
+	return Buffer.concat(chunks);
+}
+
+
+/**
+ * Escapes text for inclusion in a RegExp
+ *
+ * Adds \ character in front of special meta characters in the passsed in
+ * text so that it can be embedded into a RegExp and only match the text.
+ *
+ * See https://stackoverflow.com/a/9310752
+ *
+ * @param {string} text - Text to escape RegExp meta chars in.
+ * @returns {string} escaped text.
+ */
+function escapeRegExp(text) {
+	return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
+}
+
+
+module.exports = {
+	basicType,
+	wait,
+	timeout,
+	readStream,
+	escapeRegExp,
+};

packages/lib/is_deep_strict_equal.js

@@ -0,0 +1,8 @@
+/* eslint-disable global-require */
+"use strict";
+// eslint-disable-next-line no-process-env
+if (process.env.APP_ENV === "browser") {
+	module.exports = require("fast-deep-equal/es6");
+} else {
+	module.exports = require("util").isDeepStrictEqual;
+}

packages/lib/link/connectors.js

@@ -1,11 +1,19 @@
+// Connection adapters for Links
 "use strict";
 const events = require("events");
-const WebSocket = require("ws");
+const WebSocket = require("isomorphic-ws");
 
-const schema = require("lib/schema");
-const errors = require("lib/errors");
+const libSchema = require("@clusterio/lib/schema");
+const libErrors = require("@clusterio/lib/errors");
+const { logger } = require("@clusterio/lib/logging");
 
 
+/**
+ * Base connector for links
+ *
+ * @extends events.EventEmitter
+ * @memberof module:lib/link
+ */
 class WebSocketBaseConnector extends events.EventEmitter {
 	constructor() {
 		super();
@@ -27,7 +35,7 @@ class WebSocketBaseConnector extends events.EventEmitter {
 		let dropCount = 0;
 		for (let index = 0; index < this._sendBuffer.length; index++) {
 			if (this._sendBuffer[index].seq <= seq) {
-				dropCount++;
+				dropCount += 1;
 			} else {
 				break;
 			}
@@ -36,9 +44,9 @@ class WebSocketBaseConnector extends events.EventEmitter {
 	}
 
 	_doHeartbeat() {
-		this._check(["connected"]);
+		this._check(["connected", "closing"]);
 		if (Date.now() - this._lastHeartbeat > 2000 * this._heartbeatInterval) {
-			console.log("SOCKET | closing after heartbeat timed out");
+			logger.verbose("SOCKET | closing after heartbeat timed out");
 			this._socket.close(1008, "Heartbeat timeout");
 			clearInterval(this._heartbeatId);
 			this._heartbeatId = null;
@@ -53,10 +61,9 @@ class WebSocketBaseConnector extends events.EventEmitter {
 	}
 
 	_processHeartbeat(message) {
-		if (!schema.heartbeat(message)) {
-			console.log("SOCKET | closing after received invalid heartbeat:", message);
+		if (!libSchema.heartbeat(message)) {
+			logger.warn("SOCKET | closing after received invalid heartbeat");
 			this._socket.close(1002, "Invalid heartbeat");
-			console.log(schema.heartbeat.errors);
 			return;
 		}
 
@@ -87,16 +94,22 @@ class WebSocketBaseConnector extends events.EventEmitter {
 	/**
 	 * Send a message over the socket
 	 *
-	 * @returns the sequence number of the message sent
+	 * @param {string} type - Message type to send.
+	 * @param {Object} data - Data to send with message.
+	 * @returns {number} the sequence number of the message sent
 	 */
 	send(type, data) {
-		this._check(["handshake", "connected", "closing"]);
-		let message = { seq: this._seq, type, data };
+		if (!["handshake", "connected", "closing"].includes(this._state)) {
+			throw new libErrors.SessionLost("No session");
+		}
+		let seq = this._seq;
+		this._seq += 1;
+		let message = { seq, type, data };
 		this._sendBuffer.push(message);
 		if (["connected", "closing"].includes(this._state)) {
 			this._socket.send(JSON.stringify(message));
 		}
-		return this._seq++;
+		return seq;
 	}
 
 	/**
@@ -108,16 +121,34 @@ class WebSocketBaseConnector extends events.EventEmitter {
 	setClosing() {
 		this._state = "closing";
 	}
+
+	/**
+	 * True if the connection is established and active
+	 */
+	get connected() {
+		return this._state === "connected";
+	}
+
+	/**
+	 * True if the connection is closing down or closed
+	 */
+	get closing() {
+		return ["closing", "closed"].includes(this._state);
+	}
 }
 
 /**
  * Connector for master server clients
+ *
+ * @extends module:lib/link.WebSocketBaseConnector
+ * @memberof module:lib/link
  */
 class WebSocketClientConnector extends WebSocketBaseConnector {
-	constructor(url, reconnectDelay) {
+	constructor(url, reconnectDelay, tlsCa = null) {
 		super();
 
 		this._url = url;
+		this._tlsCa = tlsCa;
 
 		// The following states are used in the client connector
 		// new: Not connected
@@ -129,6 +160,8 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 		this._sessionToken = null;
 		this._connected = false;
 		this._reconnectDelay = reconnectDelay;
+		this._timeout = 15 * 60;
+		this._startedReconnect = null;
 	}
 
 	_check(expectedStates) {
@@ -139,11 +172,23 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 		}
 	}
 
+	setTimeout(timeout) {
+		this._timeout = timeout;
+
+		if (this._reconnectId && this.timeout < this._reconnectDelay / 2) {
+			clearTimeout(this._reconnectId);
+			this.reconnect();
+		}
+	}
+
 	/**
 	 * Send a handshake message over the socket
 	 *
 	 * Used in the register function in order to send the register handshake
 	 * message over the WebSocket.
+	 *
+	 * @param {string} type - Message type to send.
+	 * @param {Object} data - Data to send with message.
 	 */
 	sendHandshake(type, data) {
 		this._check(["handshake"]);
@@ -154,8 +199,11 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 	 * Close the connection with the given reason.
 	 *
 	 * Sends a close frame and disconnects the connector.
+	 *
+	 * @param {number} code - WebSocket close code.
+	 * @param {string} reason - WebSocket close reason.
 	 */
-	close(code, reason) {
+	async close(code, reason) {
 		if (this._state === "new") {
 			return;
 		}
@@ -168,6 +216,7 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 		this.stopHeartbeat();
 		this._state = "closing";
 		this._socket.close(code, reason);
+		await events.once(this, "close");
 	}
 
 	/**
@@ -180,7 +229,7 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 		this._doConnect();
 
 		// Wait for the connection to be ready
-		await events.once(this, "connected");
+		await events.once(this, "connect");
 	}
 
 	async _doConnect() {
@@ -188,13 +237,17 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 		url.pathname += "api/socket";
 
 		// Open WebSocket to master
-		console.log(`SOCKET | connecting to ${url}`)
+		logger.verbose(`SOCKET | connecting to ${url}`);
 
-		this._socket = new WebSocket(url, {
-			// For now we do not verify TLS certificates since the default setup is
-			// to create a self-signed certificate.
-			rejectUnauthorized: false,
-		});
+		// eslint-disable-next-line no-process-env
+		if (process.env.APP_ENV === "browser") {
+			this._socket = new WebSocket(url);
+
+		} else {
+			let options = {};
+			if (this._tlsCa) { options.ca = this._tlsCa; }
+			this._socket = new WebSocket(url, options);
+		}
 
 		this._attachSocketHandlers();
 	}
@@ -207,8 +260,25 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 			return;
 		}
 
+		if (this._startedReconnect === null) {
+			this._startedReconnect = Date.now();
+		}
+
+		if (this._startedReconnect + this._timeout * 1000 < Date.now()) {
+			logger.error("SOCKET | Timed out trying to reconnect");
+			this._heartbeatInterval = null;
+			this._lastReceivedSeq = null;
+			this._sessionToken = null;
+			this._sendBuffer.length = 0;
+			this._connected = false;
+			this.emit("close");
+			return;
+		}
+
 		let delay = Math.random() * this._reconnectDelay;
-		console.log(`SOCKET | waiting ${delay.toLocaleString("en", { maximumFractionDigits: 2 })} seconds for reconnect`)
+		logger.verbose(
+			`SOCKET | waiting ${delay.toLocaleString("en", { maximumFractionDigits: 2 })} seconds for reconnect`
+		);
 		this._reconnectId = setTimeout(() => {
 			this._reconnectId = null;
 			this._doConnect();
@@ -216,11 +286,11 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 	}
 
 	_attachSocketHandlers() {
-		this._socket.on("close", (code, reason) => {
-			console.log(`SOCKET | Close (code: ${code}, reason: ${reason})`);
+		this._socket.onclose = event => {
+			logger.verbose(`SOCKET | Close (code: ${event.code}, reason: ${event.reason})`);
 			// Authentication failed
-			if (code === 4003) {
-				this.emit("error", new errors.AuthenticationFailed("Authentication failed"));
+			if (event.code === 4003) {
+				this.emit("error", new libErrors.AuthenticationFailed(event.reason));
 				this._state = "closing";
 			}
 
@@ -229,55 +299,45 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 				this._sessionToken = null;
 				this._sendBuffer.length = 0;
 				this._state = "new";
-
-				if (this._connected) {
-					this.emit("close");
-					this._connected = false;
-				}
+				this._connected = false;
+				this.emit("close");
 
 			} else {
 				this._state = "handshake";
 				this.reconnect();
-
 				if (this._connected) {
-					this.emit("dropped");
 					this._connected = false;
+					this.emit("drop");
 				}
 			}
 
 			this.stopHeartbeat();
-		});
+		};
 
-		this._socket.on("error", err => {
+		this._socket.onerror = event => {
 			// It's assumed that close is always called by ws
-			console.error("SOCKET | Error:", err);
-		});
+			let code = !event.error ? "" : `, code: ${event.error.code}`;
+			logger.error(`SOCKET | Error: ${event.message || "unknown error"}${code}`);
+			if (event.error) {
+				// Abort connection if certificate validation failed
+				if ([
+					"CERT_HAS_EXPIRED",
+					"DEPTH_ZERO_SELF_SIGNED_CERT",
+					"ERR_TLS_CERT_ALTNAME_INVALID",
+				].includes(event.error.code)) {
+					this.emit("error", new libErrors.AuthenticationFailed(event.error.message));
+					this._state = "closing";
+				}
+			}
+		};
 
-		/*
-		this._socket.on("unexpected-response", (req, res) => {
-			let data = "";
-			res.on('data', (chunk) => { data += chunk; });
-			res.on('end', () => {
-				console.error(`SOCKET | Unexpeced Response ${res.statusCode}`);
-				console.error(data);
-				this._socket.emit("error", new Error(`Unexpected Response ${res.statusCode}`));
-			});
-		});
-		*/
-
-		this._socket.on("open", () => {
-			console.log("SOCKET | Open");
-		});
-		this._socket.on("ping", data => {
-			console.log(`SOCKET | Ping (data: ${data}`);
-		});
-		this._socket.on("pong", data => {
-			console.log(`SOCKET | Pong (data: ${data}`);
-		});
+		this._socket.onopen = () => {
+			logger.verbose("SOCKET | Open");
+		};
 
 		// Handle messages
-		this._socket.on("message", data => {
-			let message = JSON.parse(data);
+		this._socket.onmessage = event => {
+			let message = JSON.parse(event.data);
 			if (this._state === "handshake") {
 				this._processHandshake(message);
 
@@ -296,21 +356,22 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 			} else {
 				throw new Error(`Received message in unexpected state ${this._state}`);
 			}
-		});
+		};
 	}
 
 	_processHandshake(message) {
-		if (!schema.serverHandshake(message)) {
-			console.log("SOCKET | closing after received invalid handshake:", message);
+		if (!libSchema.serverHandshake(message)) {
+			logger.verbose("SOCKET | closing after received invalid handshake");
 			this._socket.close(1002, "Invalid handshake");
 			return;
 		}
 
 		let { seq, type, data } = message;
-		if (type == 'hello') {
-			console.log(`SOCKET | received hello from master version ${data.version}`);
+		if (type === "hello") {
+			logger.verbose(`SOCKET | received hello from master version ${data.version}`);
+			this.emit("hello", data);
 			if (this._sessionToken) {
-				console.log("SOCKET | Attempting resume");
+				logger.verbose("SOCKET | Attempting resume");
 				this.sendHandshake("resume", {
 					session_token: this._sessionToken,
 					last_seq: this._lastReceivedSeq,
@@ -319,26 +380,31 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 				this.register();
 			}
 
-		} else if (type === 'ready') {
-			console.log("SOCKET | received ready from master");
+		} else if (type === "ready") {
+			logger.verbose("SOCKET | received ready from master");
 			this._state = "connected";
 			this._sessionToken = data.session_token;
 			this._heartbeatInterval = data.heartbeat_interval;
 			this.startHeartbeat();
+			for (let bufferedMessage of this._sendBuffer) {
+				this._socket.send(JSON.stringify(bufferedMessage));
+			}
+			this._startedReconnect = null;
 			this._connected = true;
-			this.emit("connected");
+			this.emit("connect");
 
 		} else if (type === "continue") {
-			console.log("SOCKET | resuming existing session");
+			logger.verbose("SOCKET | resuming existing session");
 			this._state = "connected";
 			this._heartbeatInterval = data.heartbeat_interval;
 			this.startHeartbeat();
 			this._dropSendBufferSeq(data.last_seq);
-			for (let message of this._sendBuffer) {
-				this._socket.send(JSON.stringify(message));
+			for (let bufferedMessage of this._sendBuffer) {
+				this._socket.send(JSON.stringify(bufferedMessage));
 			}
+			this._startedReconnect = null;
 			this._connected = true;
-			this.emit("connected");
+			this.emit("resume");
 
 		} else if (type === "invalidate") {
 			this._heartbeatInterval = null;
@@ -361,6 +427,12 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 	}
 }
 
+/**
+ * Connector for in-memory local links
+ *
+ * @extends events.EventEmitter
+ * @memberof module:lib/link
+ */
 class VirtualConnector extends events.EventEmitter {
 	constructor() {
 		super();
@@ -388,11 +460,15 @@ class VirtualConnector extends events.EventEmitter {
 	/**
 	 * Send a message to the other end of the connector
 	 *
-	 * @returns the sequence number of the message sent
+	 * @param {string} type - Message type to send.
+	 * @param {Object} data - Data to send with message.
+	 * @returns {number} the sequence number of the message sent
 	 */
 	send(type, data) {
-		this.other.emit('message', { seq: this._seq, type, data });
-		return this._seq++;
+		let seq = this._seq;
+		this._seq += 1;
+		this.other.emit("message", { seq, type, data });
+		return seq;
 	}
 }
 

packages/lib/link/index.js

@@ -0,0 +1,12 @@
+/* eslint-disable global-require */
+/**
+ * Library for message based communication between nodes
+ * @module lib/link
+ * @author Hornwitser
+ */
+"use strict";
+module.exports = {
+	...require("./link"),
+	...require("./messages"),
+	...require("./connectors"),
+};

packages/lib/link/link.js

@@ -1,8 +1,10 @@
+// Implementation of Link class
 "use strict";
 
-const schema = require("lib/schema");
-const errors = require("lib/errors");
-const messages = require("lib/link/messages");
+const libSchema = require("@clusterio/lib/schema");
+const libErrors = require("@clusterio/lib/errors");
+const { logger } = require("@clusterio/lib/logging");
+const messages = require("./messages");
 
 // Some definitions for the terminology used here:
 // link: Either side of a master - client connection
@@ -15,6 +17,8 @@ const messages = require("lib/link/messages");
 
 /**
  * Common interface for server and client connections
+ *
+ * @memberof module:lib/link
  */
 class Link {
 	constructor(source, target, connector) {
@@ -27,17 +31,29 @@ class Link {
 		this._validators = new Map();
 
 		// Process messages received by the connector
-		connector.on('message', payload => {
+		connector.on("message", payload => {
 			try {
 				this.processMessage(payload);
 			} catch (err) {
-				if (err instanceof errors.InvalidMessage) {
-					console.error(`Invalid message on ${this.source}-${this.target} link: ${err.message}`)
+				if (err instanceof libErrors.InvalidMessage) {
+					logger.error(`Invalid message on ${this.source}-${this.target} link: ${err.message}`);
 					if (err.errors) {
-						console.error(err.errors);
+						logger.error(JSON.stringify(err.errors, null, 4));
+					}
+
+					// Send back an error response if this was a request.
+					if (
+						typeof payload.type === "string"
+						&& payload.type.endsWith("_request")
+						&& Number.isInteger(payload.seq)
+					) {
+						this.connector.send(
+							`${payload.type.slice(0, -8)}_response`,
+							{ seq: payload.seq, error: err.message }
+						);
 					}
 				} else {
-					this.connector.emit('error', err);
+					this.connector.emit("error", err);
 				}
 			}
 		});
@@ -45,7 +61,17 @@ class Link {
 		connector.on("invalidate", () => {
 			for (let waiterType of this._waiters.values()) {
 				for (let waiter of waiterType) {
-					waiter.reject(new Error("Session lost"));
+					waiter.reject(new libErrors.SessionLost("Session Lost"));
+				}
+			}
+
+			this._waiters.clear();
+		});
+
+		connector.on("close", () => {
+			for (let waiterType of this._waiters.values()) {
+				for (let waiter of waiterType) {
+					waiter.reject(new libErrors.SessionLost("Session Closed"));
 				}
 			}
 
@@ -59,22 +85,23 @@ class Link {
 	 * Validates and invokes the handler and/or waiters for a message that has
 	 * been received.  An unhandled message is considered to be an error.
 	 *
-	 * @throws {InvalidMessage}
+	 * @param {Object} message - Message to process.
+	 * @throws {module:lib/errors.InvalidMessage}
 	 *     if validation failed, is missing, the message is invalid, or no
 	 *     handlers/waiters were invoked to handle the message.
 	 */
 	processMessage(message) {
-		if (!schema.message(message)) {
-			throw new errors.InvalidMessage("Malformed", schema.message.errors);
+		if (!libSchema.message(message)) {
+			throw new libErrors.InvalidMessage("Malformed", libSchema.message.errors);
 		}
 
 		let validator = this._validators.get(message.type);
 		if (!validator) {
-			throw new errors.InvalidMessage(`No validator for ${message.type} on ${this.source}-${this.target}`);
+			throw new libErrors.InvalidMessage(`No validator for ${message.type} on ${this.source}-${this.target}`);
 		}
 
 		if (!validator(message)) {
-			throw new errors.InvalidMessage(`Validation failed for ${message.type}`, validator.errors);
+			throw new libErrors.InvalidMessage(`Validation failed for ${message.type}`, validator.errors);
 		}
 
 		let hadHandlers = this._processHandler(message);
@@ -82,13 +109,14 @@ class Link {
 
 		if (!hadWaiters && !hadHandlers) {
 			// XXX console.error(`Unhandled message: ${JSON.stringify(message, null, 4)}`);
-			throw new errors.InvalidMessage(`Unhandled message ${message.type}`);
+			throw new libErrors.InvalidMessage(`Unhandled message ${message.type}`);
 		}
 	}
 
 	/**
 	 * Invoke the handler for this message if it exists
 	 *
+	 * @param {Object} message - Message to invoke handlers on.
 	 * @returns {Boolean} true if a handler was invoked.
 	 */
 	_processHandler(message) {
@@ -96,14 +124,14 @@ class Link {
 		if (handler) {
 			handler(message);
 			return true;
-		} else {
-			return false;
 		}
+		return false;
 	}
 
 	/**
 	 * Invokes the waiters for this message if any
 	 *
+	 * @param {Object} message - Message to invoke waiters on.
 	 * @returns {Boolean} true if a waiter was triggered.
 	 */
 	_processWaiters(message) {
@@ -183,6 +211,9 @@ class Link {
 	 * as any properties specified in data.  For example specifying { foo:
 	 * 21 } as the data will cause it to wait until a message with the right
 	 * type and a foo property value of 21 in the data payload.
+	 *
+	 * @param {string} type - Message type to wait for.
+	 * @param {Object} data - Properties to match in the message data.
 	 */
 	async waitFor(type, data) {
 		if (!this._validators.has(type)) {
@@ -208,14 +239,14 @@ class Link {
 	async pingRequestHandler() { }
 
 	/**
-	 * Prepare connection for close
+	 * Prepare connection for disconnect
 	 *
 	 * Waits for all pending requests on the connection to resolve.
 	 * Sub-classes should implement this handler with code that prevents
 	 * additional requests to be sent out before calling the super class
 	 * method.
 	 */
-	async shutdownConnectionRequestHandler() {
+	async prepareDisconnectRequestHandler() {
 		let promises = [];
 		for (let waiterType of this._waiters.values()) {
 			for (let waiter of waiterType) {
@@ -224,11 +255,10 @@ class Link {
 		}
 
 		await Promise.all(promises);
-		this.connector.setClosing();
 	}
 }
 
 
 module.exports = {
 	Link,
-}
+};

packages/lib/logging.js

@@ -0,0 +1,94 @@
+"use strict";
+const winston = require("winston");
+const Transport = require("winston-transport");
+const { LEVEL, MESSAGE } = require("triple-beam");
+
+const levels = Object.freeze({
+	fatal: 0,
+	error: 1,
+	warn: 2,
+	audit: 3,
+	info: 4,
+	server: 5,
+	verbose: 6,
+});
+
+const colors = {
+	fatal: "inverse red",
+	error: "bold brightRed",
+	warn: "bold brightYellow",
+	audit: "bold brightGreen",
+	info: "bold brightBlue",
+	server: "bold",
+	verbose: "bold grey",
+};
+winston.addColors(colors);
+
+// The Console Transport is replicated here because it's overly complicated
+// in winston, writes to stdout/err instead of console and doesn't work in
+// the browser.
+class ConsoleTransport extends Transport {
+	constructor(options = {}) {
+		super(options);
+		this.errorLevels = new Set(options.errorLevels || ["fatal", "error"]);
+		this.warnLevels = new Set(options.warnLevels || ["warn"]);
+	}
+
+	log(info, callback) {
+		/* eslint-disable no-console */
+		if (this.errorLevels.has(info[LEVEL])) {
+			console.error(info[MESSAGE]);
+		} else if (this.warnLevels.has(info[LEVEL])) {
+			console.warn(info[MESSAGE]);
+		} else {
+			console.log(info[MESSAGE]);
+		}
+		/* eslint-enable no-console */
+		callback();
+	}
+}
+
+/**
+ * Formats winston log messages for the web console
+ */
+class WebConsoleFormat {
+	constructor(options = {}) {
+		this.options = options;
+	}
+
+	transform(info, options) {
+		let src = " ";
+		if (info.slave_id) {
+			let slaveName = info.slave_id;
+			src += `s:${slaveName} - `;
+		}
+		if (info.instance_id) {
+			let instanceName = info.instance_id;
+			src += `i:${instanceName}`;
+		}
+		if (info.plugin) {
+			src += `${info.plugin}: `;
+		}
+
+		if (info.stack) {
+			info[MESSAGE] = `[${info.level}]${src}${info.stack}`;
+		} else {
+			info[MESSAGE] = `[${info.level}]${src}${info.message}`;
+		}
+		return info;
+	}
+}
+
+const logger = winston.createLogger({
+	level: "verbose",
+	levels,
+	format: winston.format.timestamp(),
+});
+
+module.exports = {
+	ConsoleTransport,
+	WebConsoleFormat,
+	colors,
+	levels,
+	logger,
+};

packages/lib/logging_utils.js

@@ -0,0 +1,131 @@
+"use strict";
+const winston = require("winston");
+const Transport = require("winston-transport");
+const { LEVEL, MESSAGE } = require("triple-beam");
+const chalk = require("chalk");
+
+const libLink = require("@clusterio/lib/link");
+const libErrors = require("@clusterio/lib/errors");
+
+
+/**
+ * Format a parsed Factorio message with colors
+ *
+ * Formats a parsed Factorio output from lib/factorio into a readable
+ * colorized output using terminal escape codes that can be printed.
+ *
+ * @param {Object} parsed - Parsed Factorio server output.
+ * @returns {string} terminal colorized message.
+ * @private
+ */
+function formatServerOutput(parsed) {
+	let time = "";
+	if (parsed.format === "seconds") {
+		time = `${chalk.yellow(parsed.time.padStart(8))} `;
+	} else if (parsed.format === "date") {
+		time = `${chalk.yellow(parsed.time)} `;
+	}
+
+	let info = "";
+	if (parsed.type === "log") {
+		let level = parsed.level;
+		if (level === "Script") {
+			level = chalk.bold.greenBright(level);
+		} else if (level === "Verbose") {
+			level = chalk.bold.gray(level);
+		} else if (level === "Info") {
+			level = chalk.bold.blueBright(level);
+		} else if (parsed.level === "Warning") {
+			level = chalk.bold.yellowBright(level);
+		} else if (parsed.level === "Error") {
+			level = chalk.bold.redBright(level);
+		}
+
+		info = `${level} ${chalk.gray(parsed.file)}: `;
+
+	} else if (parsed.type === "action") {
+		info = `[${chalk.yellow(parsed.action)}] `;
+	}
+
+	return time + info + parsed.message;
+}
+
+// These are defined here to avoid circular dependencies and pulling them
+// into the web interface code.
+
+/**
+ * Formats winston log messages for a character terminal.
+ */
+class TerminalFormat {
+	constructor(options = {}) {
+		this.options = options;
+		this.colorize = winston.format.colorize(options);
+	}
+
+	transform(info, options) {
+		info = this.colorize.transform(info, this.colorize.options);
+		let ts = "";
+		if (options.showTimestamp && info.timestamp) {
+			ts = `${info.timestamp.replace("T", " ")} `;
+		}
+
+		let src = " ";
+		if (info.instance_name) {
+			src += `i:${info.instance_name} `;
+
+		} else if (info.slave_name) {
+			src += `s:${info.slave_name} `;
+		}
+		if (info.plugin) {
+			src += `${info.plugin}: `;
+		}
+
+		if (info[LEVEL] === "server" && info.parsed) {
+			info.message = formatServerOutput(info.parsed);
+		}
+
+		if (info.stack) {
+			info[MESSAGE] = `${ts}[${info.level}]${src}${info.stack}`;
+		} else {
+			info[MESSAGE] = `${ts}[${info.level}]${src}${info.message}`;
+		}
+		return info;
+	}
+}
+
+/**
+ * Sends logs over a lib/link connection.
+ */
+class LinkTransport extends Transport {
+	constructor(options) {
+		super(options);
+
+		this.link = options.link;
+		this.filter = options.filter || null;
+	}
+
+	log(info, callback) {
+		if (this.filter && !this.filter(info)) {
+			return callback();
+		}
+
+		try {
+			libLink.messages.logMessage.send(this.link, { info });
+		} catch (err) {
+			// Ignore session lost errors.
+			if (!(err instanceof libErrors.SessionLost)) {
+				throw err;
+			}
+		}
+		return callback();
+	}
+}
+
+
+module.exports = {
+	TerminalFormat,
+	LinkTransport,
+
+	// for testing only
+	_formatServerOutput: formatServerOutput,
+};

packages/lib/lua_tools.js

@@ -1,5 +1,14 @@
+/**
+ * Utilities for dealing with Lua
+ * @module lib/lua_tools
+ */
+"use strict";
+
 /**
  * Escapes a string for inclusion into a lua string
+ *
+ * @param {string} content - String to escape.
+ * @returns {string} Escaped string.
  */
 function escapeString(content) {
 	return content
@@ -14,4 +23,4 @@ function escapeString(content) {
 
 module.exports = {
 	escapeString,
-}
+};

packages/lib/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@clusterio/lib",
+  "description": "Shared library for Clusterio",
+  "version": "2.0.0-alpha.4",
+  "repository": "https://github.com/clusterio/factorioClusterio",
+  "license": "MIT",
+  "keywords": [
+    "factorio"
+  ],
+  "engines": {
+    "node": ">=10.16 <11 || >= 12"
+  },
+  "dependencies": {
+    "ajv": "^6.12.6",
+    "fs-extra": "^8.1.0",
+    "ini": "^1.3.5",
+    "isomorphic-ws": "^4.0.1",
+    "jimp": "^0.12.1",
+    "jsonwebtoken": "^8.2.1",
+    "jszip": "^3.2.2",
+    "klaw": "^3.0.0",
+    "node-forge": "^0.10.0",
+    "rcon-client": "^4.2.0",
+    "semver": "^7.1.1",
+    "set-blocking": "^2.0.0",
+    "triple-beam": "^1.3.0",
+    "winston": "^3.3.3",
+    "winston-transport": "^4.4.0",
+    "ws": "^7.2.3",
+    "yargs": "^14.2.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/lib/plugin.js

@@ -0,0 +1,495 @@
+/**
+ * Plugin interfaces and utilities.
+ * @module lib/plugin
+ */
+"use strict";
+
+
+/**
+ * Conceptual base for master and instance plugins.
+ * @typedef {
+ *     module:lib/plugin.BaseMasterPlugin
+ *     |module:lib/plugin.BaseInstancePlugin
+ *     |module:lib/plugin.BaseControlPlugin
+ * } module:lib/plugin.BasePlugin
+ */
+
+/**
+ * Base class for instance plugins
+ *
+ * Instance plugins are subclasses of this class which get instantiated by
+ * the slave when it brings up an instance with the plugin enabled in the
+ * config.  To be discovered the class must be exported under the name
+ * `InstancePlugin` in the module specified by the `instanceEntrypoint` in
+ * the plugin's info.js file.
+ *
+ * Instances may be started and stopped many times, and many instances may
+ * be running at the same time, each of which will have their own instance
+ * of the InstancePlugin class.
+ *
+ * @static
+ */
+class BaseInstancePlugin {
+	constructor(info, instance, slave) {
+		/**
+		 * The plugin's own info module
+		 */
+		this.info = info;
+
+		/**
+		 * Instance the plugin started for
+		 * @type {module:slave/slave~Instance}
+		 */
+		this.instance = instance;
+
+		/**
+		 * Slave running the instance
+		 *
+		 * With the exepction of accessing the slave's config you should
+		 * avoid ineracting with the slave object directly.
+		 *
+		 * @type {module:slave/slave~Slave}
+		 */
+		this.slave = slave;
+
+		/**
+		 * Logger for this plugin
+		 *
+		 * Instance of winston Logger for sending log messages from this
+		 * plugin.  Supported methods and their corresponding log levels are
+		 * `error`, `warn`, `audit`, `info` and `verbose`.
+		 */
+		this.logger = instance.logger.child({ plugin: this.info.name });
+	}
+
+	/**
+	 * Called immediately after the class is instantiated
+	 */
+	async init() { }
+
+	/**
+	 * Called when the value of a config field changed.
+	 *
+	 * Invoked after the value of the config field given by `field` has
+	 * changed.
+	 *
+	 * @param {module:lib/config.ConfigGroup} group -
+	 *     The group who's field got changed on.
+	 * @param {string} field - Name of the field that changed.
+	 * @param {*} prev - The previous value of the field.
+	 */
+	async onInstanceConfigFieldChanged(group, field, prev) { }
+
+	/**
+	 * Called before collecting Prometheus metrics
+	 *
+	 * Invoked before the default metrics of prometheus is collected.  Note
+	 * that since this is done while replying to /metrics this should not
+	 * take a long time, otherwise the requst will time out.
+	 *
+	 * Although you can return the results of collecting your prometheus
+	 * collectors directly, all gauges in the default registry is already
+	 * automatically collected.  It's therefore recommended to only update
+	 * the Gauges/Counters/etc in onMetric and let Clusterio deal will
+	 * collecting the values.
+	 *
+	 * @returns {*} an async iterator of prometheus metric results or undefined.
+	 */
+	async onMetrics() { }
+
+	/**
+	 * Called after the Factorio server is started
+	 */
+	async onStart() { }
+
+	/**
+	 * Called before the Factorio server is stopped
+	 *
+	 * This will not be called if for example the Factorio server crashes or
+	 * is killed.
+	 */
+	async onStop() { }
+
+	/**
+	 * Called when the instance exits
+	 *
+	 * Invoked when the instance is shut down.  This may occur after init
+	 * has been called if an error occurs during startup.  Note that if
+	 * the plugin's init() throws this method will still be invoked.
+	 */
+	onExit() { }
+
+	/**
+	 * Called when the Factorio outputs a line
+	 *
+	 * Called when the Factorio server outputs a line on either stdout or
+	 * stderr.  The output is a parsed form of the line and contains the
+	 * following properties:
+	 *
+	 * @param {Object} parsed - parsed server output.
+	 * @param {string} parsed.source -
+	 *     Where the message came from, one of "stdout" and "stderr".
+	 * @param {string} parsed.format -
+	 *     Timestamp format, one of "date", "seconds" and "none".
+	 * @param {string=} parsed.time -
+	 *     Timestamp of the message.  Not present if format is "none".
+	 * @param {string} parsed.type -
+	 *     Type of message, one of "log", "action" and "generic".
+	 * @param {string=} parsed.level -
+	 *     Log level for "log" type.  i.e "Info" normally.
+	 * @param {string=} parsed.file - File reported for "log" type.
+	 * @param {string=} parsed.action -
+	 *     Kind of action for "action" type. i.e "CHAT" for chat.
+	 * @param {string} parsed.message - Main content of the line.
+	 * @param {string} line - raw line of server output.
+	 */
+	async onOutput(parsed, line) { }
+
+	/**
+	 * Called when an event on the master connection happens
+	 *
+	 * The event param may be one of connect, drop, resume and close and has
+	 * the following meaning:
+	 *
+	 * ##### connect
+	 *
+	 * Invoked when a new connection to the master has been established.
+	 *
+	 * ##### drop
+	 *
+	 * Invoked when a connection loss is detected between the slave and the
+	 * master.  Plugins should respond to this event by throtteling messages
+	 * it is sending to the master to an absolute minimum.
+	 *
+	 * Messages sent over a dropped master connection will get queued up in
+	 * memory on the slave and sent all in one go when the connection is
+	 * re-established again.
+	 *
+	 * ##### resume
+	 *
+	 * Invoked when the connection that had previously dropped is
+	 * re-established.
+	 *
+	 * ##### close
+	 *
+	 * Invoked when the connection to the master has been closed.  This
+	 * typically means the master server has shut down.  Plugins should not
+	 * send any messages that goes to or via the master server after the
+	 * connection has been closed and before a new one is established.
+	 *
+	 * @param {string} event - one of connect, drop, resume and close
+	 */
+	onMasterConnectionEvent(event) { }
+
+	/**
+	 * Called when the master is preparing to disconnect from the slave
+	 *
+	 * Invoked when the master server has requested the disconnection from
+	 * the slave.  This typically happens when it is in the process of
+	 * shutting down.
+	 *
+	 * Plugins must stop sending messages to the master, or are forwarded
+	 * via the master server after the prepare disconnect has been handled.
+	 *
+	 * @param {module:master/master~SlaveConnection} connection -
+	 *     The connection to the slave preparing to disconnect.
+	 */
+	async onPrepareMasterDisconnect(connection) { }
+
+	/**
+	 * Called when a player joins or leaves the game
+	 *
+	 * Invoked when a player either joins or leaves the instance.
+	 *
+	 * @param {Object} event - Information about the event.
+	 * @param {string} event.type - Either "join" or "leave".
+	 * @param {string} event.name - Name of the player that joined/left.
+	 */
+	async onPlayerEvent(event) { }
+
+	async sendRcon(message, expectEmpty) {
+		return await this.instance.sendRcon(message, expectEmpty, this.info.name);
+	}
+}
+
+/**
+ * Base class for master plugins
+ *
+ * Master plugins are subclasses of this class which get instantiated by
+ * the master server on startup when the plugin is enabled in the config.
+ * To be discovered the class must be exported under the name `MasterPlugin`
+ * in the module specified by the `masterEntrypoint` in the plugin's info.js
+ * file.
+ *
+ * @static
+ */
+class BaseMasterPlugin {
+	constructor(info, master, metrics, logger) {
+		this.info = info;
+		this.master = master;
+		this.metrics = metrics;
+
+		/**
+		 * Logger for this plugin
+		 *
+		 * Instance of winston Logger for sending log messages from this
+		 * plugin.  Supported methods and their corresponding log levels are
+		 * `error`, `warn`, `audit`, `info` and `verbose`.
+		 */
+		this.logger = logger.child({ plugin: this.info.name });
+	}
+
+	/**
+	 * Called immediately after the class is instantiated
+	 */
+	async init() { }
+
+	/**
+	 * Called when the status of an instance changes
+	 *
+	 * Invoked when the master server has received notice from a slave that
+	 * the running status of an instance has changed.  The possible statuses
+	 * that can be notified about are:
+	 * - `stopped`: Instance is no longer running.
+	 * - `starting`: Instance is in the process of starting up.
+	 * - `running`: Instance startup completed and is now running.
+	 * - `creating_save`: Instance is in the process of creating a save.
+	 * - `exporting_data`: Instance is in the process of exporting item
+	 *   icons and locale data.
+	 *
+	 * On master startup all known instances gets the status `unknown` if
+	 * they are assigned to a slave, when the slave then connects to the
+	 * master and informs of the current status of its instances this hook
+	 * is invoked for all those instances.  You can detect this situation by
+	 * checking if prev equals `unknown`.
+	 *
+	 * Note that it's possible for status change notification to get lost in
+	 * the case of network outages if reconnect fails to re-establish the
+	 * session between the master server and the slave.
+	 *
+	 * @param {Object} instance - the instance that changed.
+	 * @param {string} prev - the previous status of the instance.
+	 */
+	async onInstanceStatusChanged(instance, prev) { }
+
+	/**
+	 * Called before collecting Prometheus metrics
+	 *
+	 * Invoked before the default metrics of prometheus is collected.  Note
+	 * that since this is done while replying to /metrics this should not
+	 * take a long time, otherwise the requst will time out.
+	 *
+	 * Although you can return the results of collecting your prometheus
+	 * collectors directly, all gauges in the default registry is already
+	 * automatically collected.  It's therefore recommended to only update
+	 * the Gauges/Counters/etc in onMetric and let Clusterio deal will
+	 * collecting the values.
+	 *
+	 * @returns {*} an async iterator of prometheus metric results or undefined.
+	 */
+	async onMetrics() { }
+
+	/**
+	 * Called when the master server is shutting down
+	 */
+	async onShutdown() { }
+
+	/**
+	 * Called when an event on a slave connection happens
+	 *
+	 * The event param may be one of connect, drop, resume and close and has
+	 * the following meaning:
+	 *
+	 * ##### connect
+	 *
+	 * Invoked when a new connection to the slave has been established.
+	 *
+	 * ##### drop
+	 *
+	 * Invoked when a connection loss is detected between the master and a
+	 * slave.  Plugins should respond to this event by throtteling messages
+	 * it is sending to the given slave connection to an absolute minimum.
+	 *
+	 * Messages sent over a dropped slave connection will get queued up in
+	 * memory on the master and sent all in one go when the connection is
+	 * re-established again.
+	 *
+	 * ##### resume
+	 *
+	 * Invoked when a connection that had previously been dropped is
+	 * re-established
+	 *
+	 * ##### close
+	 *
+	 * Invoked when the connection to the slave has been closed.
+	 *
+	 * @param {module:master/master~SlaveConnection} connection -
+	 *     The connection the event occured on.
+	 * @param {string} event - one of connect, drop, resume and close
+	 */
+	onSlaveConnectionEvent(connection, event) { }
+
+	/**
+	 * Called when a slave is preparing to disconnect from the master
+	 *
+	 * Invoked when a slave has requested the disconnection from the master.
+	 * This typically happens when it is in the process of shutting down.
+	 *
+	 * Plugins must stop sending messages to the slave in question after the
+	 * prepare disconnect has been handled.
+	 *
+	 * @param {module:master/master~SlaveConnection} connection -
+	 *     The connection to the slave preparing to disconnect.
+	 */
+	async onPrepareSlaveDisconnect(connection) { }
+
+	/**
+	 * Called when a player joins or leaves an instance
+	 *
+	 * Invoked when a player either joins or leaves an instance in the
+	 * cluster.
+	 *
+	 * @param {Object} instance - the instance it occured on.
+	 * @param {Object} event - Information about the event.
+	 * @param {string} event.type - Either "join" or "leave".
+	 * @param {string} event.name - Name of the player that joined/left.
+	 */
+	async onPlayerEvent(instance, event) { }
+
+	/**
+	 * Broadcast event to all connected slaves
+	 *
+	 * Sends the given event to all slaves connected to the master server.
+	 * This does not include slaves that are in the process of closing the
+	 * connection, which typically happens when they are shutting down.
+	 *
+	 * @param {module:lib/link.Event} event - Event to send
+	 * @param {Object} data - Data ta pass with the event.
+	 */
+	broadcastEventToSlaves(event, data={}) {
+		for (let slaveConnection of this.master.slaveConnections.values()) {
+			if (
+				!slaveConnection.connector.closing
+				&& (!event.plugin || slaveConnection.plugins.has(event.plugin))
+			) {
+				event.send(slaveConnection, data);
+			}
+		}
+	}
+}
+
+/**
+ * Base class for clusterioctl plugins
+ *
+ * Control plugins are subclasses of this class which get instantiated by
+ * clusterioctl in order to extend its functionallity.  To be discovered the
+ * class must be exported under the name `ControlPlugin` in the module
+ * specified by the `controlEntrypoint` in the plugin's info.js file.
+ *
+ * @static
+ */
+class BaseControlPlugin {
+	constructor(info, logger) {
+		/**
+		 * The plugin's own info module
+		 */
+		this.info = info;
+
+		/**
+		 * Logger for this plugin
+		 *
+		 * Instance of winston Logger for sending log messages from this
+		 * plugin.  Supported methods and their corresponding log levels are
+		 * `error`, `warn`, `audit`, `info` and `verbose`.
+		 */
+		this.logger = logger.child({ plugin: this.info.name });
+	}
+
+	/**
+	 * Called immediately after the class is instantiated
+	 */
+	async init() { }
+
+	/**
+	 * Called to add commands to the command line interface.
+	 *
+	 * Invoked by clusterioctl to let plugins add commands.  `rootCommand` is
+	 * the top level command node which the plugin should add its own {@link
+	 * module:lib/command.CommandTree} to.
+	 *
+	 * @param {module:lib/command.CommandTree} rootCommand -
+	 *     Root of the clusterioctl command tree.
+	 */
+	async addCommands(rootCommand) { }
+}
+
+/**
+ * Attach plugin messages
+ *
+ * Attaches all messages defined in the `.info.message` property of the
+ * `plugin` passed with handlers taken from the `pluin`.
+ *
+ * @param {module:lib/link.Link} link - Link to attach handlers to.
+ * @param {module:lib/plugin.BasePlugin} plugin -
+ *     The instance of the plugin attach messages and handlers from.
+ */
+function attachPluginMessages(link, plugin) {
+	let messageDefinitions = plugin.info.messages || [];
+	for (let [name, messageFormat] of Object.entries(messageDefinitions)) {
+		if (messageFormat.plugin !== plugin.info.name) {
+			throw new Error(`Type of ${name} message must start with "${plugin.info.name}:"`);
+		}
+
+		let handler = name + messageFormat.handlerSuffix;
+		try {
+			if (plugin === null || !plugin[handler]) {
+				messageFormat.attach(link);
+
+			} else {
+				messageFormat.attach(link, async (message, format) => await plugin[handler](message, format));
+			}
+
+		} catch (err) {
+			if (err.code === "MISSING_LINK_HANDLER") {
+				err.plugin = plugin.info.name;
+				err.handler = handler;
+			}
+
+			throw err;
+		}
+	}
+}
+
+/**
+ * Invokes the given hook on all plugins
+ *
+ * @param {Map<string, Object>} plugins -
+ *     Mapping of plugin names to plugins to invoke the hook on.
+ * @param {string} hook - Name of hook to invoke.
+ * @param {...*} args - Arguments to pass on to the hook.
+ * @returns {Array} Non-empty return values from the hooks.
+ */
+async function invokeHook(plugins, hook, ...args) {
+	let results = [];
+	for (let [name, plugin] of plugins) {
+		try {
+			let result = await plugin[hook](...args);
+			if (result !== undefined) {
+				results.push(result);
+			}
+		} catch (err) {
+			plugin.logger.error(`Ignoring error from plugin ${name} in ${hook}:\n${err.stack}`);
+		}
+	}
+	return results;
+}
+
+
+module.exports = {
+	BaseInstancePlugin,
+	BaseMasterPlugin,
+	BaseControlPlugin,
+
+	attachPluginMessages,
+	invokeHook,
+};

packages/lib/plugin_loader.js

@@ -0,0 +1,98 @@
+/* eslint-disable global-require */
+/**
+ * Helpers for loading plugins in Node.js
+ * @module lib/plugin_loader
+ */
+"use strict";
+
+const libErrors = require("@clusterio/lib/errors");
+const path = require("path");
+
+
+/**
+ * Load plugin information
+ *
+ * Loads plugin info modules for the paths to the given plugins.  Once
+ * loaded the info modules will not be reloaded should this function be
+ * called again.
+ *
+ * @param {Map<string, string>} pluginList -
+ *     Mapping of plugin name to require path for the plugins to load.
+ * @returns {Array<Object>} Array of plugin info modules.
+ * @static
+ */
+async function loadPluginInfos(pluginList) {
+	let plugins = [];
+	for (let [pluginName, pluginPath] of pluginList) {
+		let pluginInfo;
+		let pluginPackage;
+
+		try {
+			pluginInfo = require(path.posix.join(pluginPath, "info"));
+			pluginPackage = require(path.posix.join(pluginPath, "package.json"));
+
+		} catch (err) {
+			throw new libErrors.PluginError(pluginName, err);
+		}
+
+		if (pluginInfo.name !== pluginName) {
+			throw new libErrors.EnvironmentError(
+				`Expected plugin at ${pluginPath} to be named ${pluginName} but got ${pluginInfo.name}`
+			);
+		}
+
+		pluginInfo.requirePath = pluginPath;
+		pluginInfo.version = pluginPackage.version;
+		plugins.push(pluginInfo);
+	}
+	return plugins;
+}
+
+/**
+ * Load master plugin class of a plugin
+ *
+ * @param {Object} pluginInfo -
+ *     Plugin info object returned from {@link
+ *     module:lib/plugin_loader.loadPluginInfos} to load class from.
+ * @returns {function} plugin class
+ * @static
+ */
+async function loadMasterPluginClass(pluginInfo) {
+	let entrypoint = require(path.posix.join(pluginInfo.requirePath, pluginInfo.masterEntrypoint));
+	return entrypoint.MasterPlugin;
+}
+
+/**
+ * Load instance plugin class of a plugin
+ *
+ * @param {Object} pluginInfo -
+ *     Plugin info object returned from {@link
+ *     module:lib/plugin_loader.loadPluginInfos} to load class from.
+ * @returns {function} plugin class
+ * @static
+ */
+async function loadInstancePluginClass(pluginInfo) {
+	let entrypoint = require(path.posix.join(pluginInfo.requirePath, pluginInfo.instanceEntrypoint));
+	return entrypoint.InstancePlugin;
+}
+
+/**
+ * Load control plugin class of a plugin
+ *
+ * @param {Object} pluginInfo -
+ *     Plugin info object returned from {@link
+ *     module:lib/plugin_loader.loadPluginInfos} to load class from.
+ * @returns {function} plugin class
+ * @static
+ */
+async function loadControlPluginClass(pluginInfo) {
+	let entrypoint = require(path.posix.join(pluginInfo.requirePath, pluginInfo.controlEntrypoint));
+	return entrypoint.ControlPlugin;
+}
+
+module.exports = {
+	loadPluginInfos,
+	loadMasterPluginClass,
+	loadInstancePluginClass,
+	loadControlPluginClass,
+};

packages/lib/schema.js

@@ -1,4 +1,10 @@
-const ajv = new (require('ajv'))({
+/**
+ * JSON schemas used for validating link messages
+ * @module lib/schema
+ */
+"use strict";
+const Ajv = require("ajv");
+const ajv = new Ajv({
 	verbose: true,
 	format: "full",
 	extendRefs: "fail",
@@ -24,8 +30,20 @@ const messageSchema = {
 	},
 };
 
+/**
+ * Validator for message schemas
+ * @function
+ * @param {Object} obj - Object to validate
+ * @returns {boolean} True if object passed validation.
+ */
 const message = ajv.compile(messageSchema);
 
+/**
+ * Validator for heartbeat messages
+ * @function
+ * @param {Object} obj - Object to validate
+ * @returns {boolean} True if object passed validation.
+ */
 const heartbeat = ajv.compile({
 	$schema: "http://json-schema.org/draft-07/schema#",
 	title: "Heartbeat",
@@ -47,21 +65,31 @@ const heartbeat = ajv.compile({
 	},
 });
 
+/**
+ * Validator for server sent part of handshake
+ * @function
+ * @param {Object} obj - Object to validate
+ * @returns {boolean} True if object passed validation.
+ */
 const serverHandshake = ajv.compile({
 	$schema: "http://json-schema.org/draft-07/schema#",
 	title: "Server Handshake",
 	description: "Message sent from the server to a client during the handshake",
-	allOf: [ messageSchema ],
+	allOf: [messageSchema],
 	anyOf: [
 		{
 			properties: {
 				"type": { const: "hello" },
 				"data": {
 					additionalProperties: false,
-					required: ["version"],
+					required: ["version", "plugins"],
 					properties: {
 						"version": { type: "string" },
-					}
+						"plugins": {
+							type: "object",
+							additionalProperties: { type: "string" },
+						},
+					},
 				},
 			},
 		},
@@ -100,24 +128,34 @@ const serverHandshake = ajv.compile({
 	],
 });
 
+/**
+ * Validator for client sent part of handshake
+ * @function
+ * @param {Object} obj - Object to validate
+ * @returns {boolean} True if object passed validation.
+ */
 const clientHandshake = ajv.compile({
 	$schema: "http://json-schema.org/draft-07/schema#",
 	title: "Client Message",
 	description: "Message sent from a client to the server",
-	allOf: [ messageSchema ],
+	allOf: [messageSchema],
 	anyOf: [
 		{
 			properties: {
 				"type": { const: "register_slave" },
 				"data": {
 					additionalProperties: false,
-					required: ["token", "agent", "version", "name", "id"],
+					required: ["token", "agent", "version", "name", "id", "plugins"],
 					properties: {
 						"token": { type: "string" },
 						"agent": { type: "string" },
 						"version": { type: "string" },
 						"name": { type: "string" },
 						"id": { type: "integer" },
+						"plugins": {
+							type: "object",
+							additionalProperties: { type: "string" },
+						},
 					},
 				},
 			},
@@ -132,7 +170,7 @@ const clientHandshake = ajv.compile({
 						"token": { type: "string" },
 						"agent": { type: "string" },
 						"version": { type: "string" },
-					}
+					},
 				},
 			},
 		},
@@ -154,6 +192,12 @@ const clientHandshake = ajv.compile({
 
 
 module.exports = {
+	/**
+	 * Compile JSON schema into validator
+	 * @function
+	 * @param {Object} schema - JSON schema to create validator for.
+	 * @returns {Function} Validator for the schema.
+	 */
 	compile: ajv.compile.bind(ajv),
 
 	message,

packages/lib/shared_commands.js

@@ -0,0 +1,176 @@
+/**
+ * Implementation of commands shared between master/slave/ctl.
+ * @module lib/shared_commands
+ */
+"use strict";
+const fs = require("fs-extra");
+const path = require("path");
+
+const libConfig = require("@clusterio/lib/config");
+const { logger } = require("@clusterio/lib/logging");
+const libHelpers = require("@clusterio/lib/helpers");
+
+
+function print(...content) {
+	// eslint-disable-next-line no-console
+	console.log(...content);
+}
+
+/**
+ * Yargs plugin command
+ *
+ * Can be passed to yargs.command to implement a plugin list management
+ * command.  Use handlePluginCommand to do the requested action.
+ *
+ * @param {Object} yargs - yargs command builder.
+ */
+function pluginCommand(yargs) {
+	yargs
+		.command("add <path>", "Add plugin by require path")
+		.command("remove <name>", "Remove plugin by name")
+		.command("list", "List all plugins and their path")
+		.demandCommand(1, "You need to specify a command to run")
+		.help()
+		.strict()
+	;
+}
+
+/**
+ * Handle yargs plugin command
+ *
+ * Handle the actions that are made available by pluginCommand.
+ *
+ * @param {Object} args - yargs args object.
+ * @param {Map<string, string>} pluginList - Current list of plugins.
+ * @param {string} pluginListPath - Path to plugin list config file.
+ */
+async function handlePluginCommand(args, pluginList, pluginListPath) {
+	let command = args._[1];
+
+	if (command === "add") {
+		let pluginPath = args.path;
+		if (/^\.\.?[\/\\]/.test(pluginPath)) {
+			pluginPath = path.resolve(pluginPath);
+		}
+
+		let pluginInfo;
+		try {
+			// eslint-disable-next-line global-require
+			pluginInfo = require(path.posix.join(pluginPath, "info"));
+		} catch (err) {
+			logger.error(`Unable to import plugin info from ${args.path}:\n${err.stack}`);
+			process.exitCode = 1;
+			return;
+		}
+
+		if (pluginList.has(pluginInfo.name)) {
+			logger.error(`Plugin with the same ${pluginInfo.name} already exists`);
+			process.exitCode = 1;
+			return;
+		}
+
+		pluginList.set(pluginInfo.name, pluginPath);
+		await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
+		print(`Added ${pluginInfo.name}`);
+
+	} else if (command === "remove") {
+		if (!pluginList.delete(args.name)) {
+			logger.error(`Plugin with name ${args.name} does not exist`);
+			process.exitCode = 1;
+			return;
+		}
+
+		await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
+		print(`Removed ${args.name}`);
+
+	} else if (command === "list") {
+		for (let [pluginName, pluginPath] of pluginList) {
+			print(`${pluginName} - ${pluginPath}`);
+		}
+	}
+}
+
+
+/**
+ * Yargs config command
+ *
+ * Can be passed to yargs.command to implement a config command.  Use
+ * handleConfigCommand to do the requested action.
+ *
+ * @param {Object} yargs - yargs command builder.
+ */
+function configCommand(yargs) {
+	yargs
+		.command("set <field> [value]", "Set config field", yargs => {
+			yargs.options({
+				"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+			});
+		})
+		.command("show <field>", "Show value of the given config field")
+		.command("list", "List all configuration fields and their values")
+		.demandCommand(1, "You need to specify a command to run")
+		.help()
+		.strict()
+	;
+}
+
+/**
+ * Handle yargs command
+ *
+ * Handle the actions that are made available by configCommand.
+ *
+ * @param {Object} args - yargs args object.
+ * @param {module:lib/config.Config} instance - Config instance.
+ * @param {string} configPath - Path to configuration file.
+ */
+async function handleConfigCommand(args, instance, configPath) {
+	let command = args._[1];
+
+	if (command === "list") {
+		for (let GroupClass of instance.constructor.groups.values()) {
+			for (let def of GroupClass.definitions.values()) {
+				let value = instance.get(def.fullName);
+				print(`${def.fullName} ${JSON.stringify(value)}`);
+			}
+		}
+
+	} else if (command === "show") {
+		try {
+			print(instance.get(args.field));
+		} catch (err) {
+			if (err instanceof libConfig.InvalidField) {
+				logger.error(err.message);
+			} else {
+				throw err;
+			}
+		}
+
+	} else if (command === "set") {
+		if (args.stdin) {
+			args.value = (await libHelpers.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+
+		} else if (args.value === undefined) {
+			args.value = null;
+		}
+
+		try {
+			instance.set(args.field, args.value);
+			await fs.outputFile(configPath, JSON.stringify(instance.serialize(), null, 4));
+		} catch (err) {
+			if (err instanceof libConfig.InvalidField || err instanceof libConfig.InvalidValue) {
+				logger.error(err.message);
+			} else {
+				throw err;
+			}
+		}
+	}
+}
+
+
+module.exports = {
+	pluginCommand,
+	handlePluginCommand,
+
+	configCommand,
+	handleConfigCommand,
+};

packages/lib/users.js

@@ -0,0 +1,484 @@
+/**
+ * User and permissions library
+ *
+ * Defines data types for users, roles and permissions.
+ *
+ * @author Hornwitser
+ * @module lib/users
+ */
+"use strict";
+const jwt = require("jsonwebtoken");
+
+/**
+ * Represents a permission that can be granted
+ * @static
+ */
+class Permission {
+	constructor(name, title, description, grantByDefault) {
+		this.name = name;
+		this.title = title;
+		this.description = description;
+		this.grantByDefault = grantByDefault;
+	}
+}
+const permissions = new Map();
+
+/**
+ * Define a new user permission for Clusterio
+ *
+ * Defines a permission for use in Clusterio.  Defined permissions can be
+ * granted to roles, and checked on users.  Name should be `lower_case`
+ * style and start with the plugin name followed by a dot.
+ *
+ * @param {Object} def - The definition for the permission.
+ * @param {string} def.name - The internal name for the permission.
+ * @param {string} def.title - User facing name for permission.
+ * @param {string} def.description - User facing description.
+ * @param {boolean} def.grantByDefault -
+ *     If true this role is granted by the generated Player role by default.
+ *
+ */
+function definePermission({ name, title, description, grantByDefault=false }) {
+	if (typeof name !== "string" || !name.length) {
+		throw new Error("Expected name to be a non-empty string");
+	}
+	if (typeof title !== "string" || !title.length) {
+		throw new Error("Expected title to be a non-empty string");
+	}
+	if (typeof description !== "string" || !description.length) {
+		throw new Error("Expected description to be a non-empty string");
+	}
+	if (typeof grantByDefault !== "boolean") {
+		throw new Error("Expected grantByDefault to be a boolean");
+	}
+
+	if (permissions.has(name)) {
+		throw new Error(`Permission '${name}' is already defined`);
+	}
+
+	permissions.set(name, new Permission(name, title, description, grantByDefault));
+}
+
+/**
+ * Represents a collection of granted permissions
+ * @static
+ */
+class Role {
+	constructor(serializedRole) {
+		this.id = serializedRole.id;
+		this.name = serializedRole.name;
+		this.description = serializedRole.description;
+		this.permissions = new Set(serializedRole.permissions);
+	}
+
+	serialize() {
+		return {
+			id: this.id,
+			name: this.name,
+			description: this.description,
+			permissions: [...this.permissions],
+		};
+	}
+
+	grantDefaultPermissions() {
+		for (let permission of permissions.values()) {
+			if (permission.grantByDefault) {
+				this.permissions.add(permission.name);
+			}
+		}
+	}
+}
+
+
+/**
+ * Ensure the default admin role has full access
+ *
+ * Ensures the role with id 0 exisits and has the core.admin permission.
+ *
+ * @param {Map<number, module:lib/user.Role>} roles - role storage to modify.
+ * @returns {module:lib/permission.Role} the default admin role
+ * @static
+ */
+function ensureDefaultAdminRole(roles) {
+	let admin = roles.get(0);
+	if (!admin) {
+		admin = new Role({ id: 0, name: "Cluster Admin", description: "Cluster wide administrator." });
+		roles.set(0, admin);
+	}
+	admin.permissions.add("core.admin");
+	return admin;
+}
+
+/**
+ * Ensure the default player role has default accesses
+ *
+ * Ensures the role with id 1 exists and has all the permissions that are
+ * assigned by default.  Note that this may not be the same role as the one
+ * that is assigned to new users by default.
+ *
+ * @param {Map<number, module:lib/user.Role>} roles - role storage to modify.
+ * @static
+ */
+function ensureDefaultPlayerRole(roles) {
+	let player = roles.get(1);
+	if (!player) {
+		player = new Role({ id: 1, name: "Player", description: "Default player role." });
+		roles.set(1, player);
+	}
+	player.grantDefaultPermissions();
+}
+
+
+/**
+ * Represeents a user in the cluster
+ *
+ * Holds data about a Factorio user in the cluster.
+ * @static
+ */
+class User {
+	constructor(serializedUser, loadedRoles) {
+		/**
+		 * Factorio user name.
+		 * @type {string}
+		 */
+		this.name = serializedUser.name;
+
+		/**
+		 * Instances this user is online on.
+		 * @type {Set<number>}
+		 */
+		this.instances = new Set();
+
+		/**
+		 * Unix time in seconds the user token must be issued after to be valid.
+		 * @type {number}
+		 */
+		this.tokenValidAfter = serializedUser.token_valid_after || 0;
+
+		/**
+		 * True if the user is promoted to admin on the Factorio instances.
+		 * @type {boolean}
+		 */
+		this.isAdmin = Boolean(serializedUser.is_admin);
+
+		/**
+		 * True if the user is whitelisted on the Factorio instances.
+		 * @type {boolean}
+		 */
+		this.isWhitelisted = Boolean(serializedUser.is_whitelisted);
+
+		/**
+		 * True if the user is banned from Factorio instances.
+		 * @type {boolean}
+		 */
+		this.isBanned = Boolean(serializedUser.is_banned);
+
+		/**
+		 * Reason for being banned.  Ignored if isBanned is false.
+		 * @type {string}
+		 */
+		this.banReason = serializedUser.ban_reason || "";
+
+		/**
+		 * Roles this user has
+		 * @type {Set<module:lib/permissions.Role>}
+		 */
+		this.roles = new Set();
+		if (serializedUser.roles) {
+			for (let roleId of serializedUser.roles) {
+				let role = loadedRoles.get(roleId);
+				if (role) {
+					this.roles.add(role);
+				}
+			}
+		}
+	}
+
+	serialize() {
+		let serialized = {
+			name: this.name,
+		};
+
+		if (this.roles.size) {
+			serialized.roles = [...this.roles].map(role => role.id);
+		}
+
+		if (this.tokenValidAfter) {
+			serialized.token_valid_after = this.tokenValidAfter;
+		}
+
+		if (this.isAdmin) {
+			serialized.is_admin = true;
+		}
+
+		if (this.isWhitelisted) {
+			serialized.is_whitelisted = true;
+		}
+
+		if (this.isBanned) {
+			serialized.is_banned = true;
+		}
+
+		if (this.banReason) {
+			serialized.ban_reason = this.banReason;
+		}
+
+		return serialized;
+	}
+
+	/**
+	 * Generate access token for this user
+	 *
+	 * @param {string} secret - Secret to sign token with.
+	 * @returns {string} JWT access token for the user.
+	 */
+	createToken(secret) {
+		return jwt.sign({ aud: "user", user: this.name }, secret);
+	}
+
+	/**
+	 * Invalidate current tokens for the user
+	 *
+	 * Sets the tokenValidAfter property to the current time, which causes
+	 * all currently issued tokens for the user to become invalid.
+	 */
+	invalidateToken() {
+		this.tokenValidAfter = Math.floor(Date.now() / 1000);
+	}
+
+	/**
+	 * Check if a given permission is granted
+	 *
+	 * Checks the roles the user is member of for one that grants the given
+	 * permission.  If the permission is not granted for the user a
+	 * "Permission denied" error is thrown.
+	 *
+	 * @param {string} permission - The permission to check for.
+	 * @throws {Error} if the user does noh have the given permission.
+	 */
+	checkPermission(permission) {
+		if (!permissions.has(permission)) {
+			throw new Error(`permission ${permission} does not exist`);
+		}
+
+		for (let role of this.roles) {
+			if (role.permissions.has("core.admin") || role.permissions.has(permission)) {
+				return;
+			}
+		}
+
+		throw new Error("Permission denied");
+	}
+
+	notifyJoin(instance_id) {
+		this.instances.add(instance_id);
+		User.onlineUsers.add(this);
+	}
+
+	notifyLeave(instance_id) {
+		this.instances.delete(instance_id);
+		if (!this.instances.size) {
+			User.onlineUsers.delete(this);
+		}
+	}
+}
+/**
+ * Set of users currently online in the cluster.
+ * @type {module:lib/users.User}
+ */
+User.onlineUsers = new Set();
+
+// Definitions for the built in permissions used in Clusterio.
+// description should answer "this permission allows you to ___"
+definePermission({
+	name: "core.admin",
+	title: "Administrator",
+	description: "Bypass all permission checks.",
+});
+definePermission({
+	name: "core.control.connect",
+	title: "Connect to WebSocket",
+	description: "Connect to the master's WebSocket control interface.",
+	grantByDefault: true,
+});
+
+definePermission({
+	name: "core.master.get_config",
+	title: "Get master config",
+	description: "Get the config of master.",
+});
+definePermission({
+	name: "core.master.update_config",
+	title: "Modify master config",
+	description: "Modify the master config or entries of the master config.",
+});
+definePermission({
+	name: "core.slave.list",
+	title: "List slaves",
+	description: "Get the full list of slaves in the cluster.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.slave.generate_token",
+	title: "Generate slave token",
+	description: "Generate tokens for slaves to connect to the cluster with.",
+});
+definePermission({
+	name: "core.slave.create_config",
+	title: "Create slave config",
+	description: "Create slave configs via the master server.",
+});
+
+definePermission({
+	name: "core.instance.list",
+	title: "List instances",
+	description: "Get the full list of instances in the cluster.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.instance.create",
+	title: "Create instance",
+	description: "Create new instances on the master server.",
+});
+definePermission({
+	name: "core.instance.get_config",
+	title: "Get instance config",
+	description: "Get the config of instances.",
+});
+definePermission({
+	name: "core.instance.update_config",
+	title: "Modify instance config",
+	description: "Modify the config or entries of the config for instances.",
+});
+definePermission({
+	name: "core.instance.assign",
+	title: "Assign instance",
+	description: "Assign or reassign instances to a slave.",
+});
+definePermission({
+	name: "core.instance.create_save",
+	title: "Create new instance save",
+	description: "Create new savegames on instances.",
+});
+definePermission({
+	name: "core.instance.export_data",
+	title: "Export locale and icons",
+	description: "Export the the locale and icons from an instance and upload it to the master.",
+});
+definePermission({
+	name: "core.instance.start",
+	title: "Start instance",
+	description: "Start instances.",
+});
+definePermission({
+	name: "core.instance.load_scenario",
+	title: "Load scenario",
+	description: "Start instances by loading a scenario.",
+});
+definePermission({
+	name: "core.instance.stop",
+	title: "Stop instance",
+	description: "Stop instances.",
+});
+definePermission({
+	name: "core.instance.delete",
+	title: "Delete instance",
+	description: "Delete instances (includes config and all files stored like saves and logs).",
+});
+definePermission({
+	name: "core.instance.send_rcon",
+	title: "Send RCON",
+	description: "Send arbitrary RCON commands to instances.",
+});
+
+definePermission({
+	name: "core.permission.list",
+	title: "List permissions",
+	description: "Get the full list of permissions in the cluster.",
+	grantByDefault: true,
+});
+
+definePermission({
+	name: "core.role.list",
+	title: "List roles",
+	description: "Get the full list of roles and their permissions in the cluster.",
+	grantByDefault: true,
+});
+definePermission({
+	name: "core.role.create",
+	title: "Create role",
+	description: "Create new roles with permissions.",
+});
+definePermission({
+	name: "core.role.update",
+	title: "Update role",
+	description: "Edit existing roles including permissions they grant.",
+});
+definePermission({
+	name: "core.role.delete",
+	title: "Delete role",
+	description: "Permanently delete roles.",
+});
+
+definePermission({
+	name: "core.user.list",
+	title: "List users",
+	description: "Get the full list of users in the cluster.",
+});
+definePermission({
+	name: "core.user.create",
+	title: "Create users",
+	description: "Create user acounts with a given name.",
+});
+definePermission({
+	name: "core.user.update_roles",
+	title: "Update user roles",
+	description: "Add or remove any role to any user.",
+});
+definePermission({
+	name: "core.user.set_admin",
+	title: "Set user admin status",
+	description: "Promote or demote any user to Factorio admin.",
+});
+definePermission({
+	name: "core.user.set_banned",
+	title: "Set user ban status",
+	description: "Ban or unban any user.",
+});
+definePermission({
+	name: "core.user.set_whitelisted",
+	title: "Set user whitelist status",
+	description: "Add or remove any user to/from the whitelist.",
+});
+definePermission({
+	name: "core.user.delete",
+	title: "Delete user",
+	description: "Delete users and all data stored for them.",
+});
+definePermission({
+	name: "core.log.follow",
+	title: "Follow cluster log",
+	description: "Receive new entries in the cluster log.  Required to see instance console.",
+});
+definePermission({
+	name: "core.log.query",
+	title: "Query cluster log",
+	description: "Query past entries in the cluster log.  Required to see past entries in instance console.",
+});
+
+definePermission({
+	name: "core.debug.dump_ws",
+	title: "Dump WebSocket",
+	description: "Dump all WebSocket communicatation from the master.",
+});
+
+module.exports = {
+	Permission,
+	Role,
+	User,
+
+	permissions,
+	definePermission,
+	ensureDefaultAdminRole,
+	ensureDefaultPlayerRole,
+};

packages/master/README.md

@@ -0,0 +1,135 @@
+Clusterio Master Server
+=======================
+
+Communication hub for Clusterio clusters.  The master server forwards
+data between Clusterio slaves connected to it and allows the cluster to
+be remotely managed through WebSocket connections to it either by using
+the included web interface or the Clusterio ctl command line utility.
+
+
+Usage
+-----
+
+    npx clusteriomaster <command>
+
+Common options:
+
+ * `--plugin-list <file>` JSON file to use for storing the list of
+   plugins that are available to the master server.  Defaults to
+   `plugin-list.json` and will be created if it does not exist. See the
+   `plugin` command for managing this list.
+
+ * `--config <file>` JSON file to use for storing configuration for the
+   master server.  Defaults to `config-master.json` and will be created
+   if it does not exist.  See the `config` command for inspecting and
+   modifying the configuration.
+
+
+### `plugin <command>`
+
+Configure plugins available to be loaded by the master server.  The
+available plugins will be loaded unless they have been disabled in the
+configuration, see the config command for disabling plugins.
+
+
+#### `plugin add <path>`
+
+Add plugin either by require path or relative/absolute path to the
+plugin directory.  A relative path must start with ./ or ../ (or .\ and
+..\ on Windows) otherwise it will be assumed to be a require path for an
+installed package in node_modules.
+
+For example, installing the Subspace Storage plugin:
+
+    npm install @clusterio/plugin-subspace_storage
+    npx clusteriomaster plugin add @clusterio/plugin-subspace_storage
+
+Since the `plugin-list.json` is shared between master, slave and ctl you
+usually only need to do this once per machine.
+
+
+#### `plugin remove <name>`
+
+Remove a plugin by its name.  This should be done before uninstalling
+the plugin, otherwise there will be an error when Clusterio tries to
+load the info from the plugin.  Removing and unistalling a plugin is
+usually not neccessary as the functions provided by the plugin can be
+disabled in the config.
+
+For example, uninstalling the Subspace Storage plugin:
+
+    npx clusteriomaster plugin remove subspace_storage
+    npm uninstall @clusterio/plugin-subspace_storage
+
+Since the `plugin-list.json` is shared between master, slave and ctl you
+usually only need to do this once per machine.
+
+
+#### `plugin list`
+
+Lists the plugins set up to be available by name followed by path.
+
+
+### `config`
+
+Manage the master server configuration offline.  This should only be
+used when the master server is stopped, otherwise the config read might
+be out of date and config changes will be overwritten when the master
+server shuts down.
+
+
+#### `config set <config-entry> [value]`
+
+Set a config entry to the given value.  If value is not provided the
+entry is set to null.  If the config entry is of type object the value
+must be a valid JSON serialization of an object.
+
+See docs/configuration.md in the main repositiory for the available
+configuration entries.
+
+
+#### `config show <config-entry>`
+
+Shows the value for a single config entry.
+
+
+#### `config list`
+
+Lists up all configuration entries with their currently configured values.
+
+
+### `bootstrap`
+
+Commands for setting up initial admin acount and access to the cluster.
+
+
+#### `bootstrap create-admin <name>`
+
+Creates a new user account with the given name if it does not already
+exist in the cluster and grant it full cluster admin permission.  You
+should be using your own Factorio multiplayer username here.
+
+
+#### `bootstrap generate-user-token <name>`
+
+Generate an access token for logging in and possibly managing the
+cluster from the given user account.  This can be used both in the web
+interface and as the `master_token` to connect with clusterioctl.
+
+
+#### `bootstrap create-ctl-config <name>`
+
+Creates a clusterioctl config for the given user with url and token set
+up for connecting to the cluster.
+
+
+### `run`
+
+Runs the master server.
+
+
+## See Also
+
+[The Clusterio
+repository](https://github.com/clusterio/factorioClusterio)
+for instructions on how to set up a cluster.

packages/master/package.json

@@ -0,0 +1,56 @@
+{
+  "name": "@clusterio/master",
+  "description": "Implementation of Clusterio master server",
+  "author": "Hornwitser",
+  "license": "MIT",
+  "version": "2.0.0-alpha.4",
+  "main": "master.js",
+  "scripts": {
+    "build": "webpack-cli --env.production",
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "bin": {
+    "clusteriomaster": "./master.js"
+  },
+  "keywords": [
+    "factorio"
+  ],
+  "engines": {
+    "node": ">=10.16 <11 || >= 12"
+  },
+  "dependencies": {
+    "@clusterio/lib": "^2.0.0-alpha.4",
+    "body-parser": "^1.18.2",
+    "compression": "^1.7.2",
+    "cookie-parser": "^1.4.3",
+    "express": "^4.16.2",
+    "express-fileupload": "1.2.0",
+    "fs-extra": "^8.1.0",
+    "jsonwebtoken": "^8.2.1",
+    "jszip": "^3.2.2",
+    "node-factorio-api": "^0.3.8",
+    "node-forge": "^0.10.0",
+    "set-blocking": "^2.0.0",
+    "winston": "^3.3.3",
+    "ws": "^7.2.3",
+    "yargs": "^14.2.0"
+  },
+  "devDependencies": {
+    "@clusterio/web_ui": "^2.0.0-alpha.4",
+    "babel-loader": "^8.1.0",
+    "css-loader": "^4.3.0",
+    "fast-deep-equal": "^3.1.3",
+    "file-loader": "^6.2.0",
+    "null-loader": "^4.0.0",
+    "react": "^16.13.1",
+    "react-dom": "^16.13.1",
+    "react-router-dom": "^5.2.0",
+    "style-loader": "^1.2.1",
+    "webpack-cli": "^3.3.12",
+    "webpack-dev-middleware": "^3.7.2",
+    "webpack-merge": "^5.2.0"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

packages/master/src/HttpCloser.js

@@ -0,0 +1,109 @@
+"use strict";
+const http = require("http");
+const https = require("https");
+const util = require("util");
+
+/**
+ * Gracefully close a Node.js HTTP(S) server
+ *
+ * Helper class to properly close down a Node.js HTTP(S) server.  This is
+ * necessary due to connections being able to linger around forever with the
+ * keep-alive logic in HTTP.
+ *
+ * @example
+ * // during server initialization
+ * const server = http.createServer();
+ * const serverCloser = new HttpCloser(server);
+ *
+ * // in the shutdown logic
+ * await serverCloser.close();
+ *
+ * @alias module:master/src/HttpCloser
+ */
+class HttpCloser {
+	/**
+	 * Server to attach this closer to.
+	 * @param {https.Server | http.Server} server - HTTP(S) to attach to
+	 */
+	constructor(server) {
+		this._server = server;
+		this._sockets = new Set();
+		this._responses = new Set();
+
+		if (this._server instanceof http.Server) {
+			this._server.on("connection", this._addSocket.bind(this));
+		} else if (this._server instanceof https.Server) {
+			this._server.on("secureConnection", this._addSocket.bind(this));
+		} else {
+			throw Error("server must be an instance of either http.Server or https.Server.");
+		}
+		this._server.on("request", this._addRequest.bind(this));
+	}
+
+	_addSocket(socket) {
+		this._sockets.add(socket);
+		socket.once("close", () => {
+			this._sockets.delete(socket);
+		});
+	}
+
+	_addRequest(request, response) {
+		this._responses.add(response);
+		response.once("close", () => this._responses.delete(response));
+		response.once("finish", () => this._responses.delete(response));
+	}
+
+	/**
+	 * Gracefully close down the attached HTTP(S) server.
+	 *
+	 * Closes open connections that do not have a request in flight, signals
+	 * connections to pending requests should close, and initiates the
+	 * close operation on the attached server.
+	 *
+	 * If there's requests in progress that still haven't been completed
+	 * after the timeout value passed they will be aborted.
+	 *
+	 * @param {number} timeout -
+	 *     Time in ms to wait for connections to close.
+	 */
+	async close(timeout = 5000) {
+		let activeSockets = new Set();
+		for (let response of this._responses) {
+			if (!response.headersSent) {
+				response.setHeader("Connection", "close");
+			} else {
+				let socket = response.socket;
+				response.once("finish", () => {
+					socket.end();
+				});
+			}
+			activeSockets.add(response.socket);
+		}
+		for (let socket of this._sockets) {
+			if (!activeSockets.has(socket)) {
+				socket.end();
+			}
+		}
+
+		return new Promise((resolve, reject) => {
+			let timer = setTimeout(() => {
+				for (let socket of this._sockets) {
+					// Ideally a RST would be sent here to indicate the the
+					// transfer failed, but we can't reliably send that from
+					// Node.js, see nodejs/node#27428.
+					socket.destroy();
+				}
+			}, timeout);
+			this._server.close(err => {
+				clearTimeout(timer);
+				if (err) {
+					reject(err);
+				} else {
+					resolve();
+				}
+			});
+		});
+	}
+};
+
+module.exports = HttpCloser;

packages/master/src/routes.js

@@ -0,0 +1,15 @@
+"use strict";
+// Routes used in the web interface and served by the master server
+module.exports = [
+	"/",
+	"/master",
+	"/slaves",
+	"/slaves/:id/view",
+	"/instances",
+	"/instances/:id/view",
+	"/users",
+	"/users/:id/view",
+	"/roles",
+	"/roles/:id/view",
+	"/settings",
+];

packages/master/web/index.html

@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>Clusterio</title>
+    <script>
+        window.webRoot = "__WEB_ROOT__";
+    </script>
+  </head>
+  <body>
+    <noscript>You need to enable JavaScript to run this app.</noscript>
+    <div id="root"></div>
+    <script src="__WEB_ROOT__bundle.js"></script>
+  </body>
+</html>

packages/master/web/index.jsx

@@ -0,0 +1 @@
+import("@clusterio/web_ui/src/bootstrap");

packages/master/webpack.config.js

@@ -0,0 +1,31 @@
+"use strict";
+const path = require("path");
+const webpack = require("webpack");
+const { merge } = require("webpack-merge");
+
+const common = require("@clusterio/web_ui/webpack.common");
+
+module.exports = (env = {}) => merge(common(env), {
+	context: __dirname,
+	entry: "./web/index.jsx",
+	devServer: {
+		contentBase: "./dist/web",
+	},
+	output: {
+		filename: "bundle.js",
+		path: path.resolve(__dirname, "dist", "web"),
+	},
+	plugins: [
+		new webpack.container.ModuleFederationPlugin({
+			name: "master",
+			shared: {
+				"@clusterio/lib": { singleton: true },
+				"@clusterio/web_ui": { singleton: true },
+				"ajv": {},
+				"antd": { singleton: true },
+				"react": { singleton: true },
+				"react-dom": { singleton: true },
+			},
+		}),
+	],
+});

packages/slave/README.md

@@ -0,0 +1,111 @@
+Clusterio Slave
+===============
+
+Node hosting Factorio servers in a Clusterio cluster.  Clusterio slaves
+connect to the master server and waits for commands from the master
+server to start up and stop instances.  A cluster can have any number of
+slaves in it located on different computers, and each slave can host any
+number of instances each of which is a Factorio server that talks with
+the rest of the cluster.
+
+
+Usage
+-----
+
+    npx clusterioslave <command>
+
+Common options:
+
+ * `--plugin-list <file>` JSON file to use for storing the list of
+   plugins that are available to the slave.  Defaults to
+   `plugin-list.json` and will be created if it does not exist. See the
+   `plugin` command for managing this list.
+
+ * `--config <file>` JSON file to use for storing configuration for the
+   slave.  Defaults to `config-slave.json` and will be created if it
+   does not exist.  See the `config` command for inspecting and
+   modifying the configuration.
+
+
+### `plugin <command>`
+
+Configure plugins available to be loaded by the slave.  The available
+plugins will be loaded unless they have been disabled in the
+configuration, see the config command for disabling plugins.
+
+
+#### `plugin add <path>`
+
+Add plugin either by require path or relative/absolute path to the
+plugin directory.  A relative path must start with ./ or ../ (or .\ and
+..\ on Windows) otherwise it will be assumed to be a require path for an
+installed package in node_modules.
+
+For example, installing the Subspace Storage plugin:
+
+    npm install @clusterio/plugin-subspace_storage
+    npx clusterioslave plugin add @clusterio/plugin-subspace_storage
+
+Since the `plugin-list.json` is shared between master, slave and ctl you
+usually only need to do this once per machine.
+
+
+#### `plugin remove <name>`
+
+Remove a plugin by its name.  This should be done before uninstalling
+the plugin, otherwise there will be an error when Clusterio tries to
+load the info from the plugin.  Removing and unistalling a plugin is
+usually not neccessary as the functions provided by the plugin can be
+disabled in the config.
+
+For example, uninstalling the Subspace Storage plugin:
+
+    npx clusterioslave plugin remove subspace_storage
+    npm uninstall @clusterio/plugin-subspace_storage
+
+Since the `plugin-list.json` is shared between master, slave and ctl you
+usually only need to do this once per machine.
+
+
+#### `plugin list`
+
+Lists the plugins set up to be available by name followed by path.
+
+
+### `config`
+
+Manage the slave configuration offline.  This should only be used when
+the slave is stopped, otherwise the config read might be out of date and
+config changes will be overwritten when the slave shuts down.
+
+
+#### `config set <config-entry> [value]`
+
+Set a config entry to the given value.  If value is not provided the
+entry is set to null.  If the config entry is of type object the value
+must be a valid JSON serialization of an object.
+
+See docs/configuration.md in the main repositiory for the available
+configuration.
+
+
+#### `config show <config-entry>`
+
+Shows the value for a single config entry.
+
+
+#### `config list`
+
+Lists up all configuration entries with their currently configured values.
+
+
+### `run`
+
+Runs the slave.
+
+
+## See Also
+
+[The Clusterio
+repository](https://github.com/clusterio/factorioClusterio)
+for instructions on how to set up a cluster.

packages/slave/lua/clusterio_lib/changelog.txt

@@ -0,0 +1,13 @@
+---------------------------------------------------------------------------------------------------
+Version: 0.1.2
+  Features:
+    - Added support for Factorio 1.1.
+---------------------------------------------------------------------------------------------------
+Version: 0.1.1
+  Features:
+    - Added support for Factorio 0.18.x
+---------------------------------------------------------------------------------------------------
+Version: 0.1.0
+  Major Features:
+    - API for mods to interact with Clusterio
+    - Serialization library for converting items to JSON compatible tables.

packages/slave/lua/clusterio_lib/info.json

@@ -0,0 +1,24 @@
+{
+    "name": "clusterio_lib",
+    "variants": [
+        {
+            "version": "0.1.0",
+            "factorio_version": "0.17"
+        },
+        {
+            "version": "0.1.1",
+            "factorio_version": "0.18"
+        },
+        {
+            "version": "0.1.2",
+            "factorio_version": "1.1"
+        }
+    ],
+    "title": "Clusterio Library (Alpha)",
+    "author": "Clusterio Team",
+    "homepage": "https://github.com/clusterio/factorioClusterio",
+    "description": "Library for interfacing with Clusterio.  Warning: work in progress alpha.",
+    "additional_files": {
+        "serialize.lua": ["..", "..", "modules", "clusterio", "serialize.lua"]
+    }
+}

packages/slave/lua/export/control.lua

@@ -0,0 +1,8 @@
+local function send_json(channel, data)
+    data = game.table_to_json(data)
+    print("\f$ipc:" .. channel .. "?j" .. data)
+end
+
+script.on_init(function()
+    send_json("mod_list", game.active_mods)
+end)

packages/slave/lua/export/data-final-fixes.lua

@@ -0,0 +1,101 @@
+local item_types = {
+    -- Item categories
+    "item",
+    "ammo",
+    "capsule",
+    "gun",
+    "item-with-entity-data",
+    "item-with-label",
+    "item-with-inventory",
+    "blueprint-book",
+    "item-with-tags",
+    "selection-tool",
+    "blueprint",
+    "deconstruction-item",
+    "copy-paste-tool",
+    "upgrade-item",
+    "module",
+    "rail-planner",
+    "tool",
+    "armor",
+    "mining-tool",
+    "repair-tool",
+
+    -- Fluids
+    "fluid",
+}
+
+local function is_array(t)
+    local count = 0
+    for k in pairs(t) do
+        if type(k) ~= "number" then
+            return false
+        end
+        count = count + 1
+    end
+
+    for i = 1, count do
+        if not t[i] then
+            return false
+        end
+    end
+
+    return count ~= 0
+end
+
+-- no game.table_to_json here :(
+local function table_to_json(data)
+    if type(data) == "number" then
+        return string.format("%.17g", data)
+
+    elseif type(data) == "string" then
+        data = data:gsub('([\x00-\x1f\\"])', function(match)
+            return "\\x" .. string.format("%02x", match:byte())
+        end)
+        return '"' .. data .. '"'
+
+    elseif data == nil then
+        return "null"
+
+    elseif data == true then
+        return "true"
+    elseif data == false then
+        return "false"
+
+    elseif type(data) == "table" then
+        local r
+        if is_array(data) then
+            r = {"["}
+            for i=1, #data do
+                r[#r+1] = table_to_json(data[i])
+                r[#r+1] = ","
+            end
+
+            -- overwrite last comma, empty table is an object
+            r[#r] = "]"
+
+        else
+            r = {"{"}
+            for k, v in pairs(data) do
+                r[#r+1] = table_to_json(tostring(k))
+                r[#r+1] = ":"
+                r[#r+1] = table_to_json(v)
+                r[#r+1] = ","
+            end
+            r[#r + (r[#r] == "," and 0 or 1)] = "}"
+        end
+
+        return table.concat(r)
+    end
+end
+
+local function send_json(channel, data)
+    data = table_to_json(data)
+    print("\f$ipc:" .. channel .. "?j" .. data)
+end
+
+for _, group in ipairs(item_types) do
+    for _, item in pairs(data.raw[group]) do
+        send_json("item_export", item)
+    end
+end

packages/slave/lua/export/info.json

@@ -0,0 +1,8 @@
+{
+    "name": "export",
+    "version": "0.0.0",
+    "title": "Icon exporter",
+    "author": "Clusterio Team",
+    "homepage": "https://github.com/clusterio/factorioClusterio",
+    "description": "Exports item prototypes on stdout during the data stage."
+}

packages/slave/modules/clusterio/impl.lua

@@ -18,6 +18,19 @@ impl.events[api.events.on_server_startup] = function()
             instance_name = nil,
         }
     end
+
+    -- Replay will just desync after the save is patched
+    game.disable_replay()
+end
+
+impl.events[defines.events.on_player_joined_game] = function(event)
+    local player = game.players[event.player_index]
+    api.send_json("player_event", { type = "join", name = player.name })
+end
+
+impl.events[defines.events.on_player_left_game] = function(event)
+    local player = game.players[event.player_index]
+    api.send_json("player_event", { type = "leave", name = player.name })
 end
 
 -- Internal API

packages/slave/modules/clusterio/serialize.lua

@@ -2,6 +2,18 @@
 -- Based on code from playerManager and trainTeleports
 local serialize = {}
 
+-- 0.17 compatibility
+local supports_bar, get_bar, has_bar
+if (pcall(function() local mods = script.active_mods end)) then
+    supports_bar = "supports_bar"
+    get_bar = "get_bar"
+    set_bar = "set_bar"
+else
+    supports_bar = "hasbar"
+    get_bar = "getbar"
+    set_bar = "setbar"
+end
+
 -- Equipment Grids are serialized into an array of equipment entries
 -- where ench entry is a table with the following fields:
 --   n: name
@@ -181,8 +193,8 @@ end
 -- It's also possible that the slot is empty but has a slot filter.
 function serialize.serialize_inventory(inventory)
     local serialized = {}
-    if inventory.hasbar() and inventory.getbar() <= #inventory then
-        serialized.b = inventory.getbar()
+    if inventory[supports_bar]() and inventory[get_bar]() <= #inventory then
+        serialized.b = inventory[get_bar]()
     end
 
     serialized.i = {}
@@ -227,8 +239,8 @@ function serialize.serialize_inventory(inventory)
 end
 
 function serialize.deserialize_inventory(inventory, serialized)
-    if serialized.b and inventory.hasbar() then
-        inventory.setbar(serialized.b)
+    if serialized.b and inventory[supports_bar]() then
+        inventory[set_bar](serialized.b)
     end
 
     local last_slot_index = 0