v2.0.0-alpha.4

Remote master config management

.dockerignore

@@ -0,0 +1,23 @@
+# No need to invalidate cache just because we try speeding up the build
+.dockerignore
+Dockerfile
+docker-compose*
+
+# Install dependencies during the build
+node_modules
+packages/**/node_modules
+plugins/**/node_modules
+
+# Download factorio during the build
+factorio
+factorio.tar.gz
+
+# We build internal mods during build, mods from local test env should not be applied
+sharedMods
+sharedModsDisabled
+
+# These files should never be required during the build
+.git
+.github
+dist
+temp

.eslintignore

@@ -1,2 +1 @@
-packages/master/static
-plugins/*/static
+dist/

.eslintrc.js

@@ -45,7 +45,7 @@ module.exports = {
 		"arrow-spacing": ["error", { "after": true, "before": true }],
 		"block-scoped-var": "error",
 		"block-spacing": "error",
-		"brace-style": ["warn", "1tbs", { "allowSingleLine": true }],
+		"brace-style": ["error", "1tbs", { "allowSingleLine": true }],
 		"callback-return": "off",
 		"camelcase": "off",
 		"capitalized-comments": "off",
@@ -60,18 +60,18 @@ module.exports = {
 				"functions": "only-multiline",
 			},
 		],
-		"comma-spacing": "warn",
+		"comma-spacing": "error",
 		"comma-style": ["error", "last"],
-		"complexity": "warn",
+		"complexity": "error",
 		"computed-property-spacing": ["error", "never"],
 		"consistent-return": "error",
 		"consistent-this": "error",
-		"curly": "warn",
+		"curly": "error",
 		"default-case": "error",
 		"default-case-last": "error",
 		"default-param-last": "error",
 		"dot-location": ["error", "property"],
-		"dot-notation": ["warn", { "allowPattern": "^[a-z]+(_[a-z]+)*$" }],
+		"dot-notation": ["error", { "allowPattern": "^[a-z]+(_[a-z]+)*$" }],
 		"eol-last": "error",
 		"eqeqeq": "error",
 		"func-call-spacing": "error",
@@ -81,7 +81,7 @@ module.exports = {
 		"function-call-argument-newline": "off",
 		"function-paren-newline": "off",
 		"generator-star-spacing": ["error", { "before": false, "after": true }],
-		"global-require": "warn",
+		"global-require": "error",
 		"grouped-accessor-pairs": "error",
 		"guard-for-in": "error",
 		"handle-callback-err": "error",
@@ -94,22 +94,22 @@ module.exports = {
 		"init-declarations": "off",
 		"jsx-quotes": "error",
 		"key-spacing": "error",
-		"keyword-spacing": "warn",
+		"keyword-spacing": "error",
 		"line-comment-position": "off",
 		"linebreak-style": ["error", "unix"],
 		"lines-around-comment": "off",
 		"lines-around-directive": "off",
 		"lines-between-class-members": ["error", "always"],
 		"max-classes-per-file": "off",
-		"max-depth": "warn",
-		"max-len": ["warn", { "code": 120 }],
+		"max-depth": "error",
+		"max-len": ["error", { "code": 120 }],
 		"max-lines": "off",
 		"max-lines-per-function": "off",
 		"max-nested-callbacks": "error",
 		"max-params": "off",
 		"max-statements": "off",
 		"max-statements-per-line": "off",
-		"multiline-comment-style": ["warn", "separate-lines"],
+		"multiline-comment-style": ["error", "separate-lines"],
 		"multiline-ternary": "off",
 		"new-cap": "error",
 		"new-parens": "error",
@@ -124,13 +124,13 @@ module.exports = {
 		"no-caller": "error",
 		"no-catch-shadow": "error",
 		"no-confusing-arrow": "error",
-		"no-console": "off",
+		"no-console": "error",
 		"no-constant-condition": ["error", { "checkLoops": false }],
 		"no-constructor-return": "error",
 		"no-continue": "off",
 		"no-div-regex": "error",
 		"no-duplicate-imports": "error",
-		"no-else-return": "warn",
+		"no-else-return": "error",
 		"no-empty-function": "off",
 		"no-eq-null": "error",
 		"no-eval": "error",
@@ -143,7 +143,7 @@ module.exports = {
 		"no-implicit-globals": "error",
 		"no-implied-eval": "error",
 		"no-inline-comments": "off",
-		"no-invalid-this": "warn",
+		"no-invalid-this": "error",
 		"no-iterator": "error",
 		"no-label-var": "error",
 		"no-lone-blocks": "error",
@@ -179,7 +179,7 @@ module.exports = {
 		"no-path-concat": "error",
 		"no-plusplus": ["error", { "allowForLoopAfterthoughts": true }],
 		"no-process-env": "error",
-		"no-process-exit": "warn",
+		"no-process-exit": "error",
 		"no-proto": "error",
 		"no-restricted-exports": "error",
 		"no-restricted-globals": "error",
@@ -205,7 +205,7 @@ module.exports = {
 		"no-unmodified-loop-condition": "error",
 		"no-unneeded-ternary": "error",
 		"no-unused-expressions": "error",
-		"no-use-before-define": "warn",
+		"no-use-before-define": "error",
 		"no-useless-backreference": "error",
 		"no-useless-call": "error",
 		"no-useless-computed-key": "error",
@@ -213,8 +213,8 @@ module.exports = {
 		"no-useless-constructor": "error",
 		"no-useless-rename": "error",
 		"no-useless-return": "error",
-		"no-var": "warn",
-		"no-void": "error",
+		"no-var": "error",
+		"no-void": ["error", { "allowAsStatement": true }],
 		"no-warning-comments": "off",
 		"no-whitespace-before-property": "error",
 		"nonblock-statement-body-position": "error",
@@ -234,7 +234,7 @@ module.exports = {
 		}],
 		"padded-blocks": "off",
 		"padding-line-between-statements": "error",
-		"prefer-arrow-callback": "warn",
+		"prefer-arrow-callback": "error",
 		"prefer-const": "off",
 		"prefer-destructuring": "off",
 		"prefer-exponentiation-operator": "error",
@@ -246,7 +246,7 @@ module.exports = {
 		"prefer-regex-literals": "error",
 		"prefer-rest-params": "error",
 		"prefer-spread": "error",
-		"prefer-template": "warn",
+		"prefer-template": "error",
 		"quote-props": "off",
 		"quotes": ["error", "double", { "avoidEscape": true }],
 		"radix": ["error", "always"],
@@ -261,12 +261,12 @@ module.exports = {
 		"sort-imports": "off",
 		"sort-keys": "off",
 		"sort-vars": "error",
-		"space-before-blocks": "warn",
+		"space-before-blocks": "error",
 		"space-before-function-paren": "off",
 		"space-in-parens": ["error", "never"],
 		"space-infix-ops": "off",
 		"space-unary-ops": "error",
-		"spaced-comment": ["warn", "always"],
+		"spaced-comment": ["error", "always"],
 		"strict": "error",
 		"switch-colon-spacing": "error",
 		"symbol-description": "error",
@@ -274,7 +274,7 @@ module.exports = {
 		"template-tag-spacing": "error",
 		"unicode-bom": ["error", "never"],
 		"valid-jsdoc": ["error", { "requireReturn": false }],
-		"vars-on-top": "warn",
+		"vars-on-top": "error",
 		"wrap-iife": "error",
 		"wrap-regex": "off",
 		"yield-star-spacing": "error",

.gitattributes

@@ -0,0 +1 @@
+package-lock.json -diff

.github/workflows/docker-publish.yml

@@ -0,0 +1,159 @@
+name: Docker
+
+on:
+  push:
+    # Publish `master` as Docker `latest` image.
+    branches:
+      - '**'
+
+    # Publish `v1.2.3` tags as releases.
+    tags:
+      - v*
+
+  # Run tests for any PRs.
+  pull_request:
+
+  release:
+    types: [released, prereleased]
+
+env:
+  # TODO: Change variable to your image's name.
+  IMAGE_NAME: clusterio
+
+jobs:
+  # Build dependencies and clusterio image
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build docker image
+        run: DOCKER_BUILDKIT=1 docker build . -t $IMAGE_NAME --target clusterio_final
+
+      - name: Export docker image
+        run: docker save -o clusterio_docker_image $IMAGE_NAME
+
+      - name: Upload docker image
+        uses: actions/upload-artifact@master
+        with:
+          name: clusterio_docker_image
+          path: clusterio_docker_image
+  # Build dependencies and clusterio image with test tooling
+  build_testing:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build docker image
+        run: DOCKER_BUILDKIT=1 docker build . -t $IMAGE_NAME --target clusterio_testing
+
+      - name: Export docker image
+        run: docker save -o clusterio_docker_image_testing $IMAGE_NAME
+
+      - name: Upload docker image
+        uses: actions/upload-artifact@master
+        with:
+          name: clusterio_docker_image_testing
+          path: clusterio_docker_image_testing
+  # Upload clusterio_lib and subspace_storage versions
+  upload_mods:
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download docker image
+        uses: actions/download-artifact@master
+        with:
+          name: clusterio_docker_image
+          path: .
+
+      - name: Load docker image
+        run: docker load -i clusterio_docker_image
+      
+      - name: Create container
+        run: docker create --name clusterio_container $IMAGE_NAME
+
+      - name: Move mods out of container
+        run: docker cp clusterio_container:/clusterio/sharedMods ./mods
+
+      - name: Remove container
+        run: docker rm clusterio_container
+
+      - name: List extracted mods
+        run: ls mods
+
+      - name: Upload mods
+        uses: actions/upload-artifact@master
+        with:
+          name: mods
+          path: mods
+  # Run tests.
+  # See also https://docs.docker.com/docker-hub/builds/automated-testing/
+  test:
+    needs: build_testing
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Download docker image
+        uses: actions/download-artifact@master
+        with:
+          name: clusterio_docker_image_testing
+          path: .
+      - name: ls
+        run: ls && ls clusterio_docker_image_testing && ls /home/runner/work/factorioClusterio && ls /home/runner/work/factorioClusterio/factorioClusterio
+      - name: Load docker image
+        run: docker load -i clusterio_docker_image_testing
+
+      - name: Run tests
+        run: |
+          if [ -f docker-compose.test.yml ]; then
+            docker-compose --file docker-compose.test.yml run unit_test
+          else
+            docker build . --file Dockerfile
+          fi
+  # Push image to GitHub Packages.
+  # See also https://docs.docker.com/docker-hub/builds/
+  push:
+    # Ensure test job passes before pushing image.
+    needs: [test, build]
+
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Download artifact
+        uses: actions/download-artifact@master
+        with:
+          name: clusterio_docker_image
+          path: .
+
+      - name: Load docker image
+        run: docker load -i clusterio_docker_image
+
+      - name: Log into registry
+        run: echo "${{ secrets.CR_PAT }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+
+      - name: Push image
+        run: |
+          IMAGE_ID=ghcr.io/${{ github.actor }}/$IMAGE_NAME
+
+          # Change all uppercase to lowercase
+          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
+
+          # Strip git ref prefix from version
+          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+
+          # Strip "v" prefix from tag name
+          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
+
+          # Use Docker `latest` tag convention
+          [ "$VERSION" == "master" ] && VERSION=latest
+
+          echo IMAGE_ID=$IMAGE_ID
+          echo VERSION=$VERSION
+
+          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
+          docker push $IMAGE_ID:$VERSION

.gitignore

@@ -31,6 +31,7 @@ dist/
 node_modules
 
 Factorio*
+factorio*
 /factorio
 /instances
 /sharedMods
@@ -45,10 +46,6 @@ static/export
 saves
 /temp/
 
-# Web build
-/packages/master/static
-/plugins/*/static
-
 # IDE
 .idea/
 .vs/

.travis.yml

@@ -7,8 +7,8 @@ before_install:
   - npm install -g codecov
 script:
   - npx lerna bootstrap --no-ci
-  - node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --bump-patch --factorio-version 0.18 --output-dir temp/test/sharedMods
-  - curl -o temp/test/sharedMods/subspace_storage_1.99.1.zip -L https://github.com/clusterio/factorioClusterioMod/releases/download/1.99.1/subspace_storage_1.99.1.zip
+  - node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --output-dir temp/test/sharedMods
+  - curl -o temp/test/sharedMods/subspace_storage_1.99.8.zip -L https://github.com/clusterio/factorioClusterioMod/releases/download/1.99.6/subspace_storage_1.99.8.zip
   - npx lerna run build
   - npm run-script ci-cover
   - codecov -f coverage/lcov.info

CHANGELOG.md

@@ -27,6 +27,8 @@ Version 2.0.0
   configuration remotely.
 - Packaged and uploaded to npm again, greatly simplifying distribution and
   installation of both Clusterio and plugins.
+- Added centralized logging for the cluster.  All logs from both Clusterio
+  and Factorio is stored in a shared log which can be inspected and queried.
 
 ### Features
 
@@ -46,11 +48,16 @@ Version 2.0.0
   player data.
 - Added synchronization of in-game adminlist, banlist and whitelist to
   Clusterio core.  Previously this was handled by the playerManager plugin.
+- Added stripping of long paths in the Factorio server log.
+- Added option to configure maximum number of commands to send in parallel.
+- Added option to auto start instances on slave startup.
+- Added metric mapping of slave and instance ids to their names, allowing
+  them to be displayed by name in queries that join with the mapping.
+- Added option to configure timeout for exported Prometheus metrics.
+- Added option to set address to bind HTTP(S) port to.
 
 ### Changes
 
-- Fixed sslCert and sslPrivKey entries being ignored in the config.
-- Changed ssl key creation to be done at startup instead of on npm install.
 - Added error handling during master startup.
 - Factorio game and rcon port now defaults to a random port above 49151.
 - Removed unimplemented mods update command.
@@ -172,6 +179,7 @@ Version 2.0.0
 - Removed usage of socket.io entirely in favor of a plain WebSocket connection.
 - Renamed clusterioMod plugin to subspace_storage.
 - Master no longer creates secret-api-token.txt.
+- Removed automatic creation of self-signed TLS certificate.
 
 
 Version 1.2.4

Dockerfile

@@ -1,23 +1,64 @@
-# Warning: This is currently broken as a result of
-# changes done in the 2.0 refactoring.
-FROM node:12
-RUN apt-get update && apt install git curl tar -y
-RUN mkdir factorioClusterio
+FROM node:12 as subspace_storage_builder
+RUN apt update && apt install -y git
+WORKDIR /
+RUN git clone https://github.com/clusterio/factorioClusterioMod.git
+WORKDIR /factorioClusterioMod
+RUN git checkout clusterio-2.0 \
+&& npm install \
+&& node build
 
-RUN git clone -b master https://github.com/clusterio/factorioClusterio.git && cd factorioClusterio && npm install --only=production
-RUN cd factorioClusterio && curl -o factorio.tar.gz -L https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz
+FROM node:12 as clusterio_builder
+RUN apt update \
+&& apt install -y wget \
+&& mkdir /clusterio
+WORKDIR /clusterio
+RUN wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
+# Copy project files into the container
+COPY . .
 
-WORKDIR factorioClusterio
-RUN mkdir instances sharedMods
-RUN cp config.json.dist config.json
+RUN npm install \
+&& npx lerna bootstrap --hoist
 
-RUN node client manage shared mods add clusterio
+# Install plugins. This is intended as a reasonable default, enabling plugins to make for fun gameplay.
+# If you want a different set of plugins, consider using this as the base image for your own.
+#RUN npm install @clusterio/plugin-subspace_storage
+#RUN npx clusteriomaster plugin add @clusterio/plugin-subspace_storage
 
-LABEL maintainer "Sir3lit@gmail.com"
+COPY --from=subspace_storage_builder /factorioClusterioMod/dist/ /clusterio/sharedMods/
 
-EXPOSE 8080 34167
-VOLUME /factorioClusterio/instances
-VOLUME /factorioClusterio/sharedMods
-VOLUME /factorioClusterio/sharedPlugins
+# Build submodules (web UI, libraries, plugins etc)
+RUN npx lerna run build
+# Build Lua library mod
+RUN node packages/lib/build_mod --build --source-dir packages/slave/lua/clusterio_lib \
+&& mv dist/* sharedMods/ \
+&& mkdir temp \
+&& mkdir temp/test \
+&& cp sharedMods/ temp/test/ -r \
+&& ls sharedMods
 
-CMD MODE="$MODE" node $MODE start $INSTANCE
+# Remove node_modules
+RUN find . -name 'node_modules' -type d -prune -print -exec rm -rf '{}' \;
+
+FROM frolvlad/alpine-glibc AS clusterio_final
+
+RUN apk add --update bash nodejs npm
+
+COPY --from=clusterio_builder /clusterio /clusterio
+WORKDIR /clusterio
+
+# Install runtime dependencies
+RUN npm install --production
+RUN npx lerna bootstrap -- --production --no-optional
+LABEL maintainer "danielv@danielv.no"
+
+FROM frolvlad/alpine-glibc AS clusterio_testing
+
+RUN apk add --update bash nodejs npm
+
+COPY --from=clusterio_builder /clusterio /clusterio
+WORKDIR /clusterio
+
+# Install runtime dependencies
+RUN npm install
+RUN npx lerna bootstrap
+RUN npm install chalk semver

README.md

@@ -80,7 +80,7 @@ into a Clusterio cluster.
 
 ## Plugins
 
-The heart of Clusterio is its plugin syste.  Plugins add functionality
+The heart of Clusterio is its plugin system.  Plugins add functionality
 to Factorio servers, Clusterio itself or both.  These are the plugins
 supported and maintained by the Clusterio developers:
 
@@ -206,21 +206,21 @@ with either . or .. (which will then be resolved to an absolute path).
 
 ## Configure Master Server
 
-By default the master server will create a self signed TLS certificate
-and listen on HTTPS on port 8443.  You can change the port used with the
-command
+By default the master server will listen for HTTP on port 8080.  You can
+change the port used with the command
 
-    npx clusteriomaster config set master.https_port 1234
+    npx clusteriomaster config set master.http_port 1234
 
 If you plan to make your cluster available externally set the address
 that it will be accessible under with, for example
 
-    npx clusteriomaster config set master.external_address https://192.0.2.4:8443/
+    npx clusteriomaster config set master.external_address http://203.0.113.4:1234/
 
 Change the url to reflect the IP, protocol, and port the master server
-is accessible under, dns names are also supported.  If proxyed under a
-sub-path include the sub path as well and make sure the URL ends with a
-slash.
+is accessible under, dns names are also supported.  If you're planning
+on making the master server accessible on the internet it's recommended
+to set up TLS, see the [Setting Up TLS](/docs/setting-up-tls.md)
+document for more details.
 
 You can list the config of the master server with the `npx
 clusteriomaster config list` command.  See the [readme for
@@ -228,43 +228,6 @@ clusteriomaster config list` command.  See the [readme for
 information.
 
 
-### Hosting Behind Proxy
-
-If you want to host the master server behind a reverse proxy the
-recommended setup is to disable HTTPS and host it on a high port
-over HTTP to which you point your proxy to.  For example to configure
-port 8080 as the HTTP port for the master server.
-
-    npx clusteriomaster config set master.https_port # unsets https_port
-    npx clusteriomaster config set master.http_port 8080
-
-The proxy can then forward requests to a sub-path like `/cluster/` to
-`http://localhost:8080/`.
-
-Additianlly WebSocket connections to api/socket relative to the sub-path
-the interface is hosted under also needs to be forwarded.  See below for
-the specifics of setting this up on Apache.
-
-
-#### Apache Config
-
-To proxy with Apache make sure that proxy_module, proxy_http_module and
-proxy_wstunnel_module is loaded in the config, then add the following
-directives to wherever is the approriate place for your flavour of
-Apache config organization.
-
-```apache
-<Location /cluster/>
-    ProxyPass "http://localhost:8080/"
-</location>
-<Location /cluster/api/socket>
-    ProxyPass "ws://localhost:8080/api/socket"
-</Location>
-```
-
-Pay attention to the use of trailing slashes, they are important.
-
-
 ### Setting up an admin account
 
 Before you can manage the cluster you need to bootstrap an admin account

docker-compose.test.yml

@@ -0,0 +1,7 @@
+version: '3'
+
+services:
+  unit_test:
+    image: clusterio
+    build: .
+    command: npm run test

docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3'
+
+services:
+  master:
+    build: .
+    command: npx clusteriomaster run
+  slave:
+    build: .
+    command: npx clusterioslave run

docs/config-system.md

@@ -28,6 +28,16 @@ after the groupName have been set and before the class has been
 finalized.  The `define` method takes an object as argument with the
 following properties:
 
+**access**:
+    The locations this config field can be read and modified from.  This
+    is an array of strings which can contain the values `"master"`,
+    `"slave"` and `"control"`.  If a party is missing in this array then
+    that party will not receive the value of this field when the config
+    is shared, and will be unable to modify the field.  Takes the value
+    of `defaultAccess` set on the config group class if not passed.
+    Note that instance config groups should have both `"master"` and
+    `"slave"` for all of its fields or unexpected behaviour may occur.
+
 **type**:
     The type of config value this field will support.  The supported
     values are boolean, string, number, and object, and if an attept is

docs/configuration.md

@@ -5,6 +5,14 @@ Configuration
 Master Configuration
 --------------------
 
+### master.name
+
+Name of the cluster.  Used to distinguish it from other clusters and is
+prepended to the name of servers in the server list.
+
+Defaluts to "Your Cluster".
+
+
 ### master.database_directory
 
 Directory used to store cluster data to.  Needs to be writeable by the
@@ -19,7 +27,7 @@ Port to host HTTP server on.  If set to null no HTTP server will be
 exposed.  At least one of this and master.https_port needs to be set to
 a port.
 
-Defaults to null.
+Defaults to 8080.
 
 
 ### master.https_port
@@ -29,7 +37,21 @@ master.tls_private_key as the certificate and private key for the HTTPS
 server.  If set to null no HTTPS server will be exposed.  At least one
 of this and master.http_port needs to be set to a port.
 
-Defaults to 8443.
+See the [Setting up TLS](/docs/setting-up-tls.md) document for a guide
+to setting up HTTPS with Clusterio.
+
+Defaults to null.
+
+
+### master.bind_address
+
+IP address to bind the HTTP and HTTPS ports on.  Useful to limit which
+interface to accept connections from.  If set to null the unspecified
+IPv6 address will be used, which on most systems means it will also
+listen on the unspecified IPv4 address and accept connections from all
+interfaces.
+
+Defaults to null.
 
 
 ### master.external_address
@@ -43,29 +65,20 @@ Defaults to null meaning assume localhost.
 
 ### master.tls_certificate
 
-TLS certificate to use for the HTTPS server when master.https_port is
-configured.  If neither this nor the configured master.tls_private_key
-exists a self signed certificate is automatically created.
+Path to TLS certificate to use for the HTTPS server when
+master.https_port is configured.
 
 Defaults to "database/certificates/cert.crt".
 
 
 ### master.tls_private_key
 
-TLS private key to use for the HTTPS server when master.https_port is
-configured.  If neither this nor the configured master.tls_certificate
-exists a self signed certificate is automatically created.
+Path to TLS private key to use for the HTTPS server when
+master.https_port is configured.
 
 Defaults to "database/certificates/cert.key".
 
 
-### master.tls_bits
-
-Number of bits to use when creating a self signed certificate.
-
-Defaults to 2048.
-
-
 ### master.auth_secret
 
 Base64 encoded authentication secret used to verify tokens issued to
@@ -84,7 +97,7 @@ heartbeat interval it will be considered stale and closed.  A lower
 value means less time between connections going stale and them being
 closed.
 
-Defaults to 60
+Defaults to 45
 
 
 ### master.connector_shutdown_timeout
@@ -101,6 +114,15 @@ unavoidable.
 Defaults to 30.
 
 
+### master.metrics_timeout
+
+Timeout in seconds before a call to gather metrics from a slave times
+out.  This should be less than both the timeout and collection interval
+configured for the collection job in Prometheus.
+
+Defaults to 8.
+
+
 ### master.default_role_id
 
 Role to automatically grant to new users.  If null no role is granted.
@@ -129,7 +151,7 @@ Defaults to "New Slave".
 
 ### slave.id
 
-Imutable numeric id of the slave which uniquely identifies this slave in
+Immutable numeric id of the slave which uniquely identifies this slave in
 the cluster.
 
 Defaults to a random 31 bit number.
@@ -158,7 +180,7 @@ Defaults to "instances".
 
 URL to connect to the master server to.
 
-Defaults to "https://localhost:8443/".
+Defaults to "http://localhost:8080/".
 
 
 ### slave.master_token
@@ -171,6 +193,17 @@ create a new slave config with the correct url and token in it.
 Defaults to "enter token here".
 
 
+### slave.tls_ca
+
+Path to certificate in PEM format to use for validating a TLS connection
+to the master server.  If you have a self signed certificate on the
+master server you will need to copy the certificate to your slaves and
+set it as the certificate authority with this option.
+
+Defaults to null meaning use Node.js's default set of trusted
+certificate authorities.
+
+
 ### slave.public_address
 
 External address instances hosted on this slave can be accessed on.
@@ -191,7 +224,7 @@ Defaults to 5.
 
 
 Instance Configuration
-======================
+----------------------
 
 ### instance.name
 
@@ -218,6 +251,15 @@ directory for the instance is created on the slave.
 Defaults to null meaning not assigned.
 
 
+### instance.auto_start
+
+If enabled start this instance when the slave it is assigned to is
+started up.  Does not affect start or stop of the instance while the
+slave is running.
+
+Defaults to false
+
+
 ### factorio.version
 
 Version of Factorio to use for this instance.  Can be a version like
@@ -277,9 +319,45 @@ Factorio server.  The settings in `data/server-settings.example.json` of
 the Factorio installation is used as the base and properties specified
 here overrides the properties there.
 
+Changes to the following properties will be applied live if the instance
+is running while it is changed: `afk_autokick_interval`,
+`allow_commands`, `autosave_interval`, `autosave_only_on_server`,
+`description`, `ignore_player_limit_for_returning_players`,
+`max_players`, `max_upload_slots`, `max_upload_in_kilobytes_per_second`,
+`name`, `only_admins_can_pause_the_game`, `game_password`,
+`require_user_verification`, `tags`, `visibility`.
+
 Defaults to {"tags":["clusterio"],"auto_pause":false}.
 
 
+### factorio.verbose_logging
+
+Pass the `--verbose` flag to Factorio when starting the server.  This
+prints more messages to the log.
+
+Defaults to false
+
+
+### factorio.strip_paths
+
+Strip down absolute paths in the server log going to files inside the
+instance directory such that they are relative to the instance
+directory.  This improves the signal to noise ratio of the log.
+
+Sample output when disabled:
+
+    0.487 Loading map /srv/clusterio/instances/test/saves/world.zip: 2615888 bytes.
+    0.614 Checksum for script /srv/clusterio/instances/test/temp/currently-playing/control.lua: 2390553941
+    1.277 Script @/srv/clusterio/instances/test/temp/currently-playing/modules/example/test.lua:7: Example log line.
+
+Sample output when enabled:
+
+    0.487 Loading map saves/world.zip: 2615888 bytes.
+    0.614 Checksum for script temp/currently-playing/control.lua: 2390553941
+    1.277 Script @modules/example/test.lua:7: Example log line.
+
+Defaults to true
+
 ### factorio.sync_adminlist
 
 Synchronize in-game admin list with admin status of the users in
@@ -314,6 +392,39 @@ banned status while the instance is running.
 Defaults to true
 
 
+### factorio.max_concurrent_commands
+
+Maximum number of commands being transmitted into the game in parallel.
+Since the rate at which long commands are streamed into the game is by
+default limeted to 100 bytes/tick and scales down to 25 bytes/tick when
+there's 20 players connected, long commands can hold up the command
+interface and cause large latencies in getting data pushed into the
+game. (The rates are controlled by the `segment_size` options in the
+server settings.)  To counteract this Clusterio sends up to this
+configured value number of commands in parallel over the RCON interface.
+This causes the game updates sent to clients to have this many command
+streams in parallel contained in them, drastically increasing the
+maxiumum size they can become.  Since commands are only processed so
+ofter the maximum rate of commands that can be sustained is roughly 30
+times this configured value per second, though note that large commands
+will lower this value.
+
+The game updates sent by the server is split into roughly 500 byte
+packets, and if any one of those parts are lost the entire update is
+resent.  This means a high number of concurrent commands can amplify
+resends due to packet loss and degrade the player experience up to the
+point of becomming completely unplayable.
+
+Prior to 0.18.7 the maximum practical size of game update sent where
+arround 8 kB due to the UDP receive buffer on windows defaulting to this
+value, larger updates would not get through the receive buffer and cause
+the connection to drop.  To stay below the 8 kB limit the maximum safe
+value for this option would be around 7000 / (3 * game_speed *
+maximum_segment_size), which is around 20 in normal circumstances.
+
+Defaults to 5
+
+
 ### <plugin_name>.enabled
 
 Whether to enable the given plugin on the instance.  Note that plugins
@@ -321,3 +432,44 @@ need to be enabled on the master server in order for them to be loaded
 on instances.
 
 Defaults to true
+
+
+Control Configuration
+---------------------
+
+### control.master_url
+
+URL to connect to the master server to.
+
+Defaults to null meaning complain about it not being set and exit.
+
+
+### control.master_token
+
+Access token used for authenticating with the master server.  You can
+generate an access token with `clusteriomaster bootstrap
+generate-user-token <username>`, or use the `clusteriomaster bootstrap
+create-ctl-config <username>` to create a new ctl config with the
+correct url and token in it.
+
+Defaults to null meaning complain about it not being set and exit.
+
+
+### control.tls_ca
+
+Path to certificate in PEM format to use for validating a TLS connection
+to the master server.  If you have a self signed certificate on the
+master server you will need to copy the certificate to the computer you
+run clusterctl from and set it as the certificate authority with this
+option.
+
+Defaults to null meaning use Node.js's default set of trusted
+certificate authorities.
+
+
+### control.reconnect_delay
+
+Duration in seconds to wait before attempting to reconnect with the
+master server after the connection is dropped.
+
+Defaults to 2.

docs/contributing.md

@@ -223,8 +223,8 @@ should be named the same as the class in PascalCase.
 Imported modules from the lib package are prefixed with lib to make them
 easier to distinguish, as the names tend to be very generic.  E.g.:
 
-    const libLink = require("lib/link");
-    const libLuaTools = require("lib/lua_tools");
+    const libLink = require("@clusterio/lib/link");
+    const libLuaTools = require("@clusterio/lib/lua_tools");
 
 
 For lua code lowercase_underscore is used for everything.

docs/developing-for-clusterio.md

@@ -161,11 +161,13 @@ which accepts a table and calls `add_lib` for each value in the table.
 Communicating with Clusterio
 ----------------------------
 
-There's a poorly to virtuarly undocumented HTTP interface that can be
-used to communicate with Clusterio to do things like get the status of
-instances, grab items from the store, run commands on instances, etc,
-etc.  For now you can read the source code, and maybe ask on the Discord
-for some pointers if you get stuck.
+Clusterio uses a homebrew protocol based on sending JSON payloads over
+a WebSocket connection.  See [socket.md](socket.md) for the
+implementation details of it.
+
+It's also possible to write a plugin for Clusterio that exposes a custom
+interface over HTTP, WebSocket or any other technology supported by
+Node.js.
 
 
 Clusterio Modules
@@ -177,7 +179,9 @@ modules that are loaded into the game if you don't need the capabilites
 of the plugin system.  The save patcher puts modules into the `modules`
 folder of the Factorio save and adds code to `control.lua` to load the
 module according to the `load` and `require` options to the module.json
-file.
+file.  Like with scenarios for Clusterio the [event_handler
+interface](#event_handler-interface) has to be used for any event
+subscriptions in modules.
 
 Stand alone modules are placed into the modules folder of Clusterio,
 plugin modules are located in the module folder of the plugin.  In

docs/how-it-works.md

@@ -27,4 +27,4 @@ that log into instances under its control.  These accounts are used to
 store per player data and authorize management of the cluster.  By
 default users accounts are assigned a Player role that gives a limited
 read access to the cluster resources, and admins of the cluster can be
-assigned an Admin role to get access to manage it.
+assigned a Cluster Admin role to get access to manage it.

docs/managing-a-cluster.md

@@ -107,7 +107,7 @@ Replaces the roles the user has in the cluster with a new list of roles.
 Calling with an empty roles argument will remove all roles from the
 user.
 
-By default there's a roled named Admin which grants access to
+By default there's a roled named Cluster Admin which grants access to
 everything and a role named Player which grants a limited read access to
 the cluster, see [the section on roles](#roles) for more information
 about setting up roles and permissions.

docs/setting-up-tls.md

@@ -0,0 +1,112 @@
+Setting up TLS
+==============
+
+By default Clusterio will listen for connections over HTTP which is fine
+for local networks but over the Internet it should idealy be over HTTPS
+as there are some credentials that could be exposed to eavesdroppers.
+The recommended setup is to have an ordinary HTTPS server software like
+Apache or Nginx to act as a reverse proxy in front of the Clusterio
+master server.  The reverse proxy would then be set up with a TLS
+certificate from a Certificate Authoritiy like [Let's
+Encrypt](https://letsencrypt.org).  Alternatively you can have Clusterio
+listen for connections over HTTPS directly by providing it with a TLS
+certificate and private key.
+
+Note that a TLS certificate signed by a recognized Certificate Authority
+in general requires that you own a domain.  See the last section for how
+to create and use a self-signed certificate for an IP address if you
+don't have a domain.
+
+
+Using a reverse proxy
+---------------------
+
+Configure the proxy to forward requests over HTTP to the port the master
+server is listening to so that for example a request to
+`https://example.com/instances/` is forwarded to
+`http://localhost:8080/instances/`.  It is possible to host the
+interface under a sub-path, in that case a request under the sub-path
+should be mapped to the master server with the sub-path stripped out.
+For example `https://example.com/sub/path/instances/` would be mapped to
+`http://localhost:8080/instances`.
+
+Additianlly WebSocket connections to `/api/socket` also needs to be
+forwarded.  Typically this requires explicit configuration and support
+for forwarding WebSocket connections.  If the interface hosted under a
+sub-path the `/api/socket` address is relative to the sub-path and the
+sub-path also needs to be stripped out when forwarded.
+
+You can also optionally set the master server to listen only on the
+loopback interface to prevent it being directly connected to from the
+outside.
+
+    npx clusteriomaster config set master.bind_address 127.0.0.1
+
+
+### Apache Config
+
+To proxy with Apache make sure that `proxy_module`, `proxy_http_module`
+and `proxy_wstunnel_module` is loaded in the config, then add the
+following directives to wherever is the approriate place for your
+flavour of Apache config organization.
+
+```apache
+<Location /sub/path/>
+    ProxyPass "http://localhost:8080/"
+</location>
+<Location /sub/path/api/socket>
+    ProxyPass "ws://localhost:8080/api/socket"
+</Location>
+```
+
+Replace `/sub/path/` with the location on the proxy you want to host the
+Clusterio interface under.  If the whole domain is to be dedicated to
+the Clusterio interface then `/sub/path/` should be just `/`.  Pay
+attention to the use of trailing slashes, as they are important.
+
+
+Hosting HTTPS directly from Node.js
+-----------------------------------
+
+Listening on HTTPS is supported by the Clusterio master server via the
+`master.https_port` option provided paths to a certificate file and a
+private key file is configured for the `master.tls_certificate` and
+`master.tls_private_key` options.  Both of these are expected to be
+unencrypted PEM files.
+
+
+Creating and using a self-signed certificate
+--------------------------------------------
+
+Create a self-signed certificate with OpenSSL by running
+
+    openssl req -x509 -newkey rsa:2048 -nodes -keyout key.pem -out cert.pem -days 3650 -subj /CN=localhost -addext "subjectAltName=IP:<ip>"
+
+substituting `<ip>` with the IP address the server will be accessed
+under, for example `IP:203.0.113.4`.  Multiple addresses can be given by
+supplying a comma sepparated list of `IP:address` pairs and DNS names
+can also be specified using the `DNS:example.com` format.  You need to
+supply every name you want to connect to the server under the
+subjectAltName field, for example to connect to the HTTPS port locally
+via an address like `https://localhost:4443/` you need to add
+`DNS:localhost`.
+
+This creates a private key file named `key.pem` and a public certificate
+file named `cert.pem` and these are used for the
+`master.tls_certificate` and `master.tls_private_key` config options
+respectively on the master server:
+
+    npx clusteriomaster config set master.tls_certificate cert.pem
+    npx clusteriomaster config set master.tls_private_key key.pem
+
+Additionally you will need copy the `cert.pem` file to all of the
+computers you want to set up as slaves and configure it as the
+`slave.tls_ca` option:
+
+    npx clusterioslave config set slave.tls_ca cert.pem
+
+You will also need to copy the `cert.pem` file to all of the computers
+you want to remotely manage the cluster via clusterctl and configure it
+as `control.tls_ca` option:
+
+    npx clusterioctl control-config set control.tls_ca cert.pem

docs/writing-plugins.md

@@ -11,6 +11,7 @@ Contents
 
 - [Plugin Structure](#plugin-structure)
 - [Defining the plugin class](#defining-the-plugin-class)
+- [Logging Messages](#logging-messages)
 - [Plugin Configuration](#plugin-configuration)
 - [Communicating with Factorio](#communicating-with-factorio)
 - [Defining Link Messages](#defining-link-messages)
@@ -170,12 +171,66 @@ result in unpredicatable behaviour.  The async init method is always called
 immediatly after the constructor, so there's little reason to do this.
 
 
-Defining Configuration
-----------------------
+Logging Messages
+----------------
+
+The base plugin classes provide a winston logger for logging messages to
+the shared cluster log.  For instances a copy of the log is also stored
+on the slave the instance is on.  To use it, pass a string to one of the
+log levels functions, for example:
+
+    async init() {
+        this.logger.info("Initializing frobbing");
+    }
+
+Metadata covering which plugin and instance (for instance plugin
+classes) the log event happened on will automatically be attached to the
+log message.
+
+The available levels are fatal, error, warn, audit, info, and verbose.
+These levels have roughly the following meanings:
+
+- fatal: Unrecoverable error that prevents continuing execution of
+  the Node.js process.  Note that Plugins shouldn't have fatal errors.
+- error: An unexpected error condition occured, usually an exception
+  thrown when it wasn't expected.  Should be investigated and fixed.
+- warn: An potential error was detected, but operation continued
+  gracefully.  These may indicate that something is running
+  sub-optimally, or is not working as intedend.
+- audit: Record of actions performed by users of the cluster.  This
+  is usually emitted in response to management requests by a control
+  link.
+- info: A general log message.  May indicae a status or operation
+  completed.  This should not be emitted by periodic tasks that run
+  often.
+- verbose: A verbose log message which might be useful when
+  investigating issues, does not show by default.  Note that logging
+  messages is not a cheap operation even when the messages do not show
+  up and spammy logging should be avoided for the verbose level.
+  If the logs are still valuable in certain cases consider putting them
+  behind a config option.
+
+For guarding unexpected errors the best option is to log a short
+description along with the`stack` property of the Error:
+
+    try {
+        // Operation that should not throw but may end up throwing
+    } catch (err) {
+        this.logger.error(`Operation failed:\n${err.stack}`);
+    )
+
+For plugin hooks expections thrown are automatically catched and logged,
+but for event handlers registered on EventEmitters it's critical that
+exceptions are catched and handled appropriately.
+
+
+Plugin Configuration
+--------------------
 
 Clusterio provides a configuration system that handles storing,
 distributing, editing and validating config fields for you.  You can
-take advantage of it by subclassing `PluginConfigGroup`, setting the
+take advantage of it by subclassing `PluginConfigGroup`, setting
+`defaultAccess` to where config entries can be accessed from, setting the
 `groupName` to your plugin name, defining fields on it, finalizing it,
 and passing it as either `MasterConfigGroup` or `InstanceConfigGroup` in
 the `info.js` export.  For example in info.js:
@@ -183,6 +238,7 @@ the `info.js` export.  For example in info.js:
     const libConfig = require("@clusterio/lib/config");
 
     class MasterConfigGroup extends libConfig.PluginConfigGroup { }
+    MasterConfigGroup.defaultAccess = ["master", "slave", "control"];
     MasterConfigGroup.groupName = "foo_frobber";
     MasterConfigGroup.define({
         name: "level",
@@ -203,11 +259,11 @@ for example in the MasterPluginClass:
 
     async init() {
         let level = this.master.config.get("foo_frobber.level");
-        console.log("I got a frobnication level of", level);
+        this.logger.info(`I got a frobnication level of ${level}`);
     }
 
 The same applies for instance configs, replace "master" with "instance"
-where appropriate.  See [Configuration System](configuration-system.md)
+where appropriate.  See [Configuration System](config-system.md)
 for more details on how this system works.
 
 
@@ -230,11 +286,11 @@ Communicating with Factorio
 
 For pushing data into Factorio there's RCON, which lets you send
 arbitrary Lua commands to invoke whatever code you want in the game.
-This is done by calling the `sendRcon` method on the instance's server object.
+This is done by calling the `sendRcon` method on the plugin object.
 For example:
 
     async onStart() {
-        let response = await this.instance.server.sendRcon(
+        let response = await this.sendRcon(
             "/sc rcon.print('data')"
         );
 
@@ -258,8 +314,8 @@ data sent by `send_json`.  For example in the plugin code:
 
     async init() {
         this.instance.server.on("ipc-my_plugin_foo", content =>
-            this.handleFoo(content).catch(err => console.log(
-                "Error handling foo:", foo
+            this.handleFoo(content).catch(err => this.logger.error(
+                `Error handling foo:\n${err.stack}`
             ))
         );
     }
@@ -327,8 +383,10 @@ the event, for example the following could be defined in `info.js`:
             type: "foo_frobber:start_frobnication",
             links: ["master-slave", "slave-instance"],
             forwardTo: "instance",
+            eventRequired: ["frobnication_type"],
             eventProperties: {
                 "frobnication_type": { type: "string" },
+                "urgent": { type: "boolean" },
             },
         }),
     },
@@ -336,8 +394,8 @@ the event, for example the following could be defined in `info.js`:
 This specifies an event that can be sent from the master to a slave,
 and from a slave to an instance.  It also specifies that the event must
 contain the property `frobnication_type`, with a string value in the
-data payload.  It will also be forwarded by slaves to a specific
-instance.
+data payload and that it may optionally contain a boolean `urgent`
+property.  It will also be forwarded by slaves to a specific instance.
 
 The following properties are recognized by the Event constructor:
 
@@ -380,12 +438,19 @@ it's sent to, but not back from where it came from. Currently, only
 that slave except for the instance it came from.  from an instance will
 cause it to be broadcast to all instances of
 
+#### eventRequired
+
+By default all properties defined in `eventProperties` are required to
+be present in the payload for the event.  This can be overriden by
+specifying an array of properties which are required here.  This is
+equivalent to using the `required` keyword in JSON schema.
+
 #### eventProperties
 
 Object with properties mapping to a JSON schema of that property that
 specifies what's valid to send in the event.  This is equivalent to
 using the `properties` keyword in JSON schema, except that the
-properties specified are implicitly required and additional properties
+properties specified are by default required and additional properties
 are not allowed.  See [this guide][guide] for an introduction to writing
 JSON schemas.
 
@@ -412,8 +477,10 @@ the event. For example, the following could be defined in `info.js`:
             type: "foo_frobber:report_frobnication",
             links: ["master-slave", "slave-instance"],
             forwardTo: "instance",
+            requestRequired: ["verbosity"],
             requestProperties: {
                 "verbosity": { type: "integer" },
+                "special": { type: "boolean" },
             },
             responseProperties: {
                 "report": {
@@ -427,7 +494,8 @@ the event. For example, the following could be defined in `info.js`:
 This specifies a request that can be sent from the master to a slave,
 and from a slave to an instance.  The request data must contain the
 property `verbosity` with an integer number as the value, as well as the
-`instance_id` property (implied by `forwardTo: "instance"`), and the
+`instance_id` property (implied by `forwardTo: "instance"`) and it may
+also contain a boolean `special` property.  It also defines that the
 response sent must contain a `report` property mapping to an array of
 strings.  When received by a slave, it will also be forwarded to the
 instance specified by `instance_id`.
@@ -466,15 +534,27 @@ instance, or `"instance"` to indicate it should be forwarded to the
 instances specified by the `instance_id` request property.  This works
 by using a default handler for the request by the links that forward it.
 
+#### requestRequired
+
+By default all properties defined in `requestProperties` are required to
+be present in the payload for the request.  This can be overriden by
+specifying an array of properties which are required here.  This is
+equivalent to using the `required` keyword in JSON schema.
+
 #### requestProperties
 
 Object with properties mapping to a JSON schema of that property that
 specifies what's valid to send in the request.  This is equivalent to
 using the `properties` keyword in JSON schema, except that the
-properties specified are implicitly required and additional properties
+properties specified are by default required and additional properties
 are not allowed.  See [this guide][guide] for an introduction to writing
 JSON schemas
 
+#### responseRequired
+
+Same as the requestRequired only for the response sent back by the
+target.
+
 #### responseProperties
 
 Same as the requestProperties only for the response sent back by the
@@ -543,7 +623,7 @@ until the connection is re-established.  This means that if your plugin
 sends a lot of events or requests, they can end up being queued up in a
 buffer for a long time and sent out all at once.  To avoid this you
 should be throtteling and/or stopping your requests/events after `drop`
-has been raised, and continue back as normal when `connect` is raised.
+has been raised, and continue back as normal when `resume` is raised.
 
 
 Collecting Statistics
@@ -582,7 +662,7 @@ set it accordingly, for example:
     );
 
     // Somewhere in the instance plugin code
-    barMetric.labels(String(this.instance.config.get("instance.id"))).set(someValue);
+    barMetric.labels(String(this.instance.id)).set(someValue);
 
 Metrics are automatically registered to the default registry, and this
 default registry is automatically polled by the master server on slaves.

lerna.json

@@ -3,5 +3,5 @@
     "packages/*",
     "plugins/*"
   ],
-  "version": "2.0.0-alpha.2"
+  "version": "2.0.0-alpha.4"
 }

package.json

@@ -2,7 +2,7 @@
   "name": "root",
   "private": true,
   "scripts": {
-    "test": "mocha --exit --check-leaks -R spec --recursive test plugins/*/test",
+    "test": "mocha --exit --check-leaks -R spec --recursive test plugins/global_chat/test plugins/research_sync/test plugins/statistics_exporter/test plugins/subspace_storage/test",
     "fast-test": "FAST_TEST=y npm test",
     "cover": "nyc npm test",
     "fast-cover": "FAST_TEST=y nyc npm test",

packages/ctl/ctl.js

@@ -9,9 +9,10 @@ const fs = require("fs-extra");
 const yargs = require("yargs");
 const version = require("./package").version;
 const asTable = require("as-table").configure({ delimiter: " | " });
-const chalk = require("chalk");
 const events = require("events");
 const path = require("path");
+const winston = require("winston");
+const setBlocking = require("set-blocking");
 
 const libLink = require("@clusterio/lib/link");
 const libErrors = require("@clusterio/lib/errors");
@@ -20,52 +21,95 @@ const libPlugin = require("@clusterio/lib/plugin");
 const libPluginLoader = require("@clusterio/lib/plugin_loader");
 const libCommand = require("@clusterio/lib/command");
 const libSharedCommands = require("@clusterio/lib/shared_commands");
+const { ConsoleTransport, levels, logger } = require("@clusterio/lib/logging");
+const libLoggingUtils = require("@clusterio/lib/logging_utils");
+const libHelpers = require("@clusterio/lib/helpers");
 
 
-/**
- * Format a parsed Factorio output message with colors
- *
- * Formats a parsed Factorio output from lib/factorio into a readable
- * colorized output using terminal escape codes that can be printed.
- *
- * @param {Object} output - Factorio server output.
- * @returns {string} terminal colorized message.
- * @private
- */
-function formatOutputColored(output) {
-	let time = "";
-	if (output.format === "seconds") {
-		time = chalk.yellow(output.time.padStart(8)) + " ";
-	} else if (output.format === "date") {
-		time = chalk.yellow(output.time) + " ";
-	}
-
-	let info = "";
-	if (output.type === "log") {
-		let level = output.level;
-		if (level === "Info") {
-			level = chalk.bold.blueBright(level);
-		} else if (output.level === "Warning") {
-			level = chalk.bold.yellowBright(level);
-		} else if (output.level === "Error") {
-			level = chalk.bold.redBright(level);
-		}
-
-		info = level + " " + chalk.gray(output.file) + ": ";
-
-	} else if (output.type === "action") {
-		info = "[" + chalk.yellow(output.action) + "] ";
-	}
-
-	return time + info + output.message;
+function print(...content) {
+	// eslint-disable-next-line no-console
+	console.log(...content);
 }
 
+
+const masterCommands = new libCommand.CommandTree({ name: "master", description: "Master management" });
+const masterConfigCommands = new libCommand.CommandTree({
+	name: "config", alias: ["c"], description: "master config management",
+});
+masterConfigCommands.add(new libCommand.Command({
+	definition: ["list", "List master configuration"],
+	handler: async function(args, control) {
+		let response = await libLink.messages.getMasterConfig.send(control);
+
+		for (let group of response.serialized_config.groups) {
+			for (let [name, value] of Object.entries(group.fields)) {
+				print(`${group.name}.${name} ${JSON.stringify(value)}`);
+			}
+		}
+	},
+}));
+
+masterConfigCommands.add(new libCommand.Command({
+	definition: ["set <field> [value]", "Set field in master config", (yargs) => {
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("value", { describe: "Value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args, control) {
+		if (args.stdin) {
+			args.value = (await libHelpers.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		} else if (args.value === undefined) {
+			args.value = "";
+		}
+		await libLink.messages.setMasterConfigField.send(control, {
+			field: args.field,
+			value: args.value,
+		});
+	},
+}));
+
+masterConfigCommands.add(new libCommand.Command({
+	definition: ["set-prop <field> <prop> [value]", "Set property of field in master config", (yargs) => {
+		yargs.positional("field", { describe: "Field to set", type: "string" });
+		yargs.positional("prop", { describe: "Property to set", type: "string" });
+		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
+	}],
+	handler: async function(args, control) {
+		if (args.stdin) {
+			args.value = (await libHelpers.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		}
+		let request = {
+			field: args.field,
+			prop: args.prop,
+		};
+		try {
+			if (args.value !== undefined) {
+				request.value = JSON.parse(args.value);
+			}
+		} catch (err) {
+			// See note for the instance version of set-prop
+			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value)) {
+				throw new libErrors.CommandError(`In parsing value '${args.value}': ${err.message}`);
+			}
+			request.value = args.value;
+		}
+		await libLink.messages.setMasterConfigProp.send(control, request);
+	},
+}));
+masterCommands.add(masterConfigCommands);
+
+
 const slaveCommands = new libCommand.CommandTree({ name: "slave", description: "Slave management" });
 slaveCommands.add(new libCommand.Command({
 	definition: [["list", "l"], "List slaves connected to the master"],
 	handler: async function(args, control) {
 		let response = await libLink.messages.listSlaves.send(control);
-		console.log(asTable(response.list));
+		print(asTable(response.list));
 	},
 }));
 
@@ -75,7 +119,7 @@ slaveCommands.add(new libCommand.Command({
 	}],
 	handler: async function(args, control) {
 		let response = await libLink.messages.generateSlaveToken.send(control, { slave_id: args.id });
-		console.log(response.token);
+		print(response.token);
 	},
 }));
 
@@ -97,9 +141,9 @@ slaveCommands.add(new libCommand.Command({
 
 		let content = JSON.stringify(response.serialized_config, null, 4);
 		if (args.output === "-") {
-			console.log(content);
+			print(content);
 		} else {
-			console.log(`Writing ${args.output}`);
+			logger.info(`Writing ${args.output}`);
 			try {
 				await fs.outputFile(args.output, content, { flag: "wx" });
 			} catch (err) {
@@ -120,7 +164,7 @@ instanceCommands.add(new libCommand.Command({
 	definition: [["list", "l"], "List instances known to the master"],
 	handler: async function(args, control) {
 		let response = await libLink.messages.listInstances.send(control);
-		console.log(asTable(response.list));
+		print(asTable(response.list));
 	},
 }));
 
@@ -133,13 +177,13 @@ instanceCommands.add(new libCommand.Command({
 		});
 	}],
 	handler: async function(args, control) {
-		let instanceConfig = new libConfig.InstanceConfig();
+		let instanceConfig = new libConfig.InstanceConfig("control");
 		await instanceConfig.init();
 		if (args.id) {
 			instanceConfig.set("instance.id", args.id);
 		}
 		instanceConfig.set("instance.name", args.name);
-		let serialized_config = instanceConfig.serialize();
+		let serialized_config = instanceConfig.serialize("master");
 		let response = await libLink.messages.createInstance.send(control, { serialized_config });
 	},
 }));
@@ -157,20 +201,28 @@ instanceConfigCommands.add(new libCommand.Command({
 
 		for (let group of response.serialized_config.groups) {
 			for (let [name, value] of Object.entries(group.fields)) {
-				console.log(`${group.name}.${name} ${JSON.stringify(value)}`);
+				print(`${group.name}.${name} ${JSON.stringify(value)}`);
 			}
 		}
 	},
 }));
 
 instanceConfigCommands.add(new libCommand.Command({
-	definition: ["set <instance> <field> <value>", "Set field in instance config", (yargs) => {
+	definition: ["set <instance> <field> [value]", "Set field in instance config", (yargs) => {
 		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
 		yargs.positional("field", { describe: "Field to set", type: "string" });
 		yargs.positional("value", { describe: "Value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
 	}],
 	handler: async function(args, control) {
 		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		if (args.stdin) {
+			args.value = (await libHelpers.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		} else if (args.value === undefined) {
+			args.value = "";
+		}
 		await libLink.messages.setInstanceConfigField.send(control, {
 			instance_id: instanceId,
 			field: args.field,
@@ -180,32 +232,61 @@ instanceConfigCommands.add(new libCommand.Command({
 }));
 
 instanceConfigCommands.add(new libCommand.Command({
-	definition: ["set-prop <instance> <field> <prop> <value>", "Set property of field in instance config", (yargs) => {
+	definition: ["set-prop <instance> <field> <prop> [value]", "Set property of field in instance config", (yargs) => {
 		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
 		yargs.positional("field", { describe: "Field to set", type: "string" });
 		yargs.positional("prop", { describe: "Property to set", type: "string" });
 		yargs.positional("value", { describe: "JSON parsed value to set", type: "string" });
+		yargs.options({
+			"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+		});
 	}],
 	handler: async function(args, control) {
 		let instanceId = await libCommand.resolveInstance(control, args.instance);
-		await libLink.messages.setInstanceConfigProp.send(control, {
+		if (args.stdin) {
+			args.value = (await libHelpers.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+		}
+		let request = {
 			instance_id: instanceId,
 			field: args.field,
 			prop: args.prop,
-			value: JSON.parse(args.value),
-		});
+		};
+		try {
+			if (args.value !== undefined) {
+				request.value = JSON.parse(args.value);
+			}
+		} catch (err) {
+			// If this is from stdin or looks like an array, object or string
+			// literal throw the parse error, otherwise assume this is a string.
+			// The resoning behind this is that correctly quoting the string
+			// with the all the layers of quote removal at play is difficult.
+			// See the following table for how to pass "That's a \" quote" in
+			// different environments:
+			// cmd              : """""That's a \\"" quote"""""
+			// cmd + npx        : """""""""""That's a \\\\"""" quote"""""""""""
+			// PowerShell       : '"""""That''s a \\"" quote"""""'
+			// PowerShell + npx : '"""""""""""That''s a \\\\"""" quote"""""""""""'
+			// bash             : '""That'\''s a \" quote""'
+			// bash + npx       : '""That'\''s a \" quote""'
+			// bash + npx -s sh : "'\"\"That'\\''s a \\\" quote\"\"'"
+			if (args.stdin || /^(\[.*]|{.*}|".*")$/.test(args.value)) {
+				throw new libErrors.CommandError(`In parsing value '${args.value}': ${err.message}`);
+			}
+			request.value = args.value;
+		}
+		await libLink.messages.setInstanceConfigProp.send(control, request);
 	},
 }));
 instanceCommands.add(instanceConfigCommands);
 
 instanceCommands.add(new libCommand.Command({
-	definition: ["assign <instance> <slave>", "Assign instance to a slave", (yargs) => {
+	definition: ["assign <instance> [slave]", "Assign instance to a slave", (yargs) => {
 		yargs.positional("instance", { describe: "Instance to assign", type: "string" });
-		yargs.positional("slave", { describe: "Slave to assign to", type: "string" });
+		yargs.positional("slave", { describe: "Slave to assign to or unassign if none", type: "string" });
 	}],
 	handler: async function(args, control) {
 		let instanceId = await libCommand.resolveInstance(control, args.instance);
-		let slaveId = await libCommand.resolveSlave(control, args.slave);
+		let slaveId = args.slave ? await libCommand.resolveSlave(control, args.slave) : null;
 		await libLink.messages.assignInstanceCommand.send(control, {
 			instance_id: instanceId,
 			slave_id: slaveId,
@@ -219,7 +300,7 @@ instanceCommands.add(new libCommand.Command({
 	}],
 	handler: async function(args, control) {
 		let instanceId = await libCommand.resolveInstance(control, args.instance);
-		await libLink.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
+		await control.setLogSubscriptions({ instance_ids: [instanceId] });
 		let response = await libLink.messages.createSave.send(control, {
 			instance_id: instanceId,
 		});
@@ -232,7 +313,7 @@ instanceCommands.add(new libCommand.Command({
 	}],
 	handler: async function(args, control) {
 		let instanceId = await libCommand.resolveInstance(control, args.instance);
-		await libLink.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
+		await control.setLogSubscriptions({ instance_ids: [instanceId] });
 		let response = await libLink.messages.exportData.send(control, {
 			instance_id: instanceId,
 		});
@@ -249,11 +330,12 @@ instanceCommands.add(new libCommand.Command({
 	}],
 	handler: async function(args, control) {
 		let instanceId = await libCommand.resolveInstance(control, args.instance);
-		await libLink.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
+		await control.setLogSubscriptions({ instance_ids: [instanceId] });
 		let response = await libLink.messages.startInstance.send(control, {
 			instance_id: instanceId,
 			save: args.save || null,
 		});
+		control.keepOpen = args.keepOpen;
 	},
 }));
 
@@ -267,11 +349,12 @@ instanceCommands.add(new libCommand.Command({
 	}],
 	handler: async function(args, control) {
 		let instanceId = await libCommand.resolveInstance(control, args.instance);
-		await libLink.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
+		await control.setLogSubscriptions({ instance_ids: [instanceId] });
 		let response = await libLink.messages.loadScenario.send(control, {
 			instance_id: instanceId,
 			scenario: args.scenario || null,
 		});
+		control.keepOpen = args.keepOpen;
 	},
 }));
 
@@ -281,7 +364,7 @@ instanceCommands.add(new libCommand.Command({
 	}],
 	handler: async function(args, control) {
 		let instanceId = await libCommand.resolveInstance(control, args.instance);
-		await libLink.messages.setInstanceOutputSubscriptions.send(control, { instance_ids: [instanceId] });
+		await control.setLogSubscriptions({ instance_ids: [instanceId] });
 		let response = await libLink.messages.stopInstance.send(control, {
 			instance_id: instanceId,
 		});
@@ -320,7 +403,7 @@ permissionCommands.add(new libCommand.Command({
 	definition: [["list", "l"], "List permissions in the cluster"],
 	handler: async function(args, control) {
 		let response = await libLink.messages.listPermissions.send(control);
-		console.log(asTable(response.list));
+		print(asTable(response.list));
 	},
 }));
 
@@ -330,7 +413,7 @@ roleCommands.add(new libCommand.Command({
 	definition: [["list", "l"], "List roles in the cluster"],
 	handler: async function(args, control) {
 		let response = await libLink.messages.listRoles.send(control);
-		console.log(asTable(response.list));
+		print(asTable(response.list));
 	},
 }));
 
@@ -348,7 +431,7 @@ roleCommands.add(new libCommand.Command({
 			description: args.description,
 			permissions: args.permissions,
 		});
-		console.log(`Created role ID ${response.id}`);
+		logger.info(`Created role ID ${response.id}`);
 	},
 }));
 
@@ -358,7 +441,11 @@ roleCommands.add(new libCommand.Command({
 		yargs.options({
 			"name": { describe: "New name for role", nargs: 1, type: "string" },
 			"description": { describe: "New description for role", nargs: 1, type: "string" },
-			"permissions": { describe: "New permissions for role", array: true, type: "string" },
+			"set-perms": { describe: "Set permissions for role", array: true, type: "string" },
+			"add-perms": { describe: "Add permissions to role", array: true, type: "string", conflicts: "set-perms" },
+			"remove-perms": {
+				describe: "Remove permissions from role", array: true, type: "string", conflicts: "set-perms",
+			},
 			"grant-default": { describe: "Add default permissions to role", nargs: 0, type: "boolean" },
 		});
 	}],
@@ -371,8 +458,18 @@ roleCommands.add(new libCommand.Command({
 		if (args.description !== undefined) {
 			role.description = args.description;
 		}
-		if (args.permissions !== undefined) {
-			role.permissions = args.permissions;
+		if (args.addPerms) {
+			role.permissions = role.permissions.concat(args.addPerms);
+		}
+		if (args.removePerms) {
+			let perms = new Set(role.permissions);
+			for (let perm of args.removePerms) {
+				perms.delete(perm);
+			}
+			role.permissions = [...perms];
+		}
+		if (args.setPerms !== undefined) {
+			role.permissions = args.setPerms;
 		}
 		await libLink.messages.updateRole.send(control, role);
 
@@ -398,7 +495,7 @@ userCommands.add(new libCommand.Command({
 	definition: [["list", "l"], "List user in the cluster"],
 	handler: async function(args, control) {
 		let response = await libLink.messages.listUsers.send(control);
-		console.log(asTable(response.list));
+		print(asTable(response.list));
 	},
 }));
 
@@ -500,12 +597,75 @@ userCommands.add(new libCommand.Command({
 	},
 }));
 
+const logCommands = new libCommand.CommandTree({ name: "log", description: "Log inspection" });
+logCommands.add(new libCommand.Command({
+	definition: ["follow", "follow cluster log", (yargs) => {
+		yargs.options({
+			"all": { describe: "Follow the whole cluster log", nargs: 0, type: "boolean", default: false },
+			"master": { describe: "Follow log of the master server", nargs: 0, type: "boolean", default: false },
+			"slave": { describe: "Follow log of given slave", nargs: 1, type: "string", default: null },
+			"instance": { describe: "Follow log of given instance", nargs: 1, type: "string", default: null },
+		});
+	}],
+	handler: async function(args, control) {
+		if (!args.all && !args.master && !args.slave && !args.instance) {
+			logger.error("At least one of --all, --master, --slave and --instance must be passed");
+			process.exitCode = 1;
+			return;
+		}
+		let instance_ids = args.instance ? [await libCommand.resolveInstance(control, args.instance)] : [];
+		let slave_ids = args.slave ? [await libCommand.resolveSlave(control, args.slave)] : [];
+		await control.setLogSubscriptions({ all: args.all, master: args.master, slave_ids, instance_ids });
+		control.keepOpen = true;
+	},
+}));
+
+logCommands.add(new libCommand.Command({
+	definition: ["query", "Query cluster log", (yargs) => {
+		yargs.options({
+			"all": { describe: "Query the whole cluster log", nargs: 0, type: "boolean", default: false },
+			"master": { describe: "Query log of the master server", nargs: 0, type: "boolean", default: false },
+			"slave": { describe: "Query log of given slave", nargs: 1, type: "string", default: null },
+			"instance": { describe: "Query log of given instance", nargs: 1, type: "string", default: null },
+			"max-level": { describe: "Maximum log level to return", nargs: 1, type: "string", default: null },
+		});
+	}],
+	handler: async function(args, control) {
+		if (!args.all && !args.master && !args.slave && !args.instance) {
+			logger.error("At least one of --all, --master, --slave and --instance must be passed");
+			process.exitCode = 1;
+			return;
+		}
+		let instance_ids = args.instance ? [await libCommand.resolveInstance(control, args.instance)] : [];
+		let slave_ids = args.slave ? [await libCommand.resolveSlave(control, args.slave)] : [];
+		let result = await libLink.messages.queryLog.send(control, {
+			all: args.all,
+			master: args.master,
+			slave_ids,
+			instance_ids,
+			max_level: args.maxLevel,
+		});
+
+		let stdoutLogger = winston.createLogger({
+			level: "verbose",
+			levels,
+			format: new libLoggingUtils.TerminalFormat({ showTimestamp: true }),
+			transports: [
+				new ConsoleTransport({ errorLevels: [], warnLevels: [] }),
+			],
+		});
+		for (let info of result.log) {
+			stdoutLogger.log(info);
+		}
+	},
+}));
+
 const debugCommands = new libCommand.CommandTree({ name: "debug", description: "Debugging utilities" });
 debugCommands.add(new libCommand.Command({
 	definition: ["dump-ws", "Dump WebSocket messages sent and received by master", (yargs) => { }],
 	handler: async function(args, control) {
 		await libLink.messages.debugDumpWs.send(control);
-		return new Promise(() => {});
+		control.keepOpen = true;
 	},
 }));
 
@@ -515,13 +675,13 @@ debugCommands.add(new libCommand.Command({
  * @private
  */
 class ControlConnector extends libLink.WebSocketClientConnector {
-	constructor(url, reconnectDelay, token) {
-		super(url, reconnectDelay);
+	constructor(url, reconnectDelay, tlsCa, token) {
+		super(url, reconnectDelay, tlsCa);
 		this._token = token;
 	}
 
 	register() {
-		console.log("SOCKET | registering control");
+		logger.verbose("SOCKET | registering control");
 		this.sendHandshake("register_control", {
 			token: this._token,
 			agent: "clusterioctl",
@@ -548,17 +708,34 @@ class Control extends libLink.Link {
 		 */
 		this.plugins = controlPlugins;
 		for (let controlPlugin of controlPlugins.values()) {
-			libPlugin.attachPluginMessages(this, controlPlugin.info, controlPlugin);
+			libPlugin.attachPluginMessages(this, controlPlugin);
 		}
+
+		/**
+		 * Keep the control connection alive after the command completes.
+		 * @type {boolean}
+		 */
+		this.keepOpen = false;
 	}
 
-	async instanceOutputEventHandler(message) {
-		let { instance_id, output } = message.data;
-		console.log(formatOutputColored(output));
+	async setLogSubscriptions({
+		all = false,
+		master = false,
+		slave_ids = [],
+		instance_ids = [],
+		max_level = null,
+	}) {
+		await libLink.messages.setLogSubscriptions.send(this, {
+			all, master, slave_ids, instance_ids, max_level,
+		});
+	}
+
+	async logMessageEventHandler(message) {
+		logger.log(message.data.info);
 	}
 
 	async debugWsMessageEventHandler(message) {
-		console.log("WS", message.data.direction, message.data.content);
+		print("WS", message.data.direction, message.data.content);
 	}
 
 	async shutdown() {
@@ -576,11 +753,60 @@ class Control extends libLink.Link {
 	}
 }
 
+async function loadPlugins(pluginList) {
+	let pluginInfos = await libPluginLoader.loadPluginInfos(pluginList);
+	libConfig.registerPluginConfigGroups(pluginInfos);
+	libConfig.finalizeConfigs();
+
+	let controlPlugins = new Map();
+	for (let pluginInfo of pluginInfos) {
+		if (!pluginInfo.controlEntrypoint) {
+			continue;
+		}
+
+		let ControlPluginClass = await libPluginLoader.loadControlPluginClass(pluginInfo);
+		let controlPlugin = new ControlPluginClass(pluginInfo, logger);
+		controlPlugins.set(pluginInfo.name, controlPlugin);
+		await controlPlugin.init();
+	}
+	return controlPlugins;
+}
+
+async function registerCommands(controlPlugins, yargs) {
+	const rootCommands = new libCommand.CommandTree({ name: "clusterioctl", description: "Manage cluster" });
+	rootCommands.add(masterCommands);
+	rootCommands.add(slaveCommands);
+	rootCommands.add(instanceCommands);
+	rootCommands.add(permissionCommands);
+	rootCommands.add(roleCommands);
+	rootCommands.add(userCommands);
+	rootCommands.add(logCommands);
+	rootCommands.add(debugCommands);
+
+	for (let controlPlugin of controlPlugins.values()) {
+		await controlPlugin.addCommands(rootCommands);
+	}
+
+	for (let [name, command] of rootCommands.subCommands) {
+		if (name === command.name) {
+			command.register(yargs);
+		}
+	}
+
+	return rootCommands;
+}
 
 async function startControl() {
 	yargs
 		.scriptName("clusterioctl")
 		.usage("$0 <command> [options]")
+		.option("log-level", {
+			nargs: 1,
+			describe: "Log level to print to stderr",
+			default: "server",
+			choices: Object.keys(levels),
+			type: "string",
+		})
 		.option("config", {
 			nargs: 1,
 			describe: "config file to get credentails from",
@@ -601,9 +827,16 @@ async function startControl() {
 	;
 
 	// Parse the args first to get the configured plugin list.
-	let args = yargs.argv;
+	let args = yargs.parse();
 
-	console.log(`Loading available plugins from ${args.pluginList}`);
+	// Log stream for the ctl session.
+	logger.add(new ConsoleTransport({
+		errorLevels: Object.keys(levels),
+		level: args.logLevel,
+		format: new libLoggingUtils.TerminalFormat(),
+	}));
+
+	logger.verbose(`Loading available plugins from ${args.pluginList}`);
 	let pluginList = new Map();
 	try {
 		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
@@ -619,54 +852,27 @@ async function startControl() {
 		return;
 	}
 
+	logger.verbose("Loading Plugins");
+	let controlPlugins = await loadPlugins(pluginList);
+
 	// Add all cluster management commands including ones from plugins
-	const rootCommands = new libCommand.CommandTree({ name: "clusterioctl", description: "Manage cluster" });
-	rootCommands.add(slaveCommands);
-	rootCommands.add(instanceCommands);
-	rootCommands.add(permissionCommands);
-	rootCommands.add(roleCommands);
-	rootCommands.add(userCommands);
-	rootCommands.add(debugCommands);
-
-	console.log("Loading Plugin info");
-	let pluginInfos = await libPluginLoader.loadPluginInfos(pluginList);
-	libConfig.registerPluginConfigGroups(pluginInfos);
-	libConfig.finalizeConfigs();
-
-	let controlPlugins = new Map();
-	for (let pluginInfo of pluginInfos) {
-		if (!pluginInfo.controlEntrypoint) {
-			continue;
-		}
-
-		let { ControlPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.controlEntrypoint));
-		let controlPlugin = new ControlPlugin(pluginInfo);
-		controlPlugins.set(pluginInfo.name, controlPlugin);
-		await controlPlugin.init();
-		await controlPlugin.addCommands(rootCommands);
-	}
-
-	for (let [name, command] of rootCommands.subCommands) {
-		if (name === command.name) {
-			command.register(yargs);
-		}
-	}
+	let rootCommands = await registerCommands(controlPlugins, yargs);
 
 	// Reparse after commands have been added with help and strict checking.
 	args = yargs
 		.help()
 		.strict()
-		.argv
+		.parse()
 	;
 
-	console.log(`Loading config from ${args.config}`);
-	let controlConfig = new libConfig.ControlConfig();
+	logger.verbose(`Loading config from ${args.config}`);
+	let controlConfig = new libConfig.ControlConfig("control");
 	try {
 		await controlConfig.load(JSON.parse(await fs.readFile(args.config)));
 
 	} catch (err) {
 		if (err.code === "ENOENT") {
-			console.log("Config not found, initializing new config");
+			logger.verbose("Config not found, initializing new config");
 			await controlConfig.init();
 
 		} else {
@@ -694,20 +900,27 @@ async function startControl() {
 
 	// The remaining commands require connecting to the master server.
 	if (!controlConfig.get("control.master_url") || !controlConfig.get("control.master_token")) {
-		console.error("Missing URL and/or token to connect with.  See README.md for setting up access.");
+		logger.error("Missing URL and/or token to connect with.  See README.md for setting up access.");
 		process.exitCode = 1;
 		return;
 	}
 
+	let tlsCa = null;
+	let tlsCaPath = controlConfig.get("control.tls_ca");
+	if (tlsCaPath) {
+		tlsCa = await fs.readFile(tlsCaPath);
+	}
+
 	let controlConnector = new ControlConnector(
 		controlConfig.get("control.master_url"),
 		controlConfig.get("control.reconnect_delay"),
+		tlsCa,
 		controlConfig.get("control.master_token")
 	);
 	let control = new Control(controlConnector, controlPlugins);
 	try {
 		await controlConnector.connect();
-	} catch(err) {
+	} catch (err) {
 		if (err instanceof libErrors.AuthenticationFailed) {
 			throw new libErrors.StartupError(err.message);
 		}
@@ -715,25 +928,26 @@ async function startControl() {
 	}
 
 	process.on("SIGINT", () => {
-		console.log("Caught interrupt signal, closing connection");
+		logger.info("Caught interrupt signal, closing connection");
 		control.shutdown().catch(err => {
-			console.error(err);
+			setBlocking(true);
+			logger.error(err.stack);
+			// eslint-disable-next-line no-process-exit
 			process.exit(1);
 		});
 	});
 
-	let keepOpen = Boolean(args.keepOpen);
 	try {
 		await targetCommand.run(args, control);
 
 	} catch (err) {
-		keepOpen = false;
+		control.keepOpen = false;
 		if (err instanceof libErrors.CommandError) {
-			console.error(`Error running command: ${err.message}`);
+			logger.error(`Error running command: ${err.message}`);
 			process.exitCode = 1;
 
 		} else if (err instanceof libErrors.RequestError) {
-			console.error(`Error sending request: ${err.message}`);
+			logger.error(`Error sending request: ${err.message}`);
 			process.exitCode = 1;
 
 		} else {
@@ -741,7 +955,7 @@ async function startControl() {
 		}
 
 	} finally {
-		if (!keepOpen) {
+		if (!control.keepOpen) {
 			await control.shutdown();
 		}
 	}
@@ -749,13 +963,11 @@ async function startControl() {
 
 module.exports = {
 	Control,
-
-	// for testing only
-	_formatOutputColored: formatOutputColored,
 };
 
 
 if (module === require.main) {
+	// eslint-disable-next-line no-console
 	console.warn(`
 +==========================================================+
 I WARNING:  This is the development branch for the 2.0     I
@@ -765,21 +977,22 @@ I           version of clusterio.  Expect things to break. I
 	);
 	startControl().catch(err => {
 		if (!(err instanceof libErrors.StartupError)) {
-			console.error(`
+			logger.fatal(`
 +----------------------------------------------------------------+
 | Unexpected error occured while starting control, please report |
 | it to https://github.com/clusterio/factorioClusterio/issues    |
-+----------------------------------------------------------------+`
++----------------------------------------------------------------+
+${err.stack}`
 			);
 		} else {
-			console.error(`
+			logger.error(`
 +---------------------------------+
 | Unable to to start clusterioctl |
-+---------------------------------+`
++---------------------------------+
+${err.stack}`
 			);
 		}
 
-		console.error(err);
 		process.exitCode = 1;
 	});
 }

packages/ctl/package.json

@@ -3,7 +3,7 @@
   "description": "Clusterio command line management tool",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.4",
   "main": "ctl.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -18,10 +18,11 @@
     "node": ">=10.16 <11 || >= 12"
   },
   "dependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.2",
+    "@clusterio/lib": "^2.0.0-alpha.4",
     "as-table": "^1.0.31",
     "chalk": "^3.0.0",
     "fs-extra": "^8.1.0",
+    "winston": "^3.3.3",
     "yargs": "^14.2.0"
   },
   "publishConfig": {

packages/lib/README.md

@@ -5,4 +5,4 @@ This package contains functionality shared between master, slave, ctl
 and/or plugins of Clusterio.
 
 See [the Clusterio repository](https://github.com/clusterio/factorioClusterio)
-for more inforation on Clusterio.
+for more information on Clusterio.

packages/lib/RateLimiter.js

@@ -0,0 +1,104 @@
+"use strict";
+
+
+/**
+ * Helper class for rate limiting actions
+ *
+ * Provides a simple interface for implementing actions that are rate
+ * limited on invocations.
+ *
+ * @example
+ * const rateLimiter = RateLimiter({
+ *     action: () => { console.log("action"); },
+ *     maxRate: 1,
+ * });
+ *
+ * for (let i = 0; i < 100; i++) {
+ *     rateLimiter.activate();
+ *     // Prints action twice to the console, once for the first activation
+ *     // and one more time 1 second later for 99 other activations that were
+ *     // exceeding the rate limit.
+ * }
+ * @alias module:lib/RateLimiter
+ */
+class RateLimiter {
+	/**
+	 * Construct RateLimiter helper
+	 *
+	 * The options passed may also be modified on the instance at runtime.
+	 *
+	 * @param {object=} options - Options for this rate limiter
+	 * @param {number=} options.maxRate -
+	 *     Maximum rate of activations per second.
+	 * @param {function()=} options.action -
+	 *     Callback invoked at maste `maxRate` per second on activation.
+	 */
+	constructor(options = {}) {
+		/**
+		 * Maximum rate of activations per second.
+		 * @type {number}
+		 */
+		this.maxRate = options.maxRate || 1;
+
+		/**
+		 * Callback invoked at most `maxRate` per second on activation.
+		 * @type {function()}
+		 */
+		this.action = options.action || null;
+
+		this._lastRun = 0;
+		this._runTimeout = null;
+		this.lastProgressBroadcast = Date.now();
+	}
+
+	/**
+	 * Activate the rate limited action
+	 *
+	 * Checks if the invocation of the rate limited action now would fall
+	 * within the rate limit.  If it is within the limit then the action is
+	 * invoked immediatly, otherwise a timeout is registered to invoke the
+	 * action at next point in time where it will be within the rate limit.
+	 *
+	 * @returns {boolean} true if activation was within the rate limit.
+	 */
+	activate() {
+		let now = Date.now();
+		if (now < this._lastRun + 1000 / this.maxRate) {
+			if (!this._runTimeout && this.action !== null) {
+				this._runTimeout = setTimeout(() => {
+					this._runTimeout = null;
+					this._lastRun = Date.now();
+					this.action();
+				}, this._lastRun + 1000 / this.maxRate - now);
+			}
+			return false;
+		}
+
+		// We may end up activating before the timeout does.
+		if (this._runTimeout) {
+			clearTimeout(this._runTimeout);
+			this._runTimeout = null;
+		}
+
+		this._lastRun = now;
+		if (this.action !== null) {
+			this.action();
+		}
+		return true;
+	}
+
+	/**
+	 * Cancel pending action
+	 *
+	 * Cancel pending action invocation.  This is necessary when shutting
+	 * down the use of the rate limit as a timeout is registered to invoke
+	 * the next action when activated faster than the rate limit allows.
+	 */
+	cancel() {
+		if (this._runTimeout) {
+			clearTimeout(this._runTimeout);
+		}
+	}
+}
+
+module.exports = RateLimiter;

packages/lib/build_mod.js

@@ -1,3 +1,4 @@
+/* eslint-disable no-console */
 "use strict";
 const events = require("events");
 const fs = require("fs-extra");
@@ -7,34 +8,64 @@ const path = require("path");
 const yargs = require("yargs");
 
 
-async function main() {
-	const args = yargs
-		.scriptName("build")
-		.options({
-			"clean": { describe: "Remove previous builds", type: "boolean", default: false },
-			"build": { describe: "Build mod", type: "boolean", default: true },
-			"pack": { describe: "Pack into zip file", type: "boolean", default: true },
-			"source-dir": { describe: "Path to mod source files", nargs: 1, type: "string", demandOption: true },
-			"output-dir": { describe: "Path to output built mod", nargs: 1, type: "string", default: "dist" },
-			"bump-patch": { describe: "Increment patch number of build", type: "boolean", default: false },
-			"factorio-version": { describe: "Override factorio_version", type: "string" },
-		})
-		.strict()
-		.argv
-	;
+async function buildMod(args, info) {
+	if (args.build) {
+		await fs.ensureDir(args.outputDir);
+		let modName = `${info.name}_${info.version}`;
 
-	await build(args);
+		if (args.pack) {
+			let zip = new JSZip();
+			let walker = klaw(args.sourceDir)
+				.on("data", item => {
+					if (item.stats.isFile()) {
+						// On Windows the path created uses backslashes as the directory sepparator
+						// but the zip file needs to use forward slashes.  We can't use the posix
+						// version of relative here as it doesn't work with Windows style paths.
+						let basePath = path.relative(args.sourceDir, item.path).replace(/\\/g, "/");
+						zip.file(path.posix.join(modName, basePath), fs.createReadStream(item.path));
+					}
+				})
+			;
+			await events.once(walker, "end");
+
+			for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) {
+				let filePath = path.join(args.sourceDir, ...pathParts);
+				if (!await fs.pathExists(filePath)) {
+					throw new Error(`Additional file ${filePath} does not exist`);
+				}
+				zip.file(path.posix.join(modName, fileName), fs.createReadStream(filePath));
+			}
+			delete info.additional_files;
+
+			zip.file(path.posix.join(modName, "info.json"), JSON.stringify(info, null, 4));
+
+			let modPath = path.join(args.outputDir, `${modName}.zip`);
+			console.log(`Writing ${modPath}`);
+			let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath));
+			await events.once(writeStream, "finish");
+
+		} else {
+			let modDir = path.join(args.outputDir, modName);
+			if (await fs.exists(modDir)) {
+				console.log(`Removing existing build ${modDir}`);
+				await fs.remove(modDir);
+			}
+			console.log(`Building ${modDir}`);
+			await fs.copy(args.sourceDir, modDir);
+			for (let [fileName, pathParts] of Object.entries(info.additional_files) || []) {
+				let filePath = path.join(...pathParts);
+				await fs.copy(filePath, path.join(modDir, fileName));
+			}
+			delete info.additional_files;
+
+			await fs.writeFile(path.join(modDir, "info.json"), JSON.stringify(info, null, 4));
+		}
+	}
 }
 
 async function build(args) {
 	let info = JSON.parse(await fs.readFile(path.join(args.sourceDir, "info.json")));
 
-	if (args.bumpPatch) {
-		let [major, minor, patch] = info.version.split(".");
-		patch = String(Number.parseInt(patch, 10) + 1);
-		info.version = [major, minor, patch].join(".");
-	}
-
 	if (args.factorioVersion) {
 		info.factorio_version = args.factorioVersion;
 	}
@@ -58,60 +89,39 @@ async function build(args) {
 		}
 	}
 
-	if (args.build) {
-		await fs.ensureDir(args.outputDir);
-		let modName = `${info.name}_${info.version}`;
-
-		if (args.pack) {
-			let zip = new JSZip();
-			let walker = klaw(args.sourceDir)
-				.on("data", item => {
-					if (item.stats.isFile()) {
-						// On Windows the path created uses backslashes as the directory sepparator
-						// but the zip file needs to use forward slashes.  We can't use the posix
-						// version of relative here as it doesn't work with Windows style paths.
-						let basePath = path.relative(args.sourceDir, item.path).replace(/\\/g, "/");
-						zip.file(path.posix.join(modName, basePath), fs.createReadStream(item.path));
-					}
-				})
-			;
-			await events.once(walker, "end");
-
-			for (let pathParts of info.additional_files || []) {
-				let filePath = path.join(args.sourceDir, ...pathParts);
-				if (!await fs.pathExists(filePath)) {
-					throw new Error(`Additional file ${filePath} does not exist`);
-				}
-				zip.file(path.posix.join(modName, path.basename(filePath)), fs.createReadStream(filePath));
-			}
-			delete info.additional_files;
-
-			zip.file(path.posix.join(modName, "info.json"), JSON.stringify(info, null, 4));
-
-			let modPath = path.join(args.outputDir, `${modName}.zip`);
-			console.log(`Writing ${modPath}`);
-			let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath));
-			await events.once(writeStream, "finish");
-
-		} else {
-			let modDir = path.join(args.outputDir, modName);
-			if (await fs.exists(modDir)) {
-				console.log(`Removing existing build ${modDir}`);
-				await fs.remove(modDir);
-			}
-			console.log(`Building ${modDir}`);
-			await fs.copy(args.sourceDir, modDir);
-			for (let pathParts of info.additional_files || []) {
-				let filePath = path.join(...pathParts);
-				await fs.copy(filePath, path.join(modDir, path.basename(filePath)));
-			}
-			delete info.additional_files;
-
-			await fs.writeFile(path.join(modDir, "info.json"), JSON.stringify(info, null, 4));
+	if (info.variants) {
+		for (let [variant, variantOverrides] of Object.entries(info.variants)) {
+			let variantInfo = {
+				...info,
+				...variantOverrides,
+			};
+			delete variantInfo.variants;
+			await buildMod(args, variantInfo);
 		}
+	} else {
+		await buildMod(args, info);
 	}
 }
 
+
+async function main() {
+	const args = yargs
+		.scriptName("build")
+		.options({
+			"clean": { describe: "Remove previous builds", type: "boolean", default: false },
+			"build": { describe: "Build mod(s)", type: "boolean", default: true },
+			"pack": { describe: "Pack into zip file", type: "boolean", default: true },
+			"source-dir": { describe: "Path to mod source files", nargs: 1, type: "string", demandOption: true },
+			"output-dir": { describe: "Path to output built mod(s)", nargs: 1, type: "string", default: "dist" },
+			"factorio-version": { describe: "Override factorio_version", type: "string" },
+		})
+		.strict()
+		.argv
+	;
+
+	await build(args);
+}
+
 if (module === require.main) {
 	main().catch(err => { console.log(err); });
 }

packages/lib/config/classes.js

@@ -7,6 +7,17 @@ const isDeepStrictEqual = require("../is_deep_strict_equal");
 const { basicType } = require("@clusterio/lib/helpers");
 
 
+/**
+ * Invalid Access exception
+ *
+ * Exception class for when an access attempt is made to a field that
+ * does not permitt this access is done.
+ *
+ * @extends Error
+ * @memberof module:lib/config
+ */
+class InvalidAccess extends Error { };
+
 /**
  * Invalid Value exception
  *
@@ -69,13 +80,24 @@ function splitOn(separator, string) {
  * @memberof module:lib/config
  */
 class Config extends events.EventEmitter {
-	constructor() {
+	/**
+	 * Create a new instance of the given config
+	 *
+	 * @param {string} location -
+	 *     Location to evaluate access for this instance from
+	 */
+	constructor(location) {
 		super();
 
 		if (!this.constructor._finalized) {
 			throw new Error(`Cannot instantiate incomplete Config class ${this.constructor.name}`);
 		}
 
+		if (typeof location !== "string") {
+			throw new Error("location must be a string");
+		}
+
+		this.location = location;
 		this._groups = new Map();
 		this._unknownGroups = new Map();
 	}
@@ -83,8 +105,8 @@ class Config extends events.EventEmitter {
 	async init() {
 		for (let [name, GroupClass] of this.constructor._groups) {
 			if (!this._groups.has(name)) {
-				let group = new GroupClass();
-				await group.init(this);
+				let group = new GroupClass(this);
+				await group.init();
 				this._groups.set(name, group);
 			}
 		}
@@ -101,7 +123,13 @@ class Config extends events.EventEmitter {
 		}
 	}
 
-	async load(serializedConfig) {
+	/**
+	 * Load config from a serialized object
+	 *
+	 * @param {object} serializedConfig - Serialized config to load.
+	 * @param {string=} location - Location used for access control.
+	 */
+	async load(serializedConfig, location = this.location) {
 		this._validate(serializedConfig);
 		for (let serializedGroup of serializedConfig.groups) {
 			let GroupClass = this.constructor._groups.get(serializedGroup.name);
@@ -110,8 +138,8 @@ class Config extends events.EventEmitter {
 				continue;
 			}
 
-			let group = new GroupClass();
-			await group.load(this, serializedGroup);
+			let group = new GroupClass(this);
+			await group.load(serializedGroup, location);
 			this._groups.set(group.name, group);
 		}
 
@@ -121,12 +149,13 @@ class Config extends events.EventEmitter {
 	/**
 	 * Serialize the config to a plain JavaScript object
 	 *
+	 * @param {string=} location - Location used for access control.
 	 * @returns {Object} JSON serializable representation of the config.
 	 */
-	serialize() {
+	serialize(location = this.location) {
 		let groups = [...this._unknownGroups.values()];
 		for (let group of this._groups.values()) {
-			groups.push(group.serialize());
+			groups.push(group.serialize(location));
 		}
 
 		return { groups };
@@ -140,28 +169,42 @@ class Config extends events.EventEmitter {
 	 *
 	 * @param {Object} serializedConfig - Output from .serialize().
 	 * @param {boolean} notify - Invoke fieldChanged events if true.
+	 * @param {string=} location - Location used for access control.
 	 */
-	update(serializedConfig, notify) {
+	update(serializedConfig, notify, location = this.location) {
 		this._validate(serializedConfig);
 		for (let serializedGroup of serializedConfig.groups) {
 			let group = this._groups.get(serializedGroup.name);
 			if (group) {
-				group.update(serializedGroup, notify);
+				group.update(serializedGroup, notify, location);
 			} else {
 				this._unknownGroups.set(serializedGroup.name, serializedGroup);
 			}
 		}
 	}
 
+	/**
+	 * Check if field can be accessed from the given location
+	 *
+	 * @param {string} name - Name of the field to check.
+	 * @param {string=} location - Location used for access control.
+	 * @returns {boolean} true if field is accessible
+	 */
+	canAccess(name, location = this.location) {
+		let [group, field] = splitOn(".", name);
+		return this.group(group).canAccess(field, location);
+	}
+
 	/**
 	 * Get the value for a config field
 	 *
 	 * @param {string} name - Name of field to get.
+	 * @param {string=} location - Location used for access control.
 	 * @returns {*} the value for the field.
 	 */
-	get(name) {
+	get(name, location = this.location) {
 		let [group, field] = splitOn(".", name);
-		return this.group(group).get(field);
+		return this.group(group).get(field, location);
 	}
 
 	/**
@@ -169,10 +212,11 @@ class Config extends events.EventEmitter {
 	 *
 	 * @param {string} name - Name of field to set.
 	 * @param {*} value - the value to set on field
+	 * @param {string=} location - Location used for access control.
 	 */
-	set(name, value) {
+	set(name, value, location = this.location) {
 		let [group, field] = splitOn(".", name);
-		this.group(group).set(field, value);
+		this.group(group).set(field, value, location);
 	}
 
 	/**
@@ -181,10 +225,11 @@ class Config extends events.EventEmitter {
 	 * @param {string} name - Name of field to set property on.
 	 * @param {string} prop - Name of property to set on field.
 	 * @param {*} value - the value to set the property to.
+	 * @param {string=} location - Location used for access control.
 	 */
-	setProp(name, prop, value) {
+	setProp(name, prop, value, location = this.location) {
 		let [group, field] = splitOn(".", name);
-		this.group(group).setProp(field, prop, value);
+		this.group(group).setProp(field, prop, value, location);
 	}
 
 	/**
@@ -280,15 +325,42 @@ class ConfigGroup {
 	 *
 	 * After the creation of the new config group you have to call
 	 * either .init() or .load() on it in order to fully initialize it.
+	 *
+	 * @param {module:lib/config.Config} config -
+	 *     Parent config for this group instance.
 	 */
-	constructor() {
+	constructor(config) {
 		if (!this.constructor._finalized) {
 			throw new Error(`Cannot instantiate incomplete ConfigGroup class ${this.constructor.name}`);
 		}
 
 		this._fields = new Map();
 		this._unknownFields = new Map();
-		this._config = null;
+		this.config = config;
+	}
+
+	/**
+	 * Check access for a field
+	 *
+	 * @param {object} def - definition for for the field to check.
+	 * @param {string} remote - location of remote access to check for.
+	 * @param {boolean} error - throw on failure to pass check.
+	 * @throws {module:lib/config.InvalidAccess}
+	 *     if access is not granted and error is true.
+	 * @returns {boolean} true if access is granted
+	 * @private
+	 */
+	_checkAccess(def, remote, error) {
+		for (let loc of [remote, this.config.location]) {
+			if (!def.access.includes(loc)) {
+				if (error) {
+					throw new InvalidAccess(`Field '${def.name}' is not accessible from ${loc}`);
+				} else {
+					return false;
+				}
+			}
+		}
+		return true;
 	}
 
 	/**
@@ -296,16 +368,15 @@ class ConfigGroup {
 	 *
 	 * Computes and assigns the initial values for all fields of the
 	 * config group.
-	 *
-	 * @param {module:lib/config.Config} config - Config group belongs to.
 	 */
-	async init(config) {
-		this._config = config;
-
+	async init() {
 		for (let [name, def] of this.constructor._definitions) {
 			if (this._fields.has(name)) {
 				continue;
 			}
+			if (!def.access.includes(this.config.location)) {
+				continue;
+			}
 
 			let value = null;
 			if (typeof def.initial_value === "function") {
@@ -326,13 +397,13 @@ class ConfigGroup {
 	 * group and then initializes any possible missing fields from it to
 	 * their default values.
 	 *
-	 * @param {module:lib/config.Config} config - Config group belongs to.
 	 * @param {Object} serializedGroup -
 	 *     Result from a previous call to .serialize().
+	 * @param {string=} location - Location used for access control.
 	 */
-	async load(config, serializedGroup) {
-		this.update(serializedGroup, false);
-		await this.init(config);
+	async load(serializedGroup, location = this.config.location) {
+		this.update(serializedGroup, false, location);
+		await this.init();
 	}
 
 	/**
@@ -344,16 +415,34 @@ class ConfigGroup {
 		return this.constructor.groupName;
 	}
 
+	/**
+	 * Check if field can be accessed from the given location
+	 *
+	 * @param {string} name - Name of field to check.
+	 * @param {string=} location - Location used for access control.
+	 * @returns {boolean} true if field is accessible
+	 */
+	canAccess(name, location = this.config.location) {
+		let def = this.constructor._definitions.get(name);
+		if (!def) {
+			throw new InvalidField(`No field named '${name}'`);
+		}
+		return this._checkAccess(def, location, false);
+	}
+
 	/**
 	 * Get value for field
 	 *
 	 * @param {string} name - Name of field to get.
+	 * @param {string=} location - Location used for access control.
 	 * @returns {*} Value stored for the field.
 	 */
-	get(name) {
-		if (!this.constructor._definitions.has(name)) {
+	get(name, location = this.config.location) {
+		let def = this.constructor._definitions.get(name);
+		if (!def) {
 			throw new InvalidField(`No field named '${name}'`);
 		}
+		this._checkAccess(def, location, true);
 
 		return this._fields.get(name);
 	}
@@ -366,14 +455,16 @@ class ConfigGroup {
 	 *
 	 * @param {string} name - Name of field to set.
 	 * @param {*} value - Value to set for field.
+	 * @param {string=} location - Location used for access control.
 	 * @throws {module:lib/config.InvalidField} if field is not defined.
 	 * @throws {module:lib/config.InvalidValue} if value is not allowed for the field.
 	 */
-	set(name, value) {
+	set(name, value, location = this.config.location) {
 		let def = this.constructor._definitions.get(name);
 		if (!def) {
 			throw new InvalidField(`No field named '${name}'`);
 		}
+		this._checkAccess(def, location, true);
 
 		// Empty strings are treates as null
 		if (value === "") {
@@ -386,8 +477,11 @@ class ConfigGroup {
 
 		if (basicType(value) === "string") {
 			if (def.type === "boolean") {
-				if (value === "true") { value = true; }
-				else if (value === "false") { value = false; }
+				if (value === "true") {
+					value = true;
+				} else if (value === "false") {
+					value = false;
+				}
 
 			} else if (def.type === "number") {
 				let numberRegExp = /^[+-]?(Infinity|\d+\.(\d+)?([eE][+-]?\d+)?|\.?\d+([eE][+-]?\d+)?)$/;
@@ -415,8 +509,8 @@ class ConfigGroup {
 
 		let prev = this._fields.get(name);
 		this._fields.set(name, value);
-		if (this._config && !isDeepStrictEqual(value, prev)) {
-			this._config.emit("fieldChanged", this, name, prev);
+		if (this.config && !isDeepStrictEqual(value, prev)) {
+			this.config.emit("fieldChanged", this, name, prev);
 		}
 	}
 
@@ -429,14 +523,16 @@ class ConfigGroup {
 	 * @param {string} name - Name of field to set property on.
 	 * @param {string} prop - Name of property to set on field.
 	 * @param {*} value - the value to set the property to.
+	 * @param {string=} location - Location used for access control.
 	 * @throws {module:lib/config.InvalidField} if field is not defined.
 	 * @throws {module:lib/config.InvalidValue} if field is not an object.
 	 */
-	setProp(name, prop, value) {
+	setProp(name, prop, value, location = this.config.location) {
 		let def = this.constructor._definitions.get(name);
 		if (!def) {
 			throw new InvalidField(`No field named '${name}'`);
 		}
+		this._checkAccess(def, location, true);
 
 		if (def.type !== "object") {
 			throw new InvalidValue(`Cannot set property on non-object field '${name}'`);
@@ -445,10 +541,14 @@ class ConfigGroup {
 		let prev = this._fields.get(name);
 		let updated = {...prev || {}};
 
-		updated[prop] = value;
+		if (value !== undefined) {
+			updated[prop] = value;
+		} else {
+			delete updated[prop];
+		}
 		this._fields.set(name, updated);
-		if (this._config && !isDeepStrictEqual(updated, prev)) {
-			this._config.emit("fieldChanged", this, name, prev);
+		if (this.config && !isDeepStrictEqual(updated, prev)) {
+			this.config.emit("fieldChanged", this, name, prev);
 		}
 	}
 
@@ -461,8 +561,9 @@ class ConfigGroup {
 	 * @param {Object} serializedGroup -
 	 *     Result from a previous call to .serialize().
 	 * @param {boolean} notify - Invoke fieldChanged events if true.
+	 * @param {string=} location - Location used for access control.
 	 */
-	update(serializedGroup, notify) {
+	update(serializedGroup, notify, location = this.config.location) {
 		if (basicType(serializedGroup) !== "object") {
 			throw new Error(`Expected object, not ${basicType(serializedGroup)} for ConfigGroup`);
 		}
@@ -482,6 +583,10 @@ class ConfigGroup {
 				continue;
 			}
 
+			if (!this._checkAccess(def, location, false)) {
+				continue;
+			}
+
 			if (def.optional) {
 				if (value !== null && basicType(value) !== def.type) {
 					continue;
@@ -497,8 +602,8 @@ class ConfigGroup {
 
 			let prev = this._fields.get(name);
 			this._fields.set(name, value);
-			if (notify && this._config && !isDeepStrictEqual(value, prev)) {
-				this._config.emit("fieldChanged", this, name, prev);
+			if (notify && this.config && !isDeepStrictEqual(value, prev)) {
+				this.config.emit("fieldChanged", this, name, prev);
 			}
 		}
 	}
@@ -506,11 +611,17 @@ class ConfigGroup {
 	/**
 	 * Serialize the config to a plain JavaScript object
 	 *
+	 * @param {string=} location - Location used for access control.
 	 * @returns {Object} JSON serializable representation of the group.
 	 */
-	serialize() {
+	serialize(location = this.config.location) {
 		let fields = {};
 		for (let [name, value] of this._fields) {
+			let def = this.constructor._definitions.get(name);
+			if (!this._checkAccess(def, location, false)) {
+				continue;
+			}
+
 			fields[name] = value;
 		}
 
@@ -550,6 +661,9 @@ class ConfigGroup {
 	 * Define a new configuration field
 	 *
 	 * @param {Object} item - Field definition.
+	 * @param {Array<string>=} item.access -
+	 *     Array of strings declaring where this config entry can be read
+	 *     and modified from.
 	 * @param {string} item.type -
 	 *     string declaring the type of the config value, supports
 	 *     boolean, string, number and object.
@@ -602,6 +716,12 @@ class ConfigGroup {
 					}
 					break;
 
+				case "access":
+					if (valueType !== "array") {
+						throw new Error("access must be an array");
+					}
+					break;
+
 				case "enum":
 					if (valueType !== "array") {
 						throw new Error("enum must be an array");
@@ -648,11 +768,16 @@ class ConfigGroup {
 		}
 
 		let computed = {
+			access: this.defaultAccess,
 			optional: false,
 			fullName: `${this.groupName}.${item.name}`,
 			...item,
 		};
 
+		if (!computed.access) {
+			throw new Error(`access not set and no defaultAccess defined for ${this.name}`);
+		}
+
 		if (!computed.optional && computed.initial_value === undefined) {
 			throw new Error(`Non-optional field ${item.name} needs an initial_value`);
 		}
@@ -699,6 +824,7 @@ class PluginConfigGroup extends ConfigGroup {
 
 
 module.exports = {
+	InvalidAccess,
 	InvalidField,
 	InvalidValue,
 

packages/lib/config/definitions.js

@@ -10,7 +10,15 @@ const classes = require("./classes");
  * @memberof module:lib/config
  */
 class MasterGroup extends classes.ConfigGroup { }
+MasterGroup.defaultAccess = ["master", "slave", "control"];
 MasterGroup.groupName = "master";
+MasterGroup.define({
+	name: "name",
+	title: "Name",
+	description: "Name of the cluster",
+	type: "string",
+	initial_value: "Your Cluster",
+});
 MasterGroup.define({
 	name: "database_directory",
 	title: "Database directory",
@@ -24,6 +32,7 @@ MasterGroup.define({
 	description: "Port to listen for HTTP connections on, set to null to not listen for HTTP connections.",
 	type: "number",
 	optional: true,
+	initial_value: 8080,
 });
 MasterGroup.define({
 	name: "https_port",
@@ -31,7 +40,13 @@ MasterGroup.define({
 	description: "Port to listen for HTTPS connection on, set to null to not listen for HTTPS connections.",
 	type: "number",
 	optional: true,
-	initial_value: 8443,
+});
+MasterGroup.define({
+	name: "bind_address",
+	title: "Bind Address",
+	description: "IP address to bind the HTTP and HTTPS ports on.",
+	type: "string",
+	optional: true,
 });
 MasterGroup.define({
 	name: "external_address",
@@ -46,7 +61,6 @@ MasterGroup.define({
 	description: "Path to the certificate to use for HTTPS.",
 	type: "string",
 	optional: true,
-	initial_value: "database/certificates/cert.crt",
 });
 MasterGroup.define({
 	name: "tls_private_key",
@@ -54,16 +68,9 @@ MasterGroup.define({
 	description: "Path to the private key to use for HTTPS.",
 	type: "string",
 	optional: true,
-	initial_value: "database/certificates/cert.key",
-});
-MasterGroup.define({
-	name: "tls_bits",
-	title: "TLS Bits",
-	description: "Number of bits to use for auto generated TLS certificate.",
-	type: "number",
-	initial_value: 2048,
 });
 MasterGroup.define({
+	access: ["master"],
 	name: "auth_secret",
 	title: "Master Authentication Secret",
 	description:
@@ -78,7 +85,7 @@ MasterGroup.define({
 	title: "Heartbeat Interval",
 	description: "Interval heartbeats are sent out on WebSocket connections",
 	type: "number",
-	initial_value: 60,
+	initial_value: 45,
 });
 MasterGroup.define({
 	name: "connector_shutdown_timeout",
@@ -87,6 +94,13 @@ MasterGroup.define({
 	type: "number",
 	initial_value: 30,
 });
+MasterGroup.define({
+	name: "metrics_timeout",
+	title: "Metrics Timeout",
+	description: "Timeout in seconds for metrics gathering from slaves",
+	type: "number",
+	initial_value: 8,
+});
 MasterGroup.define({
 	name: "default_role_id",
 	title: "Default role",
@@ -112,6 +126,7 @@ MasterConfig.registerGroup(MasterGroup);
  * @memberof module:lib/config
  */
 class SlaveGroup extends classes.ConfigGroup {}
+SlaveGroup.defaultAccess = ["master", "slave", "control"];
 SlaveGroup.groupName = "slave";
 SlaveGroup.define({
 	name: "name",
@@ -141,7 +156,7 @@ SlaveGroup.define({
 	name: "master_url",
 	description: "URL to connect to the master server at",
 	type: "string",
-	initial_value: "https://localhost:8443/",
+	initial_value: "http://localhost:8080/",
 });
 SlaveGroup.define({
 	name: "master_token",
@@ -149,6 +164,12 @@ SlaveGroup.define({
 	type: "string",
 	initial_value: "enter token here",
 });
+SlaveGroup.define({
+	name: "tls_ca",
+	description: "Path to Certificate Authority to validate TLS connection to master against.",
+	type: "string",
+	optional: true,
+});
 SlaveGroup.define({
 	name: "public_address",
 	description: "Public facing address players should connect to in order to join instances on this slave",
@@ -179,6 +200,7 @@ SlaveConfig.registerGroup(SlaveGroup);
  * @memberof module:lib/config
  */
 class InstanceGroup extends classes.ConfigGroup { }
+InstanceGroup.defaultAccess = ["master", "slave", "control"];
 InstanceGroup.groupName = "instance";
 InstanceGroup.define({
 	name: "name",
@@ -196,6 +218,12 @@ InstanceGroup.define({
 	type: "number",
 	optional: true,
 });
+InstanceGroup.define({
+	name: "auto_start",
+	description: "Automatically start this instance when the slave hosting it is started up",
+	type: "boolean",
+	initial_value: false,
+});
 InstanceGroup.finalize();
 
 /**
@@ -204,6 +232,7 @@ InstanceGroup.finalize();
  * @memberof module:lib/config
  */
 class FactorioGroup extends classes.ConfigGroup { }
+FactorioGroup.defaultAccess = ["master", "slave", "control"];
 FactorioGroup.groupName = "factorio";
 FactorioGroup.define({
 	name: "version",
@@ -245,10 +274,19 @@ FactorioGroup.define({
 	name: "settings",
 	description: "Settings overridden in server-settings.json",
 	type: "object",
-	initial_value: {
-		"tags": ["clusterio"],
-		"auto_pause": false,
-	},
+	initial_value: {}, // See create instance handler in master.
+});
+FactorioGroup.define({
+	name: "verbose_logging",
+	description: "Enable verbose logging on the Factorio server",
+	type: "boolean",
+	initial_value: false,
+});
+FactorioGroup.define({
+	name: "strip_paths",
+	description: "Strip down instance paths in the log",
+	type: "boolean",
+	initial_value: true,
 });
 FactorioGroup.define({
 	name: "sync_adminlist",
@@ -268,6 +306,12 @@ FactorioGroup.define({
 	type: "boolean",
 	initial_value: true,
 });
+FactorioGroup.define({
+	name: "max_concurrent_commands",
+	description: "Maximum number of RCON commands trasmitted in parallel",
+	type: "number",
+	initial_value: 5,
+});
 FactorioGroup.finalize();
 
 /**
@@ -286,6 +330,7 @@ InstanceConfig.registerGroup(FactorioGroup);
  * @memberof module:lib/config
  */
 class ControlGroup extends classes.ConfigGroup {}
+ControlGroup.defaultAccess = ["control"];
 ControlGroup.groupName = "control";
 ControlGroup.define({
 	name: "master_url",
@@ -299,6 +344,12 @@ ControlGroup.define({
 	type: "string",
 	optional: true,
 });
+ControlGroup.define({
+	name: "tls_ca",
+	description: "Path to Certificate Authority to validate TLS connection to master against.",
+	type: "string",
+	optional: true,
+});
 ControlGroup.define({
 	name: "reconnect_delay",
 	title: "Reconnect Delay",
@@ -345,6 +396,7 @@ function registerPluginConfigGroups(pluginInfos) {
 
 		} else {
 			class MasterConfigGroup extends classes.PluginConfigGroup { }
+			MasterConfigGroup.defaultAccess = ["master", "slave", "control"];
 			MasterConfigGroup.groupName = pluginInfo.name;
 			MasterConfigGroup.finalize();
 			MasterConfig.registerGroup(MasterConfigGroup);
@@ -357,6 +409,7 @@ function registerPluginConfigGroups(pluginInfos) {
 
 			} else {
 				class InstanceConfigGroup extends classes.PluginConfigGroup { }
+				InstanceConfigGroup.defaultAccess = ["master", "slave", "control"];
 				InstanceConfigGroup.groupName = pluginInfo.name;
 				InstanceConfigGroup.finalize();
 				InstanceConfig.registerGroup(InstanceConfigGroup);

packages/lib/config/index.js

@@ -1,3 +1,4 @@
+/* eslint-disable global-require */
 /**
  * Configuration System
  * @author Hornwitser

packages/lib/database.js

@@ -59,9 +59,8 @@ async function loadJsonAsMap(filePath) {
 			// File does not exist, return empty map
 			return new Map();
 
-		} else {
-			throw err;
 		}
+		throw err;
 	}
 }
 
@@ -121,9 +120,8 @@ async function loadJsonArrayAsMap(filePath) {
 			// File does not exist, return empty map
 			return new Map();
 
-		} else {
-			throw err;
 		}
+		throw err;
 	}
 }
 

packages/lib/factorio/export.js

@@ -99,6 +99,105 @@ async function loadFile(server, modVersions, modPath) {
 	return await file.async("nodebuffer");
 }
 
+async function loadIcon(server, modVersions, iconPath, iconSize, iconMipmaps, iconCache) {
+	let icon = iconCache.get(iconPath);
+	if (icon === undefined) {
+		let fileContent = await loadFile(server, modVersions, iconPath);
+		if (fileContent) {
+			icon = await Jimp.read(fileContent);
+			icon.crop(0, 0, iconSize, iconSize);
+			iconCache.set(iconPath, icon);
+		} else {
+			icon = null;
+			server._logger.warn(`${iconPath} not found`);
+		}
+		iconCache.set(iconPath, icon);
+	}
+	return icon;
+}
+
+async function loadSimpleIcon(server, modVersions, item, size, iconCache) {
+	let icon = await loadIcon(server, modVersions, item.icon, item.icon_size, 0, iconCache);
+	if (icon) {
+		let iconScale = size / item.icon_size;
+		if (iconScale !== 1) {
+			icon = icon.clone();
+			icon.scale(iconScale);
+		}
+	}
+	return icon;
+}
+
+async function loadLayeredIcon(server, modVersions, item, size, iconCache) {
+	let baseLayerSize = item.icons[0].icon_size || item.icon_size;
+	let icon = await Jimp.create(size, size);
+
+	// The scaling factor of the base layer
+	let baseLayerScale = item.icons[0].scale || 32 / baseLayerSize;
+
+	// The size in pixels of one unit
+	let baseUnit = size / (baseLayerSize * baseLayerScale);
+
+	for (let layer of item.icons) {
+		let layerSize = layer.icon_size || item.icon_size;
+		let iconLayer = await loadIcon(server, modVersions, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache);
+
+		if (!iconLayer) {
+			continue;
+		}
+
+		iconLayer = iconLayer.clone();
+
+		let tint;
+		if (layer.tint) {
+			let divisor = (layer.tint.r > 1 || layer.tint.g > 1 || layer.tint.b > 1) ? 255 : 1;
+			tint = {
+				r: (layer.tint.r || 0) / divisor,
+				g: (layer.tint.g || 0) / divisor,
+				b: (layer.tint.b || 0) / divisor,
+				a: layer.tint.a || 1,
+			};
+		} else {
+			tint = { r: 1, b: 1, g: 1, a: 1 };
+		}
+
+		let layerScale = layer.scale || 32 / layerSize;
+		let [xs, ys] = layer.shift || [0, 0];
+		let realScale = layerScale * baseUnit;
+		let sizeShift = (size - layerSize * realScale) / 2;
+
+		xs = xs * baseUnit + sizeShift;
+		ys = ys * baseUnit + sizeShift;
+
+		if (realScale !== 1) {
+			iconLayer.scale(realScale);
+		}
+
+		// eslint-disable-next-line no-loop-func
+		iconLayer.scan(0, 0, iconLayer.bitmap.width, iconLayer.bitmap.height, (x, y, sidx) => {
+			x += xs;
+			y += ys;
+			if (x < 0 || x >= size || y < 0 || y >= size) {
+				return;
+			}
+			let sdata = iconLayer.bitmap.data;
+			let ddata = icon.bitmap.data;
+			let didx = icon.getPixelIndex(x, y);
+
+			let sa = sdata[sidx + 3] / 255;
+			let da = ddata[didx + 3] / 255;
+			let db = da * (1 - sa * tint.a);
+			let cb = sa + da * (1 - sa);
+			ddata[didx + 0] = Math.min(255, (sdata[sidx + 0] * sa * tint.r + ddata[didx + 0] * db) / cb);
+			ddata[didx + 1] = Math.min(255, (sdata[sidx + 1] * sa * tint.g + ddata[didx + 1] * db) / cb);
+			ddata[didx + 2] = Math.min(255, (sdata[sidx + 2] * sa * tint.b + ddata[didx + 2] * db) / cb);
+			ddata[didx + 3] = Math.min(255, sdata[sidx + 3] + ddata[didx + 3] * (1 - sa));
+		});
+	}
+
+	return icon;
+}
+
 /**
  * Export item icons and data
  *
@@ -130,107 +229,21 @@ async function exportItems(server, modVersions, items) {
 	let pos = 0;
 
 	let iconCache = new Map();
-	async function loadIcon(iconPath, iconSize, iconMipmaps) {
-		let icon = iconCache.get(iconPath);
-		if (icon === undefined) {
-			let fileContent = await loadFile(server, modVersions, iconPath);
-			if (fileContent) {
-				icon = await Jimp.read(fileContent);
-				icon.crop(0, 0, iconSize, iconSize);
-				iconCache.set(iconPath, icon);
-			} else {
-				icon = null;
-				console.log(`Warning: ${iconPath} not found`);
-			}
-			iconCache.set(iconPath, icon);
-		}
-		return icon;
-	}
-
 	let simpleIcons = new Map();
 	for (let item of items) {
 		let icon;
 		let iconPos;
 		if (item.icons) {
-			let baseLayerSize = item.icons[0].icon_size || item.icon_size;
-			icon = await Jimp.create(size, size);
+			icon = await loadLayeredIcon(server, modVersions, item, size, iconCache);
 			iconPos = pos;
 
-			// The scaling factor of the base layer
-			let baseLayerScale = item.icons[0].scale || 32 / baseLayerSize;
-
-			// The size in pixels of one unit
-			let baseUnit = size / (baseLayerSize * baseLayerScale);
-
-			for (let layer of item.icons) {
-				let layerSize = layer.icon_size || item.icon_size;
-				let iconLayer = await loadIcon(layer.icon, layerSize, layer.icon_mipmaps || 0);
-
-				if (!iconLayer) {
-					continue;
-				}
-
-				iconLayer = iconLayer.clone();
-
-				let tint;
-				if (layer.tint) {
-					let divisor = (layer.tint.r > 1 || layer.tint.g > 1 || layer.tint.b > 1) ? 255 : 1;
-					tint = {
-						r: (layer.tint.r || 0) / divisor,
-						g: (layer.tint.g || 0) / divisor,
-						b: (layer.tint.b || 0) / divisor,
-						a: layer.tint.a || 1,
-					};
-				} else {
-					tint = { r: 1, b: 1, g: 1, a: 1 };
-				}
-
-				let layerScale = layer.scale || 32 / layerSize;
-				let [xs, ys] = layer.shift || [0, 0];
-				let realScale = layerScale * baseUnit;
-				let sizeShift = (size - layerSize * realScale) / 2;
-
-				xs = xs * baseUnit + sizeShift;
-				ys = ys * baseUnit + sizeShift;
-
-				if (realScale !== 1) {
-					iconLayer.scale(realScale);
-				}
-
-				// eslint-disable-next-line no-loop-func
-				iconLayer.scan(0, 0, iconLayer.bitmap.width, iconLayer.bitmap.height, function(x, y, sidx) {
-					x += xs;
-					y += ys;
-					if (x < 0 || x >= size || y < 0 || y >= size) {
-						return;
-					}
-					let sdata = this.bitmap.data;
-					let ddata = icon.bitmap.data;
-					let didx = icon.getPixelIndex(x, y);
-
-					let sa = sdata[sidx + 3] / 255;
-					let da = ddata[didx + 3] / 255;
-					let db = da * (1 - sa * tint.a);
-					let cb = sa + da * (1 - sa);
-					ddata[didx + 0] = Math.min(255, (sdata[sidx + 0] * sa * tint.r + ddata[didx + 0] * db) / cb);
-					ddata[didx + 1] = Math.min(255, (sdata[sidx + 1] * sa * tint.g + ddata[didx + 1] * db) / cb);
-					ddata[didx + 2] = Math.min(255, (sdata[sidx + 2] * sa * tint.b + ddata[didx + 2] * db) / cb);
-					ddata[didx + 3] = Math.min(255, sdata[sidx + 3] + ddata[didx + 3] * (1 - sa));
-				});
-			}
-
 		} else {
 			iconPos = simpleIcons.get(item.icon);
 			if (iconPos === undefined) {
-				icon = await loadIcon(item.icon, item.icon_size, 0);
+				icon = await loadSimpleIcon(server, modVersions, item, size, iconCache);
 				if (icon) {
 					iconPos = pos;
 					simpleIcons.set(item.icon, pos);
-					let iconScale = size / item.icon_size;
-					if (iconScale !== 1) {
-						icon = icon.clone();
-						icon.scale(iconScale);
-					}
 				}
 			}
 		}
@@ -324,9 +337,9 @@ async function exportData(server) {
 
 	server.on("ipc-item_export", data => items.push(data));
 	server.on("ipc-mod_list", data => { modVersions = new Map(Object.entries(data)); });
-	server.on("output", data => {
-		if (data.format === "seconds" && data.type === "generic") {
-			let match = /^Checksum of (.*): \d+$/.exec(data.message);
+	server.on("output", parsed => {
+		if (parsed.format === "seconds" && parsed.type === "generic") {
+			let match = /^Checksum of (.*): \d+$/.exec(parsed.message);
 			if (match) {
 				modOrder.push(match[1]);
 			}

packages/lib/factorio/index.js

@@ -1,3 +1,4 @@
+/* eslint-disable global-require */
 /**
  * Library for interfacing with Factorio Servers and saves.
  * @module lib/factorio

packages/lib/factorio/patch.js

@@ -78,7 +78,7 @@ function generateLoader(patchInfo) {
 	];
 
 	for (let moduleName of patchInfo["scenario"]["modules"]) {
-		lines.push('event_handler.add_lib(require("'+moduleName+'"))');
+		lines.push(`event_handler.add_lib(require("${moduleName}"))`);
 	}
 
 	lines.push(...[
@@ -90,10 +90,10 @@ function generateLoader(patchInfo) {
 		for (let file of module["files"]) {
 			let requirePath = `modules/${module.name}/${file["path"].slice(0, -4)}`;
 			if (file["load"]) {
-				lines.push('event_handler.add_lib(require("'+requirePath+'"))');
+				lines.push(`event_handler.add_lib(require("${requirePath}"))`);
 			}
 			if (file["require"]) {
-				lines.push('require("'+requirePath+'")');
+				lines.push(`require("${requirePath}")`);
 			}
 		}
 	}

packages/lib/factorio/server.js

@@ -10,6 +10,8 @@ const crypto = require("crypto");
 const ini = require("ini");
 const rconClient = require("rcon-client");
 const libErrors = require("@clusterio/lib/errors");
+const { logger } = require("@clusterio/lib/logging");
+const { escapeRegExp } = require("@clusterio/lib/helpers");
 
 
 /**
@@ -286,7 +288,6 @@ class LineSplitter {
 function parseOutput(line, source) {
 	let output = {
 		source,
-		received: Date.now(),
 	};
 
 	// There are three broad categories of output from Factorio, the first kind
@@ -302,13 +303,15 @@ function parseOutput(line, source) {
 		// A seconds output has two different kinds of formats: The first is a
 		// log level and source location and has a format of
 		// "Level File.cpp:123: message"
-		const secLogRegex = /^(\w+) (\w+\.cpp:\d+): (.*)$/;
+		// "Script @/path/to/file.lua:123: message"
+		// "Script =(command):123: message"
+		const secLogRegex = /^(\w+) ((\w+\.cpp:\d+)|([@=].*?:\d+)): (.*)$/;
 		let secLogMatch = secLogRegex.exec(secContent);
 		if (secLogMatch) {
 			output.type = "log";
 			output.level = secLogMatch[1];
 			output.file = secLogMatch[2];
-			output.message = secLogMatch[3];
+			output.message = secLogMatch[5];
 
 		// The other possibility is that the content is a generic message
 		} else {
@@ -365,7 +368,7 @@ const outputHeuristics = [
 	// Message indicating the RCON interface has started
 	{
 		filter: { type: "log", message: /^Starting RCON interface/ },
-		action: function(output) {
+		action: function() {
 			this._startRcon().catch((err) => { this.emit("error", err); });
 		},
 	},
@@ -373,7 +376,7 @@ const outputHeuristics = [
 	// Message indicating the server is done starting up
 	{
 		filter: { type: "log", message: /^updateTick\(\d+\) changing state from\(CreatingGame\) to\(InGame\)$/ },
-		action: function(output) {
+		action: function() {
 			this._notifyGameReady().catch((err) => { this.emit("error", err); });
 		},
 	},
@@ -384,8 +387,8 @@ const outputHeuristics = [
 			type: "log",
 			message: /^Saving to _autosave\d+ \((non-)?blocking\)\.$/,
 		},
-		action: function(output) {
-			let name = /^Saving to (_autosave\d+) /.exec(output.message)[1];
+		action: function(parsed) {
+			let name = /^Saving to (_autosave\d+) /.exec(parsed.message)[1];
 			this.emit("_autosave", name);
 		},
 	},
@@ -396,7 +399,7 @@ const outputHeuristics = [
 			type: "log",
 			message: /^Saving game as /,
 		},
-		action: function(output) {
+		action: function() {
 			this.emit("_saving");
 		},
 	},
@@ -412,7 +415,7 @@ const outputHeuristics = [
 				"Can't save to default location: Default location not known",
 			],
 		},
-		action: function(output) {
+		action: function() {
 			this.emit("_saved");
 		},
 	},
@@ -423,13 +426,13 @@ const outputHeuristics = [
 			type: "generic",
 			message: /^Quitting: /,
 		},
-		action: function(output) {
+		action: function(parsed) {
 			this.emit("_quitting");
 
 			// On windows the Factorio server will wait for input on the console input buffer
 			// when an error occurs.  Since we can't send input there we terminate the server
 			// process when this happens.
-			if (process.platform === "win32" && output.message === "Quitting: multiplayer error.") {
+			if (process.platform === "win32" && parsed.message === "Quitting: multiplayer error.") {
 				this._server.kill();
 			}
 		},
@@ -445,8 +448,8 @@ const outputHeuristics = [
 				"MultiplayerManager failed: Host address is already in use.",
 			],
 		},
-		action: function(output) {
-			this._unexpected.push(`Factorio failed to bind to game port: ${output.message}`);
+		action: function(parsed) {
+			this._unexpected.push(`Factorio failed to bind to game port: ${parsed.message}`);
 		},
 	},
 	{
@@ -458,8 +461,8 @@ const outputHeuristics = [
 				"Can't bind socket: Permission denied",
 			],
 		},
-		action: function(output) {
-			this._unexpected.push(`Factorio failed to bind to RCON port: ${output.message}`);
+		action: function(parsed) {
+			this._unexpected.push(`Factorio failed to bind to RCON port: ${parsed.message}`);
 		},
 	},
 ];
@@ -491,12 +494,21 @@ class FactorioServer extends events.EventEmitter {
 	 * @param {string} factorioDir - Directory of the Factorio install(s).
 	 * @param {string} writeDir - Directory to write runtime data to.
 	 * @param {object} options - Optional parameters.
+	 * @param {Logger} options.logger - Logger to use for reporting errors.
 	 * @param {string} options.version -
 	 *     Version of Factorio to use.  Can also be the string "latest" to
 	 *     use the latest version found in `factorioDir`.
 	 * @param {number} options.gamePort - UDP port to host game on.
 	 * @param {number} options.rconPort - TCP port to use for RCON.
 	 * @param {string} options.rconPassword - Password use for RCON.
+	 * @param {boolean} options.enableWhitelist - Turn on whitelisting.
+	 * @param {boolean} options.verboseLogging - Enable verbose logging.
+	 * @param {boolean} options.stripPaths - Strip paths in the console.
+	 * @param {number} options.maxConcurrentCommands -
+	 *     Maximum number of RCON commands transmitted in parallel on the
+	 *     RCON connection.
+	 * @param {number} options.hangTimeout -
+	 *     Timeout in ms to wait for a response to a shutdown command.
 	 */
 	constructor(factorioDir, writeDir, options) {
 		super();
@@ -508,6 +520,7 @@ class FactorioServer extends events.EventEmitter {
 		this._version = null;
 		this._dataDir = null;
 
+		this._logger = options.logger || logger;
 		this._targetVersion = options.version || "latest";
 		/** UDP port used for hosting the Factorio game server on */
 		this.gamePort = options.gamePort || randomDynamicPort();
@@ -517,12 +530,31 @@ class FactorioServer extends events.EventEmitter {
 		this.rconPassword = options.rconPassword;
 		/** Enable player whitelist */
 		this.enableWhitelist = options.enableWhitelist;
+		/** Enable verbose logging */
+		this.verboseLogging = options.verboseLogging;
+		/** Maximum number of RCON commands transmitted in parallel on the RCON connection  */
+		this.maxConcurrentCommands = options.maxConcurrentCommands || 5;
+		/** Timeout in ms to wait for a response to a shutdown command */
+		this.hangTimeout = options.hangTimeout || 5000;
 		this._state = "new";
 		this._server = null;
 		this._rconClient = null;
 		this._rconReady = false;
 		this._gameReady = false;
 
+		if (options.stripPaths) {
+			let chars = new Set(this.writePath("temp"));
+			chars.delete(":"); // Having a colon could lead to matching the line number
+			chars = [...chars].join("");
+
+			this._stripRegExp = new RegExp(
+				`(${escapeRegExp(this.writePath("temp", "currently-playing", path.sep))})|` +
+				`(${escapeRegExp(this.writePath(path.sep))})|` +
+				`(\\.\\.\\.[${escapeRegExp(chars)}]*?currently-playing\\${path.sep})`,
+				"g"
+			);
+		}
+
 		// Array of possible causes for an unexpected shutdown
 		this._unexpected = [];
 
@@ -531,12 +563,12 @@ class FactorioServer extends events.EventEmitter {
 
 		// Track autosaving
 		this._runningAutosave = null;
-		this.on("_autosave", function(name) {
+		this.on("_autosave", name => {
 			this.emit("autosave-start", name);
 			this._runningAutosave = name;
 		});
 
-		this.on("_saved", function() {
+		this.on("_saved", () => {
 			if (this._runningAutosave) {
 				this.emit("autosave-finished", this._runningAutosave);
 				this._runningAutosave = null;
@@ -594,7 +626,7 @@ class FactorioServer extends events.EventEmitter {
 		}
 
 		if (!this.emit(`ipc-${channel}`, content)) {
-			console.warn(`Warning: Unhandled ipc-${channel}`, content);
+			this._logger.warn(`Warning: Unhandled ipc-${channel}`, { content });
 		}
 	}
 
@@ -606,31 +638,37 @@ class FactorioServer extends events.EventEmitter {
 
 		this.emit(source, line);
 
-		let output = parseOutput(line.toString("utf-8"), source);
+		line = line.toString("utf-8");
+		if (this._stripRegExp) {
+			line = line.replace(this._stripRegExp, "");
+		}
+
+
+		let parsed = parseOutput(line, source);
 		heuristicLoop: for (let heuristic of outputHeuristics) {
 			for (let [name, expected] of Object.entries(heuristic.filter)) {
-				if (!Object.hasOwnProperty.call(output, name)) {
+				if (!Object.hasOwnProperty.call(parsed, name)) {
 					continue heuristicLoop;
 				}
 
 				if (expected instanceof RegExp) {
-					if (!expected.test(output[name])) {
+					if (!expected.test(parsed[name])) {
 						continue heuristicLoop;
 					}
 				} else if (expected instanceof Array) {
-					if (!expected.includes(output[name])) {
+					if (!expected.includes(parsed[name])) {
 						continue heuristicLoop;
 					}
-				} else if (expected !== output[name]) {
+				} else if (expected !== parsed[name]) {
 					continue heuristicLoop;
 				}
 			}
 
 			// If we get here the filter matched the output
-			heuristic.action.call(this, output);
+			heuristic.action.call(this, parsed);
 		}
 
-		this.emit("output", output);
+		this.emit("output", parsed, line);
 	}
 
 	async _startRcon() {
@@ -647,7 +685,7 @@ class FactorioServer extends events.EventEmitter {
 			port: this.rconPort,
 			password: this.rconPassword,
 			timeout: 200000, // 200s, should allow for commands up to 1250kB in length
-			maxPending: 5,
+			maxPending: this.maxConcurrentCommands,
 		};
 
 		this._rconClient = new rconClient.Rcon(config);
@@ -655,12 +693,32 @@ class FactorioServer extends events.EventEmitter {
 		this._rconClient.on("authenticated", () => { this._rconReady = true; this.emit("rcon-ready"); });
 		this._rconClient.on("end", () => {
 			this._rconClient = null;
-			this._rconReady = false;
-			this._startRcon().catch(() => { /* Ignore */ });
 		});
+		// XXX: Workaround to suppress bogus event listener warning
+		if (this._rconClient.emitter instanceof events.EventEmitter) {
+			this._rconClient.emitter.setMaxListeners(this.maxConcurrentCommands + 5);
+		}
 		await this._rconClient.connect();
 	}
 
+	get maxConcurrentCommands() {
+		return this._maxConcurrentCommands;
+	}
+
+	set maxConcurrentCommands(value) {
+		this._maxConcurrentCommands = value;
+		this.setMaxListeners(value + 5);
+		if (this._rconClient) {
+			if (this._rconClient.sendQueue) {
+				this._rconClient.sendQueue.maxConcurrent = value;
+			}
+			// XXX: Workaround to suppress bogus event listener warning
+			if (this._rconClient.emitter instanceof events.EventEmitter) {
+				this._rconClient.emitter.setMaxListeners(value + 5);
+			}
+		}
+	}
+
 	/**
 	 * Initialize class instance
 	 *
@@ -725,12 +783,16 @@ class FactorioServer extends events.EventEmitter {
 
 				} else {
 					this.emit("error", new Error(
-						"Factorio server unexpectedly shut down. Possible causes:\n- "+
-						this._unexpected.join("\n- ")
+						"Factorio server unexpectedly shut down. Possible causes:"+
+						`\n- ${this._unexpected.join("\n- ")}`
 					));
 				}
 			}
 
+			if (this._rconClient) {
+				this._rconClient.end().catch(() => {});
+			}
+
 			// Reset server state
 			this._state = "init";
 			this._server = null;
@@ -774,6 +836,7 @@ class FactorioServer extends events.EventEmitter {
 			[
 				"--config", this.writePath("config.ini"),
 				"--create", this.writePath("saves", name),
+				...(this.verboseLogging ? ["--verbose"] : []),
 			],
 			{
 				detached: true,
@@ -816,6 +879,7 @@ class FactorioServer extends events.EventEmitter {
 				"--rcon-port", this.rconPort,
 				"--rcon-password", this.rconPassword,
 				...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
+				...(this.verboseLogging ? ["--verbose"] : []),
 			],
 			{
 				detached: true,
@@ -850,6 +914,7 @@ class FactorioServer extends events.EventEmitter {
 				"--rcon-port", this.rconPort,
 				"--rcon-password", this.rconPassword,
 				...(this.enableWhitelist ? ["--use-server-whitelist"] : []),
+				...(this.verboseLogging ? ["--verbose"] : []),
 			],
 			{
 				detached: true,
@@ -880,6 +945,9 @@ class FactorioServer extends events.EventEmitter {
 		if (!this._rconReady) {
 			await events.once(this, "rcon-ready");
 		}
+		if (!this._rconClient) {
+			throw new Error("RCON connection lost");
+		}
 
 		let response = await this._rconClient.send(message);
 		if (expectEmpty && response !== "") {
@@ -898,17 +966,25 @@ class FactorioServer extends events.EventEmitter {
 		this._check(["running"]);
 
 		let hanged = true;
-		function setAlive(output) { hanged = false; }
+		let stopped = false;
+		function setAlive() { hanged = false; }
 
 		let timeoutId = setTimeout(() => {
 			if (hanged) {
-				console.error("Factorio appears to have hanged, sending SIGKILL");
+				this._logger.error("Factorio appears to have hanged, sending SIGKILL");
 				if (this._rconClient) {
 					this._rconClient.end().catch(() => {});
 				}
 				this._server.kill("SIGKILL");
 			}
-		}, 5000);
+		}, this.hangTimeout);
+
+		// It's possible the server decides to exit on its own, in which
+		// case the stop operation has to be cancelled.
+		this._server.once("exit", () => {
+			stopped = true;
+			clearTimeout(timeoutId);
+		});
 
 		this._state = "stopping";
 		// On Windows we need to save the map as there's no graceful shutdown.
@@ -940,6 +1016,12 @@ class FactorioServer extends events.EventEmitter {
 			await this._rconClient.end();
 		}
 
+		// The Factorio server may have decided to get ahead of us and
+		// stop by itself before we had the chance to signal it.
+		if (stopped) {
+			return;
+		}
+
 		// On linux this sends SIGTERM to the process.  On windows this
 		// will terminate the process with no cleanup.  You can send your
 		// complaint for the lack of graceful shutdown to the Factorio

packages/lib/hash.js

@@ -14,14 +14,14 @@ const fs = require("fs");
  * @returns {Promise<string>} hash of the stream.
  */
 function hashStream(stream) {
-	return new Promise(function(resolve, reject) {
+	return new Promise((resolve, reject) => {
 		let hasher = crypto.createHash("sha1");
 		hasher.setEncoding("hex");
-		hasher.on("finish", function() {
+		hasher.on("finish", () => {
 			resolve(hasher.read());
 		});
 
-		stream.on("error", function(error) {
+		stream.on("error", (error) => {
 			// the docs doesn't say anything about what's passed on error
 			reject(error);
 		});

packages/lib/helpers.js

@@ -19,6 +19,82 @@ function basicType(value) {
 	return typeof value;
 }
 
+
+/**
+ * Asynchronously wait for the given duration
+ *
+ * @param {number} duration - Time to wait for in milliseconds.
+ */
+async function wait(duration) {
+	await new Promise(resolve => { setTimeout(resolve, duration); });
+}
+
+
+/**
+ * Resolve a promise with a timeout.
+ *
+ * @param {Promise} promise - Promise to wait for.
+ * @param {number} time - Maximum time im milliseconds to wait for.
+ * @param {*=} timeoutResult - Value to return if the operation timed out.
+ */
+async function timeout(promise, time, timeoutResult) {
+	let timer;
+	try {
+		return await Promise.race([
+			promise,
+			new Promise(resolve => {
+				timer = setTimeout(() => resolve(timeoutResult), time);
+			}),
+		]);
+
+	} finally {
+		clearTimeout(timer);
+	}
+}
+
+
+/**
+ * Read stream to the end and return its content
+ *
+ * Reads the stream given asynchronously until the end is reached and
+ * returns all the data which was read from the stream.
+ *
+ * @param {Readable} stream - byte stream to read to the end.
+ * @returns {Buffer} content of the stream.
+ */
+async function readStream(stream) {
+	let chunks = [];
+	for await (let chunk of stream) {
+		// Support using ^Z to end input on Windows
+		if (process.platform === "win32" && stream.isTTY && chunk.toString() === "\x1a\r\n") {
+			break;
+		}
+		chunks.push(chunk);
+	}
+	return Buffer.concat(chunks);
+}
+
+
+/**
+ * Escapes text for inclusion in a RegExp
+ *
+ * Adds \ character in front of special meta characters in the passsed in
+ * text so that it can be embedded into a RegExp and only match the text.
+ *
+ * See https://stackoverflow.com/a/9310752
+ *
+ * @param {string} text - Text to escape RegExp meta chars in.
+ * @returns {string} escaped text.
+ */
+function escapeRegExp(text) {
+	return text.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, "\\$&");
+}
+
+
 module.exports = {
 	basicType,
+	wait,
+	timeout,
+	readStream,
+	escapeRegExp,
 };

packages/lib/is_deep_strict_equal.js

@@ -1,3 +1,4 @@
+/* eslint-disable global-require */
 "use strict";
 // eslint-disable-next-line no-process-env
 if (process.env.APP_ENV === "browser") {

packages/lib/link/connectors.js

@@ -5,6 +5,7 @@ const WebSocket = require("isomorphic-ws");
 
 const libSchema = require("@clusterio/lib/schema");
 const libErrors = require("@clusterio/lib/errors");
+const { logger } = require("@clusterio/lib/logging");
 
 
 /**
@@ -45,7 +46,7 @@ class WebSocketBaseConnector extends events.EventEmitter {
 	_doHeartbeat() {
 		this._check(["connected", "closing"]);
 		if (Date.now() - this._lastHeartbeat > 2000 * this._heartbeatInterval) {
-			console.log("SOCKET | closing after heartbeat timed out");
+			logger.verbose("SOCKET | closing after heartbeat timed out");
 			this._socket.close(1008, "Heartbeat timeout");
 			clearInterval(this._heartbeatId);
 			this._heartbeatId = null;
@@ -61,9 +62,8 @@ class WebSocketBaseConnector extends events.EventEmitter {
 
 	_processHeartbeat(message) {
 		if (!libSchema.heartbeat(message)) {
-			console.log("SOCKET | closing after received invalid heartbeat:", message);
+			logger.warn("SOCKET | closing after received invalid heartbeat");
 			this._socket.close(1002, "Invalid heartbeat");
-			console.log(libSchema.heartbeat.errors);
 			return;
 		}
 
@@ -144,10 +144,11 @@ class WebSocketBaseConnector extends events.EventEmitter {
  * @memberof module:lib/link
  */
 class WebSocketClientConnector extends WebSocketBaseConnector {
-	constructor(url, reconnectDelay) {
+	constructor(url, reconnectDelay, tlsCa = null) {
 		super();
 
 		this._url = url;
+		this._tlsCa = tlsCa;
 
 		// The following states are used in the client connector
 		// new: Not connected
@@ -236,18 +237,16 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 		url.pathname += "api/socket";
 
 		// Open WebSocket to master
-		console.log(`SOCKET | connecting to ${url}`);
+		logger.verbose(`SOCKET | connecting to ${url}`);
 
 		// eslint-disable-next-line no-process-env
 		if (process.env.APP_ENV === "browser") {
 			this._socket = new WebSocket(url);
 
 		} else {
-			this._socket = new WebSocket(url, {
-				// For now we do not verify TLS certificates since the default setup is
-				// to create a self-signed certificate.
-				rejectUnauthorized: false,
-			});
+			let options = {};
+			if (this._tlsCa) { options.ca = this._tlsCa; }
+			this._socket = new WebSocket(url, options);
 		}
 
 		this._attachSocketHandlers();
@@ -266,7 +265,7 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 		}
 
 		if (this._startedReconnect + this._timeout * 1000 < Date.now()) {
-			console.log("SOCKET | Timed out trying to reconnect");
+			logger.error("SOCKET | Timed out trying to reconnect");
 			this._heartbeatInterval = null;
 			this._lastReceivedSeq = null;
 			this._sessionToken = null;
@@ -277,7 +276,9 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 		}
 
 		let delay = Math.random() * this._reconnectDelay;
-		console.log(`SOCKET | waiting ${delay.toLocaleString("en", { maximumFractionDigits: 2 })} seconds for reconnect`);
+		logger.verbose(
+			`SOCKET | waiting ${delay.toLocaleString("en", { maximumFractionDigits: 2 })} seconds for reconnect`
+		);
 		this._reconnectId = setTimeout(() => {
 			this._reconnectId = null;
 			this._doConnect();
@@ -286,7 +287,7 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 
 	_attachSocketHandlers() {
 		this._socket.onclose = event => {
-			console.log(`SOCKET | Close (code: ${event.code}, reason: ${event.reason})`);
+			logger.verbose(`SOCKET | Close (code: ${event.code}, reason: ${event.reason})`);
 			// Authentication failed
 			if (event.code === 4003) {
 				this.emit("error", new libErrors.AuthenticationFailed(event.reason));
@@ -315,11 +316,23 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 
 		this._socket.onerror = event => {
 			// It's assumed that close is always called by ws
-			console.error("SOCKET | Error:", event);
+			let code = !event.error ? "" : `, code: ${event.error.code}`;
+			logger.error(`SOCKET | Error: ${event.message || "unknown error"}${code}`);
+			if (event.error) {
+				// Abort connection if certificate validation failed
+				if ([
+					"CERT_HAS_EXPIRED",
+					"DEPTH_ZERO_SELF_SIGNED_CERT",
+					"ERR_TLS_CERT_ALTNAME_INVALID",
+				].includes(event.error.code)) {
+					this.emit("error", new libErrors.AuthenticationFailed(event.error.message));
+					this._state = "closing";
+				}
+			}
 		};
 
 		this._socket.onopen = () => {
-			console.log("SOCKET | Open");
+			logger.verbose("SOCKET | Open");
 		};
 
 		// Handle messages
@@ -348,17 +361,17 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 
 	_processHandshake(message) {
 		if (!libSchema.serverHandshake(message)) {
-			console.log("SOCKET | closing after received invalid handshake:", message);
+			logger.verbose("SOCKET | closing after received invalid handshake");
 			this._socket.close(1002, "Invalid handshake");
 			return;
 		}
 
 		let { seq, type, data } = message;
 		if (type === "hello") {
-			console.log(`SOCKET | received hello from master version ${data.version}`);
+			logger.verbose(`SOCKET | received hello from master version ${data.version}`);
 			this.emit("hello", data);
 			if (this._sessionToken) {
-				console.log("SOCKET | Attempting resume");
+				logger.verbose("SOCKET | Attempting resume");
 				this.sendHandshake("resume", {
 					session_token: this._sessionToken,
 					last_seq: this._lastReceivedSeq,
@@ -368,7 +381,7 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 			}
 
 		} else if (type === "ready") {
-			console.log("SOCKET | received ready from master");
+			logger.verbose("SOCKET | received ready from master");
 			this._state = "connected";
 			this._sessionToken = data.session_token;
 			this._heartbeatInterval = data.heartbeat_interval;
@@ -381,7 +394,7 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 			this.emit("connect");
 
 		} else if (type === "continue") {
-			console.log("SOCKET | resuming existing session");
+			logger.verbose("SOCKET | resuming existing session");
 			this._state = "connected";
 			this._heartbeatInterval = data.heartbeat_interval;
 			this.startHeartbeat();
@@ -391,7 +404,7 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 			}
 			this._startedReconnect = null;
 			this._connected = true;
-			this.emit("connect");
+			this.emit("resume");
 
 		} else if (type === "invalidate") {
 			this._heartbeatInterval = null;

packages/lib/link/index.js

@@ -1,3 +1,4 @@
+/* eslint-disable global-require */
 /**
  * Library for message based communication between nodes
  * @module lib/link

packages/lib/link/link.js

@@ -3,6 +3,7 @@
 
 const libSchema = require("@clusterio/lib/schema");
 const libErrors = require("@clusterio/lib/errors");
+const { logger } = require("@clusterio/lib/logging");
 const messages = require("./messages");
 
 // Some definitions for the terminology used here:
@@ -35,9 +36,21 @@ class Link {
 				this.processMessage(payload);
 			} catch (err) {
 				if (err instanceof libErrors.InvalidMessage) {
-					console.error(`Invalid message on ${this.source}-${this.target} link: ${err.message}`);
+					logger.error(`Invalid message on ${this.source}-${this.target} link: ${err.message}`);
 					if (err.errors) {
-						console.error(err.errors);
+						logger.error(JSON.stringify(err.errors, null, 4));
+					}
+
+					// Send back an error response if this was a request.
+					if (
+						typeof payload.type === "string"
+						&& payload.type.endsWith("_request")
+						&& Number.isInteger(payload.seq)
+					) {
+						this.connector.send(
+							`${payload.type.slice(0, -8)}_response`,
+							{ seq: payload.seq, error: err.message }
+						);
 					}
 				} else {
 					this.connector.emit("error", err);
@@ -111,9 +124,8 @@ class Link {
 		if (handler) {
 			handler(message);
 			return true;
-		} else {
-			return false;
 		}
+		return false;
 	}
 
 	/**

packages/lib/link/messages.js

@@ -2,6 +2,26 @@
 "use strict";
 const libSchema = require("@clusterio/lib/schema");
 const libErrors = require("@clusterio/lib/errors");
+const { logger } = require("@clusterio/lib/logging");
+
+class MissingLinkHandlerError extends Error {
+	constructor(type, source, target) {
+		super();
+		this.code = "MISSING_LINK_HANDLER";
+		this.type = type;
+		this.source = source;
+		this.target = target;
+		this.handler = null;
+		this.plugin = null;
+	}
+
+	get message() {
+		let handler = this.handler ? `${this.handler}` : "handler";
+		let plugin = this.plugin ? ` on plugin ${this.plugin}` : "";
+
+		return `Missing ${handler}${plugin} for ${this.type} on ${this.source}-${this.target} link`;
+	}
+}
 
 /**
  * Represents a message that can be sent over the link
@@ -50,16 +70,24 @@ class Request extends Message {
 	 *     Optional target to forward this request to.  'instance' add
 	 *     instance_id into the requestProperties and forward to the given
 	 *     instance.  'master' forwards it to the master server.
+	 * @param {?Array<string>} requestRequired -
+	 *     List of properties that are required to be present in the data
+	 *     payload of this request.  Defaults to all.
 	 * @param {Object<string, Object>} requestProperties -
 	 *     Mapping of property values to JSON schema specifications for the
 	 *     properties that are valid in the data payload of this requst.
+	 * @param {?Array<string>} responseRequired -
+	 *     List of properties that are required to be present in the data
+	 *     payload of the repsonse to this request.  Defaults to all.
 	 * @param {Object<string, Object>} responseProperties -
 	 *     Mapping of property values to JSON schema specifications for the
 	 *     properties that are valid in the data payload of the response to
 	 *     this requst.
 	 */
 	constructor({
-		type, permission, links, forwardTo = null, requestProperties = {}, responseProperties = {},
+		type, permission, links, forwardTo = null,
+		requestRequired = null, requestProperties = {},
+		responseRequired = null, responseProperties = {},
 	}) {
 		super();
 		this.type = type;
@@ -68,8 +96,8 @@ class Request extends Message {
 		this.forwardTo = forwardTo;
 		this.handlerSuffix = "RequestHandler";
 
-		this.requestType = type + "_request";
-		this.responseType = type + "_response";
+		this.requestType = `${type}_request`;
+		this.responseType = `${type}_response`;
 
 		if (permission === undefined && links.includes("control-master")) {
 			throw new Error(`permission is required for ${this.type} request over control-master links`);
@@ -78,7 +106,12 @@ class Request extends Message {
 			throw new Error(`permission is not allowed on ${this.type} request as it is not over control-master link`);
 		}
 
+		if (!requestRequired) {
+			requestRequired = Object.keys(requestProperties);
+		}
+
 		if (forwardTo === "instance") {
+			requestRequired = ["instance_id", ...requestRequired];
 			requestProperties = {
 				"instance_id": { type: "integer" },
 				...requestProperties,
@@ -94,12 +127,16 @@ class Request extends Message {
 				"type": { const: this.requestType },
 				"data": {
 					additionalProperties: false,
-					required: Object.keys(requestProperties),
+					required: requestRequired,
 					properties: requestProperties,
 				},
 			},
 		});
 
+		if (!responseRequired) {
+			responseRequired = Object.keys(responseProperties);
+		}
+
 		this._responseValidator = libSchema.compile({
 			$schema: "http://json-schema.org/draft-07/schema#",
 			properties: {
@@ -108,7 +145,7 @@ class Request extends Message {
 					anyOf: [
 						{
 							additionalProperties: false,
-							required: ["seq", ...Object.keys(responseProperties)],
+							required: ["seq", ...responseRequired],
 							properties: {
 								"seq": { type: "integer" },
 								...responseProperties,
@@ -168,15 +205,18 @@ class Request extends Message {
 			}
 
 			if (!handler) {
-				throw new Error(`Missing handler for ${this.requestType} on ${link.source}-${link.target} link`);
+				throw new MissingLinkHandlerError(this.requestType, link.source, link.target);
 			}
 
 			// Check permission if this is a handler for a control connection on the master
 			if (this.permission !== null && `${link.source}-${link.target}` === "master-control") {
 				let origHandler = handler;
 				handler = async function(message, request) {
-					this.user.checkPermission(request.permission);
-					return await origHandler.call(this, message, request);
+					// Abuse this binding to get a hold of the ControlConnection instance
+					// eslint-disable-next-line no-invalid-this, consistent-this
+					let controlConnection = this;
+					controlConnection.user.checkPermission(request.permission);
+					return await origHandler.call(controlConnection, message, request);
 				};
 			}
 
@@ -187,16 +227,17 @@ class Request extends Message {
 						response = {};
 					}
 
-					if (!this._responseValidator({ seq: 0, type: this.responseType, response })) {
-						console.error(this._responseValidator.errors);
+					let data = { ...response, seq: message.seq };
+					if (!this._responseValidator({ seq: 0, type: this.responseType, data })) {
+						logger.error(JSON.stringify(this._responseValidator.errors, null, 4));
 						throw new Error(`Validation failed responding to ${this.requestType}`);
 					}
 
-					link.connector.send(this.responseType, { ...response, seq: message.seq });
+					link.connector.send(this.responseType, data);
 
 				}).catch(err => {
 					if (!(err instanceof libErrors.RequestError)) {
-						console.error(`Unexpected error while responding to ${this.requestType}`, err);
+						logger.error(`Unexpected error while responding to ${this.requestType}:\n${err.stack}`);
 					}
 					link.connector.send(this.responseType, { seq: message.seq, error: err.message });
 				});
@@ -217,7 +258,7 @@ class Request extends Message {
 	async send(link, data = {}) {
 		// XXX validate link target/source?
 		if (!this._requestValidator({ seq: 0, type: this.requestType, data })) {
-			console.error(this._requestValidator.errors);
+			logger.error(JSON.stringify(this._requestValidator.errors, null, 4));
 			throw new Error(`Validation failed sending ${this.requestType}`);
 		}
 		let seq = link.connector.send(this.requestType, data);
@@ -259,6 +300,37 @@ messages.debugDumpWs = new Request({
 
 
 // Management requests
+messages.getMasterConfig = new Request({
+	type: "get_master_config",
+	links: ["control-master"],
+	permission: "core.master.get_config",
+	responseProperties: {
+		"serialized_config": { type: "object" },
+	},
+});
+
+messages.setMasterConfigField = new Request({
+	type: "set_master_config_field",
+	links: ["control-master"],
+	permission: "core.master.update_config",
+	requestProperties: {
+		"field": { type: "string" },
+		"value": { type: "string" },
+	},
+});
+
+messages.setMasterConfigProp = new Request({
+	type: "set_master_config_prop",
+	links: ["control-master"],
+	permission: "core.master.update_config",
+	requestRequired: ["field", "prop"],
+	requestProperties: {
+		"field": { type: "string" },
+		"prop": { type: "string" },
+		"value": {},
+	},
+});
+
 messages.listSlaves = new Request({
 	type: "list_slaves",
 	links: ["control-master"],
@@ -330,18 +402,6 @@ messages.createSlaveConfig = new Request({
 	},
 });
 
-messages.setInstanceOutputSubscriptions = new Request({
-	type: "set_instance_output_subscriptions",
-	links: ["control-master"],
-	permission: "core.instance.follow_log",
-	requestProperties: {
-		"instance_ids": {
-			type: "array",
-			items: { type: "integer" },
-		},
-	},
-});
-
 messages.createInstance = new Request({
 	type: "create_instance",
 	links: ["control-master"],
@@ -378,6 +438,7 @@ messages.setInstanceConfigProp = new Request({
 	type: "set_instance_config_prop",
 	links: ["control-master"],
 	permission: "core.instance.update_config",
+	requestRequired: ["instance_id", "field", "prop"],
 	requestProperties: {
 		"instance_id": { type: "integer" },
 		"field": { type: "string" },
@@ -392,19 +453,10 @@ messages.assignInstanceCommand = new Request({
 	permission: "core.instance.assign",
 	requestProperties: {
 		"instance_id": { type: "integer" },
-		"slave_id": { type: "integer" },
+		"slave_id": { type: ["integer", "null"] },
 	},
 });
 
-messages.assignInstance = new Request({
-	type: "assign_instance",
-	links: ["master-slave"],
-	requestProperties: {
-		"serialized_config": { type: "object" },
-	},
-	forwardTo: "instance",
-});
-
 messages.startInstance = new Request({
 	type: "start_instance",
 	links: ["control-master", "master-slave", "slave-instance"],
@@ -636,8 +688,90 @@ messages.deleteUser = new Request({
 	},
 });
 
+messages.setLogSubscriptions = new Request({
+	type: "set_log_subscriptions",
+	links: ["control-master"],
+	permission: "core.log.follow",
+	requestProperties: {
+		"all": { type: "boolean" },
+		"master": { type: "boolean" },
+		"slave_ids": {
+			type: "array",
+			items: { type: "integer" },
+		},
+		"instance_ids": {
+			type: "array",
+			items: { type: "integer" },
+		},
+		"max_level": { type: ["string", "null"] },
+	},
+});
+
+messages.queryLog = new Request({
+	type: "query_log",
+	links: ["control-master"],
+	permission: "core.log.query",
+	requestProperties: {
+		"all": { type: "boolean" },
+		"master": { type: "boolean" },
+		"slave_ids": {
+			type: "array",
+			items: { type: "integer" },
+		},
+		"instance_ids": {
+			type: "array",
+			items: { type: "integer" },
+		},
+		"max_level": { type: ["string", "null"] },
+	},
+	responseProperties: {
+		"log": {
+			type: "array",
+			items: { type: "object" },
+		},
+	},
+});
+
+
+// Internal requests
+messages.updateInstances = new Request({
+	type: "update_instances",
+	links: ["slave-master"],
+	requestProperties: {
+		"instances": {
+			type: "array",
+			items: {
+				type: "object",
+				additionalProperties: false,
+				required: ["serialized_config", "status"],
+				properties: {
+					"serialized_config": { type: "object" },
+					"status": { enum: [
+						"stopped", "starting", "running", "creating_save", "exporting_data",
+					]},
+				},
+			},
+		},
+	},
+});
+
+messages.assignInstance = new Request({
+	type: "assign_instance",
+	links: ["master-slave"],
+	requestProperties: {
+		"serialized_config": { type: "object" },
+	},
+	forwardTo: "instance",
+});
+
+messages.unassignInstance = new Request({
+	type: "unassigne_instance",
+	links: ["master-slave"],
+	forwardTo: "instance",
+});
+
 messages.getMetrics = new Request({
-	type: "statistics_exporter:get_metrics",
+	type: "get_metrics",
 	links: ["master-slave", "slave-instance"],
 	responseProperties: {
 		"results": { type: "array" },
@@ -675,11 +809,14 @@ class Event extends Message {
 	 *     links that are downstream, so sending an event set to instance
 	 *     broadcast that's sent to a slave will only be broadcast to that
 	 *     slave's instances.
+	 * @param {?Array<string>} eventRequired -
+	 *     List of properties required to be present in the event payload.
+	 *     Defaults to all.
 	 * @param {Object<string, Object>} eventProperties -
 	 *     Mapping of property values to JSON schema specifications for the
 	 *     properties that are valid in the data payload of this event.
 	 */
-	constructor({ type, links, forwardTo = null, broadcastTo = null, eventProperties = {} }) {
+	constructor({ type, links, forwardTo = null, broadcastTo = null, eventRequired = null, eventProperties = {} }) {
 		super();
 		this.type = type;
 		this.links = links;
@@ -687,7 +824,12 @@ class Event extends Message {
 		this.broadcastTo = broadcastTo;
 		this.handlerSuffix = "EventHandler";
 
+		if (!eventRequired) {
+			eventRequired = Object.keys(eventProperties);
+		}
+
 		if (forwardTo === "instance") {
+			eventRequired = ["instance_id", ...eventRequired];
 			eventProperties = {
 				"instance_id": { type: "integer" },
 				...eventProperties,
@@ -701,14 +843,14 @@ class Event extends Message {
 			throw new Error(`Invalid broadcastTo value ${broadcastTo}`);
 		}
 
-		this.eventType = type + "_event";
+		this.eventType = `${type}_event`;
 		this._eventValidator = libSchema.compile({
 			$schema: "http://json-schema.org/draft-07/schema#",
 			properties: {
 				"type": { const: this.eventType },
 				"data": {
 					additionalProperties: false,
-					required: Object.keys(eventProperties),
+					required: eventRequired,
 					properties: eventProperties,
 				},
 			},
@@ -766,13 +908,13 @@ class Event extends Message {
 			}
 
 			if (!handler) {
-				throw new Error(`Missing handler for ${this.eventType} on ${link.source}-${link.target} link`);
+				throw new MissingLinkHandlerError(this.eventType, link.source, link.target);
 			}
 
 			link.setHandler(this.eventType, message => {
 				// XXX Should event handlers even be allowed to be async?
 				handler.call(link, message, this).catch(err => {
-					console.error(`Unexpected error while handling ${this.eventType}`, err);
+					logger.error(`Unexpected error while handling ${this.eventType}:\n${err.stack}`);
 				});
 			}, this._eventValidator);
 		}
@@ -788,7 +930,7 @@ class Event extends Message {
 	 */
 	send(link, data = {}) {
 		if (!this._eventValidator({ seq: 0, type: this.eventType, data })) {
-			console.error(this._eventValidator.errors);
+			logger.error(JSON.stringify(this._eventValidator.errors, null, 4));
 			throw new Error(`Validation failed sending ${this.eventType}`);
 		}
 
@@ -805,6 +947,21 @@ messages.debugWsMessage = new Event({
 	},
 });
 
+messages.logMessage = new Event({
+	type: "log_message",
+	links: ["slave-master", "master-control"],
+	eventProperties: {
+		"info": {
+			type: "object",
+			required: ["level", "message"],
+			properties: {
+				"level": { type: "string" },
+				"message": { type: "string" },
+			},
+		},
+	},
+});
+
 messages.instanceInitialized = new Event({
 	type: "instance_initialized",
 	links: ["instance-slave"],
@@ -827,52 +984,6 @@ messages.instanceStatusChanged = new Event({
 	},
 });
 
-messages.updateInstances = new Event({
-	type: "update_instances",
-	links: ["slave-master"],
-	eventProperties: {
-		"instances": {
-			type: "array",
-			items: {
-				type: "object",
-				additionalProperties: false,
-				required: ["serialized_config", "status"],
-				properties: {
-					"serialized_config": { type: "object" },
-					"status": { enum: [
-						"stopped", "starting", "running", "creating_save", "exporting_data",
-					]},
-				},
-			},
-		},
-	},
-});
-
-messages.instanceOutput = new Event({
-	type: "instance_output",
-	links: ["instance-slave", "slave-master", "master-control"],
-	forwardTo: "master",
-	eventProperties: {
-		"instance_id": { type: "integer" },
-		"output": {
-			type: "object",
-			additionalProperties: false,
-			required: ["source", "received", "format", "type", "message"],
-			properties: {
-				"source": { type: "string" },
-				"received": { type: "number" },
-				"format": { type: "string" },
-				"time": { type: "string" },
-				"level": { type: "string" },
-				"file": { type: "string" },
-				"type": { type: "string" },
-				"action": { type: "string" },
-				"message": { type: "string" },
-			},
-		},
-	},
-});
-
 messages.masterConnectionEvent = new Event({
 	type: "master_connection_event",
 	links: ["slave-instance"],
@@ -959,7 +1070,15 @@ messages.playerEvent = new Event({
  */
 function attachAllMessages(link) {
 	for (let [name, message] of Object.entries(messages)) {
-		message.attach(link, link[name + message.handlerSuffix]);
+		let handler = `${name}${message.handlerSuffix}`;
+		try {
+			message.attach(link, link[handler]);
+		} catch (err) {
+			if (err.code === "MISSING_LINK_HANDLER") {
+				err.handler = handler;
+			}
+			throw err;
+		}
 	}
 }
 

packages/lib/logging.js

@@ -0,0 +1,94 @@
+"use strict";
+const winston = require("winston");
+const Transport = require("winston-transport");
+const { LEVEL, MESSAGE } = require("triple-beam");
+
+const levels = Object.freeze({
+	fatal: 0,
+	error: 1,
+	warn: 2,
+	audit: 3,
+	info: 4,
+	server: 5,
+	verbose: 6,
+});
+
+const colors = {
+	fatal: "inverse red",
+	error: "bold brightRed",
+	warn: "bold brightYellow",
+	audit: "bold brightGreen",
+	info: "bold brightBlue",
+	server: "bold",
+	verbose: "bold grey",
+};
+winston.addColors(colors);
+
+// The Console Transport is replicated here because it's overly complicated
+// in winston, writes to stdout/err instead of console and doesn't work in
+// the browser.
+class ConsoleTransport extends Transport {
+	constructor(options = {}) {
+		super(options);
+		this.errorLevels = new Set(options.errorLevels || ["fatal", "error"]);
+		this.warnLevels = new Set(options.warnLevels || ["warn"]);
+	}
+
+	log(info, callback) {
+		/* eslint-disable no-console */
+		if (this.errorLevels.has(info[LEVEL])) {
+			console.error(info[MESSAGE]);
+		} else if (this.warnLevels.has(info[LEVEL])) {
+			console.warn(info[MESSAGE]);
+		} else {
+			console.log(info[MESSAGE]);
+		}
+		/* eslint-enable no-console */
+		callback();
+	}
+}
+
+/**
+ * Formats winston log messages for the web console
+ */
+class WebConsoleFormat {
+	constructor(options = {}) {
+		this.options = options;
+	}
+
+	transform(info, options) {
+		let src = " ";
+		if (info.slave_id) {
+			let slaveName = info.slave_id;
+			src += `s:${slaveName} - `;
+		}
+		if (info.instance_id) {
+			let instanceName = info.instance_id;
+			src += `i:${instanceName}`;
+		}
+		if (info.plugin) {
+			src += `${info.plugin}: `;
+		}
+
+		if (info.stack) {
+			info[MESSAGE] = `[${info.level}]${src}${info.stack}`;
+		} else {
+			info[MESSAGE] = `[${info.level}]${src}${info.message}`;
+		}
+		return info;
+	}
+}
+
+const logger = winston.createLogger({
+	level: "verbose",
+	levels,
+	format: winston.format.timestamp(),
+});
+
+module.exports = {
+	ConsoleTransport,
+	WebConsoleFormat,
+	colors,
+	levels,
+	logger,
+};

packages/lib/logging_utils.js

@@ -0,0 +1,131 @@
+"use strict";
+const winston = require("winston");
+const Transport = require("winston-transport");
+const { LEVEL, MESSAGE } = require("triple-beam");
+const chalk = require("chalk");
+
+const libLink = require("@clusterio/lib/link");
+const libErrors = require("@clusterio/lib/errors");
+
+
+/**
+ * Format a parsed Factorio message with colors
+ *
+ * Formats a parsed Factorio output from lib/factorio into a readable
+ * colorized output using terminal escape codes that can be printed.
+ *
+ * @param {Object} parsed - Parsed Factorio server output.
+ * @returns {string} terminal colorized message.
+ * @private
+ */
+function formatServerOutput(parsed) {
+	let time = "";
+	if (parsed.format === "seconds") {
+		time = `${chalk.yellow(parsed.time.padStart(8))} `;
+	} else if (parsed.format === "date") {
+		time = `${chalk.yellow(parsed.time)} `;
+	}
+
+	let info = "";
+	if (parsed.type === "log") {
+		let level = parsed.level;
+		if (level === "Script") {
+			level = chalk.bold.greenBright(level);
+		} else if (level === "Verbose") {
+			level = chalk.bold.gray(level);
+		} else if (level === "Info") {
+			level = chalk.bold.blueBright(level);
+		} else if (parsed.level === "Warning") {
+			level = chalk.bold.yellowBright(level);
+		} else if (parsed.level === "Error") {
+			level = chalk.bold.redBright(level);
+		}
+
+		info = `${level} ${chalk.gray(parsed.file)}: `;
+
+	} else if (parsed.type === "action") {
+		info = `[${chalk.yellow(parsed.action)}] `;
+	}
+
+	return time + info + parsed.message;
+}
+
+// These are defined here to avoid circular dependencies and pulling them
+// into the web interface code.
+
+/**
+ * Formats winston log messages for a character terminal.
+ */
+class TerminalFormat {
+	constructor(options = {}) {
+		this.options = options;
+		this.colorize = winston.format.colorize(options);
+	}
+
+	transform(info, options) {
+		info = this.colorize.transform(info, this.colorize.options);
+		let ts = "";
+		if (options.showTimestamp && info.timestamp) {
+			ts = `${info.timestamp.replace("T", " ")} `;
+		}
+
+		let src = " ";
+		if (info.instance_name) {
+			src += `i:${info.instance_name} `;
+
+		} else if (info.slave_name) {
+			src += `s:${info.slave_name} `;
+		}
+		if (info.plugin) {
+			src += `${info.plugin}: `;
+		}
+
+		if (info[LEVEL] === "server" && info.parsed) {
+			info.message = formatServerOutput(info.parsed);
+		}
+
+		if (info.stack) {
+			info[MESSAGE] = `${ts}[${info.level}]${src}${info.stack}`;
+		} else {
+			info[MESSAGE] = `${ts}[${info.level}]${src}${info.message}`;
+		}
+		return info;
+	}
+}
+
+/**
+ * Sends logs over a lib/link connection.
+ */
+class LinkTransport extends Transport {
+	constructor(options) {
+		super(options);
+
+		this.link = options.link;
+		this.filter = options.filter || null;
+	}
+
+	log(info, callback) {
+		if (this.filter && !this.filter(info)) {
+			return callback();
+		}
+
+		try {
+			libLink.messages.logMessage.send(this.link, { info });
+		} catch (err) {
+			// Ignore session lost errors.
+			if (!(err instanceof libErrors.SessionLost)) {
+				throw err;
+			}
+		}
+		return callback();
+	}
+}
+
+
+module.exports = {
+	TerminalFormat,
+	LinkTransport,
+
+	// for testing only
+	_formatServerOutput: formatServerOutput,
+};

packages/lib/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@clusterio/lib",
   "description": "Shared library for Clusterio",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.4",
   "repository": "https://github.com/clusterio/factorioClusterio",
   "license": "MIT",
   "keywords": [
@@ -22,6 +22,10 @@
     "node-forge": "^0.10.0",
     "rcon-client": "^4.2.0",
     "semver": "^7.1.1",
+    "set-blocking": "^2.0.0",
+    "triple-beam": "^1.3.0",
+    "winston": "^3.3.3",
+    "winston-transport": "^4.4.0",
     "ws": "^7.2.3",
     "yargs": "^14.2.0"
   },

packages/lib/plugin.js

@@ -51,6 +51,15 @@ class BaseInstancePlugin {
 		 * @type {module:slave/slave~Slave}
 		 */
 		this.slave = slave;
+
+		/**
+		 * Logger for this plugin
+		 *
+		 * Instance of winston Logger for sending log messages from this
+		 * plugin.  Supported methods and their corresponding log levels are
+		 * `error`, `warn`, `audit`, `info` and `verbose`.
+		 */
+		this.logger = instance.logger.child({ plugin: this.info.name });
 	}
 
 	/**
@@ -117,34 +126,34 @@ class BaseInstancePlugin {
 	 * stderr.  The output is a parsed form of the line and contains the
 	 * following properties:
 	 *
-	 * @param {Object} output - parsed server output.
-	 * @param {string} output.source -
+	 * @param {Object} parsed - parsed server output.
+	 * @param {string} parsed.source -
 	 *     Where the message came from, one of "stdout" and "stderr".
-	 * @param {string} output.format -
+	 * @param {string} parsed.format -
 	 *     Timestamp format, one of "date", "seconds" and "none".
-	 * @param {string=} output.time -
+	 * @param {string=} parsed.time -
 	 *     Timestamp of the message.  Not present if format is "none".
-	 * @param {string} output.type -
+	 * @param {string} parsed.type -
 	 *     Type of message, one of "log", "action" and "generic".
-	 * @param {string=} output.level -
+	 * @param {string=} parsed.level -
 	 *     Log level for "log" type.  i.e "Info" normally.
-	 * @param {string=} output.file - File reported for "log" type.
-	 * @param {string=} output.action -
+	 * @param {string=} parsed.file - File reported for "log" type.
+	 * @param {string=} parsed.action -
 	 *     Kind of action for "action" type. i.e "CHAT" for chat.
-	 * @param {string} output.message - Main content of the line.
+	 * @param {string} parsed.message - Main content of the line.
+	 * @param {string} line - raw line of server output.
 	 */
-	async onOutput(output) { }
+	async onOutput(parsed, line) { }
 
 	/**
 	 * Called when an event on the master connection happens
 	 *
-	 * The event param may be one of connect, drop and close and has the
-	 * following meaning:
+	 * The event param may be one of connect, drop, resume and close and has
+	 * the following meaning:
 	 *
 	 * ##### connect
 	 *
-	 * Invoked when a new connection to the master has been established, or
-	 * the existing one that have previously been dropped is re-established.
+	 * Invoked when a new connection to the master has been established.
 	 *
 	 * ##### drop
 	 *
@@ -156,6 +165,11 @@ class BaseInstancePlugin {
 	 * memory on the slave and sent all in one go when the connection is
 	 * re-established again.
 	 *
+	 * ##### resume
+	 *
+	 * Invoked when the connection that had previously dropped is
+	 * re-established.
+	 *
 	 * ##### close
 	 *
 	 * Invoked when the connection to the master has been closed.  This
@@ -163,7 +177,7 @@ class BaseInstancePlugin {
 	 * send any messages that goes to or via the master server after the
 	 * connection has been closed and before a new one is established.
 	 *
-	 * @param {string} event - one of connect, drop, and close
+	 * @param {string} event - one of connect, drop, resume and close
 	 */
 	onMasterConnectionEvent(event) { }
 
@@ -192,6 +206,10 @@ class BaseInstancePlugin {
 	 * @param {string} event.name - Name of the player that joined/left.
 	 */
 	async onPlayerEvent(event) { }
+
+	async sendRcon(message, expectEmpty) {
+		return await this.instance.sendRcon(message, expectEmpty, this.info.name);
+	}
 }
 
 /**
@@ -206,10 +224,19 @@ class BaseInstancePlugin {
  * @static
  */
 class BaseMasterPlugin {
-	constructor(info, master, metrics) {
+	constructor(info, master, metrics, logger) {
 		this.info = info;
 		this.master = master;
 		this.metrics = metrics;
+
+		/**
+		 * Logger for this plugin
+		 *
+		 * Instance of winston Logger for sending log messages from this
+		 * plugin.  Supported methods and their corresponding log levels are
+		 * `error`, `warn`, `audit`, `info` and `verbose`.
+		 */
+		this.logger = logger.child({ plugin: this.info.name });
 	}
 
 	/**
@@ -270,13 +297,12 @@ class BaseMasterPlugin {
 	/**
 	 * Called when an event on a slave connection happens
 	 *
-	 * The event param may be one of connect, drop and close and has the
-	 * following meaning:
+	 * The event param may be one of connect, drop, resume and close and has
+	 * the following meaning:
 	 *
 	 * ##### connect
 	 *
-	 * Invoked when a new connection to the slave has been established, or
-	 * an existing one that had previously been dropped is re-established.
+	 * Invoked when a new connection to the slave has been established.
 	 *
 	 * ##### drop
 	 *
@@ -288,13 +314,18 @@ class BaseMasterPlugin {
 	 * memory on the master and sent all in one go when the connection is
 	 * re-established again.
 	 *
+	 * ##### resume
+	 *
+	 * Invoked when a connection that had previously been dropped is
+	 * re-established
+	 *
 	 * ##### close
 	 *
 	 * Invoked when the connection to the slave has been closed.
 	 *
 	 * @param {module:master/master~SlaveConnection} connection -
 	 *     The connection the event occured on.
-	 * @param {string} event - one of connect, drop, and close
+	 * @param {string} event - one of connect, drop, resume and close
 	 */
 	onSlaveConnectionEvent(connection, event) { }
 
@@ -358,11 +389,20 @@ class BaseMasterPlugin {
  * @static
  */
 class BaseControlPlugin {
-	constructor(info) {
+	constructor(info, logger) {
 		/**
 		 * The plugin's own info module
 		 */
 		this.info = info;
+
+		/**
+		 * Logger for this plugin
+		 *
+		 * Instance of winston Logger for sending log messages from this
+		 * plugin.  Supported methods and their corresponding log levels are
+		 * `error`, `warn`, `audit`, `info` and `verbose`.
+		 */
+		this.logger = logger.child({ plugin: this.info.name });
 	}
 
 	/**
@@ -386,29 +426,36 @@ class BaseControlPlugin {
 /**
  * Attach plugin messages
  *
- * Attaches all messages defined in the `.message` property of the
- * `pluginInfo` passed with handlers taken from `pluin`.
+ * Attaches all messages defined in the `.info.message` property of the
+ * `plugin` passed with handlers taken from the `pluin`.
  *
  * @param {module:lib/link.Link} link - Link to attach handlers to.
- * @param {object} pluginInfo - Plugin info object.
  * @param {module:lib/plugin.BasePlugin} plugin -
- *     The instance of the plugin to use handlers from.
+ *     The instance of the plugin attach messages and handlers from.
  */
-function attachPluginMessages(link, pluginInfo, plugin) {
-	let messageDefinitions = pluginInfo.messages || [];
+function attachPluginMessages(link, plugin) {
+	let messageDefinitions = plugin.info.messages || [];
 	for (let [name, messageFormat] of Object.entries(messageDefinitions)) {
-		if (messageFormat.plugin !== pluginInfo.name) {
-			throw new Error(`Type of ${name} message must start with "${pluginInfo.name}:"`);
+		if (messageFormat.plugin !== plugin.info.name) {
+			throw new Error(`Type of ${name} message must start with "${plugin.info.name}:"`);
 		}
 
 		let handler = name + messageFormat.handlerSuffix;
-		if (plugin === null || !plugin[handler]) {
-			messageFormat.attach(link);
+		try {
+			if (plugin === null || !plugin[handler]) {
+				messageFormat.attach(link);
 
-		} else {
-			messageFormat.attach(link, async function(message, format) {
-				return await plugin[handler](message, format, this);
-			});
+			} else {
+				messageFormat.attach(link, async (message, format) => await plugin[handler](message, format));
+			}
+
+		} catch (err) {
+			if (err.code === "MISSING_LINK_HANDLER") {
+				err.plugin = plugin.info.name;
+				err.handler = handler;
+			}
+
+			throw err;
 		}
 	}
 }
@@ -431,7 +478,7 @@ async function invokeHook(plugins, hook, ...args) {
 				results.push(result);
 			}
 		} catch (err) {
-			console.error(`Ignoring error from plugin ${name} in ${hook}:`, err);
+			plugin.logger.error(`Ignoring error from plugin ${name} in ${hook}:\n${err.stack}`);
 		}
 	}
 	return results;

packages/lib/plugin_loader.js

@@ -1,3 +1,4 @@
+/* eslint-disable global-require */
 /**
  * Helpers for loading plugins in Node.js
  * @module lib/plugin_loader
@@ -47,6 +48,51 @@ async function loadPluginInfos(pluginList) {
 	return plugins;
 }
 
+/**
+ * Load master plugin class of a plugin
+ *
+ * @param {Object} pluginInfo -
+ *     Plugin info object returned from {@link
+ *     module:lib/plugin_loader.loadPluginInfos} to load class from.
+ * @returns {function} plugin class
+ * @static
+ */
+async function loadMasterPluginClass(pluginInfo) {
+	let entrypoint = require(path.posix.join(pluginInfo.requirePath, pluginInfo.masterEntrypoint));
+	return entrypoint.MasterPlugin;
+}
+
+/**
+ * Load instance plugin class of a plugin
+ *
+ * @param {Object} pluginInfo -
+ *     Plugin info object returned from {@link
+ *     module:lib/plugin_loader.loadPluginInfos} to load class from.
+ * @returns {function} plugin class
+ * @static
+ */
+async function loadInstancePluginClass(pluginInfo) {
+	let entrypoint = require(path.posix.join(pluginInfo.requirePath, pluginInfo.instanceEntrypoint));
+	return entrypoint.InstancePlugin;
+}
+
+/**
+ * Load control plugin class of a plugin
+ *
+ * @param {Object} pluginInfo -
+ *     Plugin info object returned from {@link
+ *     module:lib/plugin_loader.loadPluginInfos} to load class from.
+ * @returns {function} plugin class
+ * @static
+ */
+async function loadControlPluginClass(pluginInfo) {
+	let entrypoint = require(path.posix.join(pluginInfo.requirePath, pluginInfo.controlEntrypoint));
+	return entrypoint.ControlPlugin;
+}
+
 module.exports = {
 	loadPluginInfos,
+	loadMasterPluginClass,
+	loadInstancePluginClass,
+	loadControlPluginClass,
 };

packages/lib/schema.js

@@ -3,7 +3,8 @@
  * @module lib/schema
  */
 "use strict";
-const ajv = new (require("ajv"))({
+const Ajv = require("ajv");
+const ajv = new Ajv({
 	verbose: true,
 	format: "full",
 	extendRefs: "fail",

packages/lib/shared_commands.js

@@ -6,9 +6,16 @@
 const fs = require("fs-extra");
 const path = require("path");
 
-const libConfig = require("./config");
+const libConfig = require("@clusterio/lib/config");
+const { logger } = require("@clusterio/lib/logging");
+const libHelpers = require("@clusterio/lib/helpers");
 
 
+function print(...content) {
+	// eslint-disable-next-line no-console
+	console.log(...content);
+}
+
 /**
  * Yargs plugin command
  *
@@ -48,37 +55,37 @@ async function handlePluginCommand(args, pluginList, pluginListPath) {
 
 		let pluginInfo;
 		try {
+			// eslint-disable-next-line global-require
 			pluginInfo = require(path.posix.join(pluginPath, "info"));
 		} catch (err) {
-			console.error(`Unable to import plugin info from ${args.path}`);
-			console.error(err);
+			logger.error(`Unable to import plugin info from ${args.path}:\n${err.stack}`);
 			process.exitCode = 1;
 			return;
 		}
 
 		if (pluginList.has(pluginInfo.name)) {
-			console.error(`Plugin with the same ${pluginInfo.name} already exists`);
+			logger.error(`Plugin with the same ${pluginInfo.name} already exists`);
 			process.exitCode = 1;
 			return;
 		}
 
 		pluginList.set(pluginInfo.name, pluginPath);
 		await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
-		console.log(`Added ${pluginInfo.name}`);
+		print(`Added ${pluginInfo.name}`);
 
 	} else if (command === "remove") {
 		if (!pluginList.delete(args.name)) {
-			console.error(`Plugin with name ${args.name} does not exist`);
+			logger.error(`Plugin with name ${args.name} does not exist`);
 			process.exitCode = 1;
 			return;
 		}
 
 		await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
-		console.log(`Removed ${args.name}`);
+		print(`Removed ${args.name}`);
 
 	} else if (command === "list") {
 		for (let [pluginName, pluginPath] of pluginList) {
-			console.log(`${pluginName} - ${pluginPath}`);
+			print(`${pluginName} - ${pluginPath}`);
 		}
 	}
 }
@@ -94,7 +101,11 @@ async function handlePluginCommand(args, pluginList, pluginListPath) {
  */
 function configCommand(yargs) {
 	yargs
-		.command("set <field> [value]", "Set config field")
+		.command("set <field> [value]", "Set config field", yargs => {
+			yargs.options({
+				"stdin": { describe: "read value from stdin", nargs: 0, type: "boolean" },
+			});
+		})
 		.command("show <field>", "Show value of the given config field")
 		.command("list", "List all configuration fields and their values")
 		.demandCommand(1, "You need to specify a command to run")
@@ -119,23 +130,26 @@ async function handleConfigCommand(args, instance, configPath) {
 		for (let GroupClass of instance.constructor.groups.values()) {
 			for (let def of GroupClass.definitions.values()) {
 				let value = instance.get(def.fullName);
-				console.log(`${def.fullName} ${JSON.stringify(value)}`);
+				print(`${def.fullName} ${JSON.stringify(value)}`);
 			}
 		}
 
 	} else if (command === "show") {
 		try {
-			console.log(instance.get(args.field));
+			print(instance.get(args.field));
 		} catch (err) {
 			if (err instanceof libConfig.InvalidField) {
-				console.error(err.message);
+				logger.error(err.message);
 			} else {
 				throw err;
 			}
 		}
 
 	} else if (command === "set") {
-		if (args.value === undefined) {
+		if (args.stdin) {
+			args.value = (await libHelpers.readStream(process.stdin)).toString().replace(/\r?\n$/, "");
+
+		} else if (args.value === undefined) {
 			args.value = null;
 		}
 
@@ -144,7 +158,7 @@ async function handleConfigCommand(args, instance, configPath) {
 			await fs.outputFile(configPath, JSON.stringify(instance.serialize(), null, 4));
 		} catch (err) {
 			if (err instanceof libConfig.InvalidField || err instanceof libConfig.InvalidValue) {
-				console.error(err.message);
+				logger.error(err.message);
 			} else {
 				throw err;
 			}

packages/lib/users.js

@@ -102,7 +102,7 @@ class Role {
 function ensureDefaultAdminRole(roles) {
 	let admin = roles.get(0);
 	if (!admin) {
-		admin = new Role({ id: 0, name: "Admin", description: "Cluster wide administrator." });
+		admin = new Role({ id: 0, name: "Cluster Admin", description: "Cluster wide administrator." });
 		roles.set(0, admin);
 	}
 	admin.permissions.add("core.admin");
@@ -302,6 +302,16 @@ definePermission({
 	grantByDefault: true,
 });
 
+definePermission({
+	name: "core.master.get_config",
+	title: "Get master config",
+	description: "Get the config of master.",
+});
+definePermission({
+	name: "core.master.update_config",
+	title: "Modify master config",
+	description: "Modify the master config or entries of the master config.",
+});
 definePermission({
 	name: "core.slave.list",
 	title: "List slaves",
@@ -345,11 +355,6 @@ definePermission({
 	title: "Assign instance",
 	description: "Assign or reassign instances to a slave.",
 });
-definePermission({
-	name: "core.instance.follow_log",
-	title: "Follow instance log",
-	description: "Listen for the stdout log of instances.",
-});
 definePermission({
 	name: "core.instance.create_save",
 	title: "Create new instance save",
@@ -450,6 +455,16 @@ definePermission({
 	title: "Delete user",
 	description: "Delete users and all data stored for them.",
 });
+definePermission({
+	name: "core.log.follow",
+	title: "Follow cluster log",
+	description: "Receive new entries in the cluster log.  Required to see instance console.",
+});
+definePermission({
+	name: "core.log.query",
+	title: "Query cluster log",
+	description: "Query past entries in the cluster log.  Required to see past entries in instance console.",
+});
 
 definePermission({
 	name: "core.debug.dump_ws",

packages/master/package.json

@@ -3,7 +3,7 @@
   "description": "Implementation of Clusterio master server",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.4",
   "main": "master.js",
   "scripts": {
     "build": "webpack-cli --env.production",
@@ -19,7 +19,7 @@
     "node": ">=10.16 <11 || >= 12"
   },
   "dependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.2",
+    "@clusterio/lib": "^2.0.0-alpha.4",
     "body-parser": "^1.18.2",
     "compression": "^1.7.2",
     "cookie-parser": "^1.4.3",
@@ -31,11 +31,12 @@
     "node-factorio-api": "^0.3.8",
     "node-forge": "^0.10.0",
     "set-blocking": "^2.0.0",
+    "winston": "^3.3.3",
     "ws": "^7.2.3",
     "yargs": "^14.2.0"
   },
   "devDependencies": {
-    "@clusterio/web_ui": "^2.0.0-alpha.2",
+    "@clusterio/web_ui": "^2.0.0-alpha.4",
     "babel-loader": "^8.1.0",
     "css-loader": "^4.3.0",
     "fast-deep-equal": "^3.1.3",

packages/master/src/HttpCloser.js

@@ -0,0 +1,109 @@
+"use strict";
+const http = require("http");
+const https = require("https");
+const util = require("util");
+
+/**
+ * Gracefully close a Node.js HTTP(S) server
+ *
+ * Helper class to properly close down a Node.js HTTP(S) server.  This is
+ * necessary due to connections being able to linger around forever with the
+ * keep-alive logic in HTTP.
+ *
+ * @example
+ * // during server initialization
+ * const server = http.createServer();
+ * const serverCloser = new HttpCloser(server);
+ *
+ * // in the shutdown logic
+ * await serverCloser.close();
+ *
+ * @alias module:master/src/HttpCloser
+ */
+class HttpCloser {
+	/**
+	 * Server to attach this closer to.
+	 * @param {https.Server | http.Server} server - HTTP(S) to attach to
+	 */
+	constructor(server) {
+		this._server = server;
+		this._sockets = new Set();
+		this._responses = new Set();
+
+		if (this._server instanceof http.Server) {
+			this._server.on("connection", this._addSocket.bind(this));
+		} else if (this._server instanceof https.Server) {
+			this._server.on("secureConnection", this._addSocket.bind(this));
+		} else {
+			throw Error("server must be an instance of either http.Server or https.Server.");
+		}
+		this._server.on("request", this._addRequest.bind(this));
+	}
+
+	_addSocket(socket) {
+		this._sockets.add(socket);
+		socket.once("close", () => {
+			this._sockets.delete(socket);
+		});
+	}
+
+	_addRequest(request, response) {
+		this._responses.add(response);
+		response.once("close", () => this._responses.delete(response));
+		response.once("finish", () => this._responses.delete(response));
+	}
+
+	/**
+	 * Gracefully close down the attached HTTP(S) server.
+	 *
+	 * Closes open connections that do not have a request in flight, signals
+	 * connections to pending requests should close, and initiates the
+	 * close operation on the attached server.
+	 *
+	 * If there's requests in progress that still haven't been completed
+	 * after the timeout value passed they will be aborted.
+	 *
+	 * @param {number} timeout -
+	 *     Time in ms to wait for connections to close.
+	 */
+	async close(timeout = 5000) {
+		let activeSockets = new Set();
+		for (let response of this._responses) {
+			if (!response.headersSent) {
+				response.setHeader("Connection", "close");
+			} else {
+				let socket = response.socket;
+				response.once("finish", () => {
+					socket.end();
+				});
+			}
+			activeSockets.add(response.socket);
+		}
+		for (let socket of this._sockets) {
+			if (!activeSockets.has(socket)) {
+				socket.end();
+			}
+		}
+
+		return new Promise((resolve, reject) => {
+			let timer = setTimeout(() => {
+				for (let socket of this._sockets) {
+					// Ideally a RST would be sent here to indicate the the
+					// transfer failed, but we can't reliably send that from
+					// Node.js, see nodejs/node#27428.
+					socket.destroy();
+				}
+			}, timeout);
+			this._server.close(err => {
+				clearTimeout(timer);
+				if (err) {
+					reject(err);
+				} else {
+					resolve();
+				}
+			});
+		});
+	}
+};
+
+module.exports = HttpCloser;

packages/master/src/generate_ssl_cert.js

@@ -1,45 +0,0 @@
-"use strict";
-const forge = require("node-forge");
-const fs = require("fs-extra");
-const util = require("util");
-
-const hrtime = process.hrtime.bigint;
-
-
-module.exports = async function({
-	bits = 2048,
-	sslCertPath,
-	sslPrivKeyPath,
-	doLogging = false,
-} = {}){
-	function log(t) {
-		if(doLogging) console.log("src/generate_ssl_cert.js | "+t);
-	}
-	log("Generating SSL certificate with "+bits+" bits at "+sslCertPath);
-
-	let startNs = hrtime();
-	let keypair = await util.promisify(forge.pki.rsa.generateKeyPair)({ bits: bits, e: 0x10001 });
-	log("Generated 2048bit keypair in: "+(Number(hrtime()-startNs) / 1e6)+" ms");
-
-	startNs = hrtime();
-	let cert = forge.pki.createCertificate();
-	cert.publicKey = keypair.publicKey;
-	cert.serialNumber = "01";
-
-	cert.validity.notBefore = new Date();
-	cert.validity.notAfter = new Date(Date.now()+(1000*60*60*24*365)); // valid for one year
-
-	// XXX This should probably be limited to the actual hostname
-	let attrs = [{ name: "commonName", value: "*" }];
-	cert.setSubject(attrs);
-	cert.setIssuer(attrs);
-
-	// self-sign certificate
-	cert.sign(keypair.privateKey);
-
-	// Write certificate and private key to pem files
-	await fs.outputFile(sslPrivKeyPath, forge.pki.privateKeyToPem(keypair.privateKey));
-	await fs.outputFile(sslCertPath, forge.pki.certificateToPem(cert));
-
-	log("Generated certificate in: "+(Number(hrtime()-startNs) / 1e6)+" ms");
-};

packages/master/src/routes.js

@@ -2,15 +2,14 @@
 // Routes used in the web interface and served by the master server
 module.exports = [
 	"/",
-	"/instances",
-	"/instances/:id/view",
+	"/master",
 	"/slaves",
 	"/slaves/:id/view",
+	"/instances",
+	"/instances/:id/view",
 	"/users",
 	"/users/:id/view",
 	"/roles",
 	"/roles/:id/view",
-	"/permissions",
-	"/permissions/:id/view",
 	"/settings",
 ];

packages/master/webpack.config.js

@@ -9,11 +9,11 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	devServer: {
-		contentBase: "./static",
+		contentBase: "./dist/web",
 	},
 	output: {
 		filename: "bundle.js",
-		path: path.resolve(__dirname, "static"),
+		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({

packages/slave/lua/clusterio_lib/changelog.txt

@@ -1,4 +1,8 @@
 ---------------------------------------------------------------------------------------------------
+Version: 0.1.2
+  Features:
+    - Added support for Factorio 1.1.
+---------------------------------------------------------------------------------------------------
 Version: 0.1.1
   Features:
     - Added support for Factorio 0.18.x

packages/slave/lua/clusterio_lib/info.json

@@ -1,12 +1,24 @@
 {
     "name": "clusterio_lib",
-    "version": "0.1.1",
+    "variants": [
+        {
+            "version": "0.1.0",
+            "factorio_version": "0.17"
+        },
+        {
+            "version": "0.1.1",
+            "factorio_version": "0.18"
+        },
+        {
+            "version": "0.1.2",
+            "factorio_version": "1.1"
+        }
+    ],
     "title": "Clusterio Library (Alpha)",
     "author": "Clusterio Team",
     "homepage": "https://github.com/clusterio/factorioClusterio",
     "description": "Library for interfacing with Clusterio.  Warning: work in progress alpha.",
-    "factorio_version": "0.17",
-    "additional_files": [
-        ["..", "..", "modules", "clusterio", "serialize.lua"]
-    ]
+    "additional_files": {
+        "serialize.lua": ["..", "..", "modules", "clusterio", "serialize.lua"]
+    }
 }

packages/slave/package.json

@@ -3,7 +3,7 @@
   "description": "Implementation of Clusterio slave server",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.4",
   "main": "slave.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -18,11 +18,12 @@
     "node": ">=10.16 <11 || >= 12"
   },
   "dependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.2",
+    "@clusterio/lib": "^2.0.0-alpha.4",
     "fs-extra": "^8.1.0",
     "phin": "^3.4.1",
     "pidusage": "^2.0.18",
     "set-blocking": "^2.0.0",
+    "winston": "^3.3.3",
     "yargs": "^14.2.0"
   },
   "publishConfig": {

packages/web_ui/package.json

@@ -3,7 +3,7 @@
   "description": "Clusterio web interface implementation",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.4",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },
@@ -15,14 +15,18 @@
   },
   "dependencies": {
     "@ant-design/icons": "^4.2.2",
-    "@clusterio/lib": "^2.0.0-alpha.2",
+    "@clusterio/lib": "^2.0.0-alpha.4",
     "antd": "^4.6.5",
+    "buffer": "^6.0.2",
     "clean-webpack-plugin": "^3.0.0",
     "events": "^3.2.0",
+    "os-browserify": "^0.3.0",
     "path-browserify": "^1.0.1",
+    "process": "^0.11.10",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",
-    "react-router-dom": "^5.2.0"
+    "react-router-dom": "^5.2.0",
+    "util": "^0.12.3"
   },
   "publishConfig": {
     "access": "public"

packages/web_ui/src/bootstrap.jsx

@@ -4,6 +4,7 @@ import React from "react";
 import ReactDOM from "react-dom";
 
 import libConfig from "@clusterio/lib/config";
+import { ConsoleTransport, WebConsoleFormat, logger } from "@clusterio/lib/logging";
 
 import App from "./components/App";
 import { Control, ControlConnector } from "./util/websocket";
@@ -31,7 +32,7 @@ async function loadPluginInfos() {
 		pluginList = await response.json();
 
 	} else {
-		console.log("Failed to get plugin data, running without plugins");
+		logger.error("Failed to get plugin data, running without plugins");
 		pluginList = [];
 	}
 
@@ -47,14 +48,17 @@ async function loadPluginInfos() {
 			pluginInfos.push(info);
 
 		} catch (err) {
-			console.log(`Failed to load plugin info for ${plugin.name}`);
-			console.log(err);
+			logger.error(`Failed to load plugin info for ${plugin.name}`);
 		}
 	}
 	return pluginInfos;
 }
 
 async function load() {
+	logger.add(new ConsoleTransport({
+		level: "verbose",
+		format: new WebConsoleFormat(),
+	}));
 	let pluginInfos = await loadPluginInfos();
 	libConfig.registerPluginConfigGroups(pluginInfos);
 	libConfig.finalizeConfigs();
@@ -68,4 +72,4 @@ async function load() {
 	ReactDOM.render(<App control={control}/>, document.getElementById("root"));
 }
 
-load().catch(console.log);
+load().catch((err) => logger.fatal(err));

packages/web_ui/src/components/App.jsx

@@ -2,6 +2,8 @@ import events from "events";
 import React, { useEffect, useState } from "react";
 import { BrowserRouter } from "react-router-dom";
 
+import { logger } from "@clusterio/lib/logging";
+
 import basename from "../basename";
 import SiteLayout from "./SiteLayout";
 import ControlContext from "./ControlContext";
@@ -31,7 +33,7 @@ export default function App(props) {
 		}
 
 		function onError(err) {
-			console.error("Unexpected error in connector", err);
+			logger.error(`Unexpected error in connector:\n${err.stack}`);
 			clearToken();
 			setConnected(false);
 		}

packages/web_ui/src/components/AssignInstanceModal.jsx

@@ -0,0 +1,91 @@
+import React, { useContext, useState } from "react";
+import { Button, Form, Modal, Select, Typography } from "antd";
+
+import libLink from "@clusterio/lib/link";
+
+import ControlContext from "./ControlContext";
+import { notifyErrorHandler } from "../util/notify";
+
+const { Paragraph } = Typography;
+
+
+export default function AssignInstanceModal(props) {
+	let [visible, setVisible] = useState(false);
+	let [slaves, setSlaves] = useState([]);
+	let [loading, setLoading] = useState(true);
+	let [applying, setApplying] = useState(false);
+	let [form] = Form.useForm();
+	let control = useContext(ControlContext);
+
+	function open() {
+		setVisible(true);
+		libLink.messages.listSlaves.send(control).then((result) => {
+			setSlaves(result.list);
+			setLoading(false);
+		});
+	}
+
+	function handleAssign() {
+		let slaveId = form.getFieldValue("slave");
+		if (slaveId === undefined) {
+			setVisible(false);
+			return;
+		}
+
+		setApplying(true);
+		libLink.messages.assignInstanceCommand.send(control, {
+			instance_id: props.id,
+			slave_id: slaveId,
+		}).then(() => {
+			setVisible(false);
+			if (props.onFinish) {
+				props.onFinish();
+			}
+		}).catch(
+			notifyErrorHandler("Error assigning instance")
+		).finally(
+			() => setApplying(false)
+		);
+	}
+
+	function handleCancel() {
+		setVisible(false);
+	}
+
+	return <>
+		<Button {...props.buttonProps} onClick={open}>
+			{props.buttonContent || "Assign"}
+		</Button>
+		<Modal
+			title="Assign Instance"
+			okText="Assign"
+			visible={visible}
+			confirmLoading={applying}
+			onOk={handleAssign}
+			onCancel={handleCancel}
+			destroyOnClose
+		>
+			<Paragraph style={{ maxWidth: "30em" }}>
+				Select a Slave to assign this instance to.  Assignment
+				creates the necessary files on the slave to start up
+				the instance.  Note that reassigning an instance from
+				one slave to another will not move the server save over.
+			</Paragraph>
+			<Form form={form} initialValues={{ slave: props.slaveId }}>
+				<Form.Item name="slave" label="Slave">
+					<Select loading={loading}>
+						{slaves.map((slave) => <Select.Option
+							key={slave["id"]}
+							value={slave["id"]}
+							disabled={!slave["connected"]}
+						>
+							{slave["name"]}
+							{!slave["connected"] && " (offline)"}
+						</Select.Option>)}
+					</Select>
+				</Form.Item>
+			</Form>
+		</Modal>
+	</>;
+}
+

packages/web_ui/src/components/BaseConfigTree.jsx

@@ -0,0 +1,252 @@
+import React, { useEffect, useContext, useState } from "react";
+import { Button, Card, Checkbox, Form, Input, InputNumber, Space, Spin, Tree, Typography } from "antd";
+
+const { Title } = Typography;
+
+
+export default function BaseConfigTree(props) {
+	let [config, setConfig] = useState(null);
+	let [form] = Form.useForm();
+	let [changedFields, setChangedFields] = useState(new Set());
+	let [errorFields, setErrorFields] = useState(new Map());
+	let [applying, setApplying] = useState(false);
+
+	async function updateConfig() {
+		let serializedConfig = await props.retrieveConfig();
+		let newConfig = new props.ConfigClass("control");
+		await newConfig.load(serializedConfig);
+		setConfig(newConfig);
+	}
+	useEffect(() => { updateConfig(); }, [props.id]);
+
+	function renderInput(def) {
+		if (def.type === "boolean") {
+			return <Checkbox/>;
+		}
+		if (def.type === "string") {
+			return <Input/>;
+		}
+		if (def.type === "number") {
+			return <InputNumber/>;
+		}
+
+		return `Unknown type ${def.type}`;
+	}
+
+	if (config === null) {
+		return <Spin/>;
+	}
+
+	let initialValues = {};
+	function onValuesChange(changedValues) {
+		let newChangedFields = new Set(changedFields);
+		let changed = false;
+		for (let [key, value] of Object.entries(changedValues)) {
+			if (key.endsWith(":add.name") || key.endsWith(":add.value")) {
+				continue;
+			}
+			if (initialValues[key] === value) {
+				changed = newChangedFields.delete(key) || changed;
+			} else {
+				changed = !newChangedFields.has(key) || changed;
+				newChangedFields.add(key);
+			}
+		}
+
+		if (changed) {
+			setChangedFields(newChangedFields);
+		}
+	}
+
+	let treeData = [];
+	let propsMap = new Map();
+	for (let [name, GroupClass] of props.ConfigClass.groups) {
+		let group = config.group(name);
+		let treeNode = {
+			key: group.name,
+			title: group.name,
+			children: [],
+		};
+
+		for (let def of GroupClass.definitions.values()) {
+			if (!group.canAccess(def.name)) {
+				continue;
+			}
+
+			let value = group.get(def.name);
+			let fieldName = `${group.name}.${def.name}`;
+			let childNode = {
+				key: fieldName,
+				children: [],
+			};
+
+			if (def.type === "object") {
+				childNode.title = <Form.Item label={def.title || def.name} tooltip={def.description} />;
+				for (let prop of Object.keys(value)) {
+					let propPath = `${group.name}.${def.name}.${prop}`;
+					childNode.children.push({
+						key: propPath,
+						title: <Form.Item
+							name={propPath}
+							label={`"${prop}"`}
+							rules={[{ validator: (_, fieldValue) => {
+								if (!fieldValue.length) {
+									return Promise.reject(new Error("Will be removed"));
+								}
+								try {
+									JSON.parse(fieldValue);
+								} catch (err) {
+									return Promise.reject(new Error("Must be valid json"));
+								}
+								return Promise.resolve();
+							}}]}
+						>
+							<Input className="json-input" />
+						</Form.Item>,
+					});
+
+					initialValues[propPath] = JSON.stringify(value[prop]);
+					propsMap.set(propPath, [fieldName, prop]);
+				}
+
+				let newPropPath = `${group.name}.${def.name}:add`;
+				childNode.children.push({
+					key: newPropPath,
+					title: <Space>
+						<Form.Item
+							name={`${newPropPath}.name`}
+							noStyle={true}
+						>
+							<Input />
+						</Form.Item>:
+						<Form.Item
+							name={`${newPropPath}.value`}
+							noStyle={true}
+						>
+							<Input className="json-input" />
+						</Form.Item>
+						<Button
+							size="small"
+							onClick={async () => {
+								let propName = form.getFieldValue(`${newPropPath}.name`);
+								let propValue = form.getFieldValue(`${newPropPath}.value`);
+								if (!Object.prototype.hasOwnProperty.call(value, propName)) {
+									let newConfig = new props.ConfigClass("control");
+									await newConfig.load(config.serialize());
+									newConfig.setProp(fieldName, propName, null);
+									setConfig(newConfig);
+								}
+
+								let propPath = `${group.name}.${def.name}.${propName}`;
+								form.setFieldsValue({
+									[`${newPropPath}.name`]: "",
+									[`${newPropPath}.value`]: "",
+									[propPath]: propValue,
+								});
+								form.validateFields([propPath]);
+								onValuesChange({ [propPath]: propValue });
+							}}
+						>
+							Add
+						</Button>
+					</Space>,
+				});
+
+			} else {
+				childNode.title = <Form.Item
+					name={fieldName}
+					label={def.title || def.name}
+					validateStatus={errorFields.has(fieldName) ? "error" : undefined}
+					help={errorFields.get(fieldName)}
+					tooltip={def.description}
+					valuePropName={def.type === "boolean" ? "checked" : "value"}
+				>
+					{renderInput(def)}
+				</Form.Item>;
+				initialValues[fieldName] = value;
+			}
+
+			treeNode.children.push(childNode);
+		}
+
+		treeData.push(treeNode);
+	}
+
+	return <>
+		<Title level={5} style={{ marginTop: 16 }}>
+			Config
+			<Space style={{ float: "right" }}>
+				<Button
+					type="primary"
+					size="small"
+					loading={applying}
+					onClick={async () => {
+						setApplying(true);
+						for (let field of changedFields) {
+							let value = form.getFieldValue(field);
+							let request;
+							if (propsMap.has(field)) {
+								let [fieldName, prop] = propsMap.get(field);
+								request = props.setProp(fieldName, prop, value);
+
+							} else {
+								request = props.setField(field, value);
+							}
+
+							try {
+								await request;
+								setChangedFields(changed => {
+									let newChanged = new Set(changed);
+									newChanged.delete(field);
+									return newChanged;
+								});
+
+							} catch (err) {
+								setErrorFields(errors => {
+									let newErrors = new Map(errors);
+									newErrors.set(field, err.message);
+									return newErrors;
+								});
+							}
+						}
+
+						await updateConfig();
+						setApplying(false);
+
+						if (props.onApply) {
+							props.onApply();
+						}
+					}}
+					disabled={changedFields.size === 0}
+				>
+					Apply
+				</Button>
+				<Button
+					size="small"
+					onClick={() => {
+						form.resetFields();
+						setChangedFields(new Set());
+						setErrorFields(new Map());
+					}}
+				>
+					Revert
+				</Button>
+			</Space>
+		</Title>
+		<Card size="small" className="config-container">
+			<Form
+				size="small"
+				form={form}
+				initialValues={initialValues}
+				onValuesChange={onValuesChange}
+			>
+				<Tree
+					treeData={treeData}
+					selectable={false}
+					defaultExpandAll={true}
+					filterTreeNode={(node) => changedFields.has(node.key)}
+				/>
+			</Form>
+		</Card>
+	</>;
+}

packages/web_ui/src/components/ErrorBoundary.jsx

@@ -0,0 +1,20 @@
+import React from "react";
+
+
+export default class ErrorBoundary extends React.Component {
+	constructor(props) {
+		super(props);
+		this.state = { error: null };
+	}
+
+	static getDerivedStateFromError(error) {
+		return { error };
+	}
+
+	render() {
+		if (this.state.error) {
+			return <this.props.Component error={this.state.error} />;
+		}
+		return this.props.children;
+	}
+}

packages/web_ui/src/components/ErrorPage.jsx

@@ -0,0 +1,16 @@
+import React, { useContext } from "react";
+import { Typography } from "antd";
+
+import libLink from "@clusterio/lib/link";
+
+import PageLayout from "./PageLayout";
+
+const { Paragraph } = Typography;
+
+
+export default function ErrorPage(props) {
+	return <PageLayout nav={[{ name: "Error" }]}>
+		<h2>An unexpected error occured</h2>
+		<Paragraph code className="error-traceback">{props.error.stack}</Paragraph>
+	</PageLayout>;
+};

packages/web_ui/src/components/InstanceConfigTree.jsx

@@ -0,0 +1,52 @@
+import React, { useContext } from "react";
+import { Typography } from "antd";
+
+import libLink from "@clusterio/lib/link";
+import libConfig from "@clusterio/lib/config";
+
+import BaseConfigTree from "./BaseConfigTree";
+import ControlContext from "./ControlContext";
+
+
+export default function InstanceConfigTree(props) {
+	let control = useContext(ControlContext);
+
+	async function retrieveConfig() {
+		let result = await libLink.messages.getInstanceConfig.send(control, {
+			instance_id: props.id,
+		});
+		return result.serialized_config;
+	}
+
+	async function setField(field, value) {
+		await libLink.messages.setInstanceConfigField.send(control, {
+			instance_id: props.id,
+			field,
+			value: String(value),
+		});
+	}
+
+	async function setProp(field, prop, value) {
+		let data = {
+			instance_id: props.id,
+			field,
+			prop,
+		};
+		if (value) {
+			try {
+				data.value = JSON.parse(value);
+			} catch (err) {
+				return;
+			}
+		}
+		await libLink.messages.setInstanceConfigProp.send(control, data);
+	}
+
+	return <BaseConfigTree
+		ConfigClass={libConfig.InstanceConfig}
+		retrieveConfig={retrieveConfig}
+		setField={setField}
+		setProp={setProp}
+		id={props.id}
+	/>;
+}

packages/web_ui/src/components/InstanceConsole.jsx

@@ -0,0 +1,94 @@
+import React, { useEffect, useContext, useRef, useState } from "react";
+import { Typography } from "antd";
+
+import libLink from "@clusterio/lib/link";
+
+import ControlContext from "./ControlContext";
+
+const { Paragraph } = Typography;
+
+
+function formatParsedOutput(parsed, key) {
+	let time = "";
+	if (parsed.format === "seconds") {
+		time = <span className="factorio-time">{parsed.time.padStart(8)} </span>;
+	} else if (parsed.format === "date") {
+		time = <span className="factorio-time">{parsed.time} </span>;
+	}
+
+	let info = "";
+	if (parsed.type === "log") {
+		let level = parsed.level;
+		if (level === "Script") {
+			level = <span className="factorio-script">{level}</span>;
+		} else if (level === "Verbose") {
+			level = <span className="factorio-verbose">{level}</span>;
+		} else if (level === "Info") {
+			level = <span className="factorio-info">{level}</span>;
+		} else if (parsed.level === "Warning") {
+			level = <span className="factorio-warning">{level}</span>;
+		} else if (parsed.level === "Error") {
+			level = <span className="factorio-error">{level}</span>;
+		}
+
+		info = <>{level} <span className="factorio-filename">{parsed.file}: </span></>;
+
+	} else if (parsed.type === "action") {
+		info = <>[<span className="factorio-action">{parsed.action}</span>] </>;
+	}
+
+	return <span key={key}>{time}{info}{parsed.message}<br/></span>;
+}
+
+function formatLog(info, key) {
+	if (info.level === "server" && info.parsed) {
+		return formatParsedOutput(info.parsed, key);
+	}
+	let level = <span className={`log-${info.level}`}>{info.level}</span>;
+	return <span key={key}>[{level}] {info.message}<br/></span>;
+}
+
+export default function InstanceConsole(props) {
+	let control = useContext(ControlContext);
+	let anchor = useRef();
+	let [pastLines, setPastLines] = useState([<span key={0}>{"Loading past entries..."}<br/></span>]);
+	let [lines, setLines] = useState([]);
+
+	useEffect(() => {
+		// Scroll view to the anchor so it sticks to the bottom
+		let parent = anchor.current.parentElement;
+		parent.scrollTop = parent.scrollHeight - parent.clientHeight;
+
+		libLink.messages.queryLog.send(control, {
+			all: false,
+			master: false,
+			slave_ids: [],
+			instance_ids: [props.id],
+			max_level: null,
+		}).then(result => {
+			setPastLines(result.log.slice(-400).map((info, index) => formatLog(info, index)));
+		}).catch(err => {
+			setPastLines([<span key={0}>{`Error loading log: ${err.message}`}<br/></span>]);
+		});
+
+		function logHandler(info) {
+			setLines(currentLines => currentLines.concat(
+				[formatLog(info, currentLines.length)]
+			));
+		}
+
+		control.onInstanceLog(props.id, logHandler);
+		return () => {
+			control.offInstanceLog(props.id, logHandler);
+		};
+	}, [props.id]);
+
+	return <>
+		<Paragraph code className="instance-console">
+			{pastLines}
+			{lines}
+			<div className="scroll-anchor" ref={anchor} />
+		</Paragraph>
+	</>;
+}
+

packages/web_ui/src/components/InstanceRcon.jsx

@@ -0,0 +1,49 @@
+import React, { useContext, useState } from "react";
+import { Input, Typography } from "antd";
+
+import libLink from "@clusterio/lib/link";
+
+import ControlContext from "./ControlContext";
+import { notifyErrorHandler } from "../util/notify";
+
+const { Title, Paragraph } = Typography;
+
+
+export default function InstanceRcon(props) {
+	let control = useContext(ControlContext);
+	let [output, setOutput] = useState(null);
+	let [running, setRunning] = useState(false);
+
+	async function sendCommand(command) {
+		if (!command) {
+			setOutput(null);
+			return;
+		}
+
+		setRunning(true);
+		try {
+			let result = await libLink.messages.sendRcon.send(control, {
+				instance_id: props.id,
+				command: command,
+			});
+			setOutput(result.result);
+		} finally {
+			setRunning(false);
+		}
+	}
+
+	return <>
+		{output && <>
+			<Title level={5}>Rcon result</Title>
+			<Paragraph code className="rcon-result">{output}</Paragraph>
+		</>}
+		<Input.Search
+			disabled={props.disabled}
+			placeholder="Send RCON Command"
+			enterButton="Send"
+			onSearch={(command) => sendCommand(command).catch(notifyErrorHandler("Error sending command"))}
+			loading={running}
+		/>
+	</>;
+}
+

packages/web_ui/src/components/InstanceViewPage.jsx

@@ -1,506 +1,24 @@
-import React, { useEffect, useContext, useState } from "react";
+import React, { useContext, useState } from "react";
 import { useHistory, useParams } from "react-router-dom";
-import {
-	notification, Button, Card, Checkbox, Col, Descriptions, Dropdown, Form, Input, InputNumber, Modal,
-	Popconfirm, Row, Select, Space, Spin, Tree, Typography,
-} from "antd";
+import { Button, Descriptions, Popconfirm, Space, Spin, Typography } from "antd";
 import DeleteOutlined from "@ant-design/icons/DeleteOutlined";
 
 import libLink from "@clusterio/lib/link";
-import libConfig from "@clusterio/lib/config";
 
 import ControlContext from "./ControlContext";
 import PageLayout from "./PageLayout";
+import InstanceConfigTree from "./InstanceConfigTree";
+import InstanceConsole from "./InstanceConsole";
+import InstanceRcon from "./InstanceRcon";
+import AssignInstanceModal from "./AssignInstanceModal";
+import StartStopInstanceButton from "./StartStopInstanceButton";
 import { notifyErrorHandler } from "../util/notify";
+import { useInstance } from "../model/instance";
+import { useSlave } from "../model/slave";
 
 const { Title, Paragraph } = Typography;
 
 
-function useInstance(id) {
-	let control = useContext(ControlContext);
-	let [instance, setInstance] = useState({ loading: true });
-
-	function updateInstance() {
-		if (!Number.isInteger(id)) {
-			setInstance({ missing: true });
-			return;
-		}
-
-		// XXX optimize by requesting only the instance in question
-		libLink.messages.listInstances.send(control).then(result => {
-			let match = result.list.find(i => i.id === id);
-			if (!match) {
-				setInstance({ missing: true });
-			} else {
-				setInstance({ ...match, present: true });
-			}
-		});
-	}
-
-	useEffect(() => {
-		updateInstance();
-	}, [id]);
-
-	return [instance, updateInstance];
-}
-
-function useSlave(id) {
-	let control = useContext(ControlContext);
-	let [slave, setSlave] = useState({ loading: true });
-
-	function updateSlave() {
-		if (!Number.isInteger(id)) {
-			setSlave({ missing: true });
-			return;
-		}
-
-		// XXX optimize by requesting only the slave in question
-		libLink.messages.listSlaves.send(control).then(result => {
-			let match = result.list.find(i => i.id === id);
-			if (!match) {
-				setSlave({ missing: true });
-			} else {
-				setSlave({ ...match, present: true });
-			}
-		});
-	}
-
-	useEffect(() => {
-		updateSlave();
-	}, [id]);
-
-	return [slave, updateSlave];
-}
-
-function formatOutputColored(output, key) {
-	let time = "";
-	if (output.format === "seconds") {
-		time = <span className="factorio-time">{output.time.padStart(8)} </span>;
-	} else if (output.format === "date") {
-		time = <span className="factorio-time">{output.time} </span>;
-	}
-
-	let info = "";
-	if (output.type === "log") {
-		let level = output.level;
-		if (level === "Info") {
-			level = <span className="factorio-info">{level}</span>;
-		} else if (output.level === "Warning") {
-			level = <span className="factorio-warning">{level}</span>;
-		} else if (output.level === "Error") {
-			level = <span className="factorio-error">{level}</span>;
-		}
-
-		info = <>{level} <span className="factorio-filename">{output.file}: </span></>;
-
-	} else if (output.type === "action") {
-		info = <>[<span className="factorio-action">{output.action}</span>] </>;
-	}
-
-	return <span key={key}>{time}{info}{output.message}<br/></span>;
-}
-
-function InstanceConsole(props) {
-	let control = useContext(ControlContext);
-	let [lines, setLines] = useState([{ message: "<previous messages are not shown>" }]);
-
-	useEffect(() => {
-		function outputHandler(output) {
-			setLines(currentLines => currentLines.concat([output]));
-		}
-
-		control.onInstanceOutput(props.id, outputHandler);
-		return () => {
-			control.offInstanceOutput(props.id, outputHandler);
-		};
-	}, [props.id]);
-
-	return <>
-		<Paragraph code className="instance-console">
-			{lines.map(formatOutputColored)}
-			<div className="scroll-anchor" />
-		</Paragraph>
-	</>;
-}
-
-function InstanceRcon(props) {
-	let control = useContext(ControlContext);
-	let [output, setOutput] = useState(null);
-	let [running, setRunning] = useState(false);
-
-	async function sendCommand(command) {
-		if (!command) {
-			setOutput(null);
-			return;
-		}
-
-		setRunning(true);
-		try {
-			let result = await libLink.messages.sendRcon.send(control, {
-				instance_id: props.id,
-				command: command,
-			});
-			setOutput(result.result);
-		} finally {
-			setRunning(false);
-		}
-	}
-
-	return <>
-		{output && <>
-			<Title level={5}>Rcon result</Title>
-			<Paragraph code className="rcon-result">{output}</Paragraph>
-		</>}
-		<Input.Search
-			disabled={props.disabled}
-			placeholder="Send RCON Command"
-			enterButton="Send"
-			onSearch={(command) => sendCommand(command).catch(notifyErrorHandler("Error sending command"))}
-			loading={running}
-		/>
-	</>;
-}
-
-function AssignInstanceModal(props) {
-	let [visible, setVisible] = useState(false);
-	let [slaves, setSlaves] = useState([]);
-	let [loading, setLoading] = useState(true);
-	let [applying, setApplying] = useState(false);
-	let [form] = Form.useForm();
-	let control = useContext(ControlContext);
-
-	function open() {
-		setVisible(true);
-		libLink.messages.listSlaves.send(control).then((result) => {
-			setSlaves(result.list);
-			setLoading(false);
-		});
-	}
-
-	function handleAssign() {
-		let slaveId = form.getFieldValue("slave");
-		if (slaveId === undefined) {
-			setVisible(false);
-			return;
-		}
-
-		setApplying(true);
-		libLink.messages.assignInstanceCommand.send(control, {
-			instance_id: props.id,
-			slave_id: slaveId,
-		}).then(() => {
-			setVisible(false);
-			if (props.onFinish) {
-				props.onFinish();
-			}
-		}).catch(
-			notifyErrorHandler("Error assigning instance")
-		).finally(
-			() => setApplying(false)
-		);
-	}
-
-	function handleCancel() {
-		setVisible(false);
-	}
-
-	return <>
-		<Button {...props.buttonProps} onClick={open}>
-			{props.buttonContent || "Assign"}
-		</Button>
-		<Modal
-			title="Assign Instance"
-			okText="Assign"
-			visible={visible}
-			confirmLoading={applying}
-			onOk={handleAssign}
-			onCancel={handleCancel}
-			destroyOnClose
-		>
-			<Paragraph style={{ maxWidth: "30em" }}>
-				Select a Slave to assign this instance to.  Assignment
-				creates the necessary files on the slave to start up
-				the instance.  Note that reassigning an instance from
-				one slave to another will not move the server save over.
-			</Paragraph>
-			<Form form={form} initialValues={{ slave: props.slaveId }}>
-				<Form.Item name="slave" label="Slave">
-					<Select loading={loading}>
-						{slaves.map((slave) => <Select.Option
-							key={slave["id"]}
-							value={slave["id"]}
-							disabled={!slave["connected"]}
-						>
-							{slave["name"]}
-							{!slave["connected"] && " (offline)"}
-						</Select.Option>)}
-					</Select>
-				</Form.Item>
-			</Form>
-		</Modal>
-	</>;
-}
-
-function StartStopInstanceButton(props) {
-	let control = useContext(ControlContext);
-	let [switching, setSwitching] = useState(false);
-
-	function onClick() {
-		setSwitching(true);
-		let action;
-		if (props.instance["status"] === "stopped") {
-			action = libLink.messages.startInstance.send(
-				control, { instance_id: props.instance["id"], save: null }
-			).catch(
-				notifyErrorHandler("Error starting instance")
-			);
-
-		} else if (["starting", "running"].includes(props.instance["status"])) {
-			action = libLink.messages.stopInstance.send(
-				control, { instance_id: props.instance["id"] }
-			).catch(
-				notifyErrorHandler("Error stopping instance")
-			);
-
-		} else {
-			setSwitching(false);
-			return;
-		}
-
-		action.finally(() => {
-			setSwitching(false);
-			if (props.onFinish) {
-				props.onFinish();
-			}
-		});
-	}
-
-	return <Button
-		{...(props.buttonProps || {})}
-		loading={switching}
-		type="primary"
-		disabled={!["starting", "running", "stopped"].includes(props.instance["status"])}
-		onClick={onClick}
-	>
-		{props.instance["status"] === "stopped" ? "Start" : "Stop"}
-	</Button>;
-}
-
-
-function InstanceConfigTree(props) {
-	let control = useContext(ControlContext);
-	let [config, setConfig] = useState(null);
-	let [form] = Form.useForm();
-	let [changedFields, setChangedFields] = useState(new Set());
-	let [errorFields, setErrorFields] = useState(new Map());
-	let [applying, setApplying] = useState(false);
-
-	async function updateConfig() {
-		let result = await libLink.messages.getInstanceConfig.send(control, {
-			instance_id: props.id,
-		});
-
-		let newConfig = new libConfig.InstanceConfig();
-		await newConfig.load(result.serialized_config);
-		setConfig(newConfig);
-	}
-
-	useEffect(() => { updateConfig(); }, [props.id]);
-
-	function renderInput(group, def) {
-		if (def.type === "boolean") {
-			return <Checkbox/>;
-		}
-		if (def.type === "string") {
-			return <Input/>;
-		}
-		if (def.type === "number") {
-			return <InputNumber/>;
-		}
-
-		return `Unknown type ${def.type}`;
-	}
-
-	if (config === null) {
-		return <Spin/>;
-	}
-
-	let initialValues = {};
-	let treeData = [];
-	let propsMap = new Map();
-	for (let [name, GroupClass] of libConfig.InstanceConfig.groups) {
-		let group = config.group(name);
-		let treeNode = {
-			key: group.name,
-			title: group.name,
-			children: [],
-		};
-
-		for (let def of GroupClass.definitions.values()) {
-			let value = group.get(def.name);
-			let fieldName = `${group.name}.${def.name}`;
-			let childNode = {
-				key: fieldName,
-				children: [],
-			};
-
-			if (def.type === "object") {
-				childNode.title = <Form.Item label={def.title || def.name} tooltip={def.description} />;
-				for (let prop of Object.keys(value)) {
-					let propPath = `${group.name}.${def.name}.${prop}`;
-					childNode.children.push({
-						key: propPath,
-						title: <Form.Item
-							name={propPath}
-							label={`"${prop}"`}
-							rules={[{ validator: (_, fieldValue) => {
-								try {
-									JSON.parse(fieldValue);
-								} catch (err) {
-									return Promise.reject(new Error("Must be valid json"));
-								}
-								return Promise.resolve();
-							}}]}
-						>
-							<Input className="json-input" />
-						</Form.Item>,
-					});
-
-					initialValues[propPath] = JSON.stringify(value[prop]);
-					propsMap.set(propPath, [fieldName, prop]);
-				}
-
-			} else {
-				childNode.title = <Form.Item
-					name={fieldName}
-					label={def.title || def.name}
-					validateStatus={errorFields.has(fieldName) ? "error" : undefined}
-					help={errorFields.get(fieldName)}
-					tooltip={def.description}
-					valuePropName={def.type === "boolean" ? "checked" : "value"}
-				>
-					{renderInput(group, def)}
-				</Form.Item>;
-				initialValues[fieldName] = value;
-			}
-
-			treeNode.children.push(childNode);
-		}
-
-		treeData.push(treeNode);
-	}
-
-	function onValuesChange(changedValues) {
-		let newChangedFields = new Set(changedFields);
-		let changed = false;
-		for (let [key, value] of Object.entries(changedValues)) {
-			if (initialValues[key] === value) {
-				changed = newChangedFields.delete(key) || changed;
-			} else {
-				changed = !newChangedFields.has(key) || changed;
-				newChangedFields.add(key);
-			}
-		}
-
-		if (changed) {
-			setChangedFields(newChangedFields);
-		}
-	}
-
-	return <>
-		<Title level={5} style={{ marginTop: 16 }}>
-			Config
-			<Space style={{ float: "right" }}>
-				<Button
-					type="primary"
-					size="small"
-					loading={applying}
-					onClick={async () => {
-						setApplying(true);
-						for (let field of changedFields) {
-							let value = form.getFieldValue(field);
-							let request;
-							if (propsMap.has(field)) {
-								let [fieldName, prop] = propsMap.get(field);
-								try {
-									value = JSON.parse(value);
-								} catch (err) {
-									continue;
-								}
-
-								request = libLink.messages.setInstanceConfigProp.send(control, {
-									instance_id: props.id,
-									field: fieldName,
-									prop,
-									value,
-								});
-
-							} else {
-								request = libLink.messages.setInstanceConfigField.send(control, {
-									instance_id: props.id,
-									field,
-									value: String(value),
-								});
-							}
-
-							try {
-								await request;
-								setChangedFields(changed => {
-									let newChanged = new Set(changed);
-									newChanged.delete(field);
-									return newChanged;
-								});
-
-							} catch (err) {
-								setErrorFields(errors => {
-									let newErrors = new Map(errors);
-									newErrors.set(field, err.message);
-									return newErrors;
-								});
-							}
-						}
-
-						await updateConfig();
-						setApplying(false);
-
-						if (props.onApply) {
-							props.onApply();
-						}
-					}}
-					disabled={changedFields.size === 0}
-				>
-					Apply
-				</Button>
-				<Button
-					size="small"
-					onClick={() => {
-						form.resetFields();
-						setChangedFields(new Set());
-						setErrorFields(new Map());
-					}}
-				>
-					Revert
-				</Button>
-			</Space>
-		</Title>
-		<Card size="small" className="config-container">
-			<Form
-				size="small"
-				form={form}
-				initialValues={initialValues}
-				onValuesChange={onValuesChange}
-			>
-				<Tree
-					treeData={treeData}
-					selectable={false}
-					defaultExpandAll={true}
-					filterTreeNode={(node) => changedFields.has(node.key)}
-				/>
-			</Form>
-		</Card>
-	</>;
-}
-
 export default function InstanceViewPage(props) {
 	let params = useParams();
 	let instanceId = Number(params.id);

packages/web_ui/src/components/InstancesPage.jsx

@@ -33,19 +33,16 @@ export default function InstancesPage() {
 					title: "Name",
 				}],
 				insert: async args => {
-					let instanceConfig = new libConfig.InstanceConfig();
+					let instanceConfig = new libConfig.InstanceConfig("control");
 					await instanceConfig.init();
-					console.log("Base config", instanceConfig);
 					instanceConfig.set("instance.name", args.name);
-					let serialized_config = instanceConfig.serialize();
+					let serialized_config = instanceConfig.serialize("master");
 					let response = await libLink.messages.createInstance.send(control, { serialized_config });
-					console.log("Created instance", response);
 				},
 			}}
 			TableProps={{
 				onRow: (record, rowIndex) => ({
 					onClick: event => {
-						console.log(record, rowIndex);
 						history.push(`/instances/${record.key}/view`);
 					},
 				}),

packages/web_ui/src/components/MasterConfigTree.jsx

@@ -0,0 +1,44 @@
+import React, { useContext } from "react";
+import { Typography } from "antd";
+
+import libLink from "@clusterio/lib/link";
+import libConfig from "@clusterio/lib/config";
+
+import BaseConfigTree from "./BaseConfigTree";
+import ControlContext from "./ControlContext";
+
+
+export default function MasterConfigTree(props) {
+	let control = useContext(ControlContext);
+
+	async function retrieveConfig() {
+		let result = await libLink.messages.getMasterConfig.send(control);
+		return result.serialized_config;
+	}
+
+	async function setField(field, value) {
+		await libLink.messages.setMasterConfigField.send(control, {
+			field,
+			value: String(value),
+		});
+	}
+
+	async function setProp(field, prop, value) {
+		let data = { field, prop };
+		if (value) {
+			try {
+				data.value = JSON.parse(value);
+			} catch (err) {
+				return;
+			}
+		}
+		await libLink.messages.setMasterConfigProp.send(control, data);
+	}
+
+	return <BaseConfigTree
+		ConfigClass={libConfig.MasterConfig}
+		retrieveConfig={retrieveConfig}
+		setField={setField}
+		setProp={setProp}
+	/>;
+}

packages/web_ui/src/components/MasterPage.jsx

@@ -0,0 +1,12 @@
+import React, { useContext } from "react";
+
+import MasterConfigTree from "./MasterConfigTree";
+import PageLayout from "./PageLayout";
+
+
+export default function SlavesPage() {
+	return <PageLayout nav={[{ name: "Master" }]}>
+		<h2>Master</h2>
+		<MasterConfigTree />
+	</PageLayout>;
+};

packages/web_ui/src/components/RoleViewPage.jsx

@@ -78,7 +78,6 @@ export default function RoleViewPage() {
 			initialValues={initialValues}
 			onValuesChange={() => { setEdited(true); }}
 			onFinish={values => {
-				console.log(values);
 				let newPermissions = [];
 				for (let [name, value] of Object.entries(values.permissions)) {
 					if (value) { newPermissions.push(name); }

packages/web_ui/src/components/RolesPage.jsx

@@ -41,7 +41,6 @@ export default function UsersPage() {
 			TableProps={{
 				onRow: (record, rowIndex) => ({
 					onClick: event => {
-						console.log(record, rowIndex);
 						history.push(`/roles/${record.key}/view`);
 					},
 				}),

packages/web_ui/src/components/SiteLayout.jsx

@@ -3,9 +3,13 @@ import { Switch, Route, useHistory } from "react-router-dom";
 import { Layout, Menu } from "antd";
 import webUiPackage from "../../package.json";
 
+import ErrorBoundary from "./ErrorBoundary";
+import ErrorPage from "./ErrorPage";
+
 const { Header, Sider } = Layout;
 
 
+import MasterPage from "./MasterPage";
 import SlavesPage from "./SlavesPage";
 import InstancesPage from "./InstancesPage";
 import InstanceViewPage from "./InstanceViewPage";
@@ -21,6 +25,7 @@ const pages = [
 ];
 
 const sidebar = [
+	{ name: "Master", path: "/master", content: <MasterPage /> },
 	{ name: "Slaves", path: "/slaves", content: <SlavesPage /> },
 	{ name: "Instances", path: "/instances", content: <InstancesPage />},
 	{ name: "Users", path: "/users", content: <UsersPage /> },
@@ -74,7 +79,9 @@ export default function SiteLayout(props) {
 					</Route>)}
 					{pages.map(page => <Route exact path={page.path} key={page.path}>
 						<SetSidebar path={page.sidebarPath} />
-						{page.content}
+						<ErrorBoundary Component={ErrorPage}>
+							{page.content}
+						</ErrorBoundary>
 					</Route>)}
 					<Route>
 						<SetSidebar path={null} />

packages/web_ui/src/components/StartStopInstanceButton.jsx

@@ -0,0 +1,55 @@
+import React, { useContext, useState } from "react";
+import { Button } from "antd";
+
+import libLink from "@clusterio/lib/link";
+
+import ControlContext from "./ControlContext";
+import { notifyErrorHandler } from "../util/notify";
+
+
+export default function StartStopInstanceButton(props) {
+	let control = useContext(ControlContext);
+	let [switching, setSwitching] = useState(false);
+
+	function onClick() {
+		setSwitching(true);
+		let action;
+		if (props.instance["status"] === "stopped") {
+			action = libLink.messages.startInstance.send(
+				control, { instance_id: props.instance["id"], save: null }
+			).catch(
+				notifyErrorHandler("Error starting instance")
+			);
+
+		} else if (["starting", "running"].includes(props.instance["status"])) {
+			action = libLink.messages.stopInstance.send(
+				control, { instance_id: props.instance["id"] }
+			).catch(
+				notifyErrorHandler("Error stopping instance")
+			);
+
+		} else {
+			setSwitching(false);
+			return;
+		}
+
+		action.finally(() => {
+			setSwitching(false);
+			if (props.onFinish) {
+				props.onFinish();
+			}
+		});
+	}
+
+	return <Button
+		{...(props.buttonProps || {})}
+		loading={switching}
+		type="primary"
+		disabled={!["starting", "running", "stopped"].includes(props.instance["status"])}
+		onClick={onClick}
+	>
+		{props.instance["status"] === "stopped" ? "Start" : "Stop"}
+	</Button>;
+}
+
+

packages/web_ui/src/components/UsersPage.jsx

@@ -59,7 +59,6 @@ export default function UsersPage() {
 			TableProps={{
 				onRow: (record, rowIndex) => ({
 					onClick: event => {
-						console.log(record, rowIndex);
 						history.push(`/users/${record.key}/view`);
 					},
 				}),

packages/web_ui/src/components/data-table.jsx

@@ -16,12 +16,16 @@ const { Panel } = Collapse;
  * Editable - Boolean - Allow editing
  * AddRecord - Boolean - Allow adding new records
  * CustomDataFilterFunction - function(tableData) - Filter tabledata and return results
- * RecordIdField - String - By default we assume the column "Id" is a unique identifier for a record. If that isn't the case, supply the column name here
+ * RecordIdField - String -
+ *     By default we assume the column "Id" is a unique identifier for a record. If
+ *     that isn't the case, supply the column name here
  */
 class DataTable extends Component {
 	constructor(props) {
 		super(props);
-		if (!this.props.DataFunction) throw new Error("DataFunction not defined");
+		if (!this.props.DataFunction) {
+			throw new Error("DataFunction not defined");
+		}
 		this.state = {
 			pagination: {
 				pageSize: 100,
@@ -40,34 +44,21 @@ class DataTable extends Component {
 	}
 
 	async getData(pagination, filters, sorter) {
-		/*
-			pagination = {
-				pageSize: 10, // items per page
-				current: 1, // page number
-				total: 123, // Total amount of items
-			}
-			filters = {
-				ReportId: 123
-			}
-			sorter = {}
-		*/
+		// pagination = {
+		//     pageSize: 10, // items per page
+		//     current: 1, // page number
+		//     total: 123, // Total amount of items
+		// }
+		// filters = {
+		//     ReportId: 123
+		// }
+		// sorter = {}
 		pagination = pagination || this.state.pagination;
 		filters = filters || this.state.filters;
 		sorter = sorter || this.state.sorter;
-		// console.log(pagination, filters, sorter)
 
 		// eslint-disable-next-line new-cap
 		let data = await this.props.DataFunction();
-		// await fetchApi(this.state.dataUrl, "post", {
-		//     database: this.props.Database,
-		//     table: this.props.Table,
-		//     dataSource: this.props.DataSource,
-		//     pagination: this.props.RecordCount? {...pagination, pageSize: this.props.RecordCount, current: 1}:pagination,
-		//     filters,
-		//     sorter,
-		//     recordIdField: this.props.RecordIdField || "Id", // Sort by this
-		// })
-		console.log(data);
 		data = data || {};
 		this.setState({
 			// eslint-disable-next-line new-cap
@@ -96,8 +87,6 @@ class DataTable extends Component {
 	}
 
 	async onEditFinish(values) {
-		console.log("Finished editing", values);
-
 		if (this.state.inserting) { // Create new record
 			await this.props.AddRecord.insert(values);
 		} else { // Edit existing record, capable of doing sparse updates
@@ -114,22 +103,16 @@ class DataTable extends Component {
 
 	async deleteRecord(record) {
 		delete record.key;
-		// Delete "null" values as they seem to cause deletes to fail silently when the value is null and field type is float
+		// Delete "null" values as they seem to cause deletes to fail silently when
+		// the value is null and field type is float
 		for (let key in record) {
-			if (record[key] === null) delete record[key];
+			if (record[key] === null) { delete record[key]; }
 		}
-		var status = {};
-		//  await fetchApi(`/api/${this.props.Server}/tableData/record`, "delete", {
-		//     database: this.props.Database,
-		//     table: this.props.Table,
-		//     dataSource: this.props.DataSource,
-		//     record,
-		// }, undefined, true)
+		let status = {};
+		// TODO: Delete data here
 
-		console.log("Deleted record", status);
-
-		if (status.ok) notify("success", status.msg, "success");
-		if (!status.ok) notify("error", status.msg, "error");
+		if (status.ok) { notify("success", status.msg, "success"); }
+		if (!status.ok) { notify("error", status.msg, "error"); }
 
 		// Force refresh of data
 		this.getData();
@@ -139,61 +122,66 @@ class DataTable extends Component {
 		let tableProps = {
 			...this.props.TableProps || {},
 			onRow: (record, rowIndex) => {
-				/*
-					To allow for extending functionality, we take a TableProps argument from props and pass it through to the Table component.
-					Since we are also using onRow internaly here, we handle the externally provided events ourselves by passing along the event
-					object. This way the end user isn't affected by our event handlers.
-				*/
+				// To allow for extending functionality, we take a TableProps argument
+				// from props and pass it through to the Table component.  Since we
+				// are also using onRow internaly here, we handle the externally
+				// provided events ourselves by passing along the event object. This
+				// way the end user isn't affected by our event handlers.
 				let externalOnRow = {
-					...(this.props.TableProps && this.props.TableProps.onRow && this.props.TableProps.onRow(record, rowIndex)),
+					...(
+						this.props.TableProps
+						&& this.props.TableProps.onRow
+						&& this.props.TableProps.onRow(record, rowIndex)
+					),
 				};
 				return {
 					onClick: event => {
 						if (event.target.type !== "button") {
-							console.log("onRow", record, rowIndex);
-							if (this.props.Editable) this.showEditModal(record, rowIndex);
-							if (externalOnRow.onClick) externalOnRow.onClick(event);
+							if (this.props.Editable) { this.showEditModal(record, rowIndex); }
+							if (externalOnRow.onClick) { externalOnRow.onClick(event); }
 						}
 					},
 					onDoubleClick: event => {
-						if (externalOnRow.onDoubleClick) externalOnRow.onDoubleClick(event);
+						if (externalOnRow.onDoubleClick) { externalOnRow.onDoubleClick(event); }
 					},
 					onContextMenu: event => {
-						if (externalOnRow.onContextMenu) externalOnRow.onContextMenu(event);
+						if (externalOnRow.onContextMenu) { externalOnRow.onContextMenu(event); }
 					},
 					onMouseEnter: event => {
-						if (externalOnRow.onMouseEnter) externalOnRow.onMouseEnter(event);
+						if (externalOnRow.onMouseEnter) { externalOnRow.onMouseEnter(event); }
 					},
 					onMouseLeave: event => {
-						if (externalOnRow.onMouseLeave) externalOnRow.onMouseLeave(event);
+						if (externalOnRow.onMouseLeave) { externalOnRow.onMouseLeave(event); }
 					},
 				};
 			},
 		};
 		let columns = this.props.columns || this.inferColumns(this.state.tableData);
-		if (this.props.DeleteRecord && !columns.find(x => x.dataIndex === "notInUse")) columns.push({
-			// Add delete button on the right
-			dataIndex: "notInUse",
-			title: "",
-			render: (text, record) => <Popover
-				content={
-					<Button
-						type="primary"
-						danger
-						onClick={() => {
-							this.deleteRecord(record);
-						}}
-					>
-						Slett permanent
+		if (this.props.DeleteRecord && !columns.find(x => x.dataIndex === "notInUse")) {
+			columns.push({
+				// Add delete button on the right
+				dataIndex: "notInUse",
+				title: "",
+				render: (text, record) => <Popover
+					content={
+						<Button
+							type="primary"
+							danger
+							onClick={() => {
+								this.deleteRecord(record);
+							}}
+						>
+							Slett permanent
+						</Button>
+					}
+					trigger="click"
+				>
+					<Button danger style={{ float: "right", fontSize: "16px" }}>
+						<DeleteOutlined />
 					</Button>
-				}
-				trigger="click"
-			>
-				<Button danger style={{ float: "right", fontSize: "16px" }}>
-					<DeleteOutlined />
-				</Button>
-			</Popover>,
-		});
+				</Popover>,
+			});
+		}
 
 		return <>
 			<Row>
@@ -210,7 +198,9 @@ class DataTable extends Component {
 						{...tableProps}
 						columns={columns.map(x => ({
 							...x, // Add a 4th parameter to the stock render function with all of our table data
-							render: x.render ? (...originalParams) => x.render(...originalParams, this.state.tableData) : undefined,
+							render: x.render
+								? (...originalParams) => x.render(...originalParams, this.state.tableData)
+								: undefined,
 						}))}
 						dataSource={this.state.tableData}
 						pagination={this.state.pagination.total > 10 ? this.state.pagination : false}
@@ -242,20 +232,21 @@ class DataTable extends Component {
 						onFinish={this.onEditFinish.bind(this)}
 						initialValues={this.state.record}
 					>
-						{
-
-							(
-								(this.state.inserting && this.props.AddRecord && this.props.AddRecord.fields) // Use externally supplied form input fields
-								|| this.props.columns.filter && this.props.columns.filter(x => x.dataIndex !== "notInUse") // Generate input fields from external columns
-								|| this.inferColumns(this.state.tableData)).map(col => { // Generate input fields from autogenerated columns (only works with existing data)
-								console.log(this.state.record);
-								return <Form.Item key={col.dataIndex} label={col.title} name={col.dataIndex}>
-									{col.renderEdit && col.renderEdit(this.state.record[col.dataIndex], this.state.record, undefined /* Non spec compliant, is supposed to be index of record */, this.state.tableData) || <Input
-										placeholder={this.state.record[col.dataIndex]}
-									/>}
-								</Form.Item>;
-							})
-						}
+						{(
+							// Use externally supplied form input fields
+							(this.state.inserting && this.props.AddRecord && this.props.AddRecord.fields)
+							// Generate input fields from external columns
+							|| this.props.columns.filter && this.props.columns.filter(x => x.dataIndex !== "notInUse")
+							// Generate input fields from autogenerated columns (only works with existing data)
+							|| this.inferColumns(this.state.tableData)
+						).map(col => <Form.Item key={col.dataIndex} label={col.title} name={col.dataIndex}>
+							{col.renderEdit && col.renderEdit(
+								this.state.record[col.dataIndex],
+								this.state.record,
+								undefined /* Non spec compliant, is supposed to be index of record */,
+								this.state.tableData
+							) || <Input placeholder={this.state.record[col.dataIndex]} />}
+						</Form.Item>)}
 						<Form.Item>
 							<Button type="primary" htmlType="submit">
 								{this.state.inserting ? "Legg til" : "Lagre"}
@@ -268,7 +259,8 @@ class DataTable extends Component {
 	}
 
 	inferColumns(data) {
-		// Look at the data and find columns and how to display them. Can be overridden by props.Columns if more customization is needed
+		// Look at the data and find columns and how to display them. Can be
+		// overridden by props.Columns if more customization is needed
 		let columns = [];
 		data.forEach(row => {
 			Object.keys(row).filter(x => x !== "key").forEach(key => {

packages/web_ui/src/index.css

@@ -82,6 +82,14 @@ body {
 	background-size: 64px;
 }
 
+.error-traceback code {
+	display: block;
+	margin: 0;
+	background: #000;
+	border-color: #252525;
+	white-space: pre-wrap;
+}
+
 .rcon-result code {
 	display: block;
 	background: #000;
@@ -108,6 +116,11 @@ body {
 	border-color: #252525;
 	height: 300px;
 	overflow-y: scroll;
+	white-space: pre-wrap;
+
+	/* Make sure there's at least one pixel of scroll space, this */
+	/* is necessary for the view to stick on the scroll anchor. */
+	padding-top: 300px;
 }
 
 .instance-console .factorio-time {
@@ -118,6 +131,16 @@ body {
 	color: #808080;
 }
 
+.instance-console .factorio-script {
+	font-weight: bold;
+	color: #49c33c;
+}
+
+.instance-console .factorio-verbose {
+	font-weight: bold;
+	color: #888888;
+}
+
 .instance-console .factorio-info {
 	font-weight: bold;
 	color: #4646ea;
@@ -137,6 +160,37 @@ body {
 	color: #afaf1f;
 }
 
+.log-verbose {
+	font-weight: bold;
+	color: #888888;
+}
+
+.log-info {
+	font-weight: bold;
+	color: #4646ea;
+}
+
+.log-audit {
+	font-weight: bold;
+	color: #49c33c;
+}
+
+.log-warn {
+	font-weight: bold;
+	color: #afaf1f;
+}
+
+.log-error {
+	font-weight: bold;
+	color: #ad3939;
+}
+
+.log-fatal {
+	font-weight: bold;
+	background-color: #ad3939;
+	color: #000;
+}
+
 .config-container {
 	background-color: #111111;
 }

packages/web_ui/src/model/instance.jsx

@@ -0,0 +1,33 @@
+import React, { useEffect, useContext, useState } from "react";
+import ControlContext from "../components/ControlContext";
+
+import libLink from "@clusterio/lib/link";
+
+
+export function useInstance(id) {
+	let control = useContext(ControlContext);
+	let [instance, setInstance] = useState({ loading: true });
+
+	function updateInstance() {
+		if (!Number.isInteger(id)) {
+			setInstance({ missing: true });
+			return;
+		}
+
+		// XXX optimize by requesting only the instance in question
+		libLink.messages.listInstances.send(control).then(result => {
+			let match = result.list.find(i => i.id === id);
+			if (!match) {
+				setInstance({ missing: true });
+			} else {
+				setInstance({ ...match, present: true });
+			}
+		});
+	}
+
+	useEffect(() => {
+		updateInstance();
+	}, [id]);
+
+	return [instance, updateInstance];
+}

packages/web_ui/src/model/slave.jsx

@@ -0,0 +1,33 @@
+import React, { useEffect, useContext, useState } from "react";
+import ControlContext from "../components/ControlContext";
+
+import libLink from "@clusterio/lib/link";
+
+
+export function useSlave(id) {
+	let control = useContext(ControlContext);
+	let [slave, setSlave] = useState({ loading: true });
+
+	function updateSlave() {
+		if (!Number.isInteger(id)) {
+			setSlave({ missing: true });
+			return;
+		}
+
+		// XXX optimize by requesting only the slave in question
+		libLink.messages.listSlaves.send(control).then(result => {
+			let match = result.list.find(i => i.id === id);
+			if (!match) {
+				setSlave({ missing: true });
+			} else {
+				setSlave({ ...match, present: true });
+			}
+		});
+	}
+
+	useEffect(() => {
+		updateSlave();
+	}, [id]);
+
+	return [slave, updateSlave];
+}

packages/web_ui/src/util/notify.jsx

@@ -1,4 +1,5 @@
 import { notification } from "antd";
+import { logger } from "@clusterio/lib/logging";
 
 
 export default function notify(message, type = "info", description = undefined) {
@@ -7,7 +8,9 @@ export default function notify(message, type = "info", description = undefined)
 		description,
 		placement: "bottomRight",
 	});
-	if (typeof message !== "string") console.error(message);
+	if (typeof message !== "string") {
+		logger.error(message.stack);
+	}
 }
 
 /**
@@ -22,7 +25,7 @@ export default function notify(message, type = "info", description = undefined)
  */
 export function notifyErrorHandler(message) {
 	return function handler(err) {
-		console.error(err);
+		logger.error(err.stack);
 		notification.error({
 			message,
 			description: err.message,

packages/web_ui/src/util/websocket.jsx

@@ -1,7 +1,8 @@
-const libLink = require("@clusterio/lib/link");
-const libPlugin = require("@clusterio/lib/plugin");
-const libErrors = require("@clusterio/lib/errors");
-const version = require("../../package.json").version;
+import libLink from "@clusterio/lib/link";
+import libPlugin from "@clusterio/lib/plugin";
+import libErrors from "@clusterio/lib/errors";
+import { logger } from "@clusterio/lib/logging";
+import packageJson from "../../package.json";
 
 
 /**
@@ -19,11 +20,11 @@ export class ControlConnector extends libLink.WebSocketClientConnector {
 			throw new Error("Token not set");
 		}
 
-		console.log("SOCKET | registering control");
+		logger.verbose("SOCKET | registering control");
 		this.sendHandshake("register_control", {
 			token: this.token,
 			agent: "Clusterio Web UI",
-			version,
+			version: packageJson.version,
 		});
 	}
 }
@@ -48,37 +49,46 @@ export class Control extends libLink.Link {
 			libPlugin.attachPluginMessages(this, controlPlugin.info, controlPlugin);
 		}
 
-		this.instanceOutputHandlers = new Map();
+		this.instanceLogHandlers = new Map();
+
+		this.connector.on("connect", () => {
+			this.updateLogSubscriptions().catch(err => logger.error(
+				`Unexpected error updating log subscriptions:\n${err.stack}`
+			));
+		});
 	}
 
-	async instanceOutputEventHandler(message) {
-		let { instance_id, output } = message.data;
-		let handlers = this.instanceOutputHandlers.get(instance_id);
-		for (let handler of handlers || []) {
-			handler(output);
+	async logMessageEventHandler(message) {
+		let info = message.data.info;
+
+		if (info.instance_id !== undefined) {
+			let instanceHandlers = this.instanceLogHandlers.get(info.instance_id);
+			for (let handler of instanceHandlers || []) {
+				handler(info);
+			}
 		}
 	}
 
-	async onInstanceOutput(id, handler) {
+	async onInstanceLog(id, handler) {
 		if (!Number.isInteger(id)) {
 			throw new Error("Invalid instance id");
 		}
 
-		let handlers = this.instanceOutputHandlers.get(id);
+		let handlers = this.instanceLogHandlers.get(id);
 		if (!handlers) {
 			handlers = [];
-			this.instanceOutputHandlers.set(id, handlers);
+			this.instanceLogHandlers.set(id, handlers);
 		}
 
 		handlers.push(handler);
 
 		if (handlers.length === 1) {
-			await this.updateInstanceOutputSubscriptions();
+			await this.updateLogSubscriptions();
 		}
 	}
 
-	async offInstanceOutput(id, handler) {
-		let handlers = this.instanceOutputHandlers.get(id);
+	async offInstanceLog(id, handler) {
+		let handlers = this.instanceLogHandlers.get(id);
 		if (!handlers || !handlers.length) {
 			throw new Error(`No handlers for instance ${id} exist`);
 		}
@@ -90,17 +100,22 @@ export class Control extends libLink.Link {
 
 		handlers.splice(index, 1);
 		if (!handlers.length) {
-			await this.updateInstanceOutputSubscriptions();
+			await this.updateLogSubscriptions();
 		}
 	}
 
-	async updateInstanceOutputSubscriptions() {
-		await libLink.messages.setInstanceOutputSubscriptions.send(this, {
-			instance_ids: [...this.instanceOutputHandlers.keys()],
+	async updateLogSubscriptions() {
+		await libLink.messages.setLogSubscriptions.send(this, {
+			all: false,
+			master: false,
+			slave_ids: [],
+			instance_ids: [...this.instanceLogHandlers.keys()],
+			max_level: null,
 		});
 	}
 
 	async debugWsMessageEventHandler(message) {
+		// eslint-disable-next-line no-console
 		console.log("WS", message.data.direction, message.data.content);
 	}
 

packages/web_ui/webpack.common.js

@@ -11,6 +11,13 @@ module.exports = (env = {}) => ({
 	},
 	plugins: [
 		new CleanWebpackPlugin(),
+
+		// required for winston
+		new webpack.ProvidePlugin({
+			process: require.resolve("process/browser.js"),
+			Buffer: [require.resolve("buffer/"), "Buffer"],
+		}),
+
 		new webpack.DefinePlugin({
 			"process.env.APP_ENV": JSON.stringify("browser"),
 		}),
@@ -18,7 +25,7 @@ module.exports = (env = {}) => ({
 		// Make sure ant-design icons use the ES variant
 		new webpack.NormalModuleReplacementPlugin(
 			/@ant-design\/icons\/[A-Z]/,
-			function(resource) {
+			resource => {
 				resource.request = resource.request.replace(/@ant-design\/icons/, "$&/es/icons");
 			}
 		),
@@ -33,6 +40,15 @@ module.exports = (env = {}) => ({
 				test: /node_modules.jsonwebtoken/,
 				use: "null-loader",
 			},
+			{
+				test: /node_modules.winston.dist.winston.transports.(http|file)/,
+				use: "null-loader",
+			},
+			// Colour library used by winston which doesn't work in browser.
+			{
+				test: /node_modules.colors/,
+				use: "null-loader",
+			},
 			{
 				test: /\.css$/,
 				use: [
@@ -72,6 +88,10 @@ module.exports = (env = {}) => ({
 	resolve: {
 		extensions: [".js", ".json", ".jsx"],
 		fallback: {
+			// Required for winston
+			"util": require.resolve("util/"),
+			"os": require.resolve("os-browserify/browser"),
+
 			"events": require.resolve("events/"),
 			"path": require.resolve("path-browserify"),
 		},

plugins/global_chat/instance.js

@@ -33,7 +33,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 	async chatEventHandler(message) {
 		// TODO check if cross server chat is enabled
 		let content = `[${message.data.instance_name}] ${removeTags(message.data.content)}`;
-		await this.instance.server.sendRcon(`/sc game.print('${libLuaTools.escapeString(content)}')`, true);
+		await this.sendRcon(`/sc game.print('${libLuaTools.escapeString(content)}')`, true);
 	}
 
 	sendChat(message) {

plugins/global_chat/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-global_chat",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.4",
   "description": "Clusterio plugin forwarding between Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/factorioClusterio#readme",
@@ -17,7 +17,7 @@
     "@clusterio/lib": "^2.0.0-alpha.0"
   },
   "devDependencies": {
-    "@clusterio/web_ui": "^2.0.0-alpha.2",
+    "@clusterio/web_ui": "^2.0.0-alpha.4",
     "babel-loader": "^8.1.0",
     "file-loader": "^6.2.0",
     "null-loader": "^4.0.0",

plugins/global_chat/webpack.config.js

@@ -10,7 +10,7 @@ module.exports = (env = {}) => merge(common(env), {
 	entry: "./web/index.jsx",
 	output: {
 		filename: "bundle.js",
-		path: path.resolve(__dirname, "static"),
+		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({

plugins/research_sync/instance.js

@@ -3,23 +3,21 @@ const libPlugin = require("@clusterio/lib/plugin");
 const libLuaTools = require("@clusterio/lib/lua_tools");
 
 
-function unexpectedError(err) {
-	console.log("Unexpected error in research_sync");
-	console.log("---------------------------------");
-	console.log(err);
-}
-
 class InstancePlugin extends libPlugin.BaseInstancePlugin {
+	unexpectedError(err) {
+		this.logger.error(`Unexpected error:\n${err.stack}`);
+	}
+
 	async init() {
 		if (!this.instance.config.get("factorio.enable_save_patching")) {
 			throw new Error("research_sync plugin requires save patching.");
 		}
 
 		this.instance.server.on("ipc-research_sync:contribution", (tech) => {
-			this.researchContribution(tech).catch(unexpectedError);
+			this.researchContribution(tech).catch(err => this.unexpectedError(err));
 		});
 		this.instance.server.on("ipc-research_sync:finished", (tech) => {
-			this.researchFinished(tech).catch(unexpectedError);
+			this.researchFinished(tech).catch(err => this.unexpectedError(err));
 		});
 	}
 
@@ -29,7 +27,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 
 	async progressEventHandler(message) {
 		let techsJson = libLuaTools.escapeString(JSON.stringify(message.data.technologies));
-		await this.instance.server.sendRcon(`/sc research_sync.update_progress("${techsJson}")`, true);
+		await this.sendRcon(`/sc research_sync.update_progress("${techsJson}")`, true);
 	}
 
 	async researchFinished(tech) {
@@ -38,13 +36,13 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 
 	async finishedEventHandler(message) {
 		let { name, level, researched } = message.data;
-		await this.instance.server.sendRcon(
+		await this.sendRcon(
 			`/sc research_sync.research_technology("${libLuaTools.escapeString(name)}", ${level})`, true
 		);
 	}
 
 	async onStart() {
-		let dumpJson = await this.instance.server.sendRcon("/sc research_sync.dump_technologies()");
+		let dumpJson = await this.sendRcon("/sc research_sync.dump_technologies()");
 		let techsToSend = [];
 		let instanceTechs = new Map();
 		for (let tech of JSON.parse(dumpJson)) {
@@ -74,7 +72,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 
 		if (techsToSync.length) {
 			let syncJson = libLuaTools.escapeString(JSON.stringify(techsToSync));
-			await this.instance.server.sendRcon(`/sc research_sync.sync_technologies("${syncJson}")`, true);
+			await this.sendRcon(`/sc research_sync.sync_technologies("${syncJson}")`, true);
 		}
 	}
 }

plugins/research_sync/master.js

@@ -3,39 +3,47 @@ const fs = require("fs-extra");
 const path = require("path");
 
 const libPlugin = require("@clusterio/lib/plugin");
+const RateLimiter = require("@clusterio/lib/RateLimiter");
 
 
+async function loadTechnologies(masterConfig, logger) {
+	let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json");
+	logger.verbose(`Loading ${filePath}`);
+	try {
+		return new Map(JSON.parse(await fs.readFile(filePath)));
+
+	} catch (err) {
+		if (err.code === "ENOENT") {
+			logger.verbose("Creating new technologies database");
+			return new Map();
+
+		}
+		throw err;
+	}
+}
+
+async function saveTechnologies(masterConfig, technologies, logger) {
+	let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json");
+	logger.verbose(`writing ${filePath}`);
+	await fs.outputFile(filePath, JSON.stringify([...technologies.entries()], null, 4));
+}
+
 class MasterPlugin extends libPlugin.BaseMasterPlugin {
 	async init() {
-		this.technologies = await loadTechnologies(this.master.config);
+		this.technologies = await loadTechnologies(this.master.config, this.logger);
+		this.progressRateLimiter = new RateLimiter({
+			maxRate: 1,
+			action: () => this.broadcastProgress(),
+		});
+
 		this.lastProgressBroadcast = Date.now();
 		this.progressBroadcastId = null;
 		this.progressToBroadcast = new Set();
 	}
 
 	async onShutdown() {
-		if (this.progressBroadcastId) {
-			clearTimeout(this.progressBroadcastId);
-		}
-		await saveTechnologies(this.master.config, this.technologies);
-	}
-
-	registerProgress() {
-		// Rate limit progress broadcasts to one per second
-		if (Date.now() < this.lastProgressBroadcast + 1000) {
-			if (!this.progressBroadcastId) {
-				this.progressBroadcastId = setTimeout(() => {
-					this.broadcastProgress();
-					this.progressBroadcastId = null;
-				}, 2000);
-			}
-
-		} else {
-			if (this.progressBroadcastId) {
-				clearTimeout(this.progressBroadcastId);
-			}
-			this.broadcastProgress();
-		}
+		this.progressRateLimiter.cancel();
+		await saveTechnologies(this.master.config, this.technologies, this.logger);
 	}
 
 	broadcastProgress() {
@@ -81,7 +89,7 @@ class MasterPlugin extends libPlugin.BaseMasterPlugin {
 		if (newProgress < 1) {
 			tech.progress = newProgress;
 			this.progressToBroadcast.add(name);
-			this.registerProgress();
+			this.progressRateLimiter.activate();
 
 		} else {
 			tech.researched = true;
@@ -106,9 +114,8 @@ class MasterPlugin extends libPlugin.BaseMasterPlugin {
 			let match = /-(\d+)$/.exec(name);
 			if (!match) {
 				return 1;
-			} else {
-				return Number.parseInt(match[1], 10);
 			}
+			return Number.parseInt(match[1], 10);
 		}
 
 		for (let instanceTech of message.data.technologies) {
@@ -148,7 +155,7 @@ class MasterPlugin extends libPlugin.BaseMasterPlugin {
 				}
 			}
 		}
-		this.registerProgress();
+		this.progressRateLimiter.activate();
 
 		let technologies = [];
 		for (let [name, tech] of this.technologies) {
@@ -159,29 +166,6 @@ class MasterPlugin extends libPlugin.BaseMasterPlugin {
 	}
 }
 
-async function loadTechnologies(masterConfig) {
-	let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json");
-	console.log(`Loading ${filePath}`);
-	try {
-		return new Map(JSON.parse(await fs.readFile(filePath)));
-
-	} catch (err) {
-		if (err.code === "ENOENT") {
-			console.log("Creating new technologies database");
-			return new Map();
-
-		} else {
-			throw err;
-		}
-	}
-}
-
-async function saveTechnologies(masterConfig, technologies) {
-	let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json");
-	console.log(`writing ${filePath}`);
-	await fs.outputFile(filePath, JSON.stringify([...technologies.entries()], null, 4));
-}
-
 
 module.exports = {
 	MasterPlugin,

plugins/research_sync/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-research_sync",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.4",
   "description": "Clusterio plugin for synchronizing research between Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/factorioClusterio#readme",
@@ -20,7 +20,7 @@
     "fs-extra": "^8.1.0"
   },
   "devDependencies": {
-    "@clusterio/web_ui": "^2.0.0-alpha.2",
+    "@clusterio/web_ui": "^2.0.0-alpha.4",
     "babel-loader": "^8.1.0",
     "file-loader": "^6.2.0",
     "null-loader": "^4.0.0",

plugins/research_sync/webpack.config.js

@@ -10,7 +10,7 @@ module.exports = (env = {}) => merge(common(env), {
 	entry: "./web/index.jsx",
 	output: {
 		filename: "bundle.js",
-		path: path.resolve(__dirname, "static"),
+		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({

plugins/statistics_exporter/README.md

@@ -2,12 +2,12 @@ Clusterio Statistics Exporter Plugin
 ====================================
 
 Plugin exporting item production, fluid production, kill count, entity
-build and poluttion statistics to the Prometheus integration to
+build and pollution statistics to the Prometheus integration to
 Clusterio.
 
 
 Installation
-============
+------------
 
 Run the following commands in the folder Clusterio is installed to:
 
@@ -16,3 +16,18 @@ Run the following commands in the folder Clusterio is installed to:
 
 Substitute clusteriomaster with clusterioslave or clusterioctl if this a
 dedicate slave or ctl installation respectively.
+
+
+Instance Configuration
+----------------------
+
+### statistics_exporter.command_timeout
+
+Timeout in seconds to wait for the statitics gathering command to the
+Factorio server return the result.  If the timeout is exceeded the
+previous collected results are returned instead.  Normally the command
+returns immediatly with the data, but if the RCON interface is
+overloaded then the command may take too long to timely answer the
+metrics collection request.
+
+Defaults to `1`.

plugins/statistics_exporter/info.js

@@ -1,8 +1,22 @@
 "use strict";
+const libConfig = require("@clusterio/lib/config");
+
+class InstanceConfigGroup extends libConfig.PluginConfigGroup {}
+InstanceConfigGroup.defaultAccess = ["master", "slave", "control"];
+InstanceConfigGroup.groupName = "statistics_exporter";
+InstanceConfigGroup.define({
+	name: "command_timeout",
+	title: "Command Timeout",
+	description: "Timeout in seconds of the metrics command before returning results from the previous invocation.",
+	type: "number",
+	initial_value: 1,
+});
+InstanceConfigGroup.finalize();
 
 module.exports = {
 	name: "statistics_exporter",
 	title: "Prometheus Statistics Export",
 	description: "Exports statistics to Prometheus",
 	instanceEntrypoint: "instance",
+	InstanceConfigGroup,
 };

plugins/statistics_exporter/instance.js

@@ -1,25 +1,41 @@
 "use strict";
 const libPlugin = require("@clusterio/lib/plugin");
 const { Gauge } = require("@clusterio/lib/prometheus");
+const libHelpers = require("@clusterio/lib/helpers");
 
 
 const instancePlayerCount = new Gauge(
-	"clusterio_statistics_exporter_instance_player_count", "Amount of players connected to this cluster",
+	"clusterio_statistics_exporter_instance_player_count",
+	"Amount of players connected to this cluster",
 	{ labels: ["instance_id"] }
 );
 const instanceGameTicksTotal = new Gauge(
-	"clusterio_statistics_exporter_instance_game_ticks_total", "Game tick an instance has progressed to",
+	"clusterio_statistics_exporter_instance_game_ticks_total",
+	"Game tick an instance has progressed to",
 	{ labels: ["instance_id"] }
 );
 const instanceForceFlowStatistics = new Gauge(
-	"clusterio_statistics_exporter_instance_force_flow_statistics", "Items/fluids/enemies/buildings produced/built/killed by a force",
+	"clusterio_statistics_exporter_instance_force_flow_statistics",
+	"Items/fluids/enemies/buildings produced/built/killed by a force",
 	{ labels: ["instance_id", "force", "statistic", "direction", "name"] },
 );
 const instanceGameFlowStatistics = new Gauge(
-	"clusterio_statistics_exporter_instance_game_flow_statistics", "Pollution produced/consumed in the game",
+	"clusterio_statistics_exporter_instance_game_flow_statistics",
+	"Pollution produced/consumed in the game",
 	{ labels: ["instance_id", "statistic", "direction", "name"] },
 );
 
+function setForceFlowStatistic(instanceId, forceName, statisticName, direction, item, value) {
+	instanceForceFlowStatistics.labels(String(instanceId), forceName, statisticName, direction, item).set(value);
+
+	// For item and fluid statistics it's useful to compare the input flow with the
+	// output flow, to simplify the comparison ensure both directions have a value.
+	if (["item_production_statistics", "fluid_production_statistic"].includes(statisticName)) {
+		let reverseDirection = direction === "input" ? "output" : "input";
+		instanceForceFlowStatistics.labels(String(instanceId), forceName, statisticName, reverseDirection, item);
+	}
+}
+
 
 class InstancePlugin extends libPlugin.BaseInstancePlugin {
 	async init() {
@@ -28,8 +44,8 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 		}
 	}
 
-	async onMetrics() {
-		let string = await this.instance.server.sendRcon("/sc statistics_exporter.export()");
+	async gatherMetrics() {
+		let string = await this.sendRcon("/sc statistics_exporter.export()");
 		let stats;
 		try {
 			stats = JSON.parse(string);
@@ -37,7 +53,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 			throw new Error(`Error parsing statistics JSON: ${err.message}, content "${string}"`);
 		}
 
-		let instanceId = this.instance.config.get("instance.id");
+		let instanceId = this.instance.id;
 		instanceGameTicksTotal.labels(String(instanceId)).set(stats.game_tick);
 		instancePlayerCount.labels(String(instanceId)).set(stats.player_count);
 
@@ -45,9 +61,7 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 			for (let [statisticName, statistic] of Object.entries(flowStatistics)) {
 				for (let [direction, counts] of Object.entries(statistic)) {
 					for (let [item, value] of Object.entries(counts)) {
-						instanceForceFlowStatistics.labels(
-							String(instanceId), forceName, statisticName, direction, item
-						).set(value);
+						setForceFlowStatistic(instanceId, forceName, statisticName, direction, item, value);
 					}
 				}
 			}
@@ -61,6 +75,15 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 			}
 		}
 	}
+
+	async onMetrics() {
+		// Wait configured timeout for the metrics to be collected.  It may
+		// take a long time for the command to go through if the command
+		// stream is overloaded.  Should the timeout be exceeded the
+		// previous values for the metrics will end up being sent to master.
+		let timeout = this.instance.config.get("statistics_exporter.command_timeout") * 1000;
+		await libHelpers.timeout(this.gatherMetrics(), timeout);
+	}
 }
 
 module.exports = {

plugins/statistics_exporter/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-statistics_exporter",
-  "version": "2.0.0-alpha.2",
+  "version": "2.0.0-alpha.4",
   "description": "Clusterio plugin for gathering statistics from Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/factorioClusterio#readme",
@@ -17,7 +17,7 @@
     "@clusterio/lib": "^2.0.0-alpha.0"
   },
   "devDependencies": {
-    "@clusterio/web_ui": "^2.0.0-alpha.2",
+    "@clusterio/web_ui": "^2.0.0-alpha.4",
     "babel-loader": "^8.1.0",
     "file-loader": "^6.2.0",
     "null-loader": "^4.0.0",

plugins/statistics_exporter/test/plugin.js

@@ -9,8 +9,10 @@ const info = require("../info");
 describe("statistics_exporter plugin", function() {
 	describe("class InstancePlugin", function() {
 		let instancePlugin;
+		let mockInstance = new mock.MockInstance();
+		mockInstance.mockConfigEntries.set("statistics_exporter.command_timeout", 1);
 		before(async function() {
-			instancePlugin = new instance.InstancePlugin(info, new mock.MockInstance());
+			instancePlugin = new instance.InstancePlugin(info, mockInstance);
 			await instancePlugin.init();
 		});
 

plugins/statistics_exporter/webpack.config.js

@@ -10,7 +10,7 @@ module.exports = (env = {}) => merge(common(env), {
 	entry: "./web/index.jsx",
 	output: {
 		filename: "bundle.js",
-		path: path.resolve(__dirname, "static"),
+		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({