v2.0.0-alpha.7

Alpha 7 preparations

.eslintrc.js

@@ -1,5 +1,8 @@
 "use strict";
 module.exports = {
+	"plugins": [
+		"node",
+	],
 	"env": {
 		"node": true,
 		"commonjs": true,
@@ -18,6 +21,8 @@ module.exports = {
 			"rules": {
 				"prefer-arrow-callback": "off",
 				"no-invalid-this": "off",
+				"node/no-unpublished-import": "off",
+				"node/no-unpublished-require": "off",
 			},
 		},
 		{
@@ -31,6 +36,44 @@ module.exports = {
 					"jsx": true,
 				},
 			},
+			"rules": {
+				"node/callback-return": "off",
+				"node/exports-style": "off",
+				"node/file-extension-in-import": "off",
+				"node/global-require": "off",
+				"node/handle-callback-err": "off",
+				"node/no-callback-literal": "off",
+				"node/no-deprecated-api": "off",
+				"node/no-exports-assign": "off",
+				"node/no-extraneous-import": "off",
+				"node/no-extraneous-require": "off",
+				"node/no-missing-import": "off",
+				"node/no-missing-require": "off",
+				"node/no-mixed-requires": "off",
+				"node/no-new-require": "off",
+				"node/no-path-concat": "off",
+				"node/no-process-env": "off",
+				"node/no-process-exit": "off",
+				"node/no-restricted-require": "off",
+				"node/no-sync": "off",
+				"node/no-unpublished-bin": "off",
+				"node/no-unpublished-import": "off",
+				"node/no-unpublished-require": "off",
+				"node/no-unsupported-features/es-builtins": "off",
+				"node/no-unsupported-features/es-syntax": "off",
+				"node/no-unsupported-features/node-builtins": "off",
+				"node/prefer-global/buffer": "off",
+				"node/prefer-global/console": "off",
+				"node/prefer-global/process": "off",
+				"node/prefer-global/text-decoder": "off",
+				"node/prefer-global/text-encoder": "off",
+				"node/prefer-global/url-search-params": "off",
+				"node/prefer-global/url": "off",
+				"node/prefer-promises/dns": "off",
+				"node/prefer-promises/fs": "off",
+				"node/process-exit-as-throw": "off",
+				"node/shebang": "off",
+			},
 		},
 	],
 
@@ -46,7 +89,6 @@ module.exports = {
 		"block-scoped-var": "error",
 		"block-spacing": "error",
 		"brace-style": ["error", "1tbs", { "allowSingleLine": true }],
-		"callback-return": "off",
 		"camelcase": "off",
 		"capitalized-comments": "off",
 		"class-methods-use-this": "off",
@@ -81,10 +123,8 @@ module.exports = {
 		"function-call-argument-newline": "off",
 		"function-paren-newline": "off",
 		"generator-star-spacing": ["error", { "before": false, "after": true }],
-		"global-require": "error",
 		"grouped-accessor-pairs": "error",
 		"guard-for-in": "error",
-		"handle-callback-err": "error",
 		"id-blacklist": "error",
 		"id-length": "off",
 		"id-match": "error",
@@ -160,7 +200,6 @@ module.exports = {
 				],
 			},
 		],
-		"no-mixed-requires": "error",
 		"no-multi-assign": "error",
 		"no-multi-spaces": "error",
 		"no-multi-str": "error",
@@ -172,19 +211,14 @@ module.exports = {
 		"no-new": "error",
 		"no-new-func": "error",
 		"no-new-object": "error",
-		"no-new-require": "error",
 		"no-new-wrappers": "error",
 		"no-octal-escape": "error",
 		"no-param-reassign": "off",
-		"no-path-concat": "error",
 		"no-plusplus": ["error", { "allowForLoopAfterthoughts": true }],
-		"no-process-env": "error",
-		"no-process-exit": "error",
 		"no-proto": "error",
 		"no-restricted-exports": "error",
 		"no-restricted-globals": "error",
 		"no-restricted-imports": "error",
-		"no-restricted-modules": "error",
 		"no-restricted-properties": "error",
 		"no-restricted-syntax": "error",
 		"no-return-assign": "error",
@@ -194,7 +228,6 @@ module.exports = {
 		"no-sequences": "error",
 		"no-shadow": ["error", { "allow": ["yargs"] }],
 		"no-spaced-func": "error",
-		"no-sync": "error",
 		"no-tabs": ["error", { "allowIndentationTabs": true }],
 		"no-template-curly-in-string": "error",
 		"no-ternary": "off",
@@ -279,5 +312,45 @@ module.exports = {
 		"wrap-regex": "off",
 		"yield-star-spacing": "error",
 		"yoda": ["error", "never", { "exceptRange": true }],
+
+		"node/callback-return": "off",
+		"node/exports-style": "off",
+		"node/file-extension-in-import": "off",
+		"node/global-require": "error",
+		"node/handle-callback-err": "error",
+		"node/no-callback-literal": "error",
+		"node/no-deprecated-api": "error",
+		"node/no-exports-assign": "error",
+		"node/no-extraneous-import": "error",
+		"node/no-extraneous-require": "error",
+		"node/no-missing-import": "error",
+		"node/no-missing-require": "error",
+		"node/no-mixed-requires": "error",
+		"node/no-new-require": "error",
+		"node/no-path-concat": "error",
+		"node/no-process-env": "error",
+		"node/no-process-exit": "error",
+		"node/no-restricted-require": "off",
+		"node/no-sync": "error",
+		"node/no-unpublished-bin": "error",
+		"node/no-unpublished-import": "error",
+		"node/no-unpublished-require": [
+			"error",
+			{ "allowModules": ["webpack", "webpack-merge", "webpack-dev-middleware"] },
+		],
+		"node/no-unsupported-features/es-builtins": "error",
+		"node/no-unsupported-features/es-syntax": "error",
+		"node/no-unsupported-features/node-builtins": "error",
+		"node/prefer-global/buffer": "error",
+		"node/prefer-global/console": "error",
+		"node/prefer-global/process": "error",
+		"node/prefer-global/text-decoder": "error",
+		"node/prefer-global/text-encoder": "error",
+		"node/prefer-global/url-search-params": "error",
+		"node/prefer-global/url": "error",
+		"node/prefer-promises/dns": "off",
+		"node/prefer-promises/fs": "off",
+		"node/process-exit-as-throw": "error",
+		"node/shebang": "error",
 	},
 };

.gitignore

@@ -31,7 +31,6 @@ dist/
 node_modules
 
 Factorio*
-factorio*
 /factorio
 /instances
 /sharedMods

CHANGELOG.md

@@ -19,6 +19,7 @@
 - Packaged and uploaded to npm again, greatly simplifying distribution and installation of both Clusterio and plugins.
 - Added centralized logging for the cluster.
   All logs from both Clusterio and Factorio is stored in a shared log which can be inspected and queried.
+- Added remote save management with the ability to list, create, upload, download, delete and load saves for instances.
 
 ### Features
 
@@ -76,6 +77,7 @@
   UPS statistics is exported by the statistics exporter plugin.
 - Master server now defaults to hosting on https on port 8443.
 - Renamed client to slave.
+- Removed rotation of factorio-current.log files.
 
 ### Breaking Changes
 

docs/configuration.md

@@ -109,6 +109,14 @@ This should be less than both the timeout and collection interval configured for
 Defaults to 8.
 
 
+### master.stream_timeout
+
+Timeout in seconds for proxy streams through the master that has been created but is not yet sending data to be automatically closed.
+Streams that have started sending data before this timeout is not affected.
+
+Defaults to 15.
+
+
 ### master.default_role_id
 
 Role to automatically grant to new users.

docs/contributing.md

@@ -56,7 +56,17 @@ In order to get the development environment up and running you will need to run
     npx lerna bootstrap --hoist
 
 This installs dependencies needed by tests and links the packages up so they work from the git work tree.
+To start a cluster from the repo, substitute the commands in the normal readme as follows:
 
+    npx clusteriomaster -> node packages/master
+    npx clusterioslave -> node packages/slave
+    npx clusterioctl -> node packages/ctl
+
+For web development on the master there are also the following flags:
+
+`--dev` - Start the master with webpack hot reloading of core components
+
+`--dev-plugin [name]` - Start the master with hot reloading of a plugin
 
 ### Starting a Feature Branch
 

docs/writing-plugins.md

@@ -48,18 +48,20 @@ For a plugin the most important file is the `info.js` file.
 Without it the plugin will not recognized by Clusterio.
 Here's an example of it:
 
-    const libLink = require("@clusterio/lib/link"); // For messages
+```js
+const libLink = require("@clusterio/lib/link"); // For messages
 
-    module.exports = {
-        name: "foo_frobber",
-        title: "Foo Frobber",
-        description: "Does advanced frobnication",
-        instanceEntrypoint: "instance",
-        masterEntrypoint: "master",
-        messages: {
-            /* See below */
-        },
-    };
+module.exports = {
+    name: "foo_frobber",
+    title: "Foo Frobber",
+    description: "Does advanced frobnication",
+    instanceEntrypoint: "instance",
+    masterEntrypoint: "master",
+    messages: {
+        /* See below */
+    },
+};
+```
 
 The following properties are recognized:
 
@@ -117,20 +119,22 @@ For both instanceEntrypoint and masterEntrypoint the path should not end with .j
 The plugin class should derive from its respective base class defined in `lib/plugin`.
 For example, to define a MasterPlugin class the following code can be used:
 
-    const libPlugin = require("@clusterio/lib/plugin");
+```js
+const libPlugin = require("@clusterio/lib/plugin");
 
-    class MasterPlugin extends libPlugin.BaseMasterPlugin {
-        async init() {
-            this.foo = 42;
-            await this.startFrobnication();
-        }
-
-        // ...
+class MasterPlugin extends libPlugin.BaseMasterPlugin {
+    async init() {
+        this.foo = 42;
+        await this.startFrobnication();
     }
 
-    module.exports = {
-        MasterPlugin,
-    }
+    // ...
+}
+
+module.exports = {
+    MasterPlugin,
+}
+```
 
 For the instance plugin it's exactly the same except "Master" is replaced with "Instance", and for the clusterioctl plugin "Control" is used.
 The available hooks that you can override are documented in the base class [in lib/plugin.js](/packages/lib/plugin.js).
@@ -138,11 +142,13 @@ The available hooks that you can override are documented in the base class [in l
 It's best to avoid defining a constructor, but if you insist on defining one, forward all arguments to the base class.
 E.g.:
 
-        constructor(...args) {
-            super(...args);
+```js
+constructor(...args) {
+    super(...args);
 
-            // Code here
-        }
+    // Code here
+}
+```
 
 The arguments passed may change, and attempting to modify them will result in unpredicatable behaviour.
 The async init method is always called immediatly after the constructor, so there's little reason to do this.
@@ -154,9 +160,11 @@ The base plugin classes provide a winston logger for logging messages to the sha
 For instances a copy of the log is also stored on the slave the instance is on.
 To use it, pass a string to one of the log levels functions, for example:
 
-    async init() {
-        this.logger.info("Initializing frobbing");
-    }
+```js
+async init() {
+    this.logger.info("Initializing frobbing");
+}
+```
 
 Metadata covering which plugin and instance (for instance plugin classes) the log event happened on will automatically be attached to the log message.
 
@@ -180,11 +188,13 @@ These levels have roughly the following meanings:
 
 For guarding unexpected errors the best option is to log a short description along with the`stack` property of the Error:
 
-    try {
-        // Operation that should not throw but may end up throwing
-    } catch (err) {
-        this.logger.error(`Operation failed:\n${err.stack}`);
-    )
+```js
+try {
+    // Operation that should not throw but may end up throwing
+} catch (err) {
+    this.logger.error(`Operation failed:\n${err.stack}`);
+)
+```
 
 For plugin hooks expections thrown are automatically catched and logged, but for event handlers registered on EventEmitters it's critical that exceptions are catched and handled appropriately.
 
@@ -195,30 +205,34 @@ Clusterio provides a configuration system that handles storing, distributing, ed
 You can take advantage of it by subclassing `PluginConfigGroup`, setting `defaultAccess` to where config entries can be accessed from, setting the `groupName` to your plugin name, defining fields on it, finalizing it, and passing it as either `MasterConfigGroup` or `InstanceConfigGroup` in the `info.js` export.
 For example in info.js:
 
-    const libConfig = require("@clusterio/lib/config");
+```js
+const libConfig = require("@clusterio/lib/config");
 
-    class MasterConfigGroup extends libConfig.PluginConfigGroup { }
-    MasterConfigGroup.defaultAccess = ["master", "slave", "control"];
-    MasterConfigGroup.groupName = "foo_frobber";
-    MasterConfigGroup.define({
-        name: "level",
-        description: "Level of frobnication done",
-        type: "number",
-        initial_value: 2,
-    });
-    MasterConfigGroup.finalize();
+class MasterConfigGroup extends libConfig.PluginConfigGroup { }
+MasterConfigGroup.defaultAccess = ["master", "slave", "control"];
+MasterConfigGroup.groupName = "foo_frobber";
+MasterConfigGroup.define({
+    name: "level",
+    description: "Level of frobnication done",
+    type: "number",
+    initial_value: 2,
+});
+MasterConfigGroup.finalize();
 
-    module.exports = {
-        ...
-        MasterConfigGroup: MasterConfigGroup,
-    };
+module.exports = {
+    ...
+    MasterConfigGroup: MasterConfigGroup,
+};
+```
 
 Code inside the `MasterPlugin` class will then be able to access the level config field through the `Config` object at `this.master.config`, for example in the MasterPluginClass:
 
-    async init() {
-        let level = this.master.config.get("foo_frobber.level");
-        this.logger.info(`I got a frobnication level of ${level}`);
-    }
+```js
+async init() {
+    let level = this.master.config.get("foo_frobber.level");
+    this.logger.info(`I got a frobnication level of ${level}`);
+}
+```
 
 The same applies for instance configs, replace "master" with "instance" where appropriate.
 See [Configuration System](config-system.md) for more details on how this system works.
@@ -230,11 +244,13 @@ If the plugin requires a certain feature to be enabled to function it should thr
 The most common such feature is the save patching, which can be disabled to run vanilla or scenarios not compatible with Clusterio.
 For example:
 
-    async init() {
-        if (!this.instance.config.get("factorio.enable_save_patching")) {
-            throw new Error("foo_frobber plugin requires save patching.");
-        }
+```js
+async init() {
+    if (!this.instance.config.get("factorio.enable_save_patching")) {
+        throw new Error("foo_frobber plugin requires save patching.");
     }
+}
+```
 
 
 ## Communicating with Factorio
@@ -243,14 +259,15 @@ For pushing data into Factorio there's RCON, which lets you send arbitrary Lua c
 This is done by calling the `sendRcon` method on the plugin object.
 For example:
 
-    async onStart() {
-        let response = await this.sendRcon(
-            "/sc rcon.print('data')"
-        );
-
-        // Do stuff with response.
-    }
+```js
+async onStart() {
+    let response = await this.sendRcon(
+        "/sc rcon.print('data')"
+    );
 
+    // Do stuff with response.
+}
+```
 
 Because data into Factorio is streamed at a rate of 3-6 kB/s by default, it is recommended to avoid sending large commands as much as possible, and to strip down the data on the ones you send to only what's strictly necessary.
 You can have lua code injected into the game via the module system and call that from RCON to avoid having to send code through the commands.
@@ -261,24 +278,28 @@ The `send_json` API allows sending JSON payloads on channels that plugins can li
 From a plugin you listen for an event named `ipc-channel_name` in order to get data sent by `send_json`.
 For example in the plugin code:
 
-    async init() {
-        this.instance.server.on("ipc-my_plugin_foo", content =>
-            this.handleFoo(content).catch(err => this.logger.error(
-                `Error handling foo:\n${err.stack}`
-            ))
-        );
-    }
+```js
+async init() {
+    this.instance.server.on("ipc-my_plugin_foo", content =>
+        this.handleFoo(content).catch(err => this.logger.error(
+            `Error handling foo:\n${err.stack}`
+        ))
+    );
+}
 
-    async handleFoo(content) {
-        // Do stuff with content
-    }
+async handleFoo(content) {
+    // Do stuff with content
+}
+```
 
 And then in the module for the plugin:
 
-    local clusterio_api = require("modules/clusterio/api")
+```lua
+local clusterio_api = require("modules/clusterio/api")
 
-    -- inside some event handler
-    clusterio_api.send_json("my_plugin_foo", { data = 123 })
+-- inside some event handler
+clusterio_api.send_json("my_plugin_foo", { data = 123 })
+```
 
 It's recommended to either use the plugin name as the channel name or to prefix the channel name with the name of the plugin if you need multiple channels.
 It's also important to catch any errors that might occur as they will otherwise be propogated to the instance code and kill the server.
@@ -310,18 +331,20 @@ Events are defined as properties of the messages object exported by `info.js` th
 The name of the property correspond to the handler invoked on the plugin class.
 The Event constructor takes an object of properties that define the event, for example the following could be defined in `info.js`:
 
-    messages: {
-        startFrobnication: new libLink.Event({
-            type: "foo_frobber:start_frobnication",
-            links: ["master-slave", "slave-instance"],
-            forwardTo: "instance",
-            eventRequired: ["frobnication_type"],
-            eventProperties: {
-                "frobnication_type": { type: "string" },
-                "urgent": { type: "boolean" },
-            },
-        }),
-    },
+```js
+messages: {
+    startFrobnication: new libLink.Event({
+        type: "foo_frobber:start_frobnication",
+        links: ["master-slave", "slave-instance"],
+        forwardTo: "instance",
+        eventRequired: ["frobnication_type"],
+        eventProperties: {
+            "frobnication_type": { type: "string" },
+            "urgent": { type: "boolean" },
+        },
+    }),
+},
+```
 
 This specifies an event that can be sent from the master to a slave, and from a slave to an instance.
 It also specifies that the event must contain the property `frobnication_type`, with a string value in the data payload and that it may optionally contain a boolean `urgent` property.
@@ -383,24 +406,26 @@ The name of the property corresponds to the handler invoked on the plugin class.
 The Request constructor takes an object of properties that define the event.
 For example, the following could be defined in `info.js`:
 
-    messages: {
-        reportFrobnication: new libLink.Request({
-            type: "foo_frobber:report_frobnication",
-            links: ["master-slave", "slave-instance"],
-            forwardTo: "instance",
-            requestRequired: ["verbosity"],
-            requestProperties: {
-                "verbosity": { type: "integer" },
-                "special": { type: "boolean" },
+```js
+messages: {
+    reportFrobnication: new libLink.Request({
+        type: "foo_frobber:report_frobnication",
+        links: ["master-slave", "slave-instance"],
+        forwardTo: "instance",
+        requestRequired: ["verbosity"],
+        requestProperties: {
+            "verbosity": { type: "integer" },
+            "special": { type: "boolean" },
+        },
+        responseProperties: {
+            "report": {
+                type: "array",
+                items: { type: "string" },
             },
-            responseProperties: {
-                "report": {
-                    type: "array",
-                    items: { type: "string" },
-                },
-            },
-        }),
-    },
+        },
+    }),
+},
+```
 
 This specifies a request that can be sent from the master to a slave, and from a slave to an instance.
 The request data must contain the property `verbosity` with an integer number as the value, as well as the `instance_id` property (implied by `forwardTo: "instance"`) and it may also contain a boolean `special` property.
@@ -459,10 +484,12 @@ For `InstancePlugin` code the link to the slave is the `instance` itself, which
 The `.info` property of the plugin class exposes the data exported from the plugin's `info.js` module.
 In other words:
 
-    // In an InstancePlugin class
-    async frobnicate() {
-        this.info.messages.exampleEvent.send(this.instance, { foo: "bar" });
-    }
+```js
+// In an InstancePlugin class
+async frobnicate() {
+    this.info.messages.exampleEvent.send(this.instance, { foo: "bar" });
+}
+```
 
 For the Request class the send method is async and returns the response data received from the target it was sent to, or throws an error if the request failed.
 
@@ -478,9 +505,11 @@ Plugins must respond to the prepare disconnect by stopping any processess it doe
 This can be accomplished either through listening for the prepare disconnect hook, or by checking the `connected` or `closing` property of the connector for the master/slave connection.
 For example the sending of an event from an `InstancePlugin` class can be stopped while the connection is closing by using the following code:
 
-    if (!this.slave.connector.closing) {
-        this.info.messages.frobnicate.send(this.instance, { foo: "bar" });
-    }
+```js
+if (!this.slave.connector.closing) {
+    this.info.messages.frobnicate.send(this.instance, { foo: "bar" });
+}
+```
 
 If the event or request needs to be sent to the master it can be put into a queue stored on the plugin instance and sent out when the connection is established again.
 The re-establishement of the connection is  notified to plugins via the `connect` event to the `onMasterConnectionEvent` and `onSlaveConnectionEvent` hooks.
@@ -500,29 +529,33 @@ Clusterio comes with its own Prometheus client implementation, one part due to N
 In its simplest form collecting data from plugins consists of defining the metric and updating it somewhere in the plugin code.
 For example:
 
-    const { Counter } = require("@clusterio/lib/prometheus");
+```js
+const { Counter } = require("@clusterio/lib/prometheus");
 
-    const fooMetric = new Counter(
-        "clusterio_foo_frobber_foo_metric", "Measures the level of foo",
-    );
+const fooMetric = new Counter(
+    "clusterio_foo_frobber_foo_metric", "Measures the level of foo",
+);
 
-    // Somewhere in the master plugin code
-    fooMetric.inc();
+// Somewhere in the master plugin code
+fooMetric.inc();
+```
 
 This works for master plugins, and the metric will be automatically available through the /metric HTTP endpoint.
 It's recommended that plugin metrics follow `clusterio_<plugin_name>_<metric_name>` as the naming scheme.
 
 For metrics that are per-instance, you must define an `instance_id` label and set it accordingly, for example:
 
-    const { Counter } = require("@clusterio/lib/prometheus");
+```js
+const { Counter } = require("@clusterio/lib/prometheus");
 
-    const barMetric = new Gauge(
-        "clusterio_foo_frobber_bar_metric", "Bar instance level",
-        { labels: ["instance_id"] }
-    );
+const barMetric = new Gauge(
+    "clusterio_foo_frobber_bar_metric", "Bar instance level",
+    { labels: ["instance_id"] }
+);
 
-    // Somewhere in the instance plugin code
-    barMetric.labels(String(this.instance.id)).set(someValue);
+// Somewhere in the instance plugin code
+barMetric.labels(String(this.instance.id)).set(someValue);
+```
 
 Metrics are automatically registered to the default registry, and this default registry is automatically polled by the master server on slaves.
 This means that it's important that you place the definition of the metric at module level so that it's not created more than once over the lifetime of a slave.
@@ -536,28 +569,32 @@ For statistics you need to update on collection there's an `onMetrics` hook on b
 The control entrypoint for plugins allows you to extend clustectl with your own commands.
 The creation of custom commands typically starts with defining a command tree for the plugin:
 
-    const { Command, CommandTree } = require("@clusterio/lib/command");
-    const fooFrobberCommands = new CommandTree({
-        name: "foo-frobber", description: "Foo Frobber Plugin commands"
-    });
+```js
+const { Command, CommandTree } = require("@clusterio/lib/command");
+const fooFrobberCommands = new CommandTree({
+    name: "foo-frobber", description: "Foo Frobber Plugin commands"
+});
+```
 
 Then commands are added to the the plugin's command tree:
 
-    const info = require("./info");
+```js
+const info = require("./info");
 
-    fooFrobberCommands.add(new Command({
-        definition: ["frobnicate <type>", "Do frobnications", (yargs) => {
-            yargs.positional("level", {
-                describe: "type of frobnication", type: "string"
-            });
-        }],
-        handler: async function(args, control) {
-            await info.messages.frobnicate.send(control, {
-                instance_name: "Console",
-                content: args.message,
-            });
-        },
-    }));
+fooFrobberCommands.add(new Command({
+    definition: ["frobnicate <type>", "Do frobnications", (yargs) => {
+        yargs.positional("level", {
+            describe: "type of frobnication", type: "string"
+        });
+    }],
+    handler: async function(args, control) {
+        await info.messages.frobnicate.send(control, {
+            instance_name: "Console",
+            content: args.message,
+        });
+    },
+}));
+```
 
 For a command the `definition` is the arguments to pass to [yargs.command](http://yargs.js.org/docs/#api-reference-commandcmd-desc-builder-handler) (see also [yargs.positional](http://yargs.js.org/docs/#api-reference-positionalkey-opt) and [yargs.options](http://yargs.js.org/docs/#api-reference-optionskey-opt) for setting up positional and optional arguments to commands).
 The `handler` is an async function that's invoked when the command is executed and it's passed the parsed command line arguments and a reference to the `Control` class of clusterioctl.
@@ -567,14 +604,16 @@ Note that messages sent from clusterioctl needs to have `"control-master"` as a
 
 To have the command tree become part of clusterioctl it needs to be added to the rootCommand tree in `addCommands` callback of the Control plugin:
 
-    const libPlugin = require("@clusterio/lib/plugin");
+```js
+const libPlugin = require("@clusterio/lib/plugin");
 
-    class ControlPlugin extends libPlugin.BaseControlPlugin {
-        async addCommands(rootCommand) {
-            rootCommand.add(fooFrobberCommands);
-        }
+class ControlPlugin extends libPlugin.BaseControlPlugin {
+    async addCommands(rootCommand) {
+        rootCommand.add(fooFrobberCommands);
     }
+}
 
-    module.exports = {
-        ControlPlugin,
-    }
+module.exports = {
+    ControlPlugin,
+}
+```

lerna.json

@@ -4,5 +4,5 @@
     "plugins/*",
     "test"
   ],
-  "version": "2.0.0-alpha.6"
+  "version": "2.0.0-alpha.7"
 }

package.json

@@ -22,6 +22,7 @@
   },
   "devDependencies": {
     "eslint": "^7.25.0",
+    "eslint-plugin-node": "^11.1.0",
     "jsdoc": "^3.6.6",
     "lerna": "^4.0.0",
     "mocha": "^8.4.0",

packages/ctl/ctl.js

@@ -25,6 +25,8 @@ const libSharedCommands = require("@clusterio/lib/shared_commands");
 const { ConsoleTransport, levels, logger } = require("@clusterio/lib/logging");
 const libLoggingUtils = require("@clusterio/lib/logging_utils");
 const libHelpers = require("@clusterio/lib/helpers");
+const libFactorio = require("@clusterio/lib/factorio");
+const libFileOps = require("@clusterio/lib/file_ops");
 
 
 function print(...content) {
@@ -135,10 +137,11 @@ slaveCommands.add(new libCommand.Command({
 
 slaveCommands.add(new libCommand.Command({
 	definition: ["generate-token", "Generate token for a slave", (yargs) => {
-		yargs.option("id", { type: "number", nargs: 1, describe: "Slave id", demandOption: true });
+		yargs.option("id", { type: "number", nargs: 1, describe: "Slave id" });
 	}],
 	handler: async function(args, control) {
-		let response = await libLink.messages.generateSlaveToken.send(control, { slave_id: args.id });
+		let slaveId = typeof args.id === "number" ? args.id : null;
+		let response = await libLink.messages.generateSlaveToken.send(control, { slave_id: slaveId });
 		print(response.token);
 	},
 }));
@@ -314,15 +317,175 @@ instanceCommands.add(new libCommand.Command({
 	},
 }));
 
+async function loadMapSettings(args) {
+	let seed = args.seed !== undefined ? args.seed : null;
+	let mapGenSettings = null;
+	let mapSettings = null;
+	if (args.mapExchangeString) {
+		let parsed = libFactorio.readMapExchangeString(args.mapExchangeString);
+		mapGenSettings = parsed.map_gen_settings;
+		mapSettings = parsed.map_settings;
+	}
+	if (args.mapGenSettings) {
+		mapGenSettings = JSON.parse(await fs.readFile(args.mapGenSettings));
+	}
+	if (args.mapSettings) {
+		mapSettings = JSON.parse(await fs.readFile(args.mapSettings));
+	}
+
+	return {
+		seed,
+		mapGenSettings,
+		mapSettings,
+	};
+}
+
 instanceCommands.add(new libCommand.Command({
-	definition: ["create-save <instance>", "Create a new save on an instance", (yargs) => {
-		yargs.positional("instance", { describe: "Instance to create on", type: "string" });
+	definition: ["list-saves <instance>", "list saves on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to list saves on", type: "string" });
 	}],
 	handler: async function(args, control) {
 		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		let response = await libLink.messages.listSaves.send(control, { instance_id: instanceId });
+		for (let entry of response.list) {
+			entry.mtime = new Date(entry.mtime_ms).toLocaleString();
+			delete entry.mtime_ms;
+		}
+		print(asTable(response.list));
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["create-save <instance> [name]", "Create a new save on an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to create on", type: "string" });
+		yargs.positional("name", { describe: "Name of save to create.", type: "string", default: "world.zip" });
+		yargs.options({
+			"seed": { describe: "Seed to use, takes precedence over map-gen-settings", nargs: 1, type: "number" },
+			"map-exchange-string": { describe: "Map exchange string to use for the save", nargs: 1, type: "string" },
+			"map-gen-settings": { describe: "path to file to use for map-gen-settings", nargs: 1, type: "string" },
+			"map-settings": { describe: "path to file to use for map-settings", nargs: 1, type: "string" },
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		let { seed, mapGenSettings, mapSettings } = await loadMapSettings(args);
 		await control.setLogSubscriptions({ instance_ids: [instanceId] });
 		let response = await libLink.messages.createSave.send(control, {
 			instance_id: instanceId,
+			name: args.name,
+			seed,
+			map_gen_settings: mapGenSettings,
+			map_settings: mapSettings,
+		});
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["upload-save <instance> <filepath>", "Upload a save to an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to upload to", type: "string" });
+		yargs.positional("filepath", { describe: "Path to save to upload", type: "string" });
+		yargs.options({
+			"name": { describe: "Name to give save on server", nargs: 1, type: "string" },
+		});
+	}],
+	handler: async function(args, control) {
+		let filename = args.name || path.basename(args.filepath);
+		if (!filename.endsWith(".zip")) {
+			throw new libErrors.CommandError("Save name must end with .zip");
+		}
+		// phin doesn't support streaming requests :(
+		let content = await fs.readFile(args.filepath);
+
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		let url = new URL(control.config.get("control.master_url"));
+		url.pathname += "api/upload-save";
+		url.searchParams.append("instance_id", instanceId);
+		url.searchParams.append("filename", filename);
+
+		let result = await phin({
+			url, method: "POST",
+			headers: {
+				"X-Access-Token": control.config.get("control.master_token"),
+				"Content-Type": "application/zip",
+			},
+			core: { ca: control.tlsCa },
+			data: content,
+			parse: "json",
+		});
+
+		for (let error of result.body.errors || []) {
+			logger.error(error);
+		}
+
+		for (let requestError of result.body.request_errors || []) {
+			logger.error(error);
+		}
+
+		if (result.body.saves && result.body.saves.length) {
+			logger.info(`Successfully uploaded as ${result.body.saves[0]}`);
+		}
+
+		if ((result.body.errors || []).length || (result.body.request_errors || []).length) {
+			throw new libErrors.CommandError("Uploading save failed");
+		}
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["download-save <instance> <save>", "Download a save from an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to download save from", type: "string" });
+		yargs.positional("save", { describe: "Save to download", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		let result = await libLink.messages.downloadSave.send(control, {
+			instance_id: instanceId,
+			save: args.save,
+		});
+
+		let url = new URL(control.config.get("control.master_url"));
+		url.pathname += `api/stream/${result.stream_id}`;
+		let response = await phin({
+			url, method: "GET",
+			core: { ca: control.tlsCa },
+			stream: true,
+		});
+
+		let stream;
+		let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip");
+		while (true) {
+			try {
+				stream = fs.createWriteStream(tempFilename, { flags: "wx" });
+				await events.once(stream, "open");
+				break;
+			} catch (err) {
+				if (err.code === "EEXIST") {
+					tempFilename = await libFileOps.findUnusedName(".", tempFilename, ".tmp.zip");
+				} else {
+					throw err;
+				}
+			}
+		}
+		response.pipe(stream);
+		await events.once(stream, "finish");
+
+		let filename = await libFileOps.findUnusedName(".", args.save, ".zip");
+		await fs.rename(tempFilename, filename);
+
+		logger.info(`Downloaded ${args.save}${args.save === filename ? "" : ` as ${filename}`}`);
+	},
+}));
+
+instanceCommands.add(new libCommand.Command({
+	definition: ["delete-save <instance> <save>", "Delete a save from an instance", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to delete save from", type: "string" });
+		yargs.positional("save", { describe: "Save to delete", type: "string" });
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		await libLink.messages.deleteSave.send(control, {
+			instance_id: instanceId,
+			save: args.save,
 		});
 	},
 }));
@@ -364,15 +527,23 @@ instanceCommands.add(new libCommand.Command({
 		yargs.positional("instance", { describe: "Instance to start", type: "string" });
 		yargs.positional("scenario", { describe: "Scenario to load", type: "string" });
 		yargs.options({
+			"seed": { describe: "Seed to use, takes precedence over map-gen-settings", nargs: 1, type: "number" },
+			"map-exchange-string": { describe: "Map exchange string to use for the save", nargs: 1, type: "string" },
+			"map-gen-settings": { describe: "path to file to use for map-gen-settings", nargs: 1, type: "string" },
+			"map-settings": { describe: "path to file to use for map-settings", nargs: 1, type: "string" },
 			"keep-open": { describe: "Keep console open", nargs: 0, type: "boolean", default: false },
 		});
 	}],
 	handler: async function(args, control) {
 		let instanceId = await libCommand.resolveInstance(control, args.instance);
 		await control.setLogSubscriptions({ instance_ids: [instanceId] });
+		let { seed, mapGenSettings, mapSettings } = await loadMapSettings(args);
 		let response = await libLink.messages.loadScenario.send(control, {
 			instance_id: instanceId,
-			scenario: args.scenario || null,
+			scenario: args.scenario,
+			seed,
+			map_gen_settings: mapGenSettings,
+			map_settings: mapSettings,
 		});
 		control.keepOpen = args.keepOpen;
 	},
@@ -748,6 +919,12 @@ class Control extends libLink.Link {
 		this.keepOpen = false;
 	}
 
+	async slaveUpdateEventHandler() { }
+
+	async instanceUpdateEventHandler() { }
+
+	async saveListUpdateEventHandler() { }
+
 	async setLogSubscriptions({
 		all = false,
 		master = false,
@@ -906,7 +1083,7 @@ async function startControl() {
 			await controlConfig.init();
 
 		} else {
-			throw err;
+			throw new libErrors.StartupError(`Failed to load ${args.config}: ${err.message}`);
 		}
 	}
 
@@ -962,7 +1139,7 @@ async function startControl() {
 		control.shutdown().catch(err => {
 			setBlocking(true);
 			logger.error(err.stack);
-			// eslint-disable-next-line no-process-exit
+			// eslint-disable-next-line node/no-process-exit
 			process.exit(1);
 		});
 	});

packages/ctl/package.json

@@ -3,7 +3,7 @@
   "description": "Clusterio command line management tool",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.7",
   "main": "ctl.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -18,10 +18,11 @@
     "node": ">=12"
   },
   "dependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.6",
+    "@clusterio/lib": "^2.0.0-alpha.7",
     "as-table": "^1.0.31",
     "fs-extra": "^8.1.0",
     "phin": "^3.5.1",
+    "set-blocking": "^2.0.0",
     "winston": "^3.3.3",
     "yargs": "^14.2.0"
   },

packages/lib/config/definitions.js

@@ -101,6 +101,13 @@ MasterGroup.define({
 	type: "number",
 	initial_value: 8,
 });
+MasterGroup.define({
+	name: "proxy_stream_timeout",
+	title: "Proxy Stream Timeout",
+	description: "Timeout in seconds for proxy streams to start flowing",
+	type: "number",
+	initial_value: 15,
+});
 MasterGroup.define({
 	name: "default_role_id",
 	title: "Default role",

packages/lib/config/index.js

@@ -1,11 +1,13 @@
-/* eslint-disable global-require */
 /**
  * Configuration System
  * @author Hornwitser
  * @module lib/config
  */
 "use strict";
+const classes = require("./classes");
+const definitions = require("./definitions");
+
 module.exports = {
-	...require("./classes"),
-	...require("./definitions"),
+	...classes,
+	...definitions,
 };

packages/lib/factorio/exchange_string.js

@@ -0,0 +1,393 @@
+"use strict";
+const zlib = require("zlib");
+
+
+class MapReaderState {
+	constructor(buf) {
+		this.pos = 0;
+		this.buf = buf;
+		this.last_position = { x: 0, y: 0 };
+	}
+}
+
+function readUInt8(state) {
+	let value = state.buf.readUInt8(state.pos);
+	state.pos += 1;
+	return value;
+}
+
+function readBool(state) {
+	let value = readUInt8(state) !== 0;
+	return value;
+}
+
+function readInt16(state) {
+	let value = state.buf.readInt16LE(state.pos);
+	state.pos += 2;
+	return value;
+}
+
+function readUInt16(state) {
+	let value = state.buf.readUInt16LE(state.pos);
+	state.pos += 2;
+	return value;
+}
+
+function readInt32(state) {
+	let value = state.buf.readInt32LE(state.pos);
+	state.pos += 4;
+	return value;
+}
+
+function readUInt32(state) {
+	let value = state.buf.readUInt32LE(state.pos);
+	state.pos += 4;
+	return value;
+}
+
+function readUInt32so(state) {
+	let value = readUInt8(state);
+	if (value === 0xff) {
+		return readUInt32(state);
+	}
+
+	return value;
+}
+
+function readFloat(state) {
+	let value = state.buf.readFloatLE(state.pos);
+	state.pos += 4;
+	return value;
+}
+
+function readDouble(state) {
+	let value = state.buf.readDoubleLE(state.pos);
+	state.pos += 8;
+	return value;
+}
+
+function readString(state) {
+	let size = readUInt32so(state);
+	let data = state.buf.slice(state.pos, state.pos + size).toString("utf-8");
+	state.pos += size;
+	return data;
+}
+
+function readOptional(state, readValue) {
+	let load = readUInt8(state) !== 0;
+	if (!load) {
+		return null;
+	}
+	return readValue(state);
+}
+
+function readArray(state, readItem) {
+	let size = readUInt32so(state);
+
+	let array = [];
+	for (let i = 0; i < size; i++) {
+		let item = readItem(state);
+		array.push(item);
+	}
+
+	return array;
+}
+
+function readDict(state, readKey, readValue) {
+	let size = readUInt32so(state);
+
+	let mapping = new Map();
+	for (let i = 0; i < size; i++) {
+		let key = readKey(state);
+		let value = readValue(state);
+		mapping.set(key, value);
+	}
+
+	return mapping;
+}
+
+function readVersion(state) {
+	let major = readUInt16(state);
+	let minor = readUInt16(state);
+	let patch = readUInt16(state);
+	let developer = readUInt16(state);
+	return [major, minor, patch, developer];
+}
+
+function readFrequencySizeRichness(state) {
+	return {
+		frequency: readFloat(state),
+		size: readFloat(state),
+		richness: readFloat(state),
+	};
+}
+
+function readAutoplaceSetting(state) {
+	return {
+		treat_missing_as_default: readBool(state),
+		settings: Object.fromEntries(readDict(state, readString, readFrequencySizeRichness)),
+	};
+}
+
+function readMapPosition(state) {
+	let x, y;
+	let x_diff = readInt16(state) / 256;
+	if (x_diff === 0x7fff / 256) {
+		x = readInt32(state) / 256;
+		y = readInt32(state) / 256;
+	} else {
+		let y_diff = readInt16(state) / 256;
+		x = state.last_position.x + x_diff;
+		y = state.last_position.y + y_diff;
+	}
+	state.last_position.x = x;
+	state.last_position.x = y;
+	return { x, y };
+}
+
+function readBoundingBox(state) {
+	return {
+		left_top: readMapPosition(state),
+		right_bottom: readMapPosition(state),
+		orientation: {
+			x: readInt16(state),
+			y: readInt16(state),
+		},
+	};
+}
+
+function readCliffSettings(state) {
+	return {
+		name: readString(state),
+		elevation_0: readFloat(state),
+		elevation_interval: readFloat(state),
+		richness: readFloat(state),
+	};
+}
+
+function readMapGenSettings(state) {
+	return {
+		terrain_segmentation: readFloat(state),
+		water: readFloat(state),
+		autoplace_controls: Object.fromEntries(readDict(state, readString, readFrequencySizeRichness)),
+		autoplace_settings: Object.fromEntries(readDict(state, readString, readAutoplaceSetting)),
+		default_enable_all_autoplace_controls: readBool(state),
+		seed: readUInt32(state),
+		width: readUInt32(state),
+		height: readUInt32(state),
+		area_to_generate_at_start: readBoundingBox(state),
+		starting_area: readFloat(state),
+		peaceful_mode: readBool(state),
+		starting_points: readArray(state, readMapPosition),
+		property_expression_names: Object.fromEntries(readDict(state, readString, readString)),
+		cliff_settings: readCliffSettings(state),
+	};
+}
+
+function readPollution(state) {
+	return {
+		enabled: readOptional(state, readBool),
+		diffusion_ratio: readOptional(state, readDouble),
+		min_to_diffuse: readOptional(state, readDouble),
+		ageing: readOptional(state, readDouble),
+		expected_max_per_chunk: readOptional(state, readDouble),
+		min_to_show_per_chunk: readOptional(state, readDouble),
+		min_pollution_to_damage_trees: readOptional(state, readDouble),
+		pollution_with_max_forest_damage: readOptional(state, readDouble),
+		pollution_per_tree_damage: readOptional(state, readDouble),
+		pollution_restored_per_tree_damage: readOptional(state, readDouble),
+		max_pollution_to_restore_trees: readOptional(state, readDouble),
+		enemy_attack_pollution_consumption_modifier: readOptional(state, readDouble),
+	};
+}
+
+function readSteeringValue(state) {
+	return {
+		radius: readOptional(state, readDouble),
+		separation_factor: readOptional(state, readDouble),
+		separation_force: readOptional(state, readDouble),
+		force_unit_fuzzy_goto_behavior: readOptional(state, readBool),
+	};
+
+}
+
+function readSteering(state) {
+	return {
+		default: readSteeringValue(state),
+		moving: readSteeringValue(state),
+	};
+}
+
+function readEnemyEvolution(state) {
+	return {
+		enabled: readOptional(state, readBool),
+		time_factor: readOptional(state, readDouble),
+		destroy_factor: readOptional(state, readDouble),
+		pollution_factor: readOptional(state, readDouble),
+	};
+}
+
+function readEnemyExpansion(state) {
+	return {
+		enabled: readOptional(state, readBool),
+		max_expansion_distance: readOptional(state, readUInt32),
+		friendly_base_influence_radius: readOptional(state, readUInt32),
+		enemy_building_influence_radius: readOptional(state, readUInt32),
+		building_coefficient: readOptional(state, readDouble),
+		other_base_coefficient: readOptional(state, readDouble),
+		neighbouring_chunk_coefficient: readOptional(state, readDouble),
+		neighbouring_base_chunk_coefficient: readOptional(state, readDouble),
+		max_colliding_tiles_coefficient: readOptional(state, readDouble),
+		settler_group_min_size: readOptional(state, readUInt32),
+		settler_group_max_size: readOptional(state, readUInt32),
+		min_expansion_cooldown: readOptional(state, readUInt32),
+		max_expansion_cooldown: readOptional(state, readUInt32),
+	};
+}
+
+function readUnitGroup(state) {
+	return {
+		min_group_gathering_time: readOptional(state, readUInt32),
+		max_group_gathering_time: readOptional(state, readUInt32),
+		max_wait_time_for_late_members: readOptional(state, readUInt32),
+		max_group_radius: readOptional(state, readDouble),
+		min_group_radius: readOptional(state, readDouble),
+		max_member_speedup_when_behind: readOptional(state, readDouble),
+		max_member_slowdown_when_ahead: readOptional(state, readDouble),
+		max_group_slowdown_factor: readOptional(state, readDouble),
+		max_group_member_fallback_factor: readOptional(state, readDouble),
+		member_disown_distance: readOptional(state, readDouble),
+		tick_tolerance_when_member_arrives: readOptional(state, readUInt32),
+		max_gathering_unit_groups: readOptional(state, readUInt32),
+		max_unit_group_size: readOptional(state, readUInt32),
+	};
+}
+
+function readPathFinder(state) {
+	return {
+		fwd2bwd_ratio: readOptional(state, readInt32),
+		goal_pressure_ratio: readOptional(state, readDouble),
+		use_path_cache: readOptional(state, readBool),
+		max_steps_worked_per_tick: readOptional(state, readDouble),
+		max_work_done_per_tick: readOptional(state, readUInt32),
+		short_cache_size: readOptional(state, readUInt32),
+		long_cache_size: readOptional(state, readUInt32),
+		short_cache_min_cacheable_distance: readOptional(state, readDouble),
+		short_cache_min_algo_steps_to_cache: readOptional(state, readUInt32),
+		long_cache_min_cacheable_distance: readOptional(state, readDouble),
+		cache_max_connect_to_cache_steps_multiplier: readOptional(state, readUInt32),
+		cache_accept_path_start_distance_ratio: readOptional(state, readDouble),
+		cache_accept_path_end_distance_ratio: readOptional(state, readDouble),
+		negative_cache_accept_path_start_distance_ratio: readOptional(state, readDouble),
+		negative_cache_accept_path_end_distance_ratio: readOptional(state, readDouble),
+		cache_path_start_distance_rating_multiplier: readOptional(state, readDouble),
+		cache_path_end_distance_rating_multiplier: readOptional(state, readDouble),
+		stale_enemy_with_same_destination_collision_penalty: readOptional(state, readDouble),
+		ignore_moving_enemy_collision_distance: readOptional(state, readDouble),
+		enemy_with_different_destination_collision_penalty: readOptional(state, readDouble),
+		general_entity_collision_penalty: readOptional(state, readDouble),
+		general_entity_subsequent_collision_penalty: readOptional(state, readDouble),
+		extended_collision_penalty: readOptional(state, readDouble),
+		max_clients_to_accept_any_new_request: readOptional(state, readUInt32),
+		max_clients_to_accept_short_new_request: readOptional(state, readUInt32),
+		direct_distance_to_consider_short_request: readOptional(state, readUInt32),
+		short_request_max_steps: readOptional(state, readUInt32),
+		short_request_ratio: readOptional(state, readDouble),
+		min_steps_to_check_path_find_termination: readOptional(state, readUInt32),
+		start_to_goal_cost_multiplier_to_terminate_path_find: readOptional(state, readDouble),
+		overload_levels: readOptional(state, (p) => readArray(p, readUInt32)),
+		overload_multipliers: readOptional(state, (p) => readArray(p, readDouble)),
+		negative_path_cache_delay_interval: readOptional(state, readUInt32),
+	};
+}
+
+function readDifficultySettings(state) {
+	return {
+		recipe_difficulty: readUInt8(state),
+		technology_difficulty: readUInt8(state),
+		technology_price_multiplier: readDouble(state),
+		research_queue_setting: ["always", "after-victory", "never"][readUInt8(state)],
+	};
+}
+
+function readMapSettings(state) {
+	return {
+		pollution: readPollution(state),
+		steering: readSteering(state),
+		enemy_evolution: readEnemyEvolution(state),
+		enemy_expansion: readEnemyExpansion(state),
+		unit_group: readUnitGroup(state),
+		path_finder: readPathFinder(state),
+		max_failed_behavior_count: readUInt32(state),
+		difficulty_settings: readDifficultySettings(state),
+	};
+}
+
+/**
+ * @typedef {object} module:lib/factorio.MapExchangeResult
+ * @property {array} version -
+ *     Version of Factorio the string was created with.
+ * @property {object} map_gen_settings -
+ *     Decoded map generator settings in the format the --map-gen-settings
+ *     command line option to Factorio expect.
+ * @property {object} map_settings -
+ *     Decoded map settings in the format the --map-settings command line option
+ *     to Factorio expects.
+ * @property {number} checksum - checksum for the exchange string.
+ */
+
+/**
+ * Parse a Map Exchange String
+ *
+ * Reads and decodes the data in the given map exchange string and returns
+ * data structures that can be fed into the Factorio server when creating a
+ * save to set the map gen settings and the map settings for the save.
+ *
+ * @param {string} exchangeString - Max Exchange String to parse.
+ * @returns {module:lib/factorio.MapExchangeResult} Parsed result.
+ * @memberof module:lib/factorio
+ */
+function readMapExchangeString(exchangeString) {
+	exchangeString = exchangeString.replace(/[ \t\n\r]+/g, "");
+	if (!/>>>[0-9a-zA-Z\/+]+={0,3}<<</.test(exchangeString)) {
+		throw new Error("Not a map exchange string");
+	}
+
+	let buf = Buffer.from(exchangeString.slice(3, -3), "base64");
+	try {
+		// eslint-disable-next-line node/no-sync
+		buf = zlib.inflateSync(buf);
+	} catch (err) {
+		if (err.code.startsWith("Z_")) {
+			throw new Error("Malformed map exchange string: zlib inflate failed");
+		}
+	}
+
+	let state = new MapReaderState(buf);
+	let data;
+
+	try {
+		data = {
+			version: readVersion(state),
+			unknown: readUInt8(state),
+			map_gen_settings: readMapGenSettings(state),
+			map_settings: readMapSettings(state),
+			checksum: readUInt32(state),
+		};
+	} catch (err) {
+		if (err.code === "ERR_OUT_OF_RANGE") {
+			throw new Error("Malformed map exchange string: reached end before finishing parsing");
+		}
+		throw err;
+	}
+
+	if (state.pos !== buf.length) {
+		throw new Error("Malformed map exchange string: data after end");
+	}
+
+	return data;
+}
+
+module.exports = {
+	readMapExchangeString,
+};

packages/lib/factorio/index.js

@@ -1,12 +1,17 @@
-/* eslint-disable global-require */
 /**
  * Library for interfacing with Factorio Servers and saves.
  * @module lib/factorio
  * @author Hornwitser
  */
 "use strict";
+const exchange_string = require("./exchange_string");
+const export_ = require("./export");
+const patch = require("./patch");
+const server = require("./server");
+
 module.exports = {
-	...require("./export"),
-	...require("./patch"),
-	...require("./server"),
+	...exchange_string,
+	...export_,
+	...patch,
+	...server,
 };

packages/lib/factorio/server.js

@@ -375,7 +375,7 @@ const outputHeuristics = [
 
 	// Message indicating the server is done starting up
 	{
-		filter: { type: "log", message: /^updateTick\(\d+\) changing state from\(CreatingGame\) to\(InGame\)$/ },
+		filter: { type: "log", message: /^(update|map)Tick\(\d+\) changing state from\(CreatingGame\) to\(InGame\)$/ },
 		action: function() {
 			this._notifyGameReady().catch((err) => { this.emit("error", err); });
 		},
@@ -829,17 +829,26 @@ class FactorioServer extends events.EventEmitter {
 	 *
 	 * @param {string} name -
 	 *     Name of the save to create.  Should end with ".zip".
+	 * @param {?number=} seed - Seed to pass via --map-gen-seed
+	 * @param {?object=} mapGenSettings -
+	 *     Map get settings to pass via --map-gen-settings.
+	 * @param {?object=} mapSettings -
+	 *     Map setting to pass via --map-settings.
 	 */
-	async create(name) {
+	async create(name, seed = null, mapGenSettings = null, mapSettings = null) {
 		this._check(["init"]);
 		this._state = "create";
 
 		await this._writeConfigIni();
+		await this._writeMapSettings(mapGenSettings, mapSettings);
 		this._server = child_process.spawn(
 			this.binaryPath(),
 			[
 				"--config", this.writePath("config.ini"),
 				"--create", this.writePath("saves", name),
+				...(seed !== null ? ["--map-gen-seed", seed] : []),
+				...(mapGenSettings !== null ? ["--map-gen-settings", this.writePath("map-gen-settings.json")] : []),
+				...(mapSettings !== null ? ["--map-settings", this.writePath("map-settings.json")] : []),
 				...(this.verboseLogging ? ["--verbose"] : []),
 			],
 			{
@@ -903,17 +912,26 @@ class FactorioServer extends events.EventEmitter {
 	 * argument to start the given scenario.
 	 *
 	 * @param {string} scenario - Name of the scenario to run.
+	 * @param {?number=} seed - Seed to pass via --map-gen-seed
+	 * @param {?object=} mapGenSettings -
+	 *     Map get settings to pass via --map-gen-settings.
+	 * @param {?object=} mapSettings -
+	 *     Map setting to pass via --map-settings.
 	 */
-	async startScenario(scenario) {
+	async startScenario(scenario, seed = null, mapGenSettings = null, mapSettings = null) {
 		this._check(["init"]);
 		this._state = "running";
 
 		await this._writeConfigIni();
+		await this._writeMapSettings(mapGenSettings, mapSettings);
 		this._server = child_process.spawn(
 			this.binaryPath(),
 			[
 				"--config", this.writePath("config.ini"),
 				"--start-server-load-scenario", scenario,
+				...(seed !== null ? ["--map-gen-seed", seed] : []),
+				...(mapGenSettings !== null ? ["--map-gen-settings", this.writePath("map-gen-settings.json")] : []),
+				...(mapSettings !== null ? ["--map-settings", this.writePath("map-settings.json")] : []),
 				"--port", this.gamePort,
 				"--rcon-port", this.rconPort,
 				"--rcon-password", this.rconPassword,
@@ -1153,6 +1171,19 @@ class FactorioServer extends events.EventEmitter {
 		});
 		await fs.outputFile(this.writePath("config.ini"), content);
 	}
+
+	async _writeMapSettings(mapGenSettings, mapSettings) {
+		if (mapGenSettings) {
+			await fs.outputFile(
+				this.writePath("map-gen-settings.json"), JSON.stringify(mapGenSettings, null, 4)
+			);
+		}
+		if (mapSettings) {
+			await fs.outputFile(
+				this.writePath("map-settings.json"), JSON.stringify(mapSettings, null, 4)
+			);
+		}
+	}
 }
 
 

packages/lib/file_ops.js

@@ -32,6 +32,44 @@ async function getNewestFile(directory, filter = (name) => true) {
 	return newestFile;
 }
 
+/**
+ * Modifies name in case it already exisist at the given directory
+ *
+ * Checks the directory passed if it already contains a file or folder with
+ * the given name, if it does not returns name unmodified, otherwise it
+ * returns a modified name that does not exist in the directory.
+ *
+ * Warning: this function should not be relied upon to be accurate for
+ * security sensitive applications.  The selection process is inherently
+ * racy and a file or folder may have been created in the folder by the time
+ * this function returns.
+ *
+ * @param {string} directory - directory to check in.
+ * @param {string} name - file name to check for, may have extension.
+ * @param {string} extension - dot extension used for the file name.
+ * @returns {string} modified name with extension that likely does not exist
+ *     in the folder
+ */
+async function findUnusedName(directory, name, extension = "") {
+	if (extension && name.endsWith(extension)) {
+		name = name.slice(0, -extension.length);
+	}
+
+	while (true) {
+		if (!await fs.pathExists(path.join(directory, `${name}${extension}`))) {
+			return `${name}${extension}`;
+		}
+
+		let match = /^(.*?)(-(\d+))?$/.exec(name);
+		if (!match[2]) {
+			name = `${match[1]}-2`;
+		} else {
+			name = `${match[1]}-${Number.parseInt(match[3], 10) + 1}`;
+		}
+	}
+}
+
 module.exports = {
 	getNewestFile,
+	findUnusedName,
 };

packages/lib/is_deep_strict_equal.js

@@ -1,7 +1,8 @@
-/* eslint-disable global-require */
+/* eslint-disable node/global-require */
 "use strict";
-// eslint-disable-next-line no-process-env
+// eslint-disable-next-line node/no-process-env
 if (process.env.APP_ENV === "browser") {
+	// eslint-disable-next-line node/no-extraneous-require
 	module.exports = require("fast-deep-equal/es6");
 } else {
 	module.exports = require("util").isDeepStrictEqual;

packages/lib/link/connectors.js

@@ -249,7 +249,7 @@ class WebSocketClientConnector extends WebSocketBaseConnector {
 		// Open WebSocket to master
 		logger.verbose(`SOCKET | connecting to ${url}`);
 
-		// eslint-disable-next-line no-process-env
+		// eslint-disable-next-line node/no-process-env
 		if (process.env.APP_ENV === "browser") {
 			this._socket = new WebSocket(url);
 

packages/lib/link/index.js

@@ -1,12 +1,15 @@
-/* eslint-disable global-require */
 /**
  * Library for message based communication between nodes
  * @module lib/link
  * @author Hornwitser
  */
 "use strict";
+const link = require("./link");
+const messages = require("./messages");
+const connectors = require("./connectors");
+
 module.exports = {
-	...require("./link"),
-	...require("./messages"),
-	...require("./connectors"),
+	...link,
+	...messages,
+	...connectors,
 };

packages/lib/link/messages.js

@@ -331,6 +331,14 @@ messages.setMasterConfigProp = new Request({
 	},
 });
 
+let slaveProperties = {
+	"agent": { type: "string" },
+	"version": { type: "string" },
+	"name": { type: "string" },
+	"id": { type: "integer" },
+	"connected": { type: "boolean" },
+};
+
 messages.listSlaves = new Request({
 	type: "list_slaves",
 	links: ["control-master"],
@@ -341,18 +349,45 @@ messages.listSlaves = new Request({
 			items: {
 				additionalProperties: false,
 				required: ["agent", "version", "name", "id", "connected"],
-				properties: {
-					"agent": { type: "string" },
-					"version": { type: "string" },
-					"name": { type: "string" },
-					"id": { type: "integer" },
-					"connected": { type: "boolean" },
-				},
+				properties: slaveProperties,
 			},
 		},
 	},
 });
 
+messages.setSlaveSubscriptions = new Request({
+	type: "set_slave_subscriptions",
+	links: ["control-master"],
+	permission: "core.slave.subscribe",
+	requestProperties: {
+		"all": { type: "boolean" },
+		"slave_ids": {
+			type: "array",
+			items: { type: "integer" },
+		},
+	},
+});
+
+let instanceProperties = {
+	"name": { type: "string" },
+	"id": { type: "integer" },
+	"assigned_slave": { type: ["null", "integer"] },
+	"status": { enum: [
+		"unknown", "unassigned", "stopped", "starting", "running", "stopping",
+		"creating_save", "exporting_data", "deleted",
+	]},
+};
+
+messages.getInstance = new Request({
+	type: "get_instance",
+	links: ["control-master"],
+	permission: "core.instance.get",
+	requestProperties: {
+		"id": { type: "integer" },
+	},
+	responseProperties: instanceProperties,
+});
+
 messages.listInstances = new Request({
 	type: "list_instances",
 	links: ["control-master"],
@@ -363,26 +398,31 @@ messages.listInstances = new Request({
 			items: {
 				additionalProperties: false,
 				required: ["name", "id", "assigned_slave", "status"],
-				properties: {
-					"name": { type: "string" },
-					"id": { type: "integer" },
-					"assigned_slave": { type: ["null", "integer"] },
-					"status": { enum: [
-						"unknown", "unassigned", "stopped", "starting", "running", "stopping",
-						"creating_save", "exporting_data",
-					]},
-				},
+				properties: instanceProperties,
 			},
 		},
 	},
 });
 
+messages.setInstanceSubscriptions = new Request({
+	type: "set_instance_subscriptions",
+	links: ["control-master"],
+	permission: "core.instance.subscribe",
+	requestProperties: {
+		"all": { type: "boolean" },
+		"instance_ids": {
+			type: "array",
+			items: { type: "integer" },
+		},
+	},
+});
+
 messages.generateSlaveToken = new Request({
 	type: "generate_slave_token",
 	links: ["control-master"],
 	permission: "core.slave.generate_token",
 	requestProperties: {
-		"slave_id": { type: "integer" },
+		"slave_id": { type: ["integer", "null"] },
 	},
 	responseProperties: {
 		"token": { type: "string" },
@@ -468,11 +508,103 @@ messages.startInstance = new Request({
 	forwardTo: "instance",
 });
 
+let saveList = {
+	type: "array",
+	items: {
+		type: "object",
+		additionalProperties: false,
+		required: ["type", "name", "size", "mtime_ms", "loaded"],
+		properties: {
+			"type": { enum: ["file", "directory", "special"] },
+			"name": { type: "string" },
+			"size": { type: "integer" },
+			"mtime_ms": { type: "number" },
+			"loaded": { type: "boolean" },
+			"default": { type: "boolean" },
+		},
+	},
+};
+
+messages.listSaves = new Request({
+	type: "list_saves",
+	links: ["control-master", "master-slave", "slave-instance"],
+	permission: "core.instance.save.list",
+	forwardTo: "instance",
+	responseProperties: {
+		"list": saveList,
+	},
+});
+
 messages.createSave = new Request({
 	type: "create_save",
 	links: ["control-master", "master-slave", "slave-instance"],
-	permission: "core.instance.create_save",
+	permission: "core.instance.save.create",
 	forwardTo: "instance",
+	requestProperties: {
+		"name": { type: "string" },
+		"seed": { type: ["integer", "null"] },
+		"map_gen_settings": { type: ["object", "null"] },
+		"map_settings": { type: ["object", "null"] },
+	},
+});
+
+messages.deleteSave = new Request({
+	type: "delete_save",
+	links: ["control-master", "master-slave"],
+	permission: "core.instance.save.delete",
+	forwardTo: "instance",
+	requestProperties: {
+		"save": { type: "string" },
+	},
+});
+
+messages.downloadSave = new Request({
+	type: "download_save",
+	links: ["control-master"],
+	permission: "core.instance.save.download",
+	requestProperties: {
+		"instance_id": { type: "integer" },
+		"save": { type: "string" },
+	},
+	responseProperties: {
+		"stream_id": { type: "string" },
+	},
+});
+
+messages.pullSave = new Request({
+	type: "pull_save",
+	links: ["master-slave"],
+	requestProperties: {
+		"instance_id": { type: "integer" },
+		"stream_id": { type: "string" },
+		"filename": { type: "string" },
+	},
+	responseProperties: {
+		"save": { type: "string" },
+	},
+});
+
+messages.pushSave = new Request({
+	type: "push_save",
+	links: ["master-slave"],
+	requestProperties: {
+		"instance_id": { type: "integer" },
+		"stream_id": { type: "string" },
+		"save": { type: "string" },
+	},
+});
+
+messages.setSaveListSubscriptions = new Request({
+	type: "set_save_list_subscriptions",
+	links: ["control-master"],
+	permission: "core.instance.save.list_subscribe",
+	requestProperties: {
+		"all": { type: "boolean" },
+		"instance_ids": {
+			type: "array",
+			items: { type: "integer" },
+		},
+	},
 });
 
 messages.loadScenario = new Request({
@@ -482,6 +614,9 @@ messages.loadScenario = new Request({
 	forwardTo: "instance",
 	requestProperties: {
 		"scenario": { type: "string" },
+		"seed": { type: ["integer", "null"] },
+		"map_gen_settings": { type: ["object", "null"] },
+		"map_settings": { type: ["object", "null"] },
 	},
 });
 
@@ -963,6 +1098,13 @@ messages.logMessage = new Event({
 	},
 });
 
+messages.slaveUpdate = new Event({
+	type: "slave_update",
+	links: ["master-control"],
+	eventProperties: slaveProperties,
+});
+
+
 messages.instanceInitialized = new Event({
 	type: "instance_initialized",
 	links: ["instance-slave"],
@@ -985,6 +1127,22 @@ messages.instanceStatusChanged = new Event({
 	},
 });
 
+messages.instanceUpdate = new Event({
+	type: "instance_update",
+	links: ["master-control"],
+	eventProperties: instanceProperties,
+});
+
+messages.saveListUpdate = new Event({
+	type: "save_list_update",
+	links: ["instance-slave", "slave-master", "master-control"],
+	forwardTo: "master",
+	eventProperties: {
+		"instance_id": { type: "integer" },
+		"list": saveList,
+	},
+});
+
 messages.masterConnectionEvent = new Event({
 	type: "master_connection_event",
 	links: ["slave-instance"],

packages/lib/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@clusterio/lib",
   "description": "Shared library for Clusterio",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.7",
   "repository": "https://github.com/clusterio/factorioClusterio",
   "license": "MIT",
   "keywords": [

packages/lib/plugin_loader.js

@@ -1,4 +1,4 @@
-/* eslint-disable global-require */
+/* eslint-disable node/global-require */
 /**
  * Helpers for loading plugins in Node.js
  * @module lib/plugin_loader

packages/lib/shared_commands.js

@@ -55,7 +55,7 @@ async function handlePluginCommand(args, pluginList, pluginListPath) {
 
 		let pluginInfo;
 		try {
-			// eslint-disable-next-line global-require
+			// eslint-disable-next-line node/global-require
 			pluginInfo = require(path.posix.join(pluginPath, "info"));
 		} catch (err) {
 			logger.error(`Unable to import plugin info from ${args.path}:\n${err.stack}`);

packages/lib/users.js

@@ -322,6 +322,12 @@ definePermission({
 	description: "Get the full list of slaves in the cluster.",
 	grantByDefault: true,
 });
+definePermission({
+	name: "core.slave.subscribe",
+	title: "Subscribe to slave updates",
+	description: "Subscribe to be notified on updates on the details of slaves.",
+	grantByDefault: true,
+});
 definePermission({
 	name: "core.slave.generate_token",
 	title: "Generate slave token",
@@ -333,12 +339,24 @@ definePermission({
 	description: "Create slave configs via the master server.",
 });
 
+definePermission({
+	name: "core.instance.get",
+	title: "Get instance",
+	description: "Get the details of an instance in the cluster.",
+	grantByDefault: true,
+});
 definePermission({
 	name: "core.instance.list",
 	title: "List instances",
 	description: "Get the full list of instances in the cluster.",
 	grantByDefault: true,
 });
+definePermission({
+	name: "core.instance.subscribe",
+	title: "Subscribe to instance updates",
+	description: "Subscribe to be notified on updates on the details and status of instances.",
+	grantByDefault: true,
+});
 definePermission({
 	name: "core.instance.create",
 	title: "Create instance",
@@ -360,10 +378,35 @@ definePermission({
 	description: "Assign or reassign instances to a slave.",
 });
 definePermission({
-	name: "core.instance.create_save",
+	name: "core.instance.save.list",
+	title: "List saves on instance",
+	description: "List the saves currently on the instance.",
+});
+definePermission({
+	name: "core.instance.save.create",
 	title: "Create new instance save",
 	description: "Create new savegames on instances.",
 });
+definePermission({
+	name: "core.instance.save.delete",
+	title: "Delete instance save",
+	description: "Delete savegames on instances.",
+});
+definePermission({
+	name: "core.instance.save.upload",
+	title: "Upload save to instance",
+	description: "Upload savegames to instances.",
+});
+definePermission({
+	name: "core.instance.save.download",
+	title: "Download save from instance",
+	description: "Download savegames from instances.",
+});
+definePermission({
+	name: "core.instance.save.list_subscribe",
+	title: "Subscribe to save list updatess",
+	description: "Subscribe to be notifed on updates to the list of saves of instances.",
+});
 definePermission({
 	name: "core.instance.export_data",
 	title: "Export locale and icons",

packages/master/.npmignore

@@ -0,0 +1 @@
+webpack.config.js

packages/master/master.js

@@ -22,6 +22,7 @@ const setBlocking = require("set-blocking");
 const yargs = require("yargs");
 const util = require("util");
 const winston = require("winston");
+const jwt = require("jsonwebtoken");
 
 // homebrew modules
 const libErrors = require("@clusterio/lib/errors");
@@ -135,6 +136,10 @@ async function handleBootstrapCommand(args, masterConfig) {
 		// eslint-disable-next-line no-console
 		console.log(user.createToken(masterConfig.get("master.auth_secret")));
 
+	} else if (subCommand === "generate-slave-token") {
+		// eslint-disable-next-line no-console
+		console.log(jwt.sign({ aud: "slave", slave: args.id }, masterConfig.get("master.auth_secret")));
+
 	} else if (subCommand === "create-ctl-config") {
 		let admin = userManager.users.get(args.name);
 		if (!admin) {
@@ -198,6 +203,9 @@ async function initialize() {
 			yargs
 				.command("create-admin <name>", "Create a cluster admin")
 				.command("generate-user-token <name>", "Generate authentication token for the given user")
+				.command("generate-slave-token <id>", "Generate authentication token for the given slave", yargs => {
+					yargs.positional("id", { describe: "ID of the slave", type: "number" });
+				})
 				.command("create-ctl-config <name>", "Create clusterioctl config for the given user", yargs => {
 					yargs.option("output", {
 						describe: "Path to output config (- for stdout)", type: "string",
@@ -274,7 +282,7 @@ async function initialize() {
 			await parameters.masterConfig.init();
 
 		} else {
-			throw err;
+			throw new libErrors.StartupError(`Failed to load ${parameters.masterConfigPath}: ${err.message}`);
 		}
 	}
 
@@ -322,7 +330,7 @@ async function startup() {
 		if (secondSigint) {
 			setBlocking(true);
 			logger.fatal("Caught second interrupt, terminating immediately");
-			// eslint-disable-next-line no-process-exit
+			// eslint-disable-next-line node/no-process-exit
 			process.exit(1);
 		}
 
@@ -335,7 +343,7 @@ async function startup() {
 		if (secondSigterm) {
 			setBlocking(true);
 			logger.fatal("Caught second termination, terminating immediately");
-			// eslint-disable-next-line no-process-exit
+			// eslint-disable-next-line node/no-process-exit
 			process.exit(1);
 		}
 

packages/master/package.json

@@ -3,7 +3,7 @@
   "description": "Implementation of Clusterio master server",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.7",
   "main": "master.js",
   "scripts": {
     "build": "webpack-cli --env production",
@@ -19,10 +19,10 @@
     "node": ">=12"
   },
   "dependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.6",
+    "@clusterio/lib": "^2.0.0-alpha.7",
+    "busboy": "^0.3.1",
     "compression": "^1.7.2",
     "express": "^4.16.2",
-    "express-fileupload": "1.2.0",
     "fs-extra": "^8.1.0",
     "jsonwebtoken": "^8.2.1",
     "jszip": "^3.2.2",
@@ -33,7 +33,7 @@
     "yargs": "^14.2.0"
   },
   "devDependencies": {
-    "@clusterio/web_ui": "^2.0.0-alpha.6",
+    "@clusterio/web_ui": "^2.0.0-alpha.7",
     "fast-deep-equal": "^3.1.3",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",

packages/master/src/BaseConnection.js

@@ -23,25 +23,7 @@ class BaseConnection extends libLink.Link {
 	}
 
 	async forwardRequestToInstance(message, request) {
-		let instance = this._master.instances.get(message.data.instance_id);
-		if (!instance) {
-			throw new libErrors.RequestError(`Instance with ID ${message.data.instance_id} does not exist`);
-		}
-
-		let slaveId = instance.config.get("instance.assigned_slave");
-		if (slaveId === null) {
-			throw new libErrors.RequestError("Instance is not assigned to a slave");
-		}
-
-		let connection = this._master.wsServer.slaveConnections.get(slaveId);
-		if (!connection) {
-			throw new libErrors.RequestError("Slave containing instance is not connected");
-		}
-		if (request.plugin && !connection.plugins.has(request.plugin)) {
-			throw new libErrors.RequestError(`Slave containing instance does not have ${request.plugin} plugin`);
-		}
-
-		return await request.send(connection, message.data);
+		return await this._master.forwardRequestToInstance(request, message.data);
 	}
 
 	async forwardEventToInstance(message, event) {

packages/master/src/ControlConnection.js

@@ -6,13 +6,14 @@ const winston = require("winston");
 const libConfig = require("@clusterio/lib/config");
 const libErrors = require("@clusterio/lib/errors");
 const libLink = require("@clusterio/lib/link");
-const { logger } = require("@clusterio/lib/logging");
+const { levels, logger } = require("@clusterio/lib/logging");
 const libLoggingUtils = require("@clusterio/lib/logging_utils");
 const libPlugin = require("@clusterio/lib/plugin");
 const libPrometheus = require("@clusterio/lib/prometheus");
 const libUsers = require("@clusterio/lib/users");
 
 const BaseConnection = require("./BaseConnection");
+const routes = require("./routes");
 
 const lastQueryLogTime = new libPrometheus.Gauge(
 	"clusterio_master_last_query_log_duration_seconds",
@@ -38,6 +39,21 @@ class ControlConnection extends BaseConnection {
 		 */
 		this.user = user;
 
+		this.slaveSubscriptions = {
+			all: false,
+			slave_ids: [],
+		};
+
+		this.instanceSubscriptions = {
+			all: false,
+			instance_ids: [],
+		};
+
+		this.saveListSubscriptions = {
+			all: false,
+			instance_ids: [],
+		};
+
 		this.logTransport = null;
 		this.logSubscriptions = {
 			all: false,
@@ -96,12 +112,29 @@ class ControlConnection extends BaseConnection {
 		return { list };
 	}
 
+	async setSlaveSubscriptionsRequestHandler(message) {
+		this.slaveSubscriptions = message.data;
+	}
+
+	slaveUpdated(slave, update) {
+		if (
+			this.slaveSubscriptions.all
+			|| this.slaveSubscriptions.slave_ids.includes(slave.id)
+		) {
+			libLink.messages.slaveUpdate.send(this, update);
+		}
+	}
+
 	generateSlaveToken(slaveId) {
 		return jwt.sign({ aud: "slave", slave: slaveId }, this._master.config.get("master.auth_secret"));
 	}
 
 	async generateSlaveTokenRequestHandler(message) {
-		return { token: this.generateSlaveToken(message.data.slave_id) };
+		let slaveId = message.data.slave_id;
+		if (slaveId === null) {
+			slaveId = Math.random() * 2**31 | 0;
+		}
+		return { token: this.generateSlaveToken(slaveId) };
 	}
 
 	async createSlaveConfigRequestHandler(message) {
@@ -122,6 +155,21 @@ class ControlConnection extends BaseConnection {
 		return { serialized_config: slaveConfig.serialize() };
 	}
 
+	async getInstanceRequestHandler(message) {
+		let id = message.data.id;
+		let instance = this._master.instances.get(id);
+		if (!instance) {
+			throw new libErrors.RequestError(`Instance with ID ${id} does not exist`);
+		}
+
+		return {
+			id,
+			name: instance.config.get("instance.name"),
+			assigned_slave: instance.config.get("instance.assigned_slave"),
+			status: instance.status,
+		};
+	}
+
 	async listInstancesRequestHandler(message) {
 		let list = [];
 		for (let instance of this._master.instances.values()) {
@@ -135,6 +183,24 @@ class ControlConnection extends BaseConnection {
 		return { list };
 	}
 
+	async setInstanceSubscriptionsRequestHandler(message) {
+		this.instanceSubscriptions = message.data;
+	}
+
+	instanceUpdated(instance) {
+		if (
+			this.instanceSubscriptions.all
+			|| this.instanceSubscriptions.instance_ids.includes(instance.config.get("instance.id"))
+		) {
+			libLink.messages.instanceUpdate.send(this, {
+				id: instance.config.get("instance.id"),
+				name: instance.config.get("instance.name"),
+				assigned_slave: instance.config.get("instance.assigned_slave"),
+				status: instance.status,
+			});
+		}
+	}
+
 	// XXX should probably add a hook for slave reuqests?
 	async createInstanceRequestHandler(message) {
 		let instanceConfig = new libConfig.InstanceConfig("master");
@@ -174,7 +240,7 @@ class ControlConnection extends BaseConnection {
 		let instance = { config: instanceConfig, status: "unassigned" };
 		this._master.instances.set(instanceId, instance);
 		await libPlugin.invokeHook(this._master.plugins, "onInstanceStatusChanged", instance, null);
-		this._master.addInstancePluginHooks(instance);
+		this._master.addInstanceHooks(instance);
 	}
 
 	async deleteInstanceRequestHandler(message, request) {
@@ -190,6 +256,7 @@ class ControlConnection extends BaseConnection {
 
 		let prev = instance.status;
 		instance.status = "deleted";
+		this._master.instanceUpdated(instance);
 		await libPlugin.invokeHook(this._master.plugins, "onInstanceStatusChanged", instance, prev);
 	}
 
@@ -287,6 +354,42 @@ class ControlConnection extends BaseConnection {
 		}
 	}
 
+	async setSaveListSubscriptionsRequestHandler(message) {
+		this.saveListSubscriptions = message.data;
+	}
+
+	saveListUpdate(data) {
+		if (
+			this.saveListSubscriptions.all
+			|| this.saveListSubscriptions.instance_ids.includes(data.instance_id)
+		) {
+			libLink.messages.saveListUpdate.send(this, data);
+		}
+	}
+
+	async downloadSaveRequestHandler(message) {
+		let { instance_id, save } = message.data;
+		let stream = await routes.createProxyStream(this._master.app);
+		stream.filename = save;
+
+		let ready = new Promise((resolve, reject) => {
+			stream.events.on("source", resolve);
+			stream.events.on("timeout", () => reject(
+				new libErrors.RequestError("Timed out establishing stream from slave")
+			));
+		});
+		ready.catch(() => {});
+
+		let result = await this._master.forwardRequestToInstance(libLink.messages.pushSave, {
+			instance_id,
+			stream_id: stream.id,
+			save,
+		});
+
+		await ready;
+		return { stream_id: stream.id };
+	}
+
 	async setLogSubscriptionsRequestHandler(message) {
 		this.logSubscriptions = message.data;
 		this.updateLogSubscriptions();

packages/master/src/Master.js

@@ -2,7 +2,6 @@
 const compression = require("compression");
 const events = require("events");
 const express = require("express");
-const expressFileupload = require("express-fileupload");
 const fs = require("fs-extra");
 const http = require("http");
 const https = require("https");
@@ -49,6 +48,7 @@ class Master {
 
 		this.app = express();
 		this.app.locals.master = this;
+		this.app.locals.streams = new Map();
 
 		/**
 		 * Mapping of slave id to slave info
@@ -118,7 +118,7 @@ class Master {
 		// Start webpack development server if enabled
 		if (args.dev || args.devPlugin) {
 			logger.warn("Webpack development mode enabled");
-			/* eslint-disable global-require */
+			/* eslint-disable node/global-require, node/no-unpublished-require */
 			const webpack = require("webpack");
 			const webpackDevMiddleware = require("webpack-dev-middleware");
 			const webpackConfigs = [];
@@ -136,11 +136,11 @@ class Master {
 					webpackConfigs.push(config);
 				}
 			}
-			/* eslint-enable global-require */
+			/* eslint-enable node/global-require, node/no-unpublished-require */
 
 			const compiler = webpack(webpackConfigs);
 			this.devMiddleware = webpackDevMiddleware(compiler, {});
-			app.use(this.devMiddleware);
+			this.app.use(this.devMiddleware);
 		}
 
 		let databaseDirectory = this.config.get("master.database_directory");
@@ -152,7 +152,7 @@ class Master {
 		await this.userManager.load(path.join(databaseDirectory, "users.json"));
 
 		for (let instance of this.instances.values()) {
-			this.addInstancePluginHooks(instance);
+			this.addInstanceHooks(instance);
 		}
 
 		// Make sure we're actually going to listen on a port
@@ -243,7 +243,7 @@ class Master {
 		await fs.outputFile(this.configPath, JSON.stringify(this.config.serialize(), null, 4));
 
 		if (this.devMiddleware) {
-			await new Promise((resolve, reject) => { devMiddleware.close(resolve); });
+			await new Promise((resolve, reject) => { this.devMiddleware.close(resolve); });
 		}
 
 		let databaseDirectory = this.config.get("master.database_directory");
@@ -336,7 +336,6 @@ class Master {
 			hitCounter.inc();
 			next();
 		});
-		app.use(expressFileupload());
 		app.use(compression());
 
 		// Set folder to serve static content from (the website)
@@ -361,10 +360,54 @@ class Master {
 		}
 	}
 
-	addInstancePluginHooks(instance) {
+	slaveUpdated(slave) {
+		let update = {
+			agent: slave.agent,
+			version: slave.version,
+			id: slave.id,
+			name: slave.name,
+			connected: this.wsServer.slaveConnections.has(slave.id),
+		};
+
+		for (let controlConnection of this.wsServer.controlConnections) {
+			if (controlConnection.connector.closing) {
+				continue;
+			}
+
+			controlConnection.slaveUpdated(slave, update);
+		}
+	}
+
+	addInstanceHooks(instance) {
 		instance.config.on("fieldChanged", (group, field, prev) => {
+			if (group.name === "instance" && field === "name") {
+				this.instanceUpdated(instance);
+			}
+
 			libPlugin.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, group, field, prev);
 		});
+
+		this.instanceUpdated(instance);
+	}
+
+	instanceUpdated(instance) {
+		for (let controlConnection of this.wsServer.controlConnections) {
+			if (controlConnection.connector.closing) {
+				continue;
+			}
+
+			controlConnection.instanceUpdated(instance);
+		}
+	}
+
+	saveListUpdate(data) {
+		for (let controlConnection of this.wsServer.controlConnections) {
+			if (controlConnection.connector.closing) {
+				continue;
+			}
+
+			controlConnection.saveListUpdate(data);
+		}
 	}
 
 	async loadPlugins() {
@@ -466,6 +509,41 @@ class Master {
 			});
 		};
 	}
+
+	/**
+	 * Forward the given request to the slave contaning the instance
+	 *
+	 * Finds the slave which the instance with the given instance ID is
+	 * currently assigned to and forwards the request to that instance.
+	 * The request data must have an instance_id parameter.
+	 *
+	 * @param {module:lib/link.Request} request - The request to send.
+	 * @param {object} data - Data to pass with the request.
+	 * @param {number} data.instance_id -
+	 *     ID of instance to send this request towards
+	 * @returns {object} data returned from the request.
+	 */
+	async forwardRequestToInstance(request, data) {
+		let instance = this.instances.get(data.instance_id);
+		if (!instance) {
+			throw new libErrors.RequestError(`Instance with ID ${data.instance_id} does not exist`);
+		}
+
+		let slaveId = instance.config.get("instance.assigned_slave");
+		if (slaveId === null) {
+			throw new libErrors.RequestError("Instance is not assigned to a slave");
+		}
+
+		let connection = this.wsServer.slaveConnections.get(slaveId);
+		if (!connection) {
+			throw new libErrors.RequestError("Slave containing instance is not connected");
+		}
+		if (request.plugin && !connection.plugins.has(request.plugin)) {
+			throw new libErrors.RequestError(`Slave containing instance does not have ${request.plugin} plugin`);
+		}
+
+		return await request.send(connection, data);
+	}
 }
 
 module.exports = Master;

packages/master/src/SlaveConnection.js

@@ -62,9 +62,19 @@ class SlaveConnection extends BaseConnection {
 			return;
 		}
 
+		// We may receive status changed where the status hasn't changed
+		// from our perspective if the connection was down at the time it
+		// changed.  Slaves also send status updates on assignInstance which
+		// for hacky reason is also used to push config changes and
+		// restablish status after a connection loss.
+		if (instance.status === message.data.status) {
+			return;
+		}
+
 		let prev = instance.status;
 		instance.status = message.data.status;
 		logger.verbose(`Instance ${instance.config.get("instance.name")} State: ${instance.status}`);
+		this._master.instanceUpdated(instance);
 		await libPlugin.invokeHook(this._master.plugins, "onInstanceStatusChanged", instance, prev);
 	}
 
@@ -99,6 +109,7 @@ class SlaveConnection extends BaseConnection {
 					let prev = masterInstance.status;
 					masterInstance.status = instanceData.status;
 					logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`);
+					this._master.instanceUpdated(instance);
 					await libPlugin.invokeHook(this._master.plugins, "onInstanceStatusChanged", masterInstance, prev);
 				}
 				continue;
@@ -107,7 +118,7 @@ class SlaveConnection extends BaseConnection {
 			instanceConfig.set("instance.assigned_slave", this._id);
 			let newInstance = { config: instanceConfig, status: instanceData.status };
 			this._master.instances.set(instanceConfig.get("instance.id"), newInstance);
-			this._master.addInstancePluginHooks(newInstance);
+			this._master.addInstanceHooks(newInstance);
 			await libLink.messages.assignInstance.send(this, {
 				instance_id: instanceConfig.get("instance.id"),
 				serialized_config: instanceConfig.serialize("slave"),
@@ -135,6 +146,10 @@ class SlaveConnection extends BaseConnection {
 		libLink.messages.syncUserLists.send(this, { adminlist, banlist, whitelist });
 	}
 
+	async saveListUpdateEventHandler(message) {
+		this._master.saveListUpdate(message.data);
+	}
+
 	async logMessageEventHandler(message) {
 		this._master.clusterLogger.log({
 			...message.data.info,

packages/master/src/WsServer.js

@@ -274,9 +274,11 @@ ${err.stack}`
 			connector.on("close", () => {
 				if (this.slaveConnections.get(data.id) === connection) {
 					this.slaveConnections.delete(data.id);
+					this.master.slaveUpdated(this.master.slaves.get(data.id));
 				}
 			});
 			this.slaveConnections.set(data.id, connection);
+			this.master.slaveUpdated(this.master.slaves.get(data.id));
 
 
 		} else if (type === "register_control") {

packages/master/src/routes.js

@@ -1,13 +1,18 @@
 "use strict";
+const Busboy = require("busboy");
+const crypto = require("crypto");
+const events = require("events");
 const fs = require("fs-extra");
 const JSZip = require("jszip");
 const jwt = require("jsonwebtoken");
 const path = require("path");
+const util = require("util");
 
 const libHelpers = require("@clusterio/lib/helpers");
 const libLink = require("@clusterio/lib/link");
 const libPlugin = require("@clusterio/lib/plugin");
 const libPrometheus = require("@clusterio/lib/prometheus");
+const { logger } = require("@clusterio/lib/logging");
 
 const { endpointHitCounter } = require("./metrics");
 
@@ -106,6 +111,36 @@ function validateSlaveToken(req, res, next) {
 	next();
 }
 
+function validateUserToken(req, res, next) {
+	let token = req.header("x-access-token");
+	if (!token) {
+		res.sendStatus(401);
+		return;
+	}
+
+	try {
+		let tokenPayload = jwt.verify(
+			token,
+			req.app.locals.master.config.get("master.auth_secret"),
+			{ audience: "user" }
+		);
+		let user = req.app.locals.master.userManager.users.get(tokenPayload.user);
+		if (!user) {
+			throw new Error("invalid user");
+		}
+		if (tokenPayload.iat < user.tokenValidAfter) {
+			throw new Error("invalid token");
+		}
+		res.locals.user = user;
+
+	} catch (err) {
+		res.sendStatus(401);
+		return;
+	}
+
+	next();
+}
+
 // Handle an uploaded export package.
 async function uploadExport(req, res) {
 	endpointHitCounter.labels(req.route.path).inc();
@@ -142,6 +177,218 @@ async function uploadExport(req, res) {
 	res.sendStatus(200);
 }
 
+async function createProxyStream(app) {
+	let asyncRandomBytes = util.promisify(crypto.randomBytes);
+	let id = (await asyncRandomBytes(8)).toString("hex");
+	let stream = {
+		id,
+		flowing: false,
+		size: null,
+		mime: null,
+		filename: null,
+		events: new events.EventEmitter(),
+		timeout: setTimeout(() => {
+			stream.events.emit("timeout");
+		}, app.locals.master.config.get("master.proxy_stream_timeout") * 1000),
+	};
+	stream.events.on("close", () => {
+		clearTimeout(stream.timeout);
+		app.locals.streams.delete(id);
+	});
+	stream.events.on("timeout", () => {
+		stream.events.emit("close");
+	});
+	app.locals.streams.set(id, stream);
+	return stream;
+}
+
+async function putStream(req, res) {
+	let stream = req.app.locals.streams.get(req.params.id);
+	if (!stream || stream.source) {
+		res.sendStatus(404);
+		return;
+	}
+	stream.source = req;
+	stream.mime = req.get("Content-Type");
+	stream.size = req.get("Content-Length");
+
+	stream.events.emit("source");
+	stream.events.on("close", () => {
+		if (!stream.flowing) {
+			req.resume();
+			res.sendStatus(500);
+		} else {
+			res.sendStatus(200);
+		}
+	});
+}
+
+async function getStream(req, res) {
+	let stream = req.app.locals.streams.get(req.params.id);
+	if (!stream || stream.flowing) {
+		res.sendStatus(404);
+		return;
+	}
+
+	function startStream() {
+		res.append("Content-Type", stream.mime);
+		if (stream.size) {
+			res.append("Content-Length", stream.size);
+		}
+		if (stream.filename) {
+			res.append("Content-Disposition", `attachment; filename="${stream.filename}"`);
+		} else {
+			res.append("Content-Disposition", "attachment");
+		}
+		stream.source.pipe(res);
+		stream.flowing = true;
+		res.on("finish", () => {
+			stream.events.emit("close");
+		});
+		clearTimeout(stream.timeout);
+	}
+
+	if (stream.source) {
+		startStream();
+	} else {
+		stream.events.on("source", startStream);
+		stream.events.on("timeout", () => {
+			res.sendStatus(500);
+		});
+	}
+}
+
+
+async function uploadSave(req, res) {
+	try {
+		res.locals.user.checkPermission("core.instance.save.upload");
+	} catch (err) {
+		res.status(403).json({ request_errors: [err.message] });
+		return;
+	}
+
+	let contentType = req.get("Content-Type");
+	let match = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/.exec(contentType);
+	if (!match) {
+		res.status(415).json({ request_errors: ["invalid Content-Type"] });
+		return;
+	}
+	let mimeType = match[1].toLowerCase();
+
+	let tasks = [];
+	let errors = [];
+	let requestErrors = [];
+	let saves = [];
+
+	async function handleFile(instanceId, stream, filename, streamMime) {
+		let proxyStream = await createProxyStream(req.app);
+		proxyStream.source = stream;
+		proxyStream.mime = streamMime;
+		let timeout = new Promise((_, reject) => {
+			proxyStream.events.on("timeout", () => {
+				stream.resume();
+				reject(new Error("Timed out establishing stream to slave"));
+			});
+		});
+
+		try {
+			let result = await Promise.race([
+				req.app.locals.master.forwardRequestToInstance(libLink.messages.pullSave, {
+					instance_id: instanceId,
+					stream_id: proxyStream.id,
+					filename,
+				}),
+				timeout,
+			]);
+			saves.push(result.save);
+
+		} catch (err) {
+			proxyStream.events.emit("close");
+			logger.error(`Error uploading save: ${err.message}`);
+			errors.push(err.message);
+			stream.resume();
+		}
+	}
+
+	if (mimeType === "multipart/form-data") {
+		await new Promise(resolve => {
+			let fields = {};
+			let busboy = new Busboy({ headers: req.headers });
+			busboy.on("file", (fieldname, stream, filename, transferEncoding, fileMime) => {
+				if (fields.instanceId === undefined) {
+					requestErrors.push("instance_id must come before files uploaded");
+				}
+
+				if (fileMime !== "application/zip") {
+					requestErrors.push("invalid file Content-Type");
+				}
+
+				if (!filename.endsWith(".zip")) {
+					requestErrors.push("filename must end with .zip");
+				}
+
+				if (errors.length || requestErrors.length) {
+					stream.resume();
+					return;
+				}
+
+				tasks.push(handleFile(fields.instanceId, stream, filename, fileMime));
+			});
+			busboy.on("field", (fieldname, value, fieldnameTruncated, valueTruncated, transferEncoding, valueMime) => {
+				if (fieldname === "instance_id") {
+					fields.instanceId = Number.parseInt(value, 10);
+					if (Number.isNaN(fields.instanceId)) {
+						requestErrors.push("invalid instance_id");
+					}
+				}
+			});
+			busboy.on("finish", resolve);
+			busboy.on("error", (err) => {
+				logger.error(`Error parsing multipart request in upload-save:\n${err.stack}`);
+				errors.push(err.message);
+			});
+			req.pipe(busboy);
+		});
+
+	} else if (mimeType === "application/zip") {
+		let filename = req.query.filename;
+		if (typeof filename !== "string") {
+			requestErrors.push("Missing or invalid filename parameter");
+		} else if (!filename.endsWith(".zip")) {
+			requestErrors.push("filename must end with .zip");
+		}
+		let instanceId = Number.parseInt(req.query.instance_id, 10);
+		if (Number.isNaN(instanceId)) {
+			requestErrors.push("Missing or invalid instance_id parameter");
+		}
+
+		if (errors.length || requestErrors.length) {
+			req.resume();
+		} else {
+			tasks.push(handleFile(instanceId, req, filename, mimeType));
+		}
+
+	} else {
+		res.status(415).json({ request_errors: ["invalid Content-Type"] });
+		return;
+	}
+
+	await Promise.all(tasks);
+
+	if (errors.length) {
+		res.status(500);
+		res.json({ errors, request_errors: requestErrors });
+		return;
+	}
+	if (requestErrors.length) {
+		res.status(400);
+		res.json({ request_errors: requestErrors });
+		return;
+	}
+
+	res.json({ saves });
+}
+
 function addRouteHandlers(app) {
 	app.get("/metrics", (req, res, next) => getMetrics(req, res, next).catch(next));
 	app.get("/api/plugins", getPlugins);
@@ -149,6 +396,12 @@ function addRouteHandlers(app) {
 		validateSlaveToken,
 		(req, res, next) => uploadExport(req, res).catch(next)
 	);
+	app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next));
+	app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next));
+	app.post("/api/upload-save",
+		validateUserToken,
+		(req, res, next) => uploadSave(req, res).catch(next)
+	);
 }
 
 // Routes used in the web interface and served by the master server
@@ -170,4 +423,5 @@ const webRoutes = [
 module.exports = {
 	addRouteHandlers,
 	webRoutes,
+	createProxyStream,
 };

packages/slave/modules/clusterio/serialize.lua

@@ -2,18 +2,44 @@
 -- Based on code from playerManager and trainTeleports
 local serialize = {}
 
+local function version_to_table(version)
+    local t = {}
+    for p in string.gmatch(version, "%d+") do
+        t[#t + 1] = tonumber(p)
+    end
+    return t
+end
+
 -- 0.17 compatibility
-local supports_bar, get_bar, has_bar
+local supports_bar, get_bar, has_bar, version
 if (pcall(function() local mods = script.active_mods end)) then
     supports_bar = "supports_bar"
     get_bar = "get_bar"
     set_bar = "set_bar"
+    version = version_to_table(script.active_mods.base)
 else
     supports_bar = "hasbar"
     get_bar = "getbar"
     set_bar = "setbar"
+    version = version_to_table("0.17.69")
 end
 
+-- returns true if the game version is greater than or equal to the given version
+local function version_ge(comp)
+    comp = version_to_table(comp)
+    for i=1, 3 do
+        if comp[i] > version[i] then
+            return false
+        elseif comp[i] < version[i] then
+            return true
+        end
+    end
+    return true
+end
+
+local has_create_grid = version_ge("1.1.7")
+
+
 -- Equipment Grids are serialized into an array of equipment entries
 -- where ench entry is a table with the following fields:
 --   n: name
@@ -173,7 +199,14 @@ function serialize.deserialize_item_stack(slot, entry)
             slot.allow_manual_label_change = label.a
         end
         if entry.g then
-            serialize.deserialize_equipment_grid(slot.grid, entry.g)
+            if slot.grid then
+                serialize.deserialize_equipment_grid(slot.grid, entry.g)
+            elseif slot.type == "item-with-entity-data" and has_create_grid then
+                slot.create_grid()
+                serialize.deserialize_equipment_grid(slot.grid, entry.g)
+            else
+                print("Error: Attempt to deserialize equipment grid on an unsupported entity")
+            end
         end
         if entry.i then
             local sub_inventory = slot.get_inventory(defines.inventory.item_main)

packages/slave/package.json

@@ -3,7 +3,7 @@
   "description": "Implementation of Clusterio slave server",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.7",
   "main": "slave.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -18,7 +18,7 @@
     "node": ">=12"
   },
   "dependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.6",
+    "@clusterio/lib": "^2.0.0-alpha.7",
     "fs-extra": "^8.1.0",
     "phin": "^3.5.1",
     "pidusage": "^2.0.18",

packages/slave/slave.js

@@ -37,6 +37,7 @@ const libConfig = require("@clusterio/lib/config");
 const libSharedCommands = require("@clusterio/lib/shared_commands");
 const { ConsoleTransport, levels, logger } = require("@clusterio/lib/logging");
 const libLoggingUtils = require("@clusterio/lib/logging_utils");
+const libHelpers = require("@clusterio/lib/helpers");
 
 
 const instanceRconCommandDuration = new libPrometheus.Histogram(
@@ -169,6 +170,7 @@ class Instance extends libLink.Link {
 
 		this._status = "stopped";
 		this._running = false;
+		this._loadedSave = null;
 		this.server = new libFactorio.FactorioServer(
 			factorioDir, this._dir, serverOptions
 		);
@@ -209,9 +211,50 @@ class Instance extends libLink.Link {
 		}
 	}
 
+	static async listSaves(savesDir, loadedSave) {
+		let defaultSave = null;
+		if (loadedSave === null) {
+			defaultSave = await libFileOps.getNewestFile(
+				savesDir, (name) => !name.endsWith(".tmp.zip")
+			);
+		}
+
+		let list = [];
+		for (let name of await fs.readdir(savesDir)) {
+			let type;
+			let stat = await fs.stat(path.join(savesDir, name));
+			if (stat.isFile()) {
+				type = "file";
+			} else if (stat.isDirectory()) {
+				type = "directory";
+			} else {
+				type = "special";
+			}
+
+			list.push({
+				name,
+				type,
+				size: stat.size,
+				mtime_ms: stat.mtimeMs,
+				loaded: name === loadedSave,
+				default: name === defaultSave,
+			});
+		}
+
+		return list;
+	}
+
+	async sendSaveListUpdate() {
+		libLink.messages.saveListUpdate.send(this, {
+			instance_id: this.id,
+			list: await Instance.listSaves(this.server.writePath("saves"), this._loadedSave),
+		});
+	}
+
 	async _autosave(name) {
 		let stat = await fs.stat(this.path("saves", `${name}.zip`));
 		instanceFactorioAutosaveSize.labels(String(this.id)).set(stat.size);
+		await this.sendSaveListUpdate();
 	}
 
 	notifyStatus(status) {
@@ -234,6 +277,7 @@ class Instance extends libLink.Link {
 
 	notifyExit() {
 		this._running = false;
+		this._loadedSave = null;
 		this.notifyStatus("stopped");
 
 		this.config.off("fieldChanged", this._configFieldChanged);
@@ -252,6 +296,10 @@ class Instance extends libLink.Link {
 		for (let pluginInstance of this.plugins.values()) {
 			pluginInstance.onExit();
 		}
+
+		this.sendSaveListUpdate().catch(err => {
+			this.logger.error(`Unexpected error updating save list:\n${err.stack}`);
+		});
 	}
 
 	async _loadPlugin(pluginInfo, slave) {
@@ -267,7 +315,12 @@ class Instance extends libLink.Link {
 
 	async init(pluginInfos) {
 		this.notifyStatus("starting");
-		await this.server.init();
+		try {
+			await this.server.init();
+		} catch (err) {
+			this.notifyExit();
+			throw err;
+		}
 
 		// load plugins
 		for (let pluginInfo of pluginInfos) {
@@ -384,20 +437,6 @@ class Instance extends libLink.Link {
 			);
 		}
 
-		this.logger.verbose("Rotating old logs...");
-		// clean old log file to avoid crash
-		try {
-			let logPath = this.path("factorio-current.log");
-			let stat = await fs.stat(logPath);
-			if (stat.isFile()) {
-				let logFilename = `factorio-${Math.floor(Date.parse(stat.mtime)/1000)}.log`;
-				await fs.rename(logPath, this.path(logFilename));
-				this.logger.verbose(`Log rotated as ${logFilename}`);
-			}
-		} catch (err) {
-			this.logger.error(`Error rotating logs:\n${err.stack}`);
-		}
-
 		// eslint-disable-next-line no-use-before-define
 		await symlinkMods(this, "sharedMods");
 	}
@@ -427,6 +466,23 @@ class Instance extends libLink.Link {
 			saveName = "world.zip";
 		}
 
+		// Load a copy if it's autosave to prevent overwriting the autosave
+		if (saveName.startsWith("_autosave")) {
+			this.logger.info("Copying autosave");
+			let now = new Date();
+			let newName = util.format(
+				"%s-%s-%s %s%s %s",
+				now.getUTCFullYear(),
+				(now.getUTCMonth() + 1).toLocaleString("en", { minimumIntegerDigits: 2 }),
+				now.getUTCDate().toLocaleString("en", { minimumIntegerDigits: 2 }),
+				now.getUTCHours().toLocaleString("en", { minimumIntegerDigits: 2 }),
+				now.getUTCMinutes().toLocaleString("en", { minimumIntegerDigits: 2 }),
+				saveName,
+			);
+			await fs.copy(this.path("saves", saveName), this.path("saves", newName));
+			saveName = newName;
+		}
+
 		if (!this.config.get("factorio.enable_save_patching")) {
 			return saveName;
 		}
@@ -467,7 +523,7 @@ class Instance extends libLink.Link {
 		// Find stand alone modules to load
 		// XXX for now only the included clusterio module is loaded
 		for (let entry of await fs.readdir(path.join(__dirname, "modules"), { withFileTypes: true })) {
-			if (entry.isDirectory()) {
+			if (!entry.isFile()) {
 				if (modules.has(entry.name)) {
 					throw new Error(`Module with name ${entry.name} already exists in a plugin`);
 				}
@@ -510,6 +566,7 @@ class Instance extends libLink.Link {
 		});
 
 		this.server.on("exit", () => this.notifyExit());
+		this._loadedSave = saveName;
 		await this.server.start(saveName);
 
 		if (this.config.get("factorio.enable_save_patching")) {
@@ -517,6 +574,7 @@ class Instance extends libLink.Link {
 			await this.updateInstanceData();
 		}
 
+		await this.sendSaveListUpdate();
 		await libPlugin.invokeHook(this.plugins, "onStart");
 
 		this._running = true;
@@ -530,14 +588,17 @@ class Instance extends libLink.Link {
 	 * scenario.
 	 *
 	 * @param {String} scenario - Name of scenario to load.
+	 * @param {?number} seed - seed to use.
+	 * @param {?object} mapGenSettings - MapGenSettings to use.
+	 * @param {?object} mapSettings - MapSettings to use.
 	 */
-	async startScenario(scenario) {
+	async startScenario(scenario, seed, mapGenSettings, mapSettings) {
 		this.server.on("rcon-ready", () => {
 			this.logger.verbose("RCON connection established");
 		});
 
 		this.server.on("exit", () => this.notifyExit());
-		await this.server.startScenario(scenario);
+		await this.server.startScenario(scenario, seed, mapGenSettings, mapSettings);
 
 		await libPlugin.invokeHook(this.plugins, "onStart");
 
@@ -692,15 +753,22 @@ class Instance extends libLink.Link {
 			throw err;
 		}
 
+		let { scenario, seed, map_gen_settings, map_settings } = message.data;
 		try {
-			await this.startScenario(message.data.scenario);
+			await this.startScenario(scenario, seed, map_gen_settings, map_settings);
 		} catch (err) {
 			await this.stop();
 			throw err;
 		}
 	}
 
-	async createSaveRequestHandler() {
+	async listSavesRequestHandler(message) {
+		return {
+			list: await Instance.listSaves(this.server.writePath("saves"), this._loadedSave),
+		};
+	}
+
+	async createSaveRequestHandler(message) {
 		this.notifyStatus("creating_save");
 		try {
 			this.logger.verbose("Writing server-settings.json");
@@ -716,7 +784,8 @@ class Instance extends libLink.Link {
 		}
 
 		this.server.on("exit", () => this.notifyExit());
-		await this.server.create("world");
+		let { name, seed, map_gen_settings, map_settings } = message.data;
+		await this.server.create(name, seed, map_gen_settings, map_settings);
 		this.logger.info("Successfully created save");
 	}
 
@@ -1056,13 +1125,8 @@ class Slave extends libLink.Link {
 			throw new Error(`Instance name ${err.message}`);
 		}
 
-		// For now add dashes until an unused directory name is found
-		let dir = path.join(this.config.get("slave.instances_directory"), name);
-		while (await fs.pathExists(dir)) {
-			dir += "-";
-		}
-
-		return dir;
+		let instancesDir = this.config.get("slave.instances_directory");
+		return path.join(instancesDir, await libFileOps.findUnusedName(instancesDir, name));
 	}
 
 	async forwardRequestToInstance(message, request) {
@@ -1310,12 +1374,147 @@ class Slave extends libLink.Link {
 		return await request.send(instanceConnection, message.data);
 	}
 
+	async listSavesRequestHandler(message, request) {
+		let instanceId = message.data.instance_id;
+		let instanceConnection = this.instanceConnections.get(instanceId);
+		if (instanceConnection) {
+			return await request.send(instanceConnection, message.data);
+		}
+
+		let instanceInfo = this.instanceInfos.get(instanceId);
+		if (!instanceInfo) {
+			throw new libErrors.RequestError(`Instance with ID ${instanceId} does not exist`);
+		}
+
+		return {
+			list: await Instance.listSaves(path.join(instanceInfo.path, "saves"), null),
+		};
+	}
+
 	async createSaveRequestHandler(message, request) {
 		let instanceId = message.data.instance_id;
 		let instanceConnection = await this._connectInstance(instanceId);
 		await request.send(instanceConnection, message.data);
 	}
 
+	async sendSaveListUpdate(instance_id, savesDir) {
+		let instanceConnection = this.instanceConnections.get(instance_id);
+		let saveList;
+		if (instanceConnection) {
+			saveList = (await libLink.messages.listSaves.send(instanceConnection, { instance_id })).list;
+		} else {
+			saveList = await Instance.listSaves(savesDir, null);
+		}
+
+		libLink.messages.saveListUpdate.send(this, { instance_id, list: saveList });
+	}
+
+	async deleteSaveRequestHandler(message) {
+		let { instance_id, save } = message.data;
+		try {
+			checkFilename(save);
+		} catch (err) {
+			throw new libErrors.RequestError(`Save name ${err.message}`);
+		}
+		let instanceInfo = this.instanceInfos.get(instance_id);
+		if (!instanceInfo) {
+			throw new libErrors.RequestError(`Instance with ID ${instance_id} does not exist`);
+		}
+
+		try {
+			await fs.unlink(path.join(instanceInfo.path, "saves", save));
+		} catch (err) {
+			if (err.code === "ENOENT") {
+				throw new libErrors.RequestError(`${save} does not exist`);
+			}
+			throw err;
+		}
+		await this.sendSaveListUpdate(instance_id, path.join(instanceInfo.path, "saves"));
+	}
+
+	async pullSaveRequestHandler(message) {
+		let { instance_id, stream_id, filename } = message.data;
+		try {
+			checkFilename(filename);
+		} catch (err) {
+			throw new libErrors.RequestError(`Save name ${err.message}`);
+		}
+		let instanceInfo = this.instanceInfos.get(instance_id);
+		if (!instanceInfo) {
+			throw new libErrors.RequestError(`Instance with ID ${instance_id} does not exist`);
+		}
+
+		let url = new URL(this.config.get("slave.master_url"));
+		url.pathname += `api/stream/${stream_id}`;
+		let response = await phin({
+			url, method: "GET",
+			core: { ca: this.tlsCa },
+			stream: true,
+		});
+
+		if (response.statusCode !== 200) {
+			let content = await libHelpers.readStream(response);
+			throw new libErrors.RequestError(`Stream returned ${response.statusCode}: ${content.toString()}`);
+		}
+
+		let savesDir = path.join(instanceInfo.path, "saves");
+		let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
+		let stream;
+		while (true) {
+			try {
+				stream = fs.createWriteStream(path.join(savesDir, tempFilename), { flags: "wx" });
+				await events.once(stream, "open");
+				break;
+			} catch (err) {
+				if (err.code === "EEXIST") {
+					tempFilename = await libFileOps.findUnusedName(savesDir, tempFilename, ".tmp.zip");
+				} else {
+					throw err;
+				}
+			}
+		}
+		response.pipe(stream);
+		await events.once(stream, "finish");
+
+		filename = await libFileOps.findUnusedName(savesDir, filename, ".zip");
+		await fs.rename(path.join(savesDir, tempFilename), path.join(savesDir, filename));
+
+		await this.sendSaveListUpdate(instance_id, savesDir);
+		return { save: filename };
+	}
+
+	async pushSaveRequestHandler(message) {
+		let { instance_id, stream_id, save } = message.data;
+		try {
+			checkFilename(save);
+		} catch (err) {
+			throw new libErrors.RequestError(`Save name ${err.message}`);
+		}
+		let instanceInfo = this.instanceInfos.get(instance_id);
+		if (!instanceInfo) {
+			throw new libErrors.RequestError(`Instance with ID ${instance_id} does not exist`);
+		}
+
+		let content;
+		try {
+			// phin doesn't support streaming requests :(
+			content = await fs.readFile(path.join(instanceInfo.path, "saves", save));
+		} catch (err) {
+			if (err.code === "ENOENT") {
+				throw new libErrors.RequestError(`${save} does not exist`);
+			}
+			throw err;
+		}
+
+		let url = new URL(this.config.get("slave.master_url"));
+		url.pathname += `api/stream/${stream_id}`;
+		phin({
+			url, method: "PUT",
+			core: { ca: this.tlsCa },
+			data: content,
+		}).catch(err => logger.error(`Error pushing save to master:\n${err.stack}`));
+	}
+
 	async exportDataRequestHandler(message, request) {
 		let instanceId = message.data.instance_id;
 		let instanceConnection = await this._connectInstance(instanceId);
@@ -1427,7 +1626,7 @@ class Slave extends libLink.Link {
 +--------------------------------------------------------------------+
 ${err.stack}`
 			);
-			// eslint-disable-next-line no-process-exit
+			// eslint-disable-next-line node/no-process-exit
 			process.exit(1);
 		}
 	}
@@ -1578,7 +1777,7 @@ async function startSlave() {
 			await slaveConfig.init();
 
 		} else {
-			throw err;
+			throw new libErrors.StartupError(`Failed to load ${args.config}: ${err.message}`);
 		}
 	}
 
@@ -1633,7 +1832,7 @@ async function startSlave() {
 		if (secondSigint) {
 			setBlocking(true);
 			logger.fatal("Caught second interrupt, terminating immediately");
-			// eslint-disable-next-line no-process-exit
+			// eslint-disable-next-line node/no-process-exit
 			process.exit(1);
 		}
 
@@ -1646,7 +1845,7 @@ async function startSlave() {
 		if (secondSigterm) {
 			setBlocking(true);
 			logger.fatal("Caught second termination, terminating immediately");
-			// eslint-disable-next-line no-process-exit
+			// eslint-disable-next-line node/no-process-exit
 			process.exit(1);
 		}
 
@@ -1688,6 +1887,14 @@ I           version of clusterio.  Expect things to break. I
 		if (err instanceof libErrors.AuthenticationFailed) {
 			logger.fatal(err.message);
 
+		} else if (err instanceof libErrors.StartupError) {
+			logger.fatal(`
++-----------------------------------+
+| Unable to to start clusterioslave |
++-----------------------------------+
+${err.stack}`
+			);
+
 		} else if (err instanceof libErrors.PluginError) {
 			logger.fatal(`
 ${err.pluginName} plugin threw an unexpected error

packages/web_ui/package.json

@@ -3,7 +3,7 @@
   "description": "Clusterio web interface implementation",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.7",
   "browser": "src/index.jsx",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -18,9 +18,11 @@
     "@ant-design/icons": "^4.2.2",
     "@babel/core": "^7.13.8",
     "@babel/preset-react": "^7.12.13",
-    "@clusterio/lib": "^2.0.0-alpha.6",
-    "antd": "^4.6.5",
+    "@clusterio/lib": "^2.0.0-alpha.7",
+    "antd": "^4.16.0",
+    "assert": "^2.0.0",
     "babel-loader": "^8.2.2",
+    "browserify-zlib": "^0.2.0",
     "buffer": "^6.0.2",
     "clean-webpack-plugin": "^3.0.0",
     "css-loader": "^5.1.0",
@@ -33,6 +35,7 @@
     "react": "^16.13.1",
     "react-dom": "^16.13.1",
     "react-router-dom": "^5.2.0",
+    "stream-browserify": "^3.0.0",
     "style-loader": "^2.0.0",
     "util": "^0.12.3",
     "webpack": "^5.36.2"

packages/web_ui/src/bootstrap.jsx

@@ -90,5 +90,5 @@ export default async function bootstrap() {
 	controlConnector.setTimeout(15);
 	let control = new Control(controlConnector, plugins);
 
-	ReactDOM.render(<App control={control} plugins={plugins}/>, document.getElementById("root"));
+	ReactDOM.render(<App control={control}/>, document.getElementById("root"));
 }

packages/web_ui/src/components/App.jsx

@@ -7,7 +7,6 @@ import { logger } from "@clusterio/lib/logging";
 import ErrorBoundary from "./ErrorBoundary";
 import SiteLayout from "./SiteLayout";
 import ControlContext from "./ControlContext";
-import PluginsContext from "./PluginsContext";
 import LoginForm from "./LoginForm";
 
 import { Card, Spin, Typography } from "antd";
@@ -95,11 +94,9 @@ export default function App(props) {
 	return (
 		<ErrorBoundary Component={ErrorCard}>
 			<ControlContext.Provider value={props.control}>
-				<PluginsContext.Provider value={props.plugins}>
-					<BrowserRouter basename={webRoot}>
-						{page}
-					</BrowserRouter>
-				</PluginsContext.Provider>
+				<BrowserRouter basename={webRoot}>
+					{page}
+				</BrowserRouter>
 			</ControlContext.Provider>
 		</ErrorBoundary>
 	);

packages/web_ui/src/components/AssignInstanceModal.jsx

@@ -5,24 +5,20 @@ import libLink from "@clusterio/lib/link";
 
 import ControlContext from "./ControlContext";
 import { notifyErrorHandler } from "../util/notify";
+import { useSlaveList } from "../model/slave";
 
 const { Paragraph } = Typography;
 
 
 export default function AssignInstanceModal(props) {
 	let [visible, setVisible] = useState(false);
-	let [slaves, setSlaves] = useState([]);
-	let [loading, setLoading] = useState(true);
+	let [slaveList] = useSlaveList();
 	let [applying, setApplying] = useState(false);
 	let [form] = Form.useForm();
 	let control = useContext(ControlContext);
 
 	function open() {
 		setVisible(true);
-		libLink.messages.listSlaves.send(control).then((result) => {
-			setSlaves(result.list);
-			setLoading(false);
-		});
 	}
 
 	function handleAssign() {
@@ -73,8 +69,8 @@ export default function AssignInstanceModal(props) {
 			</Paragraph>
 			<Form form={form} initialValues={{ slave: props.slaveId }}>
 				<Form.Item name="slave" label="Slave">
-					<Select loading={loading}>
-						{slaves.map((slave) => <Select.Option
+					<Select>
+						{slaveList.map((slave) => <Select.Option
 							key={slave["id"]}
 							value={slave["id"]}
 							disabled={!slave["connected"]}

packages/web_ui/src/components/CreateSaveModal.jsx

@@ -0,0 +1,141 @@
+import React, { useContext, useState } from "react";
+import { Button, Col, Form, Input, Modal, Row, Typography } from "antd";
+
+import libLink from "@clusterio/lib/link";
+import { readMapExchangeString } from "@clusterio/lib/factorio/exchange_string";
+
+import ControlContext from "./ControlContext";
+import { notifyErrorHandler } from "../util/notify";
+
+
+export default function CreateSaveModal(props) {
+	let [visible, setVisible] = useState(false);
+	let [creatingSave, setCreatingSave] = useState(false);
+	let [form] = Form.useForm();
+	let control = useContext(ControlContext);
+
+	function createSave() {
+		let values = form.getFieldsValue();
+		let name = values.saveName || "world.zip";
+		let seed = values.seed === undefined ? null : Number.parseInt(values.seed, 10);
+		let mapGenSettings = null;
+		let mapSettings = null;
+		if (values.exchangeString && values.exchangeString.trim()) {
+			let result;
+			try {
+				result = readMapExchangeString(values.exchangeString);
+			} catch (err) {
+				form.setFields([{ name: "exchangeString", errors: [err.message] }]);
+				return;
+			}
+			form.setFields([{ name: "exchangeString", errors: [] }]);
+			mapGenSettings = result.map_gen_settings;
+			mapSettings = result.map_settings;
+		}
+		if (values.mapGenSettings && values.mapGenSettings.trim()) {
+			try {
+				mapGenSettings = JSON.parse(values.mapGenSettings);
+			} catch (err) {
+				form.setFields([{ name: "mapGenSettings", errors: [err.message] }]);
+				return;
+			}
+			form.setFields([{ name: "mapGenSettings", errors: [] }]);
+		}
+		if (values.mapSettings && values.mapSettings.trim()) {
+			try {
+				mapSettings = JSON.parse(values.mapSettings);
+			} catch (err) {
+				form.setFields([{ name: "mapSettings", errors: [err.message] }]);
+				return;
+			}
+			form.setFields([{ name: "mapSettings", errors: [] }]);
+		}
+
+		setCreatingSave(true);
+		libLink.messages.createSave.send(control, {
+			instance_id: props.instance.id,
+			name,
+			seed,
+			map_gen_settings: mapGenSettings,
+			map_settings: mapSettings,
+		}).then(() => {
+			form.resetFields();
+			setVisible(false);
+		}).catch(
+			notifyErrorHandler("Error creating save")
+		).finally(() => {
+			setCreatingSave(false);
+		});
+	}
+
+	function convertExchangeString() {
+		let exchangeString = form.getFieldValue("exchangeString");
+		let result;
+		try {
+			result = readMapExchangeString(exchangeString);
+		} catch (err) {
+			form.setFields([{ name: "exchangeString", errors: [err.message] }]);
+			return;
+		}
+		form.setFields([{ name: "exchangeString", errors: [], value: "" }]);
+		form.setFieldsValue({
+			mapGenSettings: JSON.stringify(result.map_gen_settings, null, 4),
+			mapSettings: JSON.stringify(result.map_settings, null, 4),
+		});
+	}
+
+	function handleCancel() {
+		setVisible(false);
+	}
+
+	return <>
+		<Button
+			loading={creatingSave}
+			disabled={props.instance.status !== "stopped"}
+			onClick={() => {
+				setVisible(true);
+			}}
+		>
+			Create save
+		</Button>
+		<Modal
+			title="Create Save"
+			width={560}
+			okText="Create"
+			visible={visible}
+			confirmLoading={creatingSave}
+			onOk={createSave}
+			onCancel={handleCancel}
+			destroyOnClose
+		>
+			<Form form={form} layout="vertical">
+				<Form.Item name="saveName" label="Save Name">
+					<Input placeholder="world.zip" />
+				</Form.Item>
+				<Form.Item name="seed" label="Seed">
+					<Input type="number" />
+				</Form.Item>
+				<Form.Item name="exchangeString" label="Map Exchange String">
+					<Input.TextArea allowClear={true} autoSize={{ minRows: 3 }} />
+				</Form.Item>
+				<div style={{ textAlign: "right", marginTop: -8 }}>
+					<Button onClick={() => convertExchangeString()}>Convert to settings</Button>
+				</div>
+				<Form.Item
+					name="mapGenSettings"
+					label="Map Gen Settings"
+					tooltip="JSON data to pass to Factorio with --map-gen-settings, overrides Map Exchange String"
+				>
+					<Input.TextArea allowClear={true} autoSize={{ minRows: 1 }} />
+				</Form.Item>
+				<Form.Item
+					name="mapSettings"
+					label="Map Settings"
+					tooltip="JSON data to pass to Factorio with --map-settings, overrides Map Exchange String"
+				>
+					<Input.TextArea allowClear={true} autoSize={{ minRows: 1 }} />
+				</Form.Item>
+			</Form>
+		</Modal>
+	</>;
+}

packages/web_ui/src/components/InstanceViewPage.jsx

@@ -12,6 +12,8 @@ import InstanceConsole from "./InstanceConsole";
 import InstanceRcon from "./InstanceRcon";
 import AssignInstanceModal from "./AssignInstanceModal";
 import StartStopInstanceButton from "./StartStopInstanceButton";
+import LoadScenarioModal from "./LoadScenarioModal";
+import SavesList from "./SavesList";
 import { notifyErrorHandler } from "../util/notify";
 import { useInstance } from "../model/instance";
 import { useSlave } from "../model/slave";
@@ -26,10 +28,10 @@ export default function InstanceViewPage(props) {
 	let history = useHistory();
 
 	let control = useContext(ControlContext);
-	let [instance, updateInstance] = useInstance(instanceId);
+	let [instance] = useInstance(instanceId);
 	let [slave] = useSlave(Number(instance["assigned_slave"]));
 
-	let [creatingSave, setCreatingSave] = useState(false);
+	let [exportingData, setExportingData] = useState(false);
 
 	let nav = [{ name: "Instances", path: "/instances" }, { name: instance.name || "Unknown" }];
 	if (instance.loading) {
@@ -44,24 +46,24 @@ export default function InstanceViewPage(props) {
 	}
 
 	let instanceButtons = <Space>
-		<StartStopInstanceButton
-			instance={instance}
-			onFinish={updateInstance}
-		/>
-		{instance.status === "stopped" && <Button
-			loading={creatingSave}
+		<StartStopInstanceButton instance={instance} />
+		<LoadScenarioModal instance={instance} />
+		<Button
+			loading={exportingData}
+			disabled={instance.status !== "stopped"}
 			onClick={() => {
-				setCreatingSave(true);
-				libLink.messages.createSave.send(
+				setExportingData(true);
+				libLink.messages.exportData.send(
 					control, { instance_id: instanceId }
-				).catch(notifyErrorHandler("Error creating save")).finally(() => {
-					updateInstance();
-					setCreatingSave(false);
+				).catch(
+					notifyErrorHandler("Error exporting data")
+				).finally(() => {
+					setExportingData(false);
 				});
 			}}
 		>
-			Create save
-		</Button>}
+			Export data
+		</Button>
 		<Popconfirm
 			title="Permanently delete instance and server saves?"
 			okText="Delete"
@@ -107,16 +109,16 @@ export default function InstanceViewPage(props) {
 						disabled: !["unknown", "unassigned", "stopped"].includes(instance["status"]),
 					}}
 					buttonContent={assigned ? "Reassign" : "Assign"}
-					onFinish={updateInstance}
 				/>
 			</Descriptions.Item>
 			<Descriptions.Item label="Status">{instance["status"]}</Descriptions.Item>
 		</Descriptions>
 
+		<SavesList instance={instance} />
 		<Title level={5} style={{ marginTop: 16 }}>Console</Title>
 		<InstanceConsole id={instanceId} />
 		<InstanceRcon id={instanceId} disabled={instance["status"] !== "running"} />
 
-		<InstanceConfigTree id={instanceId} onApply={updateInstance} />
+		<InstanceConfigTree id={instanceId} />
 	</PageLayout>;
 }

packages/web_ui/src/components/InstancesPage.jsx

@@ -1,52 +1,105 @@
-import React, { useContext } from "react";
+import React, { useContext, useState } from "react";
 import { useHistory } from "react-router-dom";
+import { Button, Form, Input, Modal, PageHeader, Table } from "antd";
 
 import libLink from "@clusterio/lib/link";
 import libConfig from "@clusterio/lib/config";
 
-import DataTable from "./data-table";
 import ControlContext from "./ControlContext";
 import PageLayout from "./PageLayout";
+import { useInstanceList } from "../model/instance";
+import { notifyErrorHandler } from "../util/notify";
+import { useSlaveList } from "../model/slave";
 
 
-export default function InstancesPage() {
+function CreateInstanceButton(props) {
 	let control = useContext(ControlContext);
 	let history = useHistory();
+	let [visible, setVisible] = useState(false);
+	let [form] = Form.useForm();
 
-	async function listInstances() {
-		let result = await libLink.messages.listInstances.send(control);
-		return result["list"].map(item => ({
-			key: item["id"],
-			"Name": item["name"],
-			"Assigned Slave": item["assigned_slave"],
-			"Status": item["status"],
-		}));
+	async function createInstance() {
+		let values = form.getFieldsValue();
+		if (!values.instanceName) {
+			form.setFields([{ name: "instanceName", errors: ["Name is required"] }]);
+			return;
+		}
+
+		let instanceConfig = new libConfig.InstanceConfig("control");
+		await instanceConfig.init();
+		instanceConfig.set("instance.name", values.instanceName);
+		let serialized_config = instanceConfig.serialize("master");
+		let result = await libLink.messages.createInstance.send(control, { serialized_config });
+		setVisible(false);
+		history.push(`/instances/${instanceConfig.get("instance.id")}/view`);
 	}
 
+	return <>
+		<Button
+			type="primary"
+			onClick={() => {
+				setVisible(true);
+			}}
+		>Create</Button>
+		<Modal
+			title="Create Instance"
+			okText="Create"
+			visible={visible}
+			onOk={() => { createInstance().catch(notifyErrorHandler("Error creating instance")); }}
+			onCancel={() => { setVisible(false); }}
+			destroyOnClose
+		>
+			<Form form={form}>
+				<Form.Item name="instanceName" label="Name">
+					<Input/>
+				</Form.Item>
+			</Form>
+		</Modal>
+	</>;
+}
+
+export default function InstancesPage() {
+	let history = useHistory();
+	let [slaveList] = useSlaveList();
+	let [instanceList] = useInstanceList();
+
 	return <PageLayout nav={[{ name: "Instances" }]}>
-		<h2>Instances</h2>
-		<DataTable
-			DataFunction={listInstances}
-			AddRecord={{
-				fields: [{
-					dataIndex: "name",
+		<PageHeader
+			className="site-page-header"
+			title="Instances"
+			extra=<CreateInstanceButton />
+		/>
+
+		<Table
+			columns={[
+				{
 					title: "Name",
-				}],
-				insert: async args => {
-					let instanceConfig = new libConfig.InstanceConfig("control");
-					await instanceConfig.init();
-					instanceConfig.set("instance.name", args.name);
-					let serialized_config = instanceConfig.serialize("master");
-					let response = await libLink.messages.createInstance.send(control, { serialized_config });
+					dataIndex: "name",
 				},
-			}}
-			TableProps={{
-				onRow: (record, rowIndex) => ({
-					onClick: event => {
-						history.push(`/instances/${record.key}/view`);
+				{
+					title: "Assigned Slave",
+					key: "assigned_slave",
+					render: instance => {
+						let slave = slaveList.find(s => s.id === instance.assigned_slave);
+						if (slave) {
+							return slave.name;
+						}
+						return instance.assigned_slave;
 					},
-				}),
-			}}
+				},
+				{
+					title: "Status",
+					dataIndex: "status",
+				},
+			]}
+			dataSource={instanceList}
+			rowKey={instance => instance["id"]}
+			pagination={false}
+			onRow={(record, rowIndex) => ({
+				onClick: event => {
+					history.push(`/instances/${record.id}/view`);
+				},
+			})}
 		/>
 	</PageLayout>;
 }

packages/web_ui/src/components/LoadScenarioModal.jsx

@@ -0,0 +1,140 @@
+import React, { useContext, useState } from "react";
+import { Button, Col, Form, Input, Modal, Row, Typography } from "antd";
+
+import libLink from "@clusterio/lib/link";
+import { readMapExchangeString } from "@clusterio/lib/factorio/exchange_string";
+
+import ControlContext from "./ControlContext";
+import { notifyErrorHandler } from "../util/notify";
+
+
+export default function LoadScenarioModal(props) {
+	let [visible, setVisible] = useState(false);
+	let [loadingScenario, setLoadingScenario] = useState(false);
+	let [form] = Form.useForm();
+	let control = useContext(ControlContext);
+
+	function loadScenario() {
+		let values = form.getFieldsValue();
+		let scenario = values.scenario || "base/freeplay";
+		let seed = values.seed === undefined ? null : Number.parseInt(values.seed, 10);
+		let mapGenSettings = null;
+		let mapSettings = null;
+		if (values.exchangeString && values.exchangeString.trim()) {
+			let result;
+			try {
+				result = readMapExchangeString(values.exchangeString);
+			} catch (err) {
+				form.setFields([{ name: "exchangeString", errors: [err.message] }]);
+				return;
+			}
+			form.setFields([{ name: "exchangeString", errors: [] }]);
+			mapGenSettings = result.map_gen_settings;
+			mapSettings = result.map_settings;
+		}
+		if (values.mapGenSettings && values.mapGenSettings.trim()) {
+			try {
+				mapGenSettings = JSON.parse(values.mapGenSettings);
+			} catch (err) {
+				form.setFields([{ name: "mapGenSettings", errors: [err.message] }]);
+				return;
+			}
+			form.setFields([{ name: "mapGenSettings", errors: [] }]);
+		}
+		if (values.mapSettings && values.mapSettings.trim()) {
+			try {
+				mapSettings = JSON.parse(values.mapGenSettings);
+			} catch (err) {
+				form.setFields([{ name: "mapSettings", errors: [err.message] }]);
+				return;
+			}
+			form.setFields([{ name: "mapSettings", errors: [] }]);
+		}
+
+		setLoadingScenario(true);
+		libLink.messages.loadScenario.send(control, {
+			instance_id: props.instance.id,
+			scenario,
+			seed,
+			map_gen_settings: mapGenSettings,
+			map_settings: mapSettings,
+		}).then(() => {
+			setVisible(false);
+		}).catch(
+			notifyErrorHandler("Error creating save")
+		).finally(() => {
+			setLoadingScenario(false);
+		});
+	}
+
+	function convertExchangeString() {
+		let exchangeString = form.getFieldValue("exchangeString");
+		let result;
+		try {
+			result = readMapExchangeString(exchangeString);
+		} catch (err) {
+			form.setFields([{ name: "exchangeString", errors: [err.message] }]);
+			return;
+		}
+		form.setFields([{ name: "exchangeString", errors: [], value: "" }]);
+		form.setFieldsValue({
+			mapGenSettings: JSON.stringify(result.map_gen_settings, null, 4),
+			mapSettings: JSON.stringify(result.map_settings, null, 4),
+		});
+	}
+
+	function handleCancel() {
+		setVisible(false);
+	}
+
+	return <>
+		<Button
+			loading={loadingScenario}
+			disabled={props.instance.status !== "stopped"}
+			onClick={() => {
+				setVisible(true);
+			}}
+		>
+			Load scenario
+		</Button>
+		<Modal
+			title="Load Scenario"
+			width={560}
+			okText="Load"
+			visible={visible}
+			confirmLoading={loadingScenario}
+			onOk={loadScenario}
+			onCancel={handleCancel}
+			destroyOnClose
+		>
+			<Form form={form} layout="vertical">
+				<Form.Item name="scenario" label="Scenario">
+					<Input placeholder="base/freeplay" />
+				</Form.Item>
+				<Form.Item name="seed" label="Seed">
+					<Input type="number" />
+				</Form.Item>
+				<Form.Item name="exchangeString" label="Map Exchange String">
+					<Input.TextArea allowClear={true} autoSize={{ minRows: 3 }} />
+				</Form.Item>
+				<div style={{ textAlign: "right", marginTop: -8 }}>
+					<Button onClick={() => convertExchangeString()}>Convert to settings</Button>
+				</div>
+				<Form.Item
+					name="mapGenSettings"
+					label="Map Gen Settings"
+					tooltip="JSON data to pass to Factorio with --map-gen-settings, overrides Map Exchange String"
+				>
+					<Input.TextArea allowClear={true} autoSize={{ minRows: 1 }} />
+				</Form.Item>
+				<Form.Item
+					name="mapSettings"
+					label="Map Settings"
+					tooltip="JSON data to pass to Factorio with --map-settings, overrides Map Exchange String"
+				>
+					<Input.TextArea allowClear={true} autoSize={{ minRows: 1 }} />
+				</Form.Item>
+			</Form>
+		</Modal>
+	</>;
+}

packages/web_ui/src/components/LoginForm.jsx

@@ -3,7 +3,7 @@ import { Button, Card, Divider, Form, Image, Input, List, Space, Spin, Typograph
 import LockOutlined from "@ant-design/icons/LockOutlined";
 
 import logo from "../images/logo.png";
-import PluginsContext from "./PluginsContext";
+import ControlContext from "./ControlContext";
 
 const { Paragraph, Text } = Typography;
 
@@ -33,7 +33,7 @@ function TokenAuth(props) {
 }
 
 export default function LoginForm(props) {
-	let plugins = useContext(PluginsContext);
+	let plugins = useContext(ControlContext).plugins;
 
 	let loginForms = [];
 	for (let plugin of plugins.values()) {

packages/web_ui/src/components/PluginViewPage.jsx

@@ -2,14 +2,14 @@ import React, { useEffect, useContext, useState } from "react";
 import { useHistory, useParams } from "react-router-dom";
 import { Alert, Descriptions, Spin } from "antd";
 
-import PluginsContext from "./PluginsContext";
+import ControlContext from "./ControlContext";
 import PageLayout from "./PageLayout";
 import { notifyErrorHandler } from "../util/notify";
 
 
 export default function PluginViewPage() {
 	let params = useParams();
-	let plugins = useContext(PluginsContext);
+	let plugins = useContext(ControlContext).plugins;
 	let [pluginList, setPluginList] = useState(null);
 	let pluginName = params.name;
 

packages/web_ui/src/components/PluginsPage.jsx

@@ -3,12 +3,12 @@ import { useHistory } from "react-router-dom";
 import { Table } from "antd";
 
 import notify from "../util/notify";
-import PluginsContext from "./PluginsContext";
+import ControlContext from "./ControlContext";
 import PageLayout from "./PageLayout";
 
 
 export default function PluginsPage() {
-	let plugins = useContext(PluginsContext);
+	let plugins = useContext(ControlContext).plugins;
 	let history = useHistory();
 	let [pluginList, setPluginList] = useState([]);
 

packages/web_ui/src/components/SavesList.jsx

@@ -0,0 +1,135 @@
+import React, { useContext, useState } from "react";
+import { Button, Popconfirm, Space, Table, Tooltip, Upload } from "antd";
+import CaretLeftOutlined from "@ant-design/icons/CaretLeftOutlined";
+import LeftOutlined from "@ant-design/icons/LeftOutlined";
+
+import libLink from "@clusterio/lib/link";
+
+import ControlContext from "./ControlContext";
+import CreateSaveModal from "./CreateSaveModal";
+import SectionHeader from "./SectionHeader";
+import { useSaves } from "../model/saves";
+import { notifyErrorHandler } from "../util/notify";
+
+
+function formatBytes(bytes) {
+	if (bytes === 0) {
+		return "0 Bytes";
+	}
+
+	let units = [" Bytes", " kB", " MB", " GB", " TB"];
+	let factor = 1000;
+	let power = Math.min(Math.floor(Math.log(bytes) / Math.log(factor)), units.length);
+	return (power > 0 ? (bytes / factor ** power).toFixed(2) : bytes) + units[power];
+}
+
+export default function SavesList(props) {
+	let control = useContext(ControlContext);
+	let saves = useSaves(props.instance.id);
+	let [starting, setStarting] = useState(false);
+
+	const saveTable = <Table
+		size="small"
+		columns={[
+			{
+				title: "Name",
+				render: save => <>
+					{save.name}
+					{save.loaded && <Tooltip title="Currently loaded save"><CaretLeftOutlined/></Tooltip>}
+					{save.default && <Tooltip title="Save loaded by default"><LeftOutlined/></Tooltip>}
+				</>,
+				sorter: (a, b) => a.name.localeCompare(b.name),
+			},
+			{
+				title: "Size",
+				key: "size",
+				responsive: ["sm"],
+				render: save => formatBytes(save.size),
+				align: "right",
+				sorter: (a, b) => a.size - b.size,
+			},
+			{
+				title: "Last Modified",
+				key: "mtime_ms",
+				render: save => new Date(save.mtime_ms).toLocaleString(),
+				sorter: (a, b) => a.mtime_ms - b.mtime_ms,
+				defaultSortOrder: "descend",
+			},
+		]}
+		dataSource={saves}
+		rowKey={save => save.name}
+		pagination={false}
+		expandable={{
+			columnWidth: 33,
+			expandRowByClick: true,
+			expandedRowRender: save => <Space wrap style={{marginBottom: 0}}>
+				<Button
+					loading={starting}
+					disabled={props.instance.status !== "stopped"}
+					onClick={() => {
+						setStarting(true);
+						libLink.messages.startInstance.send(
+							control, { instance_id: props.instance.id, save: save.name }
+						).catch(
+							notifyErrorHandler("Error loading save")
+						).finally(
+							() => { setStarting(false); }
+						);
+					}}
+				>Load save</Button>
+				<Button
+					onClick={() => {
+						libLink.messages.downloadSave.send(
+							control, { instance_id: props.instance.id, save: save.name }
+						).then(response => {
+							let url = new URL(webRoot, document.location);
+							url.pathname += `api/stream/${response.stream_id}`;
+							document.location = url;
+						}).catch(
+							notifyErrorHandler("Error downloading save")
+						);
+					}}
+				>Download</Button>
+				<Popconfirm
+					title="Permanently delete save?"
+					okText="Delete"
+					placement="top"
+					okButtonProps={{ danger: true }}
+					onConfirm={() => {
+						libLink.messages.deleteSave.send(
+							control, { instance_id: props.instance.id, save: save.name }
+						).catch(notifyErrorHandler("Error deleting save"));
+					}}
+				>
+					<Button danger>Delete</Button>
+				</Popconfirm>
+			</Space>,
+		}}
+	/>;
+
+	let uploadProps = {
+		name: "file",
+		accept: ".zip",
+		headers: {
+			"X-Access-Token": control.connector.token,
+		},
+		data: {
+			instance_id: props.instance.id,
+		},
+		showUploadList: false,
+		action: `${webRoot}api/upload-save`,
+		// TODO: show progress to user.
+	};
+
+	return <div>
+		<SectionHeader title="Saves" extra=<Space>
+			<Upload {...uploadProps} >
+				<Button>Upload save</Button>
+			</Upload>
+			<CreateSaveModal instance={props.instance} />
+		</Space> />
+		<Upload.Dragger className="save-list-dragger" openFileDialogOnClick={false} {...uploadProps}>
+			{saveTable}
+		</Upload.Dragger>
+	</div>;
+}

packages/web_ui/src/components/SectionHeader.jsx

@@ -0,0 +1,8 @@
+import React, { useContext, useState } from "react";
+
+export default function SectionHeader(props) {
+	return <div className="section-header">
+		<div className="section-header-title">{props.title}</div>
+		{props.extra && <div className="section-header-extra">{props.extra}</div>}
+	</div>;
+}

packages/web_ui/src/components/SiteLayout.jsx

@@ -7,7 +7,7 @@ import webUiPackage from "../../package.json";
 import { useAccount } from "../model/account";
 import ErrorBoundary from "./ErrorBoundary";
 import ErrorPage from "./ErrorPage";
-import PluginsContext from "./PluginsContext";
+import ControlContext from "./ControlContext";
 import { pages } from "../pages";
 
 const { Header, Sider } = Layout;
@@ -17,7 +17,7 @@ export default function SiteLayout(props) {
 	let history = useHistory();
 	let [currentSidebarPath, setCurrentSidebarPath] = useState(null);
 	let account = useAccount();
-	let plugins = useContext(PluginsContext);
+	let plugins = useContext(ControlContext).plugins;
 
 	function SetSidebar(setSidebarProps) {
 		useEffect(() => {

packages/web_ui/src/components/SlavesPage.jsx

@@ -1,28 +1,108 @@
-import React, { useContext } from "react";
+import React, { useContext, useRef, useState } from "react";
+import { Button, Form, Input, Modal, PageHeader, Table } from "antd";
 
 import libLink from "@clusterio/lib/link";
 
-import DataTable from "./data-table";
 import ControlContext from "./ControlContext";
 import PageLayout from "./PageLayout";
+import { useSlaveList } from "../model/slave";
+import { notifyErrorHandler } from "../util/notify";
+
+
+function GenerateSlaveTokenButton(props) {
+	let control = useContext(ControlContext);
+	let [visible, setVisible] = useState(false);
+	let [token, setToken] = useState(null);
+	let [form] = Form.useForm();
+	let tokenTextAreaRef = useRef(null);
+
+	async function generateToken() {
+		let values = form.getFieldsValue();
+		let slaveId = null;
+		if (values.slaveId) {
+			slaveId = Number.parseInt(values.slaveId, 10);
+			if (Number.isNaN(slaveId)) {
+				form.setFields([{ name: "slaveId", errors: ["Must be an integer"] }]);
+				return;
+			}
+			form.setFields([{ name: "slaveId", errors: [] }]);
+		}
+
+		let result = await libLink.messages.generateSlaveToken.send(control, { slave_id: slaveId });
+		setToken(result.token);
+	}
+
+	return <>
+		<Button
+			onClick={() => { setVisible(true); }}
+		>Generate Token</Button>
+		<Modal
+			title="Generate Slave Token"
+			visible={visible}
+			footer={null}
+			onCancel={() => {
+				setVisible(false);
+				setToken(null);
+				form.resetFields();
+			}}
+		>
+			<Form form={form} layout="vertical" requiredMark="optional">
+				<Form.Item name="slaveId" label="Slave ID">
+					<Input/>
+				</Form.Item>
+				<Form.Item>
+					<Button
+						onClick={() => { generateToken().catch(notifyErrorHandler("Error generating token")); }}
+					>Generate</Button>
+				</Form.Item>
+				<Form.Item label="Token" required>
+					<Input.TextArea value={token} autoSize ref={tokenTextAreaRef} />
+				</Form.Item>
+				<Button disabled={token === null} onClick={() => {
+					let textArea = tokenTextAreaRef.current.resizableTextArea.textArea;
+					textArea.select();
+					document.execCommand("copy");
+					window.getSelection().removeAllRanges();
+				}}>Copy</Button>
+			</Form>
+		</Modal>
+	</>;
+}
 
 
 export default function SlavesPage() {
 	let control = useContext(ControlContext);
-
-	async function listSlaves() {
-		let result = await libLink.messages.listSlaves.send(control);
-		return result["list"].map((item) => ({
-			key: item["id"],
-			"Name": item["name"],
-			"Agent": item["agent"],
-			"Version": item["version"],
-			"Connected": item["connected"] && "Yes",
-		}));
-	}
+	let [slaveList] = useSlaveList();
 
 	return <PageLayout nav={[{ name: "Slaves" }]}>
-		<h2>Slaves</h2>
-		<DataTable DataFunction={listSlaves} />
+		<PageHeader
+			className="site-page-header"
+			title="Slaves"
+			extra=<GenerateSlaveTokenButton />
+		/>
+		<Table
+			columns={[
+				{
+					title: "Name",
+					dataIndex: "name",
+				},
+				{
+					title: "Agent",
+					dataIndex: "agent",
+				},
+				{
+					title: "Version",
+					dataIndex: "version",
+				},
+				{
+					title: "Connected",
+					key: "connected",
+					render: slave => slave["connected"] && "Yes",
+				},
+			]}
+			dataSource={slaveList}
+			rowKey={slave => slave["id"]}
+			pagination={false}
+		/>
 	</PageLayout>;
 };

packages/web_ui/src/components/data-table.jsx

@@ -218,7 +218,7 @@ class DataTable extends Component {
 					onCancel={this.hideEditModal.bind(this)}
 					footer={null}
 					title={
-						<h2>{this.state.inserting ? "Legg til" : "Rediger"}</h2>
+						<h2>{this.state.inserting ? "Add" : "Edit"}</h2>
 					}
 				>
 					{this.state.showEditModal && <Form
@@ -249,7 +249,7 @@ class DataTable extends Component {
 						</Form.Item>)}
 						<Form.Item>
 							<Button type="primary" htmlType="submit">
-								{this.state.inserting ? "Legg til" : "Lagre"}
+								{this.state.inserting ? "Save" : "Save"}
 							</Button>
 						</Form.Item>
 					</Form>}

packages/web_ui/src/index.css

@@ -67,6 +67,31 @@ body {
 	padding-right: 64px;
 }
 
+.site-page-header {
+	padding: 0 0 16px;
+}
+
+.section-header {
+	display: flex;
+	align-items: center;
+	margin-top: 16px;
+	margin-bottom: 8px;
+}
+
+.section-header-title {
+	flex: auto;
+	overflow: hidden;
+	font-weight: bold;
+	font-size: 16px;
+	line-height: 1.5715;
+	white-space: nowrap;
+	text-overflow: ellipsis;
+}
+
+.section-header-extra {
+	margin-left: auto;
+}
+
 .site-name {
 	position: absolute;
 	top: -10px;
@@ -143,6 +168,10 @@ body {
 	white-space: pre-wrap;
 }
 
+.save-list-dragger .ant-upload.ant-upload-btn {
+	padding: 0;
+}
+
 .rcon-result code {
 	display: block;
 	background: #000;

packages/web_ui/src/index.jsx

@@ -8,7 +8,6 @@ export * from "./model/item_metadata.jsx";
 
 
 export { default as ControlContext } from "./components/ControlContext";
-export { default as PluginsContext } from "./components/PluginsContext";
 
 export { default as BaseConfigTree } from "./components/BaseConfigTree";
 export { default as InstanceConfigTree } from "./components/InstanceConfigTree";
@@ -19,3 +18,5 @@ export { default as InstanceConsole } from "./components/InstanceConsole";
 export { default as InstanceRcon } from "./components/InstanceRcon";
 
 export { default as PageLayout } from "./components/PageLayout";
+export { default as SavesList } from "./components/SavesList";
+export { default as SectionHeader } from "./components/SectionHeader";

packages/web_ui/src/model/instance.jsx

@@ -2,6 +2,7 @@ import React, { useEffect, useContext, useState } from "react";
 import ControlContext from "../components/ControlContext";
 
 import libLink from "@clusterio/lib/link";
+import { logger } from "@clusterio/lib/logging";
 
 
 export function useInstance(id) {
@@ -9,25 +10,72 @@ export function useInstance(id) {
 	let [instance, setInstance] = useState({ loading: true });
 
 	function updateInstance() {
-		if (!Number.isInteger(id)) {
+		libLink.messages.getInstance.send(control, { id }).then(result => {
+			setInstance({ ...result, present: true });
+		}).catch(err => {
+			logger.error(`Failed to get instance: ${err}`);
 			setInstance({ missing: true });
-			return;
-		}
-
-		// XXX optimize by requesting only the instance in question
-		libLink.messages.listInstances.send(control).then(result => {
-			let match = result.list.find(i => i.id === id);
-			if (!match) {
-				setInstance({ missing: true });
-			} else {
-				setInstance({ ...match, present: true });
-			}
 		});
+
 	}
 
 	useEffect(() => {
+		if (!Number.isInteger(id)) {
+			setInstance({ missing: true });
+			return undefined;
+		}
 		updateInstance();
+
+		function updateHandler(newInstance) {
+			setInstance({ ...newInstance, present: true });
+		}
+
+		control.onInstanceUpdate(id, updateHandler);
+		return () => {
+			control.offInstanceUpdate(id, updateHandler);
+		};
 	}, [id]);
 
 	return [instance, updateInstance];
 }
+
+export function useInstanceList() {
+	let control = useContext(ControlContext);
+	let [instanceList, setInstanceList] = useState([]);
+
+	function updateInstanceList() {
+		libLink.messages.listInstances.send(control).then(result => {
+			setInstanceList(result.list);
+		}).catch(err => {
+			logger.error(`Failed to list instances:\n${err}`);
+		});
+	}
+
+	useEffect(() => {
+		updateInstanceList();
+
+		function updateHandler(newInstance) {
+			setInstanceList(oldList => {
+				let newList = oldList.concat();
+				let index = newList.findIndex(s => s.id === newInstance.id);
+				if (newInstance.status !== "deleted") {
+					if (index !== -1) {
+						newList[index] = newInstance;
+					} else {
+						newList.push(newInstance);
+					}
+				} else if (index !== -1) {
+					newList.splice(index, 1);
+				}
+				return newList;
+			});
+		}
+
+		control.onInstanceUpdate(null, updateHandler);
+		return () => {
+			control.offInstanceUpdate(null, updateHandler);
+		};
+	}, []);
+
+	return [instanceList];
+}

packages/web_ui/src/model/saves.jsx

@@ -0,0 +1,34 @@
+import React, { useEffect, useContext, useState } from "react";
+import ControlContext from "../components/ControlContext";
+
+import libLink from "@clusterio/lib/link";
+import { logger } from "@clusterio/lib/logging";
+
+
+export function useSaves(instanceId) {
+	let control = useContext(ControlContext);
+	let [saves, setSaves] = useState([]);
+
+	function updateSaves() {
+		libLink.messages.listSaves.send(control, { instance_id: instanceId }).then(result => {
+			setSaves(result.list);
+		}).catch(err => {
+			logger.error(`Failed to list instance saves: ${err}`);
+		});
+	}
+
+	useEffect(() => {
+		updateSaves();
+
+		function updateHandler(data) {
+			setSaves(data.list);
+		}
+
+		control.onSaveListUpdate(instanceId, updateHandler);
+		return () => {
+			control.offSaveListUpdate(instanceId, updateHandler);
+		};
+	}, [instanceId]);
+
+	return saves;
+}

packages/web_ui/src/model/slave.jsx

@@ -9,11 +9,6 @@ export function useSlave(id) {
 	let [slave, setSlave] = useState({ loading: true });
 
 	function updateSlave() {
-		if (!Number.isInteger(id)) {
-			setSlave({ missing: true });
-			return;
-		}
-
 		// XXX optimize by requesting only the slave in question
 		libLink.messages.listSlaves.send(control).then(result => {
 			let match = result.list.find(i => i.id === id);
@@ -26,8 +21,57 @@ export function useSlave(id) {
 	}
 
 	useEffect(() => {
+		if (!Number.isInteger(id)) {
+			setSlave({ missing: true });
+			return undefined;
+		}
 		updateSlave();
+
+		function updateHandler(newSlave) {
+			setSlave({ ...newSlave, present: true });
+		}
+
+		control.onSlaveUpdate(id, updateHandler);
+		return () => {
+			control.offSlaveUpdate(id, updateHandler);
+		};
 	}, [id]);
 
 	return [slave, updateSlave];
 }
+
+export function useSlaveList() {
+	let control = useContext(ControlContext);
+	let [slaveList, setSlaveList] = useState([]);
+
+	function updateSlaveList() {
+		libLink.messages.listSlaves.send(control).then(result => {
+			setSlaveList(result.list);
+		});
+	}
+
+	useEffect(() => {
+		updateSlaveList();
+
+		function updateHandler(newSlave) {
+			setSlaveList(oldList => {
+				let newList = oldList.concat();
+				let index = newList.findIndex(s => s.id === newSlave.id);
+				if (index !== -1) {
+					newList[index] = newSlave;
+				} else {
+					newList.push(newSlave);
+				}
+				return newList;
+			});
+		}
+
+		control.onSlaveUpdate(null, updateHandler);
+		return () => {
+			control.offSlaveUpdate(null, updateHandler);
+		};
+	}, []);
+
+
+	return [slaveList];
+}

packages/web_ui/src/util/websocket.jsx

@@ -62,10 +62,22 @@ export class Control extends libLink.Link {
 		 */
 		this.accountName = null;
 
+		this.slaveUpdateHandlers = new Map();
+		this.instanceUpdateHandlers = new Map();
+		this.saveListUpdateHandlers = new Map();
 		this.instanceLogHandlers = new Map();
 
 		this.connector.on("connect", data => {
 			this.accountName = data.account.name;
+			this.updateSlaveSubscriptions().catch(err => logger.error(
+				`Unexpected error updating slave subscriptions:\n${err.stack}`
+			));
+			this.updateInstanceSubscriptions().catch(err => logger.error(
+				`Unexpected error updating instance subscriptions:\n${err.stack}`
+			));
+			this.updateSaveListSubscriptions().catch(err => logger.error(
+				`Unexpected error updating save list subscriptions:\n${err.stack}`
+			));
 			this.updateLogSubscriptions().catch(err => logger.error(
 				`Unexpected error updating log subscriptions:\n${err.stack}`
 			));
@@ -84,6 +96,174 @@ export class Control extends libLink.Link {
 		}
 	}
 
+	async slaveUpdateEventHandler(message) {
+		let handlers = [].concat(
+			this.slaveUpdateHandlers.get(null) || [],
+			this.slaveUpdateHandlers.get(message.data.id) || [],
+		);
+		for (let handler of handlers) {
+			handler(message.data);
+		}
+	}
+
+	async onSlaveUpdate(id, handler) {
+		if (id !== null && !Number.isInteger(id)) {
+			throw new Error("Invalid slave id");
+		}
+
+		let handlers = this.slaveUpdateHandlers.get(id);
+		if (!handlers) {
+			handlers = [];
+			this.slaveUpdateHandlers.set(id, handlers);
+		}
+
+		handlers.push(handler);
+
+		if (handlers.length === 1) {
+			await this.updateSlaveSubscriptions();
+		}
+	}
+
+	async offSlaveUpdate(id, handler) {
+		let handlers = this.slaveUpdateHandlers.get(id);
+		if (!handlers || !handlers.length) {
+			throw new Error(`No handlers for slave ${id} exists`);
+		}
+
+		let index = handlers.lastIndexOf(handler);
+		if (index === -1) {
+			throw new Error(`Given handler is not registered for slave ${id}`);
+		}
+
+		handlers.slice(index, 1);
+		if (!handlers.length) {
+			this.slaveUpdateHandlers.delete(id);
+			await this.updateSlaveSubscriptions();
+		}
+	}
+
+	async updateSlaveSubscriptions() {
+		if (!this.connector.connected) {
+			return;
+		}
+
+		await libLink.messages.setSlaveSubscriptions.send(this, {
+			all: this.slaveUpdateHandlers.has(null),
+			slave_ids: [...this.slaveUpdateHandlers.keys()].filter(e => e !== null),
+		});
+	}
+
+	async instanceUpdateEventHandler(message) {
+		let handlers = [].concat(
+			this.instanceUpdateHandlers.get(null) || [],
+			this.instanceUpdateHandlers.get(message.data.id) || [],
+		);
+		for (let handler of handlers) {
+			handler(message.data);
+		}
+	};
+
+	async onInstanceUpdate(id, handler) {
+		if (id !== null && !Number.isInteger(id)) {
+			throw new Error("Invalid instance id");
+		}
+
+		let handlers = this.instanceUpdateHandlers.get(id);
+		if (!handlers) {
+			handlers = [];
+			this.instanceUpdateHandlers.set(id, handlers);
+		}
+
+		handlers.push(handler);
+
+		if (handlers.length === 1) {
+			await this.updateInstanceSubscriptions();
+		}
+	}
+
+	async offInstanceUpdate(id, handler) {
+		let handlers = this.instanceUpdateHandlers.get(id);
+		if (!handlers || !handlers.length) {
+			throw new Error(`No handlers for instance ${id} exist`);
+		}
+
+		let index = handlers.lastIndexOf(handler);
+		if (index === -1) {
+			throw new Error(`Given handler is not registered for instance ${id}`);
+		}
+
+		handlers.splice(index, 1);
+		if (!handlers.length) {
+			this.instanceUpdateHandlers.delete(id);
+			await this.updateInstanceSubscriptions();
+		}
+	}
+
+	async updateInstanceSubscriptions() {
+		if (!this.connector.connected) {
+			return;
+		}
+
+		await libLink.messages.setInstanceSubscriptions.send(this, {
+			all: this.instanceUpdateHandlers.has(null),
+			instance_ids: [...this.instanceUpdateHandlers.keys()].filter(e => e !== null),
+		});
+	}
+
+	async saveListUpdateEventHandler(message) {
+		let handlers = this.saveListUpdateHandlers.get(message.data.instance_id);
+		for (let handler of handlers || []) {
+			handler(message.data);
+		}
+	};
+
+	async onSaveListUpdate(id, handler) {
+		if (!Number.isInteger(id)) {
+			throw new Error("Invalid instance id");
+		}
+
+		let handlers = this.saveListUpdateHandlers.get(id);
+		if (!handlers) {
+			handlers = [];
+			this.saveListUpdateHandlers.set(id, handlers);
+		}
+
+		handlers.push(handler);
+
+		if (handlers.length === 1) {
+			await this.updateSaveListSubscriptions();
+		}
+	}
+
+	async offSaveListUpdate(id, handler) {
+		let handlers = this.saveListUpdateHandlers.get(id);
+		if (!handlers) {
+			throw new Error(`No handlers for instance ${id} exists`);
+		}
+
+		let index = handlers.lastIndexOf(handler);
+		if (index === -1) {
+			throw new Error(`Given handler is not registered for instance ${id}`);
+
+			handlers.splice(index, 1);
+			if (!handlers.length) {
+				this.saveListUpdateHandlers.delete(id);
+				await this.updateSaveListSubscriptions();
+			}
+		}
+	}
+
+	async updateSaveListSubscriptions() {
+		if (!this.connector.connected) {
+			return;
+		}
+
+		await libLink.messages.setSaveListSubscriptions.send(this, {
+			all: false,
+			instance_ids: [...this.saveListUpdateHandlers.keys()],
+		});
+	}
+
 	async logMessageEventHandler(message) {
 		let info = message.data.info;
 

packages/web_ui/webpack.common.js

@@ -92,8 +92,13 @@ module.exports = (env = {}) => ({
 			"util": require.resolve("util/"),
 			"os": require.resolve("os-browserify/browser"),
 
+			// Required for zlib
+			"assert": require.resolve("assert"),
+			"stream": require.resolve("stream-browserify"),
+
 			"events": require.resolve("events/"),
 			"path": require.resolve("path-browserify"),
+			"zlib": require.resolve("browserify-zlib"),
 		},
 	},
 });

plugins/global_chat/.npmignore

@@ -0,0 +1 @@
+webpack.config.js

plugins/global_chat/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-global_chat",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.7",
   "description": "Clusterio plugin forwarding between Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/factorioClusterio#readme",
@@ -20,8 +20,8 @@
     "@clusterio/lib": "^2.0.0-alpha.0"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.6",
-    "@clusterio/web_ui": "^2.0.0-alpha.6",
+    "@clusterio/lib": "^2.0.0-alpha.7",
+    "@clusterio/web_ui": "^2.0.0-alpha.7",
     "webpack": "^5.36.2",
     "webpack-cli": "^4.7.0",
     "webpack-merge": "^5.2.0"

plugins/player_auth/.npmignore

@@ -0,0 +1 @@
+webpack.config.js

plugins/player_auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-player_auth",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.7",
   "description": "Clusterio plugin authenticating logged in players to the web interface",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/factorioClusterio#readme",
@@ -24,9 +24,9 @@
     "jsonwebtoken": "^8.2.1"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.6",
-    "@clusterio/web_ui": "^2.0.0-alpha.6",
-    "antd": "^4.6.5",
+    "@clusterio/lib": "^2.0.0-alpha.7",
+    "@clusterio/web_ui": "^2.0.0-alpha.7",
+    "antd": "^4.16.0",
     "phin": "^3.5.1",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",

plugins/research_sync/.npmignore

@@ -0,0 +1 @@
+webpack.config.js

plugins/research_sync/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-research_sync",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.7",
   "description": "Clusterio plugin for synchronizing research between Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/factorioClusterio#readme",
@@ -23,8 +23,8 @@
     "fs-extra": "^8.1.0"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.6",
-    "@clusterio/web_ui": "^2.0.0-alpha.6",
+    "@clusterio/lib": "^2.0.0-alpha.7",
+    "@clusterio/web_ui": "^2.0.0-alpha.7",
     "webpack": "^5.36.2",
     "webpack-cli": "^4.7.0",
     "webpack-merge": "^5.2.0"

plugins/statistics_exporter/.npmignore

@@ -0,0 +1 @@
+webpack.config.js

plugins/statistics_exporter/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-statistics_exporter",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.7",
   "description": "Clusterio plugin for gathering statistics from Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/factorioClusterio#readme",
@@ -20,8 +20,8 @@
     "@clusterio/lib": "^2.0.0-alpha.0"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.6",
-    "@clusterio/web_ui": "^2.0.0-alpha.6",
+    "@clusterio/lib": "^2.0.0-alpha.7",
+    "@clusterio/web_ui": "^2.0.0-alpha.7",
     "webpack": "^5.36.2",
     "webpack-cli": "^4.7.0",
     "webpack-merge": "^5.2.0"

plugins/subspace_storage/.npmignore

@@ -0,0 +1 @@
+webpack.config.js

plugins/subspace_storage/instance.js

@@ -10,11 +10,19 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 	}
 
 	async init() {
+		this.stopping = false;
+		this.pendingTasks = new Set();
 		this.instance.server.on("ipc-subspace_storage:output", (output) => {
 			this.provideItems(output).catch(err => this.unexpectedError(err));
 		});
 		this.instance.server.on("ipc-subspace_storage:orders", (orders) => {
-			this.requestItems(orders).catch(err => this.unexpectedError(err));
+			if (this.stopping) {
+				return;
+			}
+
+			let task = this.requestItems(orders).catch(err => this.unexpectedError(err));
+			this.pendingTasks.add(task);
+			task.finally(() => { this.pendingTasks.delete(task); });
 		});
 	}
 
@@ -31,6 +39,12 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 		await this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}", true)`, true);
 	}
 
+	async onStop() {
+		clearInterval(this.pingId);
+		this.stopping = true;
+		await Promise.all(this.pendingTasks);
+	}
+
 	onExit() {
 		clearInterval(this.pingId);
 	}
@@ -71,13 +85,19 @@ class InstancePlugin extends libPlugin.BaseInstancePlugin {
 
 	// combinator signals ---------------------------------------------------------
 	async updateStorageEventHandler(message, event) {
+		if (this.stopping) {
+			return;
+		}
 		let items = message.data.items;
 
 		// XXX this should be moved to instance/clusterio api
 		items.push(["signal-unixtime", Math.floor(Date.now()/1000)]);
 
 		let itemsJson = libLuaTools.escapeString(JSON.stringify(items));
-		await this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`, true);
+		let task = this.sendRcon(`/sc __subspace_storage__ UpdateInvData("${itemsJson}")`, true);
+		this.pendingTasks.add(task);
+		task.finally(() => { this.pendingTasks.delete(task); });
+		await task;
 	}
 }
 

plugins/subspace_storage/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-subspace_storage",
-  "version": "2.0.0-alpha.6",
+  "version": "2.0.0-alpha.7",
   "description": "Clusterio plugin for sharing storage between Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/clusterio#readme",
@@ -23,9 +23,9 @@
     "fs-extra": "^8.1.0"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.6",
-    "@clusterio/web_ui": "^2.0.0-alpha.6",
-    "antd": "^4.6.5",
+    "@clusterio/lib": "^2.0.0-alpha.7",
+    "@clusterio/web_ui": "^2.0.0-alpha.7",
+    "antd": "^4.16.0",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",
     "webpack": "^5.36.2",

test/integration/clusterio.js

@@ -7,9 +7,10 @@ const events = require("events");
 const libLink = require("@clusterio/lib/link");
 const libUsers = require("@clusterio/lib/users");
 
+const testStrings = require("../lib/factorio/test_strings");
 const {
 	TestControl, TestControlConnector, url, controlToken, slowTest,
-	get, exec, execCtl, sendRcon, getControl, instancesDir,
+	get, exec, execCtl, sendRcon, getControl, spawn, instancesDir,
 } = require("./index");
 
 
@@ -61,6 +62,59 @@ describe("Integration of Clusterio", function() {
 		});
 	});
 
+	describe("clusterioslave", function() {
+		describe("slaveUpdateEventHandler()", function() {
+			it("should trigger when a new slave is added", async function() {
+				slowTest(this);
+				getControl().slaveUpdates = [];
+				let config = "alt-slave-config.json";
+				let configPath = path.join("temp", "test", config);
+				await fs.remove(configPath);
+				await execCtl(`slave create-config --id 5 --name alt-slave --generate-token --output ${config}`);
+				await exec(
+					`node ../../packages/slave --config ${config} config set slave.tls_ca ../../test/file/tls/cert.pem`
+				);
+
+				let slaveProcess;
+				try {
+					slaveProcess = await spawn(
+						"alt-slave:", `node ../../packages/slave run --config ${config}`, /SOCKET \| registering slave/
+					);
+				} finally {
+					if (slaveProcess) {
+						slaveProcess.kill("SIGINT");
+						await events.once(slaveProcess, "exit");
+					}
+				}
+
+				let sawUpdate = false;
+				let sawConnected = false;
+				let sawDisconnected = false;
+
+				for (let update of getControl().slaveUpdates) {
+					if (update.name !== "alt-slave") {
+						continue;
+					}
+
+					sawUpdate = true;
+					if (update.connected) {
+						sawConnected = true;
+					} else {
+						sawDisconnected = true;
+					}
+				}
+
+				assert(sawUpdate, "No slave update was sent");
+				assert(sawConnected, "No slave update with status connected was sent");
+				assert(sawDisconnected, "No slave update with status disconnected was sent");
+
+				let result = await libLink.messages.listSlaves.send(getControl());
+				let slaves = new Map(result.list.map(instance => [instance.id, instance]));
+				assert(slaves.has(5), "Slave list was not updated");
+			});
+		});
+	});
+
 	describe("clusterioctl", function() {
 		describe("master config list", function() {
 			it("runs", async function() {
@@ -113,6 +167,9 @@ describe("Integration of Clusterio", function() {
 			it("runs", async function() {
 				await execCtl("slave generate-token --id 42");
 			});
+			it("runs without an id", async function() {
+				await execCtl("slave generate-token");
+			});
 		});
 		describe("instance list", function() {
 			it("runs", async function() {
@@ -122,10 +179,18 @@ describe("Integration of Clusterio", function() {
 
 		describe("instance create", function() {
 			it("creates the instance", async function() {
+				this.timeout(6000);
 				await execCtl("instance create test --id 44");
 				let instances = await getInstances();
 				assert(instances.has(44), "instance was not created");
 				assert.equal(instances.get(44).status, "unassigned", "incorrect instance status");
+
+				// Make sure the following tests does not fail due to not having internet
+				let value = JSON.stringify({ lan: true, public: false }).replace(
+					/"/g, process.platform === "win32" ? '""' : '\\"'
+				);
+				await execCtl(`instance config set-prop test factorio.settings visibility "'${value}'"`);
+				await execCtl("instance config set-prop test factorio.settings require_user_verification false");
 			});
 		});
 
@@ -145,6 +210,21 @@ describe("Integration of Clusterio", function() {
 			});
 		});
 
+		describe("saveListUpdateEventHandler()", function() {
+			it("should have triggered for the created save", function() {
+				slowTest(this);
+				assert.equal(getControl().saveListUpdates.slice(-1)[0].list[0].name, "world.zip");
+			});
+		});
+
+		describe("instance list-saves", function() {
+			it("lists the created save", async function() {
+				slowTest(this);
+				let result = await execCtl("instance list-saves test");
+				assert(/world\.zip/.test(result.stdout), "world.zip not present in list save output");
+			});
+		});
+
 		describe("instance export-data", function() {
 			it("exports the data", async function() {
 				slowTest(this);
@@ -157,11 +237,31 @@ describe("Integration of Clusterio", function() {
 		});
 
 		describe("instance start", function() {
-			it("starts the instance", async function() {
+			it("should not hang if factorio version does not exist", async function() {
 				slowTest(this);
-				await execCtl("instance start test");
+				try {
+					await execCtl("instance config set 44 factorio.version 0.1.2");
+					await assert.rejects(execCtl("instance start test"));
+
+				} finally {
+					await execCtl("instance config set 44 factorio.version latest");
+				}
+			});
+			it("starts the given save", async function() {
+				slowTest(this);
+				await execCtl("instance start test --save world.zip");
 				await checkInstanceStatus(44, "running");
 			});
+			it("copies the save if an autosave is the target", async function() {
+				slowTest(this);
+				await execCtl("instance stop 44");
+				let savesDir = path.join("temp", "test", "instances", "test", "saves");
+				await fs.copy(path.join(savesDir, "world.zip"), path.join(savesDir, "_autosave1.zip"));
+				await execCtl("instance start test");
+				let result = await libLink.messages.listSaves.send(getControl(), { instance_id: 44 });
+				let running = result.list.find(s => s.loaded);
+				assert(running.name !== "_autosave1.zip");
+			});
 		});
 
 		describe("instance send-rcon", function() {
@@ -184,7 +284,7 @@ describe("Integration of Clusterio", function() {
 						"afk-auto-kick", "Kick if AFK for more than 2 minutes.",
 					],
 					[
-						"allow_commands", true,
+						"allow_commands", "true",
 						"allow-commands", "Allow Lua commands: Yes.",
 					],
 					[
@@ -315,6 +415,83 @@ describe("Integration of Clusterio", function() {
 			});
 		});
 
+		describe("instance load-scenario", function() {
+			it("starts the instance with the given settings", async function() {
+				slowTest(this);
+				await execCtl("instance config set 44 factorio.enable_save_patching false");
+				await execCtl("instance config set 44 player_auth.load_plugin false");
+				await execCtl("instance config set 44 research_sync.load_plugin false");
+				await execCtl("instance config set 44 statistics_exporter.load_plugin false");
+				await execCtl("instance config set 44 subspace_storage.load_plugin false");
+				try {
+					let exchangeString = testStrings.modified.replace(/[\n\r]+/g, "");
+					let args = `base/freeplay --seed 1234 --map-exchange-string "${exchangeString}"`;
+					await execCtl(`instance load-scenario test ${args}`);
+					await checkInstanceStatus(44, "running");
+					await sendRcon(44, '/c game.print("disable achievements")');
+					await sendRcon(44, '/c game.print("disable achievements")');
+					assert.equal(await sendRcon(44, "/c rcon.print(game.default_map_gen_settings.seed)"), "1234\n");
+					assert.equal(await sendRcon(44, "/c rcon.print(game.map_settings.pollution.ageing)"), "1.5\n");
+					assert.equal(
+						await sendRcon(44, "/c rcon.print(game.difficulty_settings.research_queue_setting)"), "never\n"
+					);
+
+				} finally {
+					await execCtl("instance stop test");
+				}
+			});
+		});
+
+		describe("instance upload-save", function() {
+			it("should upload a zip file", async function() {
+				await fs.outputFile(path.join("temp", "test", "upload.zip"), "a test");
+				await execCtl("instance upload-save 44 upload.zip");
+				assert(
+					await fs.pathExists(path.join("temp", "test", "instances", "test", "saves", "upload.zip")),
+					"file not uploaded to saves directory"
+				);
+			});
+			it("should reject non-zip files", async function() {
+				await fs.outputFile(path.join("temp", "test", "invalid"), "a test");
+				await assert.rejects(execCtl("instance upload-save 44 invalid"));
+			});
+			it("should reject path traversal attacks", async function() {
+				await fs.outputFile(path.join("temp", "test", "upload.zip"), "a test");
+				await assert.rejects(execCtl("instance upload-save 44 upload.zip --name ../traversal.zip"));
+			});
+		});
+
+		describe("instance download-save", function() {
+			it("should download a save", async function() {
+				await fs.remove(path.join("temp", "test", "upload.zip"));
+				await execCtl("instance download-save 44 upload.zip");
+				assert(await fs.pathExists(path.join("temp", "test", "upload.zip")));
+			});
+			it("should error if save does not exist", async function() {
+				await assert.rejects(execCtl("instance download-save 44 invalid"));
+			});
+			it("should error on path traversal attacks", async function() {
+				await assert.rejects(execCtl("instance download-save 44 ../factorio-current.log"));
+			});
+		});
+
+		describe("instance delete-save", function() {
+			it("should delete a save", async function() {
+				await execCtl("instance delete-save 44 upload.zip");
+				assert(
+					!await fs.pathExists(path.join("temp", "test", "instances", "test", "saves", "upload.zip")),
+					"file not deleted"
+				);
+			});
+			it("should error if save does not exist", async function() {
+				await assert.rejects(execCtl("instance delete-save 44 upload.zip"));
+			});
+			it("should error on path traversal attacks", async function() {
+				await fs.outputFile(path.join("temp", "test", "upload.zip"), "a test");
+				await assert.rejects(execCtl("instance delete-save 44 ../../upload.zip"));
+			});
+		});
+
 		describe("instance delete", function() {
 			it("deletes the instance", async function() {
 				slowTest(this);
@@ -325,6 +502,23 @@ describe("Integration of Clusterio", function() {
 			});
 		});
 
+		describe("instanceUpdateEventHandler()", function() {
+			it("should have triggered for the previous instance status updates", function() {
+				let statusesToCheck = new Set([
+					"unassigned", "stopped", "creating_save", "exporting_data",
+					"starting", "running", "stopping", "deleted",
+				]);
+				let statusesNotSeen = new Set(statusesToCheck);
+
+				for (let update of getControl().instanceUpdates) {
+					assert(statusesToCheck.has(update.status), `Missing check for status ${update.status}`);
+					statusesNotSeen.delete(update.status);
+				}
+
+				assert(statusesNotSeen.size === 0, `Did not see the statuses ${[...statusesNotSeen]}`);
+			});
+		});
+
 		describe("permission list", function() {
 			it("runs", async function() {
 				await execCtl("permission list");

test/integration/index.js

@@ -25,6 +25,21 @@ class TestControl extends libLink.Link {
 	constructor(connector) {
 		super("control", "master", connector);
 		libLink.attachAllMessages(this);
+		this.slaveUpdates = [];
+		this.instanceUpdates = [];
+		this.saveListUpdates = [];
+
+		this.connector.on("connect", () => {
+			libLink.messages.setSlaveSubscriptions.send(
+				this, { all: true, slave_ids: [] }
+			).catch(err => logger.error(`Error setting slave subscriptions:\n${err.stack}`));
+			libLink.messages.setInstanceSubscriptions.send(
+				this, { all: true, instance_ids: [] }
+			).catch(err => logger.error(`Error setting instance subscriptions:\n${err.stack}`));
+			libLink.messages.setSaveListSubscriptions.send(
+				this, { all: true, instance_ids: [] }
+			).catch(err => logger.error(`Error setting save list subscriptions:\n${err.stack}`));
+		});
 	}
 
 	async prepareDisconnectRequestHandler(message, request) {
@@ -34,6 +49,18 @@ class TestControl extends libLink.Link {
 
 	async debugWsMessageEventHandler() { }
 
+	async slaveUpdateEventHandler(message) {
+		this.slaveUpdates.push(message.data);
+	}
+
+	async instanceUpdateEventHandler(message) {
+		this.instanceUpdates.push(message.data);
+	}
+
+	async saveListUpdateEventHandler(message) {
+		this.saveListUpdates.push(message.data);
+	}
+
 	async logMessageEventHandler() { }
 }
 
@@ -46,7 +73,7 @@ class TestControlConnector extends libLink.WebSocketClientConnector {
 // Mark that this test takes a lot of time, or depeneds on a test
 // that takes a lot of time.
 function slowTest(test) {
-	// eslint-disable-next-line no-process-env
+	// eslint-disable-next-line node/no-process-env
 	if (process.env.FAST_TEST) {
 		test.skip();
 	}
@@ -183,7 +210,9 @@ after(async function() {
 		masterProcess.kill("SIGINT");
 		await events.once(masterProcess, "exit");
 	}
-	await control.connector.close();
+	if (control) {
+		await control.connector.close();
+	}
 });
 
 // Ensure the test processes are stopped.
@@ -202,6 +231,7 @@ module.exports = {
 	execCtl,
 	sendRcon,
 	getControl,
+	spawn,
 
 	url,
 	controlToken,

test/integration/server.js

@@ -103,6 +103,12 @@ describe("Integration of lib/factorio/server", function() {
 				slowTest(this);
 				log(".start()");
 
+				// Make sure the test does not fail due to not having internet
+				let settings = await server.exampleSettings();
+				settings.visibility = { public: false, lan: true };
+				settings.require_user_verification = false;
+				await fs.outputFile(server.writePath("server-settings.json"), JSON.stringify(settings, null, 4));
+
 				// Make sure the test does not fail due to create() failing.
 				let mapPath = server.writePath("saves", "test.zip");
 				assert(await fs.exists(mapPath), "save is missing");

test/lib/factorio/exchange_string.js

@@ -0,0 +1,73 @@
+"use strict";
+const assert = require("assert").strict;
+const zlib = require("zlib");
+
+const libFactorio = require("@clusterio/lib/factorio");
+
+const testStrings = require("./test_strings");
+
+
+// Simple linear congruential random number generator
+class Random {
+	constructor(seed) {
+		this.prev = seed;
+	}
+
+	next() {
+		let curr = (1103515245 * this.prev + 12345) & 0x7fffffff;
+		this.prev = curr;
+		return curr >> 16;
+	}
+}
+
+describe("lib/factorio/exchange_string", function() {
+	describe("readMapExchangeString", function() {
+		it("should parse a valid string", function() {
+			let result = libFactorio.readMapExchangeString(testStrings.default);
+			assert.equal(result.map_gen_settings.seed, 1234567890);
+
+			result = libFactorio.readMapExchangeString(testStrings.modified);
+			assert.equal(result.checksum, 4092204126);
+		});
+
+		it("should handle malformed strings", function() {
+			assert.throws(
+				() => libFactorio.readMapExchangeString("<<blah>>"),
+				new Error("Not a map exchange string")
+			);
+
+			assert.throws(
+				() => libFactorio.readMapExchangeString(testStrings.default.slice(0, 100)),
+				new Error("Not a map exchange string")
+			);
+
+			assert.throws(
+				() => libFactorio.readMapExchangeString(`>>>${Buffer.from("abk430ia404ah3b4").toString("base64")}<<<`),
+				new Error("Malformed map exchange string: zlib inflate failed")
+			);
+
+			for (let i = 0; i < 100; i++) {
+				let gen = new Random(i);
+				let size = gen.next() % 200 + 100;
+				let data = Buffer.alloc(size);
+				for (let j = 0; j < size; j++) {
+					data[j] = gen.next() % 256;
+				}
+				// eslint-disable-next-line node/no-sync
+				data = zlib.deflateSync(data);
+				assert.throws(
+					() => libFactorio.readMapExchangeString(`>>>${data.toString("base64")}<<<`),
+					new Error("Malformed map exchange string: reached end before finishing parsing")
+				);
+			}
+
+			let pastEnd = Buffer.from(testStrings.default.replace(/><\n/g, ""), "base64");
+			// eslint-disable-next-line node/no-sync
+			pastEnd = zlib.deflateSync(Buffer.concat([zlib.inflateSync(pastEnd), Buffer.from("junk")]));
+			assert.throws(
+				() => libFactorio.readMapExchangeString(`>>>${pastEnd.toString("base64")}<<<`),
+				new Error("Malformed map exchange string: data after end")
+			);
+		});
+	});
+});

test/lib/factorio/test_strings.js

@@ -0,0 +1,38 @@
+"use strict";
+// Exchange strings for testing with
+
+let default_ = `
+>>>eNpjZGBkUGQAgwZ7EOZgSc5PzIHxQJgrOb+gILVIN78oFVmYM
+7moNCVVNz8TVXFqXmpupW5SYjGKYo7Movw8dBNYi0vy81BFSopSU
+4uRRbhLixLzMktz0fUyMF5imubZ0CLHAML/6xkU/v8HYSDrAdAvI
+MzA2ABRCRSDAdbknMy0NAYGBUcgdgJJMzIyVousc39YNcWeEaJGz
+wHK+AAVOZAEE/GEMfwccEqpwBgmSOYYg8FnJAbE0hKgFVBVHA4IB
+kSyBSTJyNj7duuC78cu2DH+Wfnxkm9Sgj2joavIuw9G6+yAkuwgf
+zLBiVkzQWAnzCsMMDMf2EOlbtoznj0DAm/sGVlBOkRAhIMFkDjgz
+czAKMAHZC3oARIKMgwwp9nBjBFxYEwDg28wnzyGMS7bo/sDGBA2I
+MPlQMQJEAG2EO4yRgjTod+B0UEeJiuJUALUb8SA7IYUhA9Pwqw9j
+GQ/mkMwIwLZH2giKg5YooELZGEKnHjBDHcNMDwvsMN4DvMdGJlBD
+JCqL0AxCA8kAzMKQgs4gIObmQEBgGmjrO/lHgDxa6G4<<<
+`;
+
+let modified = `
+>>>eNp1UzFs00AUvZ8mbRIEypCBSlCCyFAhOQqBKULxlQUxMCCxgnCc
+S2rh2On5LAgMZOjAgISEurQLXSmCBTGwRWKhEkgIJraiMgBiaGmFOiC
+FO5/PsUz6pfv+9/6/99+/kwEBOo2k6c83NhrZtOkaNkIDXa286fZ6hG
+ouJaJIwTmT+i2iuZYdQ3GeOKTb15qGJ4oxQsMglbWo60iGYcSQ8Zjrh
+GUhwighHpcRSBHIEZ8ajuV35dmBwIPK1DRxmMX6ADnqmre1Rb/D86UF
+hNYX1lZncZpZNkGpmQ41PE87J6RFTe4YjND4hPDq943NwfIcEmv0AJV
+GI7F4tMXViYVgECgCjoWWPm66DqOurXmEMcvp1A3/br1pGV5Oq1ZqVW
+Hzk0ralCz5xDH79a5vM6tnW4Rma5XgQPVk8kTXtTzmU5Jg1g6tm0hfr
+ZwPLGPaVrstL6p0SYwEAPeLLy9/vbeigxysgsPgIESGTYVcCYMnL/Bh
+qbRKoQsq2NFBdt+LBbIp4y3CqiweBzK5LJIAu4vbD18f7Dfg77PdT1e
+bt3S4/q2w5J39IbQfFW+TitzaqrA3ahSkOLf0MPVFhw/vhf3SYUqcOC
+PczkoKQfXaNILCMb5df8Rd6QRS0hqKpoihHdgfNcm2Cj7ryTnKGC4K8
+jnh3gmXQRElVwYyxI8x4FMqOzsu4edrKK6hNZ5wU7V9G+ufEFL+7yHi
+cySQMp7wDHnRsBW571ORGn6fH2fUDj/FwV0iUbXPMbmTP7akkt8ChiL
+/BPceEs3j1M3iz71/ObT9tA==<<<`;
+
+module.exports = {
+	default: default_,
+	modified,
+};

test/lib/file_ops.js

@@ -13,6 +13,13 @@ describe("lib/file_ops", function() {
 
 		await fs.outputFile(path.join(baseDir, "test", "file.txt"), "contents");
 		await fs.outputFile(path.join(baseDir, "test", "another file.txt"), "more contents");
+
+		await fs.outputFile(path.join(baseDir, "find", "file"), "contents");
+		await fs.outputFile(path.join(baseDir, "find", "file.txt"), "contents");
+		await fs.outputFile(path.join(baseDir, "find", "foo-1"), "contents");
+		await fs.outputFile(path.join(baseDir, "find", "foo-2"), "contents");
+		await fs.outputFile(path.join(baseDir, "find", "bar-1.txt"), "contents");
+		await fs.outputFile(path.join(baseDir, "find", "bar-2.txt"), "contents");
 	}
 
 	before(setupTestingEnv);
@@ -31,4 +38,32 @@ describe("lib/file_ops", function() {
 			assert.equal(newest, null);
 		});
 	});
+
+	describe("findUnusedName()", function() {
+		it("should return named unchanged if it does not exist", async function() {
+			let cases = [
+				[["file"], "file"],
+				[["file", ".txt"], "file.txt"],
+				[["file.txt", ".txt"], "file.txt"],
+			];
+			for (let [args, expected] of cases) {
+				let actual = await libFileOps.findUnusedName(path.join(baseDir, "test", "folder"), ...args);
+				assert.equal(actual, expected);
+			}
+		});
+		it("should return changed name if it does exist", async function() {
+			let cases = [
+				[["file"], "file-2"],
+				[["file", ".txt"], "file-2.txt"],
+				[["file.txt", ".txt"], "file-2.txt"],
+				[["foo-1"], "foo-3"],
+				[["bar-1", ".txt"], "bar-3.txt"],
+				[["bar-1.txt", ".txt"], "bar-3.txt"],
+			];
+			for (let [args, expected] of cases) {
+				let actual = await libFileOps.findUnusedName(path.join(baseDir, "find"), ...args);
+				assert.equal(actual, expected);
+			}
+		});
+	});
 });

test/master.js

@@ -1,9 +0,0 @@
-"use strict";
-const assert = require("assert").strict;
-
-const mock = require("./mock");
-const master = require("@clusterio/master/master");
-
-
-describe("Master testing", function() {
-});

test/master/HttpCloser.js

@@ -144,6 +144,7 @@ function serverSuite(proto) {
 	});
 
 	it("should refuse connection if request sent after starting closure", async function() {
+		this.timeout(4000); // On Windows this test takes 2 seconds for some reason.
 		server.on("request", (request, response) => {
 			setTimeout(() => { response.end("ok-4"); }, 3*tick);
 		});

test/master/routes.js

@@ -0,0 +1,329 @@
+"use strict";
+const assert = require("assert").strict;
+const events = require("events");
+const FormData = require("form-data");
+const jwt = require("jsonwebtoken");
+const http = require("http");
+const phin = require("phin");
+
+const { wait } = require("@clusterio/lib/helpers");
+const routes = require("@clusterio/master/src/routes");
+const mock = require("../mock");
+
+
+describe("master/src/routes", function() {
+	let master;
+	let server;
+	let port;
+	beforeEach(async function() {
+		master = new mock.MockMaster();
+		routes.addRouteHandlers(master.app);
+		server = http.createServer(master.app);
+		server.listen(0, "localhost");
+		await events.once(server, "listening");
+		port = server.address().port;
+	});
+	afterEach(function() {
+		server.close();
+	});
+	describe("/api/stream/:id", function() {
+		it("should respond with 404 if stream doesn't exist", async function() {
+			let response;
+			response = await phin(`http://localhost:${port}/api/stream/01020304`);
+			assert.equal(response.statusCode, 404);
+			response = await phin({
+				url: `http://localhost:${port}/api/stream/01020304`, method: "PUT",
+				data: "test stream",
+			});
+			assert.equal(response.statusCode, 404);
+		});
+		it("should respond with 500 if stream times out", async function() {
+			let stream = await routes.createProxyStream(master.app);
+			let response = await phin(`http://localhost:${port}/api/stream/${stream.id}`);
+			assert.equal(response.statusCode, 500);
+		});
+		it("should passthrough a stream", async function() {
+			let stream;
+			let responses;
+			stream = await routes.createProxyStream(master.app);
+			responses = await Promise.all([
+				(async function() {
+					await wait(100);
+					return await phin({
+						url: `http://localhost:${port}/api/stream/${stream.id}`, method: "PUT",
+						data: "test content",
+					});
+				}()),
+				phin(`http://localhost:${port}/api/stream/${stream.id}`),
+			]);
+			assert.equal(responses[0].statusCode, 200);
+			assert.equal(responses[1].statusCode, 200);
+			assert.equal(responses[1].body.toString(), "test content");
+			stream = await routes.createProxyStream(master.app);
+			responses = await Promise.all([
+				phin({
+					url: `http://localhost:${port}/api/stream/${stream.id}`, method: "PUT",
+					data: "test content",
+				}),
+				(async function() {
+					await wait(100);
+					return await phin(`http://localhost:${port}/api/stream/${stream.id}`);
+				}()),
+			]);
+			assert.equal(responses[0].statusCode, 200);
+			assert.equal(responses[1].statusCode, 200);
+			assert.equal(responses[1].body.toString(), "test content");
+		});
+	});
+	describe("/api/upload-save", function() {
+		it("should respond with 401 with invalid token", async function() {
+			let response;
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: { "Content-Type": "application/zip" },
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 401);
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"Content-Type": "application/zip",
+					"X-Access-Token": "invalid",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 401);
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"Content-Type": "application/zip",
+					"X-Access-Token": jwt.sign({ aud: "user", user: "invalid" }, "TestSecretDoNotUse"),
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 401);
+			master.userManager.users.get("test").tokenValidAfter = Math.floor((Date.now() + 60e3) / 1000);
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"Content-Type": "application/zip",
+					"X-Access-Token": jwt.sign({ aud: "user", user: "test" }, "TestSecretDoNotUse"),
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 401);
+		});
+
+		it("should respond with 403 when user has insufficient permission", async function() {
+			let response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"Content-Type": "application/zip",
+					"X-Access-Token": jwt.sign({ aud: "user", user: "player" }, "TestSecretDoNotUse"),
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 403);
+		});
+
+		let testToken = jwt.sign({ aud: "user", user: "test" }, "TestSecretDoNotUse");
+		it("should respond with 415 if content type is missing or invalid", async function() {
+			let response;
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 415);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "invalid",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 415);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "text/plain",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 415);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] });
+		});
+
+		it("should respond with 400 if invalid multipart request", async function() {
+			let response;
+			let form;
+			form = new FormData();
+			form.append("instance_id", "123");
+			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "text/plain" });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					...form.getHeaders(),
+				},
+				data: form.getBuffer(),
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid file Content-Type"] });
+			form = new FormData();
+			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" });
+			form.append("instance_id", "123");
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					...form.getHeaders(),
+				},
+				data: form.getBuffer(),
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(
+				JSON.parse(response.body), { request_errors: ["instance_id must come before files uploaded"] }
+			);
+			form = new FormData();
+			form.append("instance_id", "123");
+			form.append("file", "totally a zip file", { filename: "test.txt", contentType: "application/zip" });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					...form.getHeaders(),
+				},
+				data: form.getBuffer(),
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(
+				JSON.parse(response.body), { request_errors: ["filename must end with .zip"] }
+			);
+			form = new FormData();
+			form.append("instance_id", "invalid");
+			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					...form.getHeaders(),
+				},
+				data: form.getBuffer(),
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(
+				JSON.parse(response.body), { request_errors: ["invalid instance_id"] }
+			);
+		});
+
+		it("should respond with 400 if invalid file request", async function() {
+			let response;
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["Missing or invalid filename parameter"] });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.txt`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["filename must end with .zip"] });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?filename=test.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(
+				JSON.parse(response.body), { request_errors: ["Missing or invalid instance_id parameter"] }
+			);
+		});
+
+		it("should respond with 500 if the transfer failed", async function() {
+			master.forwardRequestToInstance = async (data, request) => {
+				throw new Error("Something went wrong");
+			};
+			let response;
+			let form;
+			form = new FormData();
+			form.append("instance_id", "123");
+			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					...form.getHeaders(),
+				},
+				data: form.getBuffer(),
+			});
+			assert.equal(response.statusCode, 500);
+			assert.deepEqual(JSON.parse(response.body), { errors: ["Something went wrong"], request_errors: [] });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 500);
+			assert.deepEqual(JSON.parse(response.body), { errors: ["Something went wrong"], request_errors: [] });
+			master.forwardRequestToInstance = async (data, request) => {
+				await new Promise(() => {});
+			};
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 500);
+			assert.deepEqual(
+				JSON.parse(response.body), { errors: ["Timed out establishing stream to slave"], request_errors: [] }
+			);
+		});
+
+		it("should complete a valid transfer", async function() {
+			master.forwardRequestToInstance = async (request, data) => {
+				let stream = await phin({
+					url: `http://localhost:${port}/api/stream/${data.stream_id}`,
+				});
+				assert.equal(stream.body.toString(), "totally a zip file");
+				return { save: data.filename };
+			};
+			let response;
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.deepEqual(JSON.parse(response.body), { saves: ["test.zip"] });
+			assert.equal(response.statusCode, 200);
+		});
+	});
+});

test/mock.js

@@ -150,9 +150,12 @@ class MockControl extends libLink.Link {
 class MockMaster {
 	constructor() {
 		this.app = express();
+		this.app.locals.master = this;
+		this.app.locals.streams = new Map();
 		this.mockConfigEntries = new Map([
 			["master.external_address", "test"],
 			["master.auth_secret", "TestSecretDoNotUse"],
+			["master.proxy_stream_timeout", 1],
 		]);
 		this.config = {
 			get: (name) => {
@@ -227,6 +230,7 @@ module.exports = {
 	MockInstance,
 	MockSlave,
 	MockControl,
+	MockMaster,
 
 	createMasterPlugin,
 	createInstancePlugin,

test/package.json

@@ -5,11 +5,12 @@
     "node": ">=12"
   },
   "dependencies": {
-    "@clusterio/ctl": "2.0.0-alpha.5",
-    "@clusterio/lib": "2.0.0-alpha.5",
-    "@clusterio/master": "2.0.0-alpha.5",
-    "@clusterio/slave": "2.0.0-alpha.5",
+    "@clusterio/ctl": "^2.0.0-alpha.6",
+    "@clusterio/lib": "^2.0.0-alpha.6",
+    "@clusterio/master": "^2.0.0-alpha.6",
+    "@clusterio/slave": "^2.0.0-alpha.6",
     "express": "^4.16.2",
+    "form-data": "^4.0.0",
     "fs-extra": "^8.1.0",
     "jsonwebtoken": "^8.2.1",
     "jszip": "^3.2.2",