v2.0.0-alpha.11

Fix logs

.github/workflows/docker-publish.yml

@@ -1,159 +0,0 @@
-name: Docker
-
-on:
-  push:
-    # Publish `master` as Docker `latest` image.
-    branches:
-      - '**'
-
-    # Publish `v1.2.3` tags as releases.
-    tags:
-      - v*
-
-  # Run tests for any PRs.
-  pull_request:
-
-  release:
-    types: [released, prereleased]
-
-env:
-  # TODO: Change variable to your image's name.
-  IMAGE_NAME: clusterio
-
-jobs:
-  # Build dependencies and clusterio image
-  build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Build docker image
-        run: DOCKER_BUILDKIT=1 docker build . -t $IMAGE_NAME --target clusterio_final
-
-      - name: Export docker image
-        run: docker save -o clusterio_docker_image $IMAGE_NAME
-
-      - name: Upload docker image
-        uses: actions/upload-artifact@master
-        with:
-          name: clusterio_docker_image
-          path: clusterio_docker_image
-  # Build dependencies and clusterio image with test tooling
-  build_testing:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Build docker image
-        run: DOCKER_BUILDKIT=1 docker build . -t $IMAGE_NAME --target clusterio_testing
-
-      - name: Export docker image
-        run: docker save -o clusterio_docker_image_testing $IMAGE_NAME
-
-      - name: Upload docker image
-        uses: actions/upload-artifact@master
-        with:
-          name: clusterio_docker_image_testing
-          path: clusterio_docker_image_testing
-  # Upload clusterio_lib and subspace_storage versions
-  upload_mods:
-    needs: build
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download docker image
-        uses: actions/download-artifact@master
-        with:
-          name: clusterio_docker_image
-          path: .
-
-      - name: Load docker image
-        run: docker load -i clusterio_docker_image
-      
-      - name: Create container
-        run: docker create --name clusterio_container $IMAGE_NAME
-
-      - name: Move mods out of container
-        run: docker cp clusterio_container:/clusterio/sharedMods ./mods
-
-      - name: Remove container
-        run: docker rm clusterio_container
-
-      - name: List extracted mods
-        run: ls mods
-
-      - name: Upload mods
-        uses: actions/upload-artifact@master
-        with:
-          name: mods
-          path: mods
-  # Run tests.
-  # See also https://docs.docker.com/docker-hub/builds/automated-testing/
-  test:
-    needs: build_testing
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Download docker image
-        uses: actions/download-artifact@master
-        with:
-          name: clusterio_docker_image_testing
-          path: .
-      - name: ls
-        run: ls && ls clusterio_docker_image_testing && ls /home/runner/work/clusterio && ls /home/runner/work/clusterio/clusterio
-      - name: Load docker image
-        run: docker load -i clusterio_docker_image_testing
-
-      - name: Run tests
-        run: |
-          if [ -f docker-compose.test.yml ]; then
-            docker-compose --file docker-compose.test.yml run unit_test
-          else
-            docker build . --file Dockerfile
-          fi
-  # Push image to GitHub Packages.
-  # See also https://docs.docker.com/docker-hub/builds/
-  push:
-    # Ensure test job passes before pushing image.
-    needs: [test, build]
-
-    runs-on: ubuntu-latest
-    if: github.event_name == 'push'
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Download artifact
-        uses: actions/download-artifact@master
-        with:
-          name: clusterio_docker_image
-          path: .
-
-      - name: Load docker image
-        run: docker load -i clusterio_docker_image
-
-      - name: Log into registry
-        run: echo "${{ secrets.CR_PAT }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
-
-      - name: Push image
-        run: |
-          IMAGE_ID=ghcr.io/${{ github.actor }}/$IMAGE_NAME
-
-          # Change all uppercase to lowercase
-          IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]')
-
-          # Strip git ref prefix from version
-          VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-
-          # Strip "v" prefix from tag name
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//')
-
-          # Use Docker `latest` tag convention
-          [ "$VERSION" == "master" ] && VERSION=latest
-
-          echo IMAGE_ID=$IMAGE_ID
-          echo VERSION=$VERSION
-
-          docker tag $IMAGE_NAME $IMAGE_ID:$VERSION
-          docker push $IMAGE_ID:$VERSION

.github/workflows/node.js.yml

@@ -0,0 +1,39 @@
+# This workflow will do a clean install of node dependencies, cache/restore them, build the source code and run tests across different versions of node
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+
+name: Node.js CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [12.x, 14.x, 16.x]
+        # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Use Node.js ${{ matrix.node-version }}
+      uses: actions/setup-node@v2
+      with:
+        node-version: ${{ matrix.node-version }}
+    - run: npm i --no-optional
+    - run: wget -q -O factorio.tar.gz https://www.factorio.com/get-download/latest/headless/linux64 && tar -xf factorio.tar.gz && rm factorio.tar.gz
+    - run: npx lerna bootstrap --hoist
+    - run: npx lerna run build
+    - run: npm test
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: 16.x
+    - run: npm i --no-optional
+    - run: npx lerna bootstrap --hoist
+    - run: npm run lint

.gitignore

@@ -1,7 +1,6 @@
 # Logs
 logs
-*.log
-npm-debug.log*
+*debug.log*
 
 # Runtime data
 pids

lerna.json

@@ -4,5 +4,5 @@
     "plugins/*",
     "test"
   ],
-  "version": "2.0.0-alpha.10"
+  "version": "2.0.0-alpha.11"
 }

packages/create/create.js

@@ -15,6 +15,7 @@ const yargs = require("yargs");
 const { levels, logger, setLogLevel } = require("./logging");
 let dev = false;
 const scriptExt = process.platform === "win32" ? ".cmd" : "";
+const finished = util.promisify(stream.finished);
 
 
 const availablePlugins = [
@@ -85,6 +86,12 @@ class LineSplitter extends stream.Transform {
 class InstallError extends Error { }
 
 
+async function safeOutputFile(file, data, options={}) {
+	let temporary = `${file}.tmp`;
+	await fs.outputFile(temporary, data, options);
+	await fs.rename(temporary, file);
+}
+
 async function execFile(cmd, args) {
 	logger.verbose(`executing ${cmd} ${args.join(" ")}`);
 	const asyncExec = util.promisify(child_process.execFile);
@@ -119,6 +126,13 @@ async function execSlave(args) {
 	return await execFile(path.join("node_modules", ".bin", `clusterioslave${scriptExt}`), args);
 }
 
+async function execCtl(args) {
+	if (dev) {
+		return await execFile("node", [path.join("packages", "ctl"), ...args]);
+	}
+	return await execFile(path.join("node_modules", ".bin", `clusterioctl${scriptExt}`), args);
+}
+
 function validateSlaveToken(token) {
 	let parts = token.split(".");
 	if (parts.length !== 3) {
@@ -158,7 +172,7 @@ async function validateInstallDir() {
 
 async function installClusterio(mode, plugins) {
 	try {
-		await fs.writeFile("package.json", JSON.stringify({
+		await safeOutputFile("package.json", JSON.stringify({
 			name: "clusterio-install",
 			private: true,
 		}, null, 2), { flag: "wx" });
@@ -206,7 +220,7 @@ async function installClusterio(mode, plugins) {
 			}
 		}
 		try {
-			await fs.writeFile("plugin-list.json", JSON.stringify([...pluginList], null, 4));
+			await safeOutputFile("plugin-list.json", JSON.stringify([...pluginList], null, 4));
 		} catch (err) {
 			throw new InstallError(`Error writing plugin-list.json: ${err.message}`);
 		}
@@ -216,17 +230,17 @@ async function installClusterio(mode, plugins) {
 async function writeScripts(mode) {
 	if (["standalone", "master"].includes(mode)) {
 		if (process.platform === "win32") {
-			await fs.writeFile(
+			await safeOutputFile(
 				"run-master.cmd",
 				"@echo off\n.\\node_modules\\.bin\\clusteriomaster.cmd run\n"
 			);
 		} else {
-			await fs.writeFile(
+			await safeOutputFile(
 				"run-master.sh",
 				"#!/bin/sh\nexec ./node_modules/.bin/clusteriomaster run\n",
 				{ mode: 0o755 },
 			);
-			await fs.outputFile(
+			await safeOutputFile(
 				"systemd/clusteriomaster.service",
 				`[Unit]
 Description=Clusterio Master
@@ -247,17 +261,17 @@ WantedBy=multi-user.target
 
 	if (["standalone", "slave"].includes(mode)) {
 		if (process.platform === "win32") {
-			await fs.writeFile(
+			await safeOutputFile(
 				"run-slave.cmd",
 				"@echo off\n.\\node_modules\\.bin\\clusterioslave.cmd run\n"
 			);
 		} else {
-			await fs.writeFile(
+			await safeOutputFile(
 				"run-slave.sh",
 				"#!/bin/sh\nexec ./node_modules/.bin/clusterioslave run\n",
 				{ mode: 0o755 },
 			);
-			await fs.outputFile(
+			await safeOutputFile(
 				"systemd/clusterioslave.service",
 				`[Unit]
 Description=Clusterio Slave
@@ -350,7 +364,7 @@ async function inquirerMissingArgs(args) {
 	}
 
 	if (["standalone", "slave"].includes(answers.mode)) {
-		let myIp;
+		let myIp = "localhost";
 		if (args.publicAddress) {
 			answers.publicAddress = args.publicAddress;
 		} else {
@@ -420,7 +434,11 @@ async function inquirerMissingArgs(args) {
 		}
 	}
 
-	if (dev) { answers.plugins = []; }
+	if (dev) {
+		answers.plugins = [];
+	} else if (args.plugins) {
+		answers.plugins = args.plugins;
+	}
 	answers = await inquirer.prompt([
 		{
 			type: "checkbox",
@@ -463,7 +481,7 @@ async function downloadLinuxServer() {
 		const writeStream = fs.createWriteStream(tmpArchivePath);
 		res.pipe(writeStream);
 
-		await events.once(res, "end");
+		await finished(writeStream);
 
 		await fs.rename(tmpArchivePath, archivePath);
 		try {
@@ -514,7 +532,9 @@ async function main() {
 		})
 		.option("plugins", {
 			array: true, describe: "Plugins to install", type: "string",
-		});
+		})
+		.strict()
+	;
 
 	if (process.platform === "linux") {
 		args = args
@@ -566,6 +586,7 @@ async function main() {
 		let slaveToken = result.stdout.split("\n").slice(-2)[0];
 
 		await execSlave(["config", "set", "slave.master_token", slaveToken]);
+		await execSlave(["config", "set", "slave.public_address", answers.publicAddress]);
 		await execSlave(["config", "set", "slave.factorio_directory", answers.factorioDir]);
 	}
 
@@ -576,6 +597,7 @@ async function main() {
 		await execSlave(["config", "set", "slave.name", answers.slaveName]);
 		await execSlave(["config", "set", "slave.master_url", answers.masterUrl]);
 		await execSlave(["config", "set", "slave.master_token", answers.masterToken]);
+		await execSlave(["config", "set", "slave.public_address", answers.publicAddress]);
 		await execSlave(["config", "set", "slave.factorio_directory", answers.factorioDir]);
 	}
 
@@ -584,6 +606,11 @@ async function main() {
 		await writeScripts(answers.mode);
 	}
 
+	if (answers.mode === "ctl") {
+		await execCtl(["control-config", "set", "control.master_url", answers.masterUrl]);
+		await execCtl(["control-config", "set", "control.master_token", answers.masterToken]);
+	}
+
 	/* eslint-disable no-console */
 	console.log(`Successfully installed ${answers.mode}`);
 	if (adminToken) {

packages/create/package.json

@@ -3,7 +3,7 @@
   "description": "Installer for clusterio",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.9",
+  "version": "2.0.0-alpha.11",
   "main": "create.js",
   "bin": {
     "clusteriocreate": "./create.js"

packages/ctl/ctl.js

@@ -14,6 +14,10 @@ const path = require("path");
 const winston = require("winston");
 const setBlocking = require("set-blocking");
 const phin = require("phin");
+const os = require("os");
+const child_process = require("child_process");
+const stream = require("stream");
+const util = require("util");
 
 const libLink = require("@clusterio/lib/link");
 const libErrors = require("@clusterio/lib/errors");
@@ -28,12 +32,68 @@ const libHelpers = require("@clusterio/lib/helpers");
 const libFactorio = require("@clusterio/lib/factorio");
 const libFileOps = require("@clusterio/lib/file_ops");
 
+const finished = util.promisify(stream.finished);
+
 
 function print(...content) {
 	// eslint-disable-next-line no-console
 	console.log(...content);
 }
 
+async function getEditor(argsEditor) {
+	// eslint-disable-next-line
+	return argsEditor || process.env.EDITOR || process.env.VISUAL || -1
+	// needed for the process.env statements to not be flagged by eslint
+	// priority for editors is CLI argument > env.EDITOR > env.VISUAL
+}
+
+async function configToKeyVal(data) {
+	let final = {};
+	let splitData = data.split(/\r?\n/);
+	// split on newlines
+	let filtered = splitData.filter((value) => value[0] !== "#").filter((a) => a);
+	// the last filter removes empty elements left by the first. Not done on one line due to readability.
+	for (let index in filtered) {
+		if (index in filtered) {
+			filtered[index] = filtered[index].split("=");
+			let finalIndex = filtered[index][0].trim();
+			// split on the = we added earlier, giving us both value and key
+			let part = "";
+			try {
+				part = filtered[index][1].trim();
+				// it's a string if we can read it
+			} catch (err) {
+				// if we can't read it, it's a empty field and therefor null
+				part = "";
+			}
+			final[finalIndex] = part;
+		}
+	}
+	return final;
+}
+
+async function serializedConfigToString(serializedConfig, configGroup, disallowedList) {
+	let allConfigElements = "";
+	for (let group of serializedConfig.groups) {
+		for (let [name, value] of Object.entries(group.fields)) {
+			if (`${group.name}.${name}` in disallowedList) {
+				continue;
+			}
+			let desc = "";
+			try {
+				desc += configGroup.groups.get(group.name)._definitions.get(name).description;
+			} catch (err) {
+				desc += "No description found";
+			}
+			// split onto two lines for readability and es-lint
+			if (String(value) === "null") {
+				value = "";
+			}
+			allConfigElements += `${group.name}.${name} = ${value}\n\n`;
+		}
+	}
+	return allConfigElements;
+}
 
 const masterCommands = new libCommand.CommandTree({ name: "master", description: "Master management" });
 const masterConfigCommands = new libCommand.CommandTree({
@@ -104,6 +164,66 @@ masterConfigCommands.add(new libCommand.Command({
 		await libLink.messages.setMasterConfigProp.send(control, request);
 	},
 }));
+masterConfigCommands.add(new libCommand.Command({
+	definition: ["edit [editor]", "Edit master configuration", (yargs) => {
+		yargs.positional("editor", {
+			describe: "Editor to use",
+			type: "string",
+			default: "",
+		});
+	}],
+	handler: async function(args, control) {
+		let response = await libLink.messages.getMasterConfig.send(control);
+		let tmpFile = await libFileOps.getTempFile("ctl-", "-tmp", os.tmpdir());
+		let editor = await getEditor(args.editor);
+		if (editor === -1) {
+			throw new libErrors.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
+							  Try "ctl master config edit <editor of choice>"`);
+		}
+		let allConfigElements = await serializedConfigToString(response.serialized_config, libConfig.MasterConfig, {});
+		await fs.writeFile(tmpFile, allConfigElements, (err) => {
+			if (err) {
+				throw err;
+			}
+		});
+		let editorSpawn = child_process.spawn(editor, [tmpFile], {
+			stdio: "inherit",
+  			detached: false,
+		});
+		editorSpawn.on("data", (data) => {
+  			process.stdout.pipe(data);
+		});
+		let doneEmitter = new events.EventEmitter();
+		editorSpawn.on("exit", async (exit) => {
+			const data = await fs.readFile(tmpFile, "utf8");
+			const final = await configToKeyVal(data);
+			for (let index in final) {
+				if (index in final) {
+					try {
+						await libLink.messages.setMasterConfigField.send(control, {
+							field: index,
+							value: final[index],
+						});
+					} catch (err) {
+						// eslint-disable-next-line
+						print(`Attempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+						print(err);
+						// If the string is empty, it's better to just print "" instead of nothing
+					}
+				}
+			}
+			doneEmitter.emit("dot_on_done");
+		});
+		await events.once(doneEmitter, "dot_on_done");
+		await fs.unlink(tmpFile, (err) => {
+			if (err) {
+				print("err: temporary file", tmpFile, "could not be deleted.");
+				print("This is not fatal, but they may build up over time if the issue persists.");
+			}
+		});
+	},
+}));
+
 masterCommands.add(masterConfigCommands);
 
 
@@ -300,6 +420,79 @@ instanceConfigCommands.add(new libCommand.Command({
 		await libLink.messages.setInstanceConfigProp.send(control, request);
 	},
 }));
+
+instanceConfigCommands.add(new libCommand.Command({
+	definition: ["edit <instance> [editor]", "Edit instance configuration", (yargs) => {
+		yargs.positional("instance", { describe: "Instance to set config on", type: "string" });
+		yargs.positional("editor", {
+			describe: "Editor to use",
+			type: "string",
+			default: "",
+		});
+	}],
+	handler: async function(args, control) {
+		let instanceId = await libCommand.resolveInstance(control, args.instance);
+		let response = await libLink.messages.getInstanceConfig.send(control, { instance_id: instanceId });
+		let tmpFile = await libFileOps.getTempFile("ctl-", "-tmp", os.tmpdir());
+		let editor = await getEditor(args.editor);
+		if (editor === -1) {
+			throw new libErrors.CommandError(`No editor avalible. Checked CLI input, EDITOR and VISUAL env vars
+							  Try "ctl master config edit <editor of choice>"`);
+		}
+		let disallowedList = {"instance.id": 0, "instance.assigned_slave": 0, "factorio.settings": 0};
+		let allConfigElements = await serializedConfigToString(
+			response.serialized_config,
+			libConfig.InstanceConfig,
+			disallowedList
+		);
+		await fs.writeFile(tmpFile, allConfigElements, (err) => {
+			if (err) {
+				throw err;
+			}
+		});
+		let editorSpawn = child_process.spawn(editor, [tmpFile], {
+			stdio: "inherit",
+  			detached: false,
+		});
+		editorSpawn.on("data", (data) => {
+  			process.stdout.pipe(data);
+		});
+		let doneEmitter = new events.EventEmitter();
+		editorSpawn.on("exit", async (exit) => {
+			const data = await fs.readFile(tmpFile, "utf8");
+			const final = await configToKeyVal(data);
+			for (let index in final) {
+				if (index in final) {
+					try {
+						await libLink.messages.setInstanceConfigField.send(control, {
+							instance_id: instanceId,
+							field: index,
+							value: final[index],
+						});
+					} catch (err) {
+						// eslint-disable-next-line
+						print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+						// If the string is empty, it's better to just print "" instead of nothing
+						print("This message shouldn't normally appear; if the below message does not indicate it");
+						print("was a user mistake, please report it to the clustorio devs.");
+						// added this because it could be a missed entry in disallowedList
+						// i've added all the vanilla clustorio configs, but there may be
+						// some modded ones that need to be added.
+						print(err);
+					}
+				}
+			}
+			doneEmitter.emit("dot_on_done");
+		});
+		await events.once(doneEmitter, "dot_on_done");
+		await fs.unlink(tmpFile, (err) => {
+			if (err) {
+				print("err: temporary file", tmpFile, "could not be deleted.");
+				print("This is not fatal, but they may build up over time if the issue persists.");
+			}
+		});
+	},
+}));
 instanceCommands.add(instanceConfigCommands);
 
 instanceCommands.add(new libCommand.Command({
@@ -451,12 +644,12 @@ instanceCommands.add(new libCommand.Command({
 			stream: true,
 		});
 
-		let stream;
+		let writeStream;
 		let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip");
 		while (true) {
 			try {
-				stream = fs.createWriteStream(tempFilename, { flags: "wx" });
-				await events.once(stream, "open");
+				writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
+				await events.once(writeStream, "open");
 				break;
 			} catch (err) {
 				if (err.code === "EEXIST") {
@@ -466,8 +659,8 @@ instanceCommands.add(new libCommand.Command({
 				}
 			}
 		}
-		response.pipe(stream);
-		await events.once(stream, "finish");
+		response.pipe(writeStream);
+		await finished(writeStream);
 
 		let filename = await libFileOps.findUnusedName(".", args.save, ".zip");
 		await fs.rename(tempFilename, filename);

packages/ctl/package.json

@@ -3,7 +3,7 @@
   "description": "Clusterio command line management tool",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "main": "ctl.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -18,7 +18,7 @@
     "node": ">=12"
   },
   "dependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.10",
+    "@clusterio/lib": "^2.0.0-alpha.11",
     "as-table": "^1.0.31",
     "fs-extra": "^8.1.0",
     "phin": "^3.5.1",

packages/lib/build_mod.js

@@ -5,8 +5,12 @@ const fs = require("fs-extra");
 const JSZip = require("jszip");
 const klaw = require("klaw");
 const path = require("path");
+const stream = require("stream");
+const util = require("util");
 const yargs = require("yargs");
 
+const finished = util.promisify(stream.finished);
+
 
 async function buildMod(args, info) {
 	if (args.build) {
@@ -42,7 +46,7 @@ async function buildMod(args, info) {
 			let modPath = path.join(args.outputDir, `${modName}.zip`);
 			console.log(`Writing ${modPath}`);
 			let writeStream = zip.generateNodeStream().pipe(fs.createWriteStream(modPath));
-			await events.once(writeStream, "finish");
+			await finished(writeStream);
 
 		} else {
 			let modDir = path.join(args.outputDir, modName);

packages/lib/config/definitions.js

@@ -15,7 +15,7 @@ MasterGroup.groupName = "master";
 MasterGroup.define({
 	name: "name",
 	title: "Name",
-	description: "Name of the cluster",
+	description: "Name of the cluster.",
 	type: "string",
 	initial_value: "Your Cluster",
 });
@@ -83,7 +83,7 @@ MasterGroup.define({
 MasterGroup.define({
 	name: "heartbeat_interval",
 	title: "Heartbeat Interval",
-	description: "Interval heartbeats are sent out on WebSocket connections",
+	description: "Interval heartbeats are sent out on WebSocket connections.",
 	type: "number",
 	initial_value: 15,
 });
@@ -97,21 +97,21 @@ MasterGroup.define({
 MasterGroup.define({
 	name: "metrics_timeout",
 	title: "Metrics Timeout",
-	description: "Timeout in seconds for metrics gathering from slaves",
+	description: "Timeout in seconds for metrics gathering from slaves.",
 	type: "number",
 	initial_value: 8,
 });
 MasterGroup.define({
 	name: "proxy_stream_timeout",
 	title: "Proxy Stream Timeout",
-	description: "Timeout in seconds for proxy streams to start flowing",
+	description: "Timeout in seconds for proxy streams to start flowing.",
 	type: "number",
 	initial_value: 15,
 });
 MasterGroup.define({
 	name: "default_role_id",
 	title: "Default role",
-	description: "ID of role assigned by default to new users",
+	description: "ID of role assigned by default to new users.",
 	type: "number",
 	optional: true,
 	initial_value: 1,

packages/lib/database.js

@@ -8,6 +8,7 @@
 "use strict";
 const fs = require("fs-extra");
 
+const libFileOps = require("./file_ops");
 const { basicType } = require("./helpers");
 
 
@@ -78,7 +79,7 @@ async function loadJsonAsMap(filePath) {
  */
 async function saveMapAsJson(filePath, map) {
 	let obj = mapToObject(map);
-	await fs.outputFile(filePath, JSON.stringify(obj, null, 4));
+	await libFileOps.safeOutputFile(filePath, JSON.stringify(obj, null, 4));
 }
 
 /**
@@ -138,7 +139,7 @@ async function loadJsonArrayAsMap(filePath) {
  * @throws {Error} if an error occured writing to the file.
  */
 async function saveMapAsJsonArray(filePath, map) {
-	await fs.outputFile(filePath, JSON.stringify([...map.values()], null, 4));
+	await libFileOps.safeOutputFile(filePath, JSON.stringify([...map.values()], null, 4));
 }
 
 

packages/lib/factorio/patch.js

@@ -284,9 +284,11 @@ async function patch(savePath, modules) {
 	root.file("clusterio.json", JSON.stringify(patchInfo, null, 4));
 
 	// Write back the save
+	let tempSavePath = `${savePath}.tmp`;
 	let stream = zip.generateNodeStream({ compression: "DEFLATE" });
-	let pipe = stream.pipe(fs.createWriteStream(savePath));
+	let pipe = stream.pipe(fs.createWriteStream(tempSavePath));
 	await events.once(pipe, "finish");
+	await fs.rename(tempSavePath, savePath);
 }
 
 module.exports = {

packages/lib/factorio/server.js

@@ -6,12 +6,14 @@ const path = require("path");
 const events = require("events");
 const util = require("util");
 const crypto = require("crypto");
-
-const libIni = require("../ini");
 const rconClient = require("rcon-client");
+
 const libErrors = require("../errors");
+const libFileOps = require("../file_ops");
+const libIni = require("../ini");
 const { logger } = require("../logging");
 const { escapeRegExp } = require("../helpers");
+const { LineSplitter } = require("../stream");
 
 
 /**
@@ -193,82 +195,6 @@ async function generatePassword(length) {
 	}
 }
 
-/**
- * Line splitter for chunked data
- *
- * Splits a stream of bytes into lines terminated by either line feed or
- * carriage return followed by linefeed.  Each line sepparated is passed
- * to the callback one at a time without the line terminator included.
- *
- * Data is streamed via the data method, and once the stream ends the
- * end method should be called.  This ensures an unterminated line at
- * the end of the stream is passed to the callback.
- *
- * @memberof module:lib/factorio
- * @private
- * @inner
- */
-class LineSplitter {
-
-	/**
-	 * Create a line splitter
-	 *
-	 * @param {function(Buffer)} callback - Function called for every line.
-	 */
-	constructor(callback) {
-		this._callback = callback;
-		this._partial = null;
-	}
-
-	/**
-	 * Stream in bytes
-	 *
-	 * Input bytes to split lines over.  Will invoke the callback for every
-	 * line found in buf and keep any potential partial line at the end of
-	 * buf for the next invocation.
-	 *
-	 * @param {Buffer} buf - Data to stream in.
-	 */
-	data(buf) {
-		if (this._partial) {
-			buf = Buffer.concat([this._partial, buf]);
-			this._partial = null;
-		}
-
-		while (buf.length) {
-			let end = buf.indexOf("\n");
-			if (end === -1) {
-				this._partial = buf;
-				break;
-			}
-
-			let next = end + 1;
-			// Eat carriage return as well if present
-			if (end >= 1 && buf[end-1] === "\r".charCodeAt(0)) {
-				end -= 1;
-			}
-
-			let line = buf.slice(0, end);
-			buf = buf.slice(next);
-			this._callback(line);
-		}
-	}
-
-	/**
-	 * Mark end of stream
-	 *
-	 * Signal the end of the bytestream.  If the previous invocation of the
-	 * data method ended with a partial line this will invoke the callback
-	 * with this line.
-	 */
-	end() {
-		if (this._partial) {
-			this._callback(this._partial);
-			this._partial = null;
-		}
-	}
-}
-
 /**
  * Interpret lines of output from Factorio
  *
@@ -562,9 +488,6 @@ class FactorioServer extends events.EventEmitter {
 		// Array of possible causes for an unexpected shutdown
 		this._unexpected = [];
 
-		this._stdout = new LineSplitter((line) => { this._handleOutput(line, "stdout"); });
-		this._stderr = new LineSplitter((line) => { this._handleOutput(line, "stderr"); });
-
 		// Track autosaving
 		this._runningAutosave = null;
 		this.on("_autosave", name => {
@@ -689,7 +612,7 @@ class FactorioServer extends events.EventEmitter {
 		}
 
 		let config = {
-			host: "localhost",
+			host: "127.0.0.1", // localhost may resolve to ::1 while Factorio listen to only IPv4 by default.
 			port: this.rconPort,
 			password: this.rconPassword,
 			timeout: 200000, // 200s, should allow for commands up to 1250kB in length
@@ -760,10 +683,12 @@ class FactorioServer extends events.EventEmitter {
 	}
 
 	_attachStdio() {
-		this._server.stdout.on("data", chunk => { this._stdout.data(chunk); });
-		this._server.stdout.on("close", () => { this._stdout.end(); });
-		this._server.stderr.on("data", chunk => { this._stderr.data(chunk); });
-		this._server.stderr.on("close", () => { this._stderr.end(); });
+		let stdout = new LineSplitter({ readableObjectMode: true });
+		stdout.on("data", line => { this._handleOutput(line, "stdout"); });
+		this._server.stdout.pipe(stdout);
+		let stderr = new LineSplitter({ readableObjectMode: true });
+		stderr.on("data", line => { this._handleOutput(line, "stderr"); });
+		this._server.stderr.pipe(stderr);
 	}
 
 	_watchExit() {
@@ -1173,17 +1098,17 @@ class FactorioServer extends events.EventEmitter {
 				"write-data": this.writePath(),
 			},
 		});
-		await fs.outputFile(this.writePath("config.ini"), content);
+		await libFileOps.safeOutputFile(this.writePath("config.ini"), content);
 	}
 
 	async _writeMapSettings(mapGenSettings, mapSettings) {
 		if (mapGenSettings) {
-			await fs.outputFile(
+			await libFileOps.safeOutputFile(
 				this.writePath("map-gen-settings.json"), JSON.stringify(mapGenSettings, null, 4)
 			);
 		}
 		if (mapSettings) {
-			await fs.outputFile(
+			await libFileOps.safeOutputFile(
 				this.writePath("map-settings.json"), JSON.stringify(mapSettings, null, 4)
 			);
 		}
@@ -1200,6 +1125,5 @@ module.exports = {
 	_findVersion: findVersion,
 	_randomDynamicPort: randomDynamicPort,
 	_generatePassword: generatePassword,
-	_LineSplitter: LineSplitter,
 	_parseOutput: parseOutput,
 };

packages/lib/file_ops.js

@@ -4,7 +4,7 @@
 "use strict";
 const fs = require("fs-extra");
 const path = require("path");
-
+const crypto = require("crypto"); // needed for getTempFile
 /**
  * Returns the newest file in a directory
  *
@@ -68,8 +68,49 @@ async function findUnusedName(directory, name, extension = "") {
 		}
 	}
 }
+/**
+ * Warning: this function should not be relied upon to be accurate for
+ * security sensitive applications.  The selection process is inherently
+ * racy and a file or folder may have been created in the folder by the time
+ * this function returns.
+ *
+ * @param {string} prefix - Prefix for file
+ * @param {string} suffix - Suffix for file
+ * @param {string} tmpdir - Directory for temp file
+ */
+async function getTempFile(prefix, suffix, tmpdir) {
+	prefix = (typeof prefix !== "undefined") ? prefix : "tmp.";
+	suffix = (typeof suffix !== "undefined") ? suffix : "";
+	tmpdir = (typeof tmpdir !== "undefined") ? tmpdir : "./";
+	let fileName = path.join(prefix + crypto.randomBytes(16).toString("hex") + suffix);
+	let freeFile = await findUnusedName(tmpdir, fileName);
+	let fullPath = path.join(tmpdir, freeFile);
+	return fullPath;
+}
+/**
+ * Safely write data to a file
+ *
+ * Same as fs-extra.outputFile except the data is written to a temporary
+ * file that's renamed over the target file.  The name of the temporary file
+ * is the same as the target file with the suffix `.tmp` added.
+ *
+ * If the operation fails it may leave behind the temporary file.  This
+ * should not be too much of an issue as the next time the same file is
+ * written the temporary will be overwritten and renamed to the target file.
+ *
+ * @param {string} file - Path to file to write.
+ * @param {string|Buffer} data - Content to write.
+ * @param {object|string} options - see fs.writeFile, `flag` must not be set.
+ */
+async function safeOutputFile(file, data, options={}) {
+	let temporary = `${file}.tmp`;
+	await fs.outputFile(temporary, data, options);
+	await fs.rename(temporary, file);
+}
 
 module.exports = {
 	getNewestFile,
 	findUnusedName,
+	getTempFile,
+	safeOutputFile,
 };

packages/lib/link/messages.js

@@ -337,6 +337,7 @@ let slaveProperties = {
 	"name": { type: "string" },
 	"id": { type: "integer" },
 	"connected": { type: "boolean" },
+	"public_address": { type: ["null", "string"] },
 };
 
 messages.listSlaves = new Request({
@@ -372,6 +373,7 @@ let instanceProperties = {
 	"name": { type: "string" },
 	"id": { type: "integer" },
 	"assigned_slave": { type: ["null", "integer"] },
+	"game_port": { type: ["null", "integer"] },
 	"status": { enum: [
 		"unknown", "unassigned", "stopped", "starting", "running", "stopping",
 		"creating_save", "exporting_data", "deleted",
@@ -1145,11 +1147,13 @@ messages.instanceInitialized = new Event({
 messages.instanceStatusChanged = new Event({
 	type: "instance_status_changed",
 	links: ["instance-slave", "slave-master"],
+	eventRequired: ["instance_id", "status"],
 	eventProperties: {
 		"instance_id": { type: "integer" },
 		"status": { enum: [
 			"stopped", "starting", "running", "stopping", "creating_save", "exporting_data",
 		]},
+		"game_port": { type: ["null", "integer"] },
 	},
 });
 

packages/lib/logging_utils.js

@@ -1,4 +1,15 @@
+/**
+ * Utilities for logging on the Node.js side
+ *
+ * @author Hornwitser
+ * @module lib/logging_utils
+ */
 "use strict";
+const events = require("events");
+const fs = require("fs-extra");
+const path = require("path");
+const stream = require("stream");
+const util = require("util");
 const winston = require("winston");
 const Transport = require("winston-transport");
 const { LEVEL, MESSAGE } = require("triple-beam");
@@ -6,6 +17,11 @@ const chalk = require("chalk");
 
 const libLink = require("./link");
 const libErrors = require("./errors");
+const libFileOps = require("./file_ops");
+const { levels, logger } = require("./logging");
+const libStream = require("./stream");
+
+const finished = util.promisify(stream.finished);
 
 
 /**
@@ -55,6 +71,7 @@ function formatServerOutput(parsed) {
 
 /**
  * Formats winston log messages for a character terminal.
+ * @static
  */
 class TerminalFormat {
 	constructor(options = {}) {
@@ -95,6 +112,7 @@ class TerminalFormat {
 
 /**
  * Sends logs over a lib/link connection.
+ * @static
  */
 class LinkTransport extends Transport {
 	constructor(options) {
@@ -121,10 +139,385 @@ class LinkTransport extends Transport {
 	}
 }
 
+const logFileGlob = /^[a-z]+-(\d{4}-\d{2}-\d{2})\.log$/;
+/**
+ * Filter object for logs
+ * @typedef {Object} module:lib/logging_utils~LogFilter
+ * @property {string} [max_level] -
+ *     Maximum log level to include. Higher levels are more verbose.
+ * @property {boolean} [all] -
+ *     Include log entries from master, all slaves and all instances.
+ * @property {boolean} [master] -
+ *     Include log entries from the master server.
+ * @property {Array<number>} [slave_ids] -
+ *     Include log entries for the given slaves and instances of those
+ *     slaves by id.
+ * @property {Array<number>} [instance_ids] -
+ *     Include log entries for the given instances by id.
+ */
+
+/**
+ * Keeps an index over a log directory to speed up queries to it
+ * @static
+ */
+class LogIndex {
+	constructor(logDirectory, serialized) {
+		this.logDirectory = logDirectory;
+		this.index = new Map();
+
+		if (serialized.version !== 0) {
+			return;
+		}
+
+		for (let [file, serializedEntry] of Object.entries(serialized.files)) {
+			this.index.set(file, {
+				levels: new Set(serializedEntry.levels),
+				master: serializedEntry.master,
+				slave_ids: new Set(serializedEntry.slave_ids),
+				instance_ids: new Set(serializedEntry.instance_ids),
+			});
+		}
+	}
+
+	serialize() {
+		let serialized = {
+			version: 0,
+			files: {},
+		};
+
+		for (let [file, entry] of this.index) {
+			serialized.files[file] = {
+				levels: [...entry.levels],
+				master: entry.master,
+				slave_ids: [...entry.slave_ids],
+				instance_ids: [...entry.instance_ids],
+			};
+		}
+
+		return serialized;
+	}
+
+	/**
+	 * Load or create an empty index for the given directory
+	 *
+	 * Loads index.json from the given log directy expecting it to contain
+	 * data about previously indexed files saved with {@link
+	 * module:lib/logging_utils.LogIndex#save}.
+	 *
+	 * @param {string} logDirectory - Path to directory to load from.
+	 * @returns {module:lib/logging_utils.LogIndex} loaded or created log index.
+	 */
+	static async load(logDirectory) {
+		try {
+			let content = await fs.readFile(path.join(logDirectory, "index.json"));
+			return new LogIndex(logDirectory, JSON.parse(content));
+		} catch (err) {
+			if (err.code !== "ENOENT") {
+				logger.warn(`Failed to load ${path.join(logDirectory, "index.json")}: ${err}`);
+			}
+			return new LogIndex(logDirectory, {});
+		}
+	}
+
+	/**
+	 * Save index
+	 *
+	 * Saves corrently held index into index.json in the log directory it
+	 * was loaded/created from.
+	 */
+	async save() {
+		await libFileOps.safeOutputFile(
+			path.join(this.logDirectory, "index.json"),
+			JSON.stringify(this.serialize(), null, 4)
+		);
+	}
+
+	async indexFile(file) {
+		let filePath = path.join(this.logDirectory, file);
+		let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
+		let fileStream = fs.createReadStream(filePath);
+		fileStream.pipe(lineStream);
+		let entry = {
+			levels: new Set(),
+			master: false,
+			slave_ids: new Set(),
+			instance_ids: new Set(),
+		};
+		for await (let line of lineStream) {
+			let info;
+			try {
+				info = JSON.parse(line);
+			} catch (err) {
+				entry.levels.add("info");
+				entry.master = true;
+				continue;
+			}
+			if (info.level) {
+				entry.levels.add(info.level);
+			}
+			if (info.slave_id === undefined) {
+				entry.master = true;
+			} else if (info.instance_id === undefined) {
+				entry.slave_ids.add(info.slave_id);
+			}
+			if (info.instance_id !== undefined) {
+				entry.instance_ids.add(info.instance_id);
+			}
+		}
+		return entry;
+	}
+
+	/**
+	 * Build index for files currently present in the log directory
+	 */
+	async buildIndex() {
+		let today = new Date(Date.now() - 300e3).toISOString().slice(0, 10);
+
+		for (let file of await fs.readdir(this.logDirectory)) {
+			let match = logFileGlob.exec(file);
+			if (!match || match[1] >= today || this.index.has(file)) {
+				continue;
+			}
+
+			let entry = await this.indexFile(file);
+			this.index.set(file, entry);
+		}
+	}
+
+	/**
+	 * Check if a file includes entries for the given filter
+	 *
+	 * @param {string} file -
+	 *     Name of file in the log directory to check for.
+	 * @param {module:lib/logging_utils~LogFilter} filter -
+	 *     Filter to check index if file contains entries for.
+	 * @returns {boolean}
+	 *     true if the file may contain entries included by the filter.
+	 */
+	filterIncludesFile(file, { max_level, all, master, instance_ids, slave_ids }) {
+		let entry = this.index.get(file);
+		if (!entry) {
+			return true;
+		}
+
+		const hasOwnProperty = Object.prototype.hasOwnProperty;
+		if (max_level && hasOwnProperty.call(levels, max_level)) {
+			let fileLevels = [...entry.levels].map(
+				fileLevel => (hasOwnProperty.call(levels, fileLevel) ? levels[fileLevel] : levels.verbose)
+			);
+			if (fileLevels.every(level => level > levels[max_level])) {
+				return false;
+			}
+		}
+
+		if (all) {
+			return true;
+		}
+		if (master && entry.master) {
+			return true;
+		}
+		if (slave_ids && slave_ids.some(id => entry.slave_ids.has(id))) {
+			return true;
+		}
+		if (instance_ids && instance_ids.some(id => entry.instance_ids.has(id))) {
+			return true;
+		}
+		return false;
+	}
+}
+
+/**
+ * Create log filter by level and source.
+ *
+ * @param {module:lib/logging_utils~LogFilter} filter -
+ *     Filter to filter log entries by.
+ * @returns {function(object): boolean}
+ *     filter returning true for log entries that match it.
+ * @static
+ */
+function logFilter({ all, master, slave_ids, instance_ids, max_level }) {
+	return info => {
+		// Note: reversed to filter out undefined levels
+		if (max_level && !(levels[info.level] <= levels[max_level])) {
+			return false;
+		}
+
+		if (all) {
+			return true;
+		}
+		if (master && info.slave_id === undefined) {
+			return true;
+		}
+		if (
+			slave_ids
+			&& info.slave_id !== undefined
+			&& info.instance_id === undefined
+			&& slave_ids.includes(info.slave_id)
+		) {
+			return true;
+		}
+		if (instance_ids && info.instance_id !== undefined && instance_ids.includes(info.instance_id)) {
+			return true;
+		}
+		return false;
+	};
+}
+
+/**
+ * Filter object for querying logs
+ * @typedef {Object} module:lib/logging_utils~QueryLogFilter
+ * @property {number} [limit=100] - Maximum number of entries to return.
+ * @property {string} [order="asc"] -
+ *     Return entries in ascending ("asc") date order or desceding ("desc")
+ *     date order.
+ * @property {string} [max_level] -
+ *     Maximum log level to include. Higher levels are more verbose.
+ * @property {boolean} [all] -
+ *     Include log entries from master, all slaves and all instances.
+ * @property {boolean} [master] -
+ *     Include log entries from the master server.
+ * @property {Array<number>} [slave_ids] -
+ *     Include log entries for the given slaves and instances of those
+ *     slaves by id.
+ * @property {Array<number>} [instance_ids] -
+ *     Include log entries for the given instances by id.
+ */
+
+/**
+ * Query log directory
+ *
+ * @param {string} logDirectory - path to directory with logs.
+ * @param {module:lib/logging_utils~QueryLogFilter} filter -
+ *     Filter to limit logs by.
+ * @param {module:lib/logging_utils.LogIndex=} index -
+ *     Index to speed up query with.
+ * @returns {Array<Object>} log entries matching the filter
+ */
+async function queryLog(logDirectory, filter, index) {
+	let files = (await fs.readdir(logDirectory)).filter(entry => logFileGlob.test(entry));
+
+	filter = {
+		limit: 100,
+		order: "asc",
+		...filter,
+	};
+
+	files.sort();
+	if (filter.order === "desc") {
+		files.reverse();
+	}
+
+	const includeEntry = logFilter(filter);
+	let log = [];
+	for (let file of files) {
+		if (index && !index.filterIncludesFile(file, filter)) {
+			continue;
+		}
+		let filePath = path.join(logDirectory, file);
+		let fileStream;
+		let lineStream;
+		if (filter.order === "asc") {
+			lineStream = new libStream.LineSplitter({ readableObjectMode: true });
+			fileStream = fs.createReadStream(filePath);
+		} else {
+			lineStream = new libStream.ReverseLineSplitter({ readableObjectMode: true });
+			fileStream = await libStream.createReverseReadStream(filePath);
+		}
+		lineStream.on("data", line => {
+			let info;
+			try {
+				info = JSON.parse(line);
+			} catch (err) {
+				info = { level: "info", message: line.toString("utf8") };
+			}
+			if (includeEntry(info)) {
+				log.push(info);
+				if (log.length >= filter.limit) {
+					lineStream.destroy();
+					fileStream.destroy();
+				}
+			}
+		});
+		fileStream.pipe(lineStream);
+		try {
+			await finished(lineStream);
+		} catch (err) {
+			if (log.length < filter.limit || err.code !== "ERR_STREAM_PREMATURE_CLOSE") {
+				throw err;
+			}
+		}
+
+		if (log.length >= filter.limit) {
+			break;
+		}
+	}
+	return log;
+}
+
+async function migrateLogs(log, directory, pattern) {
+	const dateRegExp = /^\d{4}-\d{2}-\d{2}/;
+	let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
+	let fileStream = fs.createReadStream(log);
+	fileStream.pipe(lineStream);
+
+	let currentDate = null;
+	let output = null;
+
+	async function switchOutput(newOutputPath) {
+		if (output) {
+			output.end();
+			await finished(output);
+		}
+
+		output = fs.createWriteStream(newOutputPath, { flags: "ax" });
+		try {
+			await events.once(output, "open");
+			// eslint-disable-next-line no-console
+			console.log(`writing ${newOutputPath}`);
+		} catch (err) {
+			if (err.code === "EEXIST") {
+				// eslint-disable-next-line no-console
+				console.warn(`Discarding logs for ${newDate} due to target logfile already existing`);
+				output = null;
+			} else {
+				throw err;
+			}
+		}
+	}
+
+	for await (let line of lineStream) {
+		let info = {};
+		try {
+			info = JSON.parse(line);
+		} catch (err) {
+			// Ignore
+		}
+
+		if (typeof info.timestamp === "string" && dateRegExp.test(info.timestamp)) {
+			let newDate = info.timestamp.slice(0, 10);
+			// Only update date forward in time.
+			if (newDate !== currentDate && (!currentDate || newDate > currentDate)) {
+				await switchOutput(path.join(directory, pattern.replace("%DATE%", newDate)));
+				currentDate = newDate;
+			}
+		}
+
+		if (output) {
+			if (!output.write(Buffer.concat([line, Buffer.from("\n")]))) {
+				await events.once(output, "drain");
+			}
+		}
+	}
+}
+
 
 module.exports = {
 	TerminalFormat,
 	LinkTransport,
+	LogIndex,
+	logFilter,
+	queryLog,
+	migrateLogs,
 
 	// for testing only
 	_formatServerOutput: formatServerOutput,

packages/lib/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@clusterio/lib",
   "description": "Shared library for Clusterio",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "repository": "https://github.com/clusterio/clusterio",
   "license": "MIT",
   "browser": "browser.js",

packages/lib/plugin.js

@@ -4,6 +4,7 @@
  */
 "use strict";
 
+const helpers = require("./helpers");
 
 /**
  * Conceptual base for master and instance plugins.
@@ -733,8 +734,16 @@ async function invokeHook(plugins, hook, ...args) {
 	let results = [];
 	for (let [name, plugin] of plugins) {
 		try {
-			let result = await plugin[hook](...args);
-			if (result !== undefined) {
+			// Use an object to detect if the hook failed on timeout or tried to return a value looking like an error
+			const timeout = {};
+			let result = await helpers.timeout(
+				plugin[hook](...args),
+				15000,
+				timeout
+			);
+			if (result === timeout) {
+				throw new Error(`Invoking hook ${hook} timed out for plugin ${name}`);
+			} else if (result !== undefined) {
 				results.push(result);
 			}
 		} catch (err) {

packages/lib/schema.js

@@ -178,6 +178,7 @@ const clientHandshake = ajv.compile({
 						"version": { type: "string" },
 						"name": { type: "string" },
 						"id": { type: "integer" },
+						"public_address": { type: "string" },
 						"plugins": {
 							type: "object",
 							additionalProperties: { type: "string" },

packages/lib/shared_commands.js

@@ -7,6 +7,7 @@ const fs = require("fs-extra");
 const path = require("path");
 
 const libConfig = require("./config");
+const libFileOps = require("./file_ops");
 const { logger } = require("./logging");
 const libHelpers = require("./helpers");
 
@@ -70,7 +71,7 @@ async function handlePluginCommand(args, pluginList, pluginListPath) {
 		}
 
 		pluginList.set(pluginInfo.name, pluginPath);
-		await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
+		await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
 		print(`Added ${pluginInfo.name}`);
 
 	} else if (command === "remove") {
@@ -80,7 +81,7 @@ async function handlePluginCommand(args, pluginList, pluginListPath) {
 			return;
 		}
 
-		await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
+		await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
 		print(`Removed ${args.name}`);
 
 	} else if (command === "list") {
@@ -155,7 +156,7 @@ async function handleConfigCommand(args, instance, configPath) {
 
 		try {
 			instance.set(args.field, args.value);
-			await fs.outputFile(configPath, JSON.stringify(instance.serialize(), null, 4));
+			await libFileOps.safeOutputFile(configPath, JSON.stringify(instance.serialize(), null, 4));
 		} catch (err) {
 			if (err instanceof libConfig.InvalidField || err instanceof libConfig.InvalidValue) {
 				logger.error(err.message);

packages/lib/stream.js

@@ -0,0 +1,168 @@
+"use strict";
+const assert = require("assert");
+const fs = require("fs-extra");
+const stream = require("stream");
+
+
+/**
+ * Transform splitting a stream into lines
+ *
+ * Splits up a stream of chunks by newlines and passes on the lines one by
+ * one removing the line feed and optional carriage return from the lines.
+ *
+ * @example
+ * let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
+ * lineStream.on("data", line => { console.log(line.toString("utf8")); });
+ * let fileStream = fs.createReadStream(path);
+ * fileStream.pipe(lineStream);
+ *
+ * @memberof module:lib/stream
+ */
+class LineSplitter extends stream.Transform {
+	constructor(options) {
+		super(options);
+		this._partial = null;
+	}
+
+	_transform(chunk, encoding, callback) {
+		if (this._partial) {
+			chunk = Buffer.concat([this._partial, chunk]);
+			this._partial = null;
+		}
+
+		while (chunk.length) {
+			let end = chunk.indexOf("\n");
+			if (end === -1) {
+				this._partial = chunk;
+				break;
+			}
+
+			let next = end + 1;
+			// Eat carriage return as well if present
+			if (end >= 1 && chunk[end-1] === "\r".charCodeAt(0)) {
+				end -= 1;
+			}
+
+			let line = chunk.slice(0, end);
+			chunk = chunk.slice(next);
+			this.push(line);
+		}
+		callback();
+	}
+
+	_flush(callback) {
+		if (this._partial) {
+			this.push(this._partial);
+			this._partial = null;
+		}
+		callback();
+	}
+}
+
+/**
+ * Transform splitting a reverse stream into lines
+ *
+ * Splits up a reverse stream of chunks created by {@link
+ * module:lib/stream.createReverseReadStream} by newlines and passes on the
+ * lines one by one in reverse order removing the line feed and optional
+ * carriage return from the lines.
+ *
+ * @example
+ * let revLineStream = new libStream.ReverseLineSplitter({ readableObjectMode: true });
+ * revLineStream.on("data", line => { console.log(line.toString("utf8")); });
+ * let revFileStream = await libStream.createReverseReadStream(path);
+ * revFileStream.pipe(revLineStream);
+ *
+ * @memberof module:lib/stream
+ */
+class ReverseLineSplitter extends stream.Transform {
+	constructor(options) {
+		super(options);
+		this._partial = null;
+	}
+
+	_transform(chunk, encoding, callback) {
+		if (this._partial) {
+			chunk = Buffer.concat([chunk, this._partial]);
+			this._partial = null;
+		}
+
+		while (chunk.length) {
+			let next = chunk.lastIndexOf("\n", -2);
+			if (next === -1) {
+				this._partial = chunk;
+				break;
+			}
+
+			let end = chunk.length - 1;
+			if (chunk[end] === "\n".charCodeAt(0)) {
+				end -= 1;
+				// Eat carriage return as well if present
+				if (end - next > 0 && chunk[end] === "\r".charCodeAt(0)) {
+					end -= 1;
+				}
+			}
+
+			let line = chunk.slice(next + 1, end + 1);
+			chunk = chunk.slice(0, next + 1);
+			this.push(line);
+		}
+		callback();
+	}
+
+	_flush(callback) {
+		if (this._partial) {
+			let end = this._partial.length - 1;
+			if (this._partial[end] === "\n".charCodeAt(0)) {
+				end -= 1;
+				// Eat carriage return as well if present
+				if (end > 0 && this._partial[end] === "\r".charCodeAt(0)) {
+					end -= 1;
+				}
+			}
+			this.push(this._partial.slice(0, end + 1));
+			this._partial = null;
+		}
+		callback();
+	}
+}
+
+
+/**
+ * Create a file read stream running in reverse
+ *
+ * Like fs.createReadStream but provides data chunks starting from the end
+ * of the file and progressing towards the start of the file.  The chunks
+ * are not reversed, thus the file content can be recreated by concatenating
+ * the chunks together in the reverse order they are read.
+ *
+ * @param {string} path -
+ *     Path to file to open. May be anything accepted by fs.open.
+ * @param {object} options -
+ *     Options to pass to fs.createReadStream.  Values passed for `fd` and
+ *     `fs` will be ignored.
+ * @memberof module:lib/stream
+ */
+async function createReverseReadStream(path, options) {
+	const fileFd = await fs.open(path, "r");
+	const fileSize = (await fs.fstat(fileFd)).size;
+	let filePosition = fileSize;
+	const reverseFs = {
+		read(fd, buffer, offset, length, position, callback) {
+			assert(fd === fileFd);
+			assert(position === undefined);
+			length = Math.min(length, filePosition);
+			filePosition -= length;
+			return fs.read(fd, buffer, offset, length, filePosition, callback);
+		},
+		open() { assert(false); },
+		close(fd, callback) { return fs.close(fd, callback); },
+	};
+	return fs.createReadStream("", { ...options, fd: fileFd, fs: reverseFs });
+}
+
+module.exports = {
+	LineSplitter,
+	ReverseLineSplitter,
+	createReverseReadStream,
+};

packages/master/master.js

@@ -22,10 +22,12 @@ const setBlocking = require("set-blocking");
 const yargs = require("yargs");
 const util = require("util");
 const winston = require("winston");
+require("winston-daily-rotate-file");
 const jwt = require("jsonwebtoken");
 
 // homebrew modules
 const libErrors = require("@clusterio/lib/errors");
+const libFileOps = require("@clusterio/lib/file_ops");
 const libPluginLoader = require("@clusterio/lib/plugin_loader");
 const libPrometheus = require("@clusterio/lib/prometheus");
 const libConfig = require("@clusterio/lib/config");
@@ -162,7 +164,7 @@ async function handleBootstrapCommand(args, masterConfig) {
 			console.log(content);
 		} else {
 			logger.info(`Writing ${args.output}`);
-			await fs.outputFile(args.output, content);
+			await libFileOps.safeOutputFile(args.output, content);
 		}
 	}
 }
@@ -188,6 +190,12 @@ async function initialize() {
 			choices: ["none"].concat(Object.keys(levels)),
 			type: "string",
 		})
+		.option("log-directory", {
+			nargs: 1,
+			describe: "Directory to place logs in",
+			default: "logs",
+			type: "string",
+		})
 		.option("config", {
 			nargs: 1,
 			describe: "master config file to use",
@@ -226,20 +234,45 @@ async function initialize() {
 		.argv
 	;
 
+	{
+		// Migration from alpha-10 single file logs, note that we can't use
+		// the logger here as it's not initalized yet.
+		/* eslint-disable no-console */
+		let clusterLogDirectory = path.join(parameters.args.logDirectory, "cluster");
+		if (!await fs.pathExists(clusterLogDirectory) && await fs.pathExists("cluster.log")) {
+			console.log("Migrating cluster log...");
+			await fs.ensureDir(clusterLogDirectory);
+			await libLoggingUtils.migrateLogs("cluster.log", clusterLogDirectory, "cluster-%DATE%.log");
+			console.log("Migration complete, you should delete cluster.log now");
+		}
+
+		let masterLogDirectory = path.join(parameters.args.logDirectory, "master");
+		if (!await fs.pathExists(masterLogDirectory) && await fs.pathExists("master.log")) {
+			console.log("Migrating master log...");
+			await fs.ensureDir(masterLogDirectory);
+			await libLoggingUtils.migrateLogs("master.log", masterLogDirectory, "master-%DATE%.log");
+			console.log("Migration complete, you should delete master.log now");
+		}
+		/* eslint-enable no-console */
+	}
+
 	// Combined log stream of the whole cluster.
 	parameters.clusterLogger = winston.createLogger({
 		format: winston.format.json(),
 		level: "verbose",
 		levels,
 	});
-	parameters.clusterLogger.add(new winston.transports.File({
-		filename: "cluster.log",
+	parameters.clusterLogger.add(new winston.transports.DailyRotateFile({
+		filename: "cluster-%DATE%.log",
+		utc: true,
+		dirname: path.join(parameters.args.logDirectory, "cluster"),
 	}));
 
 	// Log stream for the master server.
-	logger.add(new winston.transports.File({
+	logger.add(new winston.transports.DailyRotateFile({
 		format: winston.format.json(),
-		filename: "master.log",
+		filename: "master-%DATE%.log",
+		dirname: path.join(parameters.args.logDirectory, "master"),
 	}));
 	logger.add(new winston.transports.Stream({
 		stream: parameters.clusterLogger,
@@ -294,7 +327,9 @@ async function initialize() {
 		let asyncRandomBytes = util.promisify(crypto.randomBytes);
 		let bytes = await asyncRandomBytes(256);
 		parameters.masterConfig.set("master.auth_secret", bytes.toString("base64"));
-		await fs.outputFile(parameters.masterConfigPath, JSON.stringify(parameters.masterConfig.serialize(), null, 4));
+		await libFileOps.safeOutputFile(
+			parameters.masterConfigPath, JSON.stringify(parameters.masterConfig.serialize(), null, 4)
+		);
 	}
 
 	if (command === "config") {

packages/master/package.json

@@ -3,7 +3,7 @@
   "description": "Implementation of Clusterio master server",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "main": "master.js",
   "scripts": {
     "build": "webpack-cli --env production",
@@ -19,7 +19,7 @@
     "node": ">=12"
   },
   "dependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.10",
+    "@clusterio/lib": "^2.0.0-alpha.11",
     "busboy": "^0.3.1",
     "compression": "^1.7.2",
     "express": "^4.16.2",
@@ -28,11 +28,12 @@
     "jszip": "^3.2.2",
     "set-blocking": "^2.0.0",
     "winston": "^3.3.3",
+    "winston-daily-rotate-file": "^4.5.5",
     "ws": "^7.2.3",
     "yargs": "^14.2.0"
   },
   "devDependencies": {
-    "@clusterio/web_ui": "^2.0.0-alpha.10",
+    "@clusterio/web_ui": "^2.0.0-alpha.11",
     "fast-deep-equal": "^3.1.3",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",

packages/master/src/ControlConnection.js

@@ -1,12 +1,10 @@
 "use strict";
 const jwt = require("jsonwebtoken");
-const util = require("util");
-const winston = require("winston");
 
 const libConfig = require("@clusterio/lib/config");
 const libErrors = require("@clusterio/lib/errors");
 const libLink = require("@clusterio/lib/link");
-const { levels, logger } = require("@clusterio/lib/logging");
+const { logger } = require("@clusterio/lib/logging");
 const libLoggingUtils = require("@clusterio/lib/logging_utils");
 const libPlugin = require("@clusterio/lib/plugin");
 const libPrometheus = require("@clusterio/lib/prometheus");
@@ -106,6 +104,7 @@ class ControlConnection extends BaseConnection {
 				version: slave.version,
 				id: slave.id,
 				name: slave.name,
+				public_address: slave.public_address || null,
 				connected: this._master.wsServer.slaveConnections.has(slave.id),
 			});
 		}
@@ -169,6 +168,7 @@ class ControlConnection extends BaseConnection {
 			id,
 			name: instance.config.get("instance.name"),
 			assigned_slave: instance.config.get("instance.assigned_slave"),
+			game_port: instance.game_port || null,
 			status: instance.status,
 		};
 	}
@@ -180,6 +180,7 @@ class ControlConnection extends BaseConnection {
 				id: instance.config.get("instance.id"),
 				name: instance.config.get("instance.name"),
 				assigned_slave: instance.config.get("instance.assigned_slave"),
+				game_port: instance.game_port || null,
 				status: instance.status,
 			});
 		}
@@ -199,6 +200,7 @@ class ControlConnection extends BaseConnection {
 				id: instance.config.get("instance.id"),
 				name: instance.config.get("instance.name"),
 				assigned_slave: instance.config.get("instance.assigned_slave"),
+				game_port: instance.game_port || null,
 				status: instance.status,
 			});
 		}
@@ -398,29 +400,6 @@ class ControlConnection extends BaseConnection {
 		this.updateLogSubscriptions();
 	}
 
-	static logFilter({ all, master, slave_ids, instance_ids, max_level }) {
-		return info => {
-			// Note: reversed to filter out undefined levels
-			if (max_level && !(levels[info.level] <= levels[max_level])) {
-				return false;
-			}
-
-			if (all) {
-				return true;
-			}
-			if (master && info.slave_id === undefined) {
-				return true;
-			}
-			if (info.slave_id && slave_ids.includes(info.slave_id)) {
-				return true;
-			}
-			if (info.instance_id && instance_ids.includes(info.instance_id)) {
-				return true;
-			}
-			return false;
-		};
-	}
-
 	updateLogSubscriptions() {
 		let { all, master, slave_ids, instance_ids } = this.logSubscriptions;
 		if (all || master || slave_ids.length || instance_ids.length) {
@@ -428,7 +407,7 @@ class ControlConnection extends BaseConnection {
 				this.logTransport = new libLoggingUtils.LinkTransport({ link: this });
 				this._master.clusterLogger.add(this.logTransport);
 			}
-			this.logTransport.filter = this.constructor.logFilter(this.logSubscriptions);
+			this.logTransport.filter = libLoggingUtils.logFilter(this.logSubscriptions);
 
 		} else if (this.logTransport) {
 			this._master.clusterLogger.remove(this.logTransport);
@@ -437,34 +416,16 @@ class ControlConnection extends BaseConnection {
 	}
 
 	async queryLogRequestHandler(message) {
-		let transport = new winston.transports.File({ filename: "cluster.log" });
-		let query = util.promisify((options, callback) => transport.query(options, callback));
-
-		// Available options and defaults inferred from reading the source
-		// rows: number = limit || 10
-		// limit: number // alias of rows.
-		// start: number = 0
-		// until: Date = now
-		// from: Date = until - 24h
-		// order: "asc"|"desc" = "desc"
-		// fields: Array<string>
-		// level: string
-		// This interface is junk:
-		// - The level option filters by exact match
-		// - No way to add your own filter
-		// - No way to stream results
-		// - The log file is read starting from the beginning
 		let setDuration = lastQueryLogTime.startTimer();
-		let log = await query({
-			order: message.data.order,
-			limit: Infinity,
-			from: new Date(0),
-		});
+		let { all, master, slave_ids, instance_ids } = message.data;
 
-		log = log.filter(this.constructor.logFilter(message.data));
-		if (message.data.limit) {
-			log = log.slice(0, message.data.limit);
+		let log;
+		if (!all && master && !slave_ids.length && !instance_ids.length) {
+			log = await this._master.queryMasterLog(message.data);
+		} else {
+			log = await this._master.queryClusterLog(message.data);
 		}
+
 		setDuration();
 		return { log };
 	}

packages/master/src/Master.js

@@ -10,7 +10,9 @@ const path = require("path");
 
 const libConfig = require("@clusterio/lib/config");
 const libErrors = require("@clusterio/lib/errors");
+const libFileOps = require("@clusterio/lib/file_ops");
 const libLink = require("@clusterio/lib/link");
+const libLoggingUtils = require("@clusterio/lib/logging_utils");
 const libPlugin = require("@clusterio/lib/plugin");
 const libPluginLoader = require("@clusterio/lib/plugin_loader");
 const libPrometheus = require("@clusterio/lib/prometheus");
@@ -116,6 +118,14 @@ class Master {
 	}
 
 	async _startInternal(args) {
+		this.logDirectory = args.logDirectory;
+		this.clusterLogIndex = await libLoggingUtils.LogIndex.load(path.join(this.logDirectory, "cluster"));
+		this.clusterLogBuildInterval = setInterval(() => {
+			this.clusterLogIndex.buildIndex().catch(
+				err => logger.error(`Error building cluster log index:\n${err.stack}`)
+			);
+		}, 600e3);
+
 		// Start webpack development server if enabled
 		if (args.dev || args.devPlugin) {
 			logger.warn("Webpack development mode enabled");
@@ -239,9 +249,13 @@ class Master {
 		// This function should never throw.
 		this._state = "stopping";
 		logger.info("Stopping master");
+		clearInterval(this.clusterLogBuildInterval);
+		if (this.clusterLogIndex) {
+			await this.clusterLogIndex.save();
+		}
 
 		logger.info("Saving config");
-		await fs.outputFile(this.configPath, JSON.stringify(this.config.serialize(), null, 4));
+		await libFileOps.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, 4));
 
 		if (this.devMiddleware) {
 			await new Promise((resolve, reject) => { this.devMiddleware.close(resolve); });
@@ -297,7 +311,7 @@ class Master {
 	}
 
 	static async saveSlaves(filePath, slaves) {
-		await fs.outputFile(filePath, JSON.stringify([...slaves.entries()], null, 4));
+		await libFileOps.safeOutputFile(filePath, JSON.stringify([...slaves.entries()], null, 4));
 	}
 
 	static async loadInstances(filePath) {
@@ -329,7 +343,34 @@ class Master {
 			serialized.push(instance.config.serialize());
 		}
 
-		await fs.outputFile(filePath, JSON.stringify(serialized, null, 4));
+		await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4));
+	}
+
+	/**
+	 * Query master log
+	 *
+	 * @param {module:lib/logging_utils~QueryLogFilter} filter -
+	 *     Filter to limit entries with. Note that only the master log can
+	 *     be queried from this function.
+	 * @returns {Array<Object>} log entries matching the filter
+	 */
+	async queryMasterLog(filter) {
+		return await libLoggingUtils.queryLog(
+			path.join(this.logDirectory, "master"), filter,
+		);
+	}
+
+	/**
+	 * Query cluster log
+	 *
+	 * @param {module:lib/logging_utils~QueryLogFilter} filter -
+	 *     Filter to limit entries with.
+	 * @returns {Array<Object>} log entries matching the filter
+	 */
+	async queryClusterLog(filter) {
+		return await libLoggingUtils.queryLog(
+			path.join(this.logDirectory, "cluster"), filter, this.clusterLogIndex,
+		);
 	}
 
 	static addAppRoutes(app, pluginInfos) {
@@ -367,6 +408,7 @@ class Master {
 			version: slave.version,
 			id: slave.id,
 			name: slave.name,
+			public_address: slave.public_address || null,
 			connected: this.wsServer.slaveConnections.has(slave.id),
 		};
 

packages/master/src/SlaveConnection.js

@@ -28,6 +28,7 @@ class SlaveConnection extends BaseConnection {
 			id: this._id,
 			name: this._name,
 			version: this._version,
+			public_address: registerData.public_address,
 			plugins: registerData.plugins,
 		});
 
@@ -87,6 +88,7 @@ class SlaveConnection extends BaseConnection {
 
 		let prev = instance.status;
 		instance.status = message.data.status;
+		instance.game_port = message.data.game_port;
 		logger.verbose(`Instance ${instance.config.get("instance.name")} State: ${instance.status}`);
 		this._master.instanceUpdated(instance);
 		await libPlugin.invokeHook(this._master.plugins, "onInstanceStatusChanged", instance, prev);

packages/master/src/UserManager.js

@@ -1,6 +1,7 @@
 "use strict";
 const fs = require("fs-extra");
 
+const libFileOps = require("@clusterio/lib/file_ops");
 const libUsers = require("@clusterio/lib/users");
 
 /**
@@ -62,7 +63,7 @@ class UserManager {
 			users: serializedUsers,
 			roles: serializedRoles,
 		};
-		await fs.outputFile(filePath, JSON.stringify(serialized, null, 4));
+		await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4));
 	}
 
 	/**

packages/master/src/routes.js

@@ -2,13 +2,13 @@
 const Busboy = require("busboy");
 const crypto = require("crypto");
 const events = require("events");
-const fs = require("fs-extra");
 const JSZip = require("jszip");
 const jwt = require("jsonwebtoken");
 const path = require("path");
 const util = require("util");
 
 const libErrors = require("@clusterio/lib/errors");
+const libFileOps = require("@clusterio/lib/file_ops");
 const libHelpers = require("@clusterio/lib/helpers");
 const libLink = require("@clusterio/lib/link");
 const libPlugin = require("@clusterio/lib/plugin");
@@ -187,7 +187,7 @@ async function uploadExport(req, res) {
 		}
 
 		let name = path.posix.basename(filePath);
-		await fs.outputFile(path.join("static", "export", name), await file.async("nodebuffer"));
+		await libFileOps.safeOutputFile(path.join("static", "export", name), await file.async("nodebuffer"));
 	}
 
 	res.sendStatus(200);

packages/master/webpack.config.js

@@ -25,6 +25,8 @@ module.exports = (env = {}) => merge(common(env), {
 				"antd": { singleton: true },
 				"react": { singleton: true },
 				"react-dom": { singleton: true },
+				"react-router": { singleton: true },
+				"react-router-dom": { singleton: true },
 			},
 		}),
 	],

packages/slave/modules/clusterio/serialize.lua

@@ -68,14 +68,12 @@ function serialize.serialize_equipment_grid(grid)
 					}
 					if equipment.shield > 0 then entry.s = equipment.shield end
 					if equipment.energy > 0 then entry.e = equipment.energy end
-					-- TODO: Test with Industrial Revolution
 					if equipment.burner then
 						local burner = equipment.burner
 						entry.i = serialize.serialize_inventory(burner.inventory)
 						entry.r = serialize.serialize_inventory(burner.burnt_result_inventory)
-						if burner.curently_burning then
-							entry.b = {}
-							serialize.serialize_item_stack(burner.curently_burning, entry.b)
+						if burner.currently_burning then
+							entry.b = burner.currently_burning.name
 							entry.f = burner.remaining_burning_fuel
 						end
 					end
@@ -98,7 +96,8 @@ function serialize.deserialize_equipment_grid(grid, serialized)
 			if entry.s then equipment.shield = entry.s end
 			if entry.e then equipment.energy = entry.e end
 			if entry.i then
-				if entry.b then serialize.deserialize_item_stack(burner.currently_burning, entry.b) end
+				local burner = equipment.burner
+				if entry.b then burner.currently_burning = entry.b end
 				if entry.f then burner.remaining_burning_fuel = entry.f end
 				serialize.deserialize_inventory(burner.burnt_result_inventory, entry.r)
 				serialize.deserialize_inventory(burner.inventory, entry.i)

packages/slave/package.json

@@ -3,7 +3,7 @@
   "description": "Implementation of Clusterio slave server",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "main": "slave.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -18,12 +18,13 @@
     "node": ">=12"
   },
   "dependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.10",
+    "@clusterio/lib": "^2.0.0-alpha.11",
     "fs-extra": "^8.1.0",
     "phin": "^3.5.1",
     "pidusage": "^2.0.18",
     "set-blocking": "^2.0.0",
     "winston": "^3.3.3",
+    "winston-daily-rotate-file": "^4.5.5",
     "yargs": "^14.2.0"
   },
   "publishConfig": {

packages/slave/slave.js

@@ -20,9 +20,11 @@ const events = require("events");
 const pidusage = require("pidusage");
 const setBlocking = require("set-blocking");
 const phin = require("phin");
+const stream = require("stream");
 const util = require("util");
 const version = require("./package").version;
 const winston = require("winston");
+require("winston-daily-rotate-file");
 
 // internal libraries
 const libFileOps = require("@clusterio/lib/file_ops");
@@ -39,6 +41,8 @@ const { ConsoleTransport, levels, logger } = require("@clusterio/lib/logging");
 const libLoggingUtils = require("@clusterio/lib/logging_utils");
 const libHelpers = require("@clusterio/lib/helpers");
 
+const finished = util.promisify(stream.finished);
+
 
 const instanceRconCommandDuration = new libPrometheus.Histogram(
 	"clusterio_instance_rcon_command_duration_seconds",
@@ -260,7 +264,9 @@ class Instance extends libLink.Link {
 	notifyStatus(status) {
 		this._status = status;
 		libLink.messages.instanceStatusChanged.send(this, {
-			instance_id: this.id, status,
+			instance_id: this.id,
+			status,
+			game_port: this.server && this.server.gamePort || this.config.get("factorio.game_port") || null,
 		});
 	}
 
@@ -376,7 +382,7 @@ class Instance extends libLink.Link {
 	 */
 	async writeServerSettings() {
 		let serverSettings = await this.resolveServerSettings(this.config.get("factorio.settings"));
-		await fs.writeFile(
+		await libFileOps.safeOutputFile(
 			this.server.writePath("server-settings.json"),
 			JSON.stringify(serverSettings, null, 4)
 		);
@@ -410,7 +416,7 @@ class Instance extends libLink.Link {
 
 		if (this.config.get("factorio.sync_adminlist")) {
 			this.logger.verbose("Writing server-adminlist.json");
-			fs.outputFile(
+			libFileOps.safeOutputFile(
 				this.server.writePath("server-adminlist.json"),
 				JSON.stringify([...this._slave.adminlist], null, 4)
 			);
@@ -418,7 +424,7 @@ class Instance extends libLink.Link {
 
 		if (this.config.get("factorio.sync_banlist")) {
 			this.logger.verbose("Writing server-banlist.json");
-			fs.outputFile(
+			libFileOps.safeOutputFile(
 				this.server.writePath("server-banlist.json"),
 				JSON.stringify([...this._slave.banlist].map(
 					([username, reason]) => ({ username, reason })
@@ -428,7 +434,7 @@ class Instance extends libLink.Link {
 
 		if (this.config.get("factorio.sync_whitelist")) {
 			this.logger.verbose("Writing server-whitelist.json");
-			fs.outputFile(
+			libFileOps.safeOutputFile(
 				this.server.writePath("server-whitelist.json"),
 				JSON.stringify([...this._slave.whitelist], null, 4)
 			);
@@ -886,6 +892,11 @@ async function discoverInstances(instancesDir) {
 				continue;
 			}
 
+			if (instanceInfos.has(instanceConfig.get("instance.id"))) {
+				logger.warn(`Ignoring instance with duplicate id in folder ${entry.name}`);
+				continue;
+			}
+
 			let instancePath = path.join(instancesDir, entry.name);
 			logger.verbose(`found instance ${instanceConfig.get("instance.name")} in ${instancePath}`);
 			instanceInfos.set(instanceConfig.get("instance.id"), {
@@ -999,6 +1010,7 @@ class SlaveConnector extends libLink.WebSocketClientConnector {
 			version,
 			id: this.slaveConfig.get("slave.id"),
 			name: this.slaveConfig.get("slave.name"),
+			public_address: this.slaveConfig.get("slave.public_address"),
 			plugins,
 		});
 	}
@@ -1092,6 +1104,10 @@ class Slave extends libLink.Link {
 			}
 
 			this.updateInstances().catch((err) => {
+				if (err instanceof libErrors.SessionLost) {
+					return undefined;
+				}
+
 				logger.fatal(`Unexpected error updating instances:\n${err.stack}`);
 				return this.shutdown();
 			});
@@ -1294,7 +1310,7 @@ class Slave extends libLink.Link {
 			_warning: "Changes to this file will be overwritten by the master server's copy.",
 			...instanceInfo.config.serialize(),
 		};
-		await fs.outputFile(
+		await libFileOps.safeOutputFile(
 			path.join(instanceInfo.path, "instance.json"),
 			JSON.stringify(warnedOutput, null, 4)
 		);
@@ -1465,11 +1481,11 @@ class Slave extends libLink.Link {
 
 		let savesDir = path.join(instanceInfo.path, "saves");
 		let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
-		let stream;
+		let writeStream;
 		while (true) {
 			try {
-				stream = fs.createWriteStream(path.join(savesDir, tempFilename), { flags: "wx" });
-				await events.once(stream, "open");
+				writeStream = fs.createWriteStream(path.join(savesDir, tempFilename), { flags: "wx" });
+				await events.once(writeStream, "open");
 				break;
 			} catch (err) {
 				if (err.code === "EEXIST") {
@@ -1479,8 +1495,8 @@ class Slave extends libLink.Link {
 				}
 			}
 		}
-		response.pipe(stream);
-		await events.once(stream, "finish");
+		response.pipe(writeStream);
+		await finished(writeStream);
 
 		filename = await libFileOps.findUnusedName(savesDir, filename, ".zip");
 		await fs.rename(path.join(savesDir, tempFilename), path.join(savesDir, filename));
@@ -1722,6 +1738,12 @@ async function startSlave() {
 			choices: ["none"].concat(Object.keys(levels)),
 			type: "string",
 		})
+		.option("log-directory", {
+			nargs: 1,
+			describe: "Directory to place logs in",
+			default: "logs",
+			type: "string",
+		})
 		.option("config", {
 			nargs: 1,
 			describe: "slave config file to use",
@@ -1742,11 +1764,25 @@ async function startSlave() {
 		.argv
 	;
 
-	logger.add(new winston.transports.File({
-		format: winston.format.combine(
-			winston.format.json(),
-		),
-		filename: "slave.log",
+	{
+		// Migration from alpha-10 single file logs, note that we can't use
+		// the logger here as it's not initalized yet.
+		/* eslint-disable no-console */
+		let slaveLogDirectory = path.join(args.logDirectory, "slave");
+		if (!await fs.pathExists(slaveLogDirectory) && await fs.pathExists("slave.log")) {
+			console.log("Migrating slave log...");
+			await fs.ensureDir(slaveLogDirectory);
+			await libLoggingUtils.migrateLogs("slave.log", slaveLogDirectory, "slave-%DATE%.log");
+			console.log("Migration complete, you should delete slave.log now");
+		}
+		/* eslint-enable no-console */
+	}
+
+	logger.add(new winston.transports.DailyRotateFile({
+		format: winston.format.json(),
+		filename: "slave-%DATE%.log",
+		utc: true,
+		dirname: path.join(args.logDirectory, "slave"),
 	}));
 	if (args.logLevel !== "none") {
 		logger.add(new ConsoleTransport({

packages/web_ui/package.json

@@ -3,7 +3,7 @@
   "description": "Clusterio web interface implementation",
   "author": "Hornwitser",
   "license": "MIT",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "browser": "src/index.jsx",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
@@ -18,7 +18,7 @@
     "@ant-design/icons": "^4.2.2",
     "@babel/core": "^7.13.8",
     "@babel/preset-react": "^7.12.13",
-    "@clusterio/lib": "^2.0.0-alpha.10",
+    "@clusterio/lib": "^2.0.0-alpha.11",
     "antd": "^4.16.0",
     "assert": "^2.0.0",
     "babel-loader": "^8.2.2",

packages/web_ui/src/components/BaseConfigTree.jsx

@@ -3,6 +3,30 @@ import { Button, Card, Checkbox, Form, Input, InputNumber, Space, Spin, Tree, Ty
 
 const { Title } = Typography;
 
+function getInitialValues(config, props) {
+	let initialValues = {};
+	for (let [name, GroupClass] of props.ConfigClass.groups) {
+		let group = config.group(name);
+		for (let def of GroupClass.definitions.values()) {
+			if (!group.canAccess(def.name)) {
+				continue;
+			}
+
+			let value = group.get(def.name);
+			let fieldName = `${group.name}.${def.name}`;
+
+			if (def.type === "object") {
+				for (let prop of Object.keys(value)) {
+					let propPath = `${group.name}.${def.name}.${prop}`;
+					initialValues[propPath] = JSON.stringify(value[prop]);
+				}
+			} else {
+				initialValues[fieldName] = value;
+			}
+		}
+	}
+	return initialValues;
+}
 
 export default function BaseConfigTree(props) {
 	let [config, setConfig] = useState(null);
@@ -16,8 +40,13 @@ export default function BaseConfigTree(props) {
 		let newConfig = new props.ConfigClass("control");
 		await newConfig.load(serializedConfig);
 		setConfig(newConfig);
+		return newConfig;
 	}
-	useEffect(() => { updateConfig(); }, [props.id]);
+	useEffect(() => {
+		updateConfig().then(updatedConfig => {
+			form.setFieldsValue(getInitialValues(updatedConfig, props));
+		});
+	}, [props.id]);
 
 	function renderInput(def) {
 		if (def.type === "boolean") {

packages/web_ui/src/components/InstanceList.jsx

@@ -0,0 +1,91 @@
+import React from "react";
+import { useHistory } from "react-router-dom";
+import { message, Button, Table } from "antd";
+import CopyOutlined from "@ant-design/icons/CopyOutlined";
+
+import { useSlaveList } from "../model/slave";
+import InstanceStatusTag from "./InstanceStatusTag";
+
+const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare;
+
+export default function InstanceList(props) {
+	let history = useHistory();
+	let [slaveList] = useSlaveList();
+
+	function slaveName(slaveId) {
+		let slave = slaveList.find(s => s.id === slaveId);
+		if (slave) {
+			return slave.name;
+		}
+		return String(slaveId);
+	}
+
+	function instancePublicAddress(instance) {
+		let slave = slaveList.find(s => s.id === instance.assigned_slave);
+		if (slave && slave.public_address) {
+			if (instance.game_port) {
+				return `${slave.public_address}:${instance.game_port}`;
+			}
+			return slave.public_address;
+		}
+		return null;
+	}
+
+	let columns = [
+		{
+			title: "Name",
+			dataIndex: "name",
+			defaultSortOrder: "ascend",
+			sorter: (a, b) => strcmp(a["name"], b["name"]),
+		},
+		{
+			title: "Assigned Slave",
+			key: "assigned_slave",
+			render: instance => slaveName(instance["assigned_slave"]),
+			sorter: (a, b) => strcmp(slaveName(a["assigned_slave"]), slaveName(b["assigned_slave"])),
+		},
+		{
+			title: "Public address",
+			key: "public_address",
+			render: instance => {
+				let public_address = instancePublicAddress(instance);
+				return public_address ? <>
+					{public_address}
+					<Button
+						type="text"
+						icon={<CopyOutlined/>}
+						onClick={(e) => {
+							e.stopPropagation();
+							navigator.clipboard.writeText(public_address);
+							message.success("Copied public address!");
+						}}
+					/>
+				</> : "";
+			},
+			sorter: (a, b) => strcmp(slavePublicAddress(a["assigned_slave"]), slavePublicAddress(b["assigned_slave"])),
+		},
+		{
+			title: "Status",
+			dataIndex: "status",
+			render: status => <InstanceStatusTag status={status} />,
+			sorter: (a, b) => strcmp(a["status"], b["status"]),
+		},
+	];
+
+	if (props.hideAssignedSlave) {
+		columns.splice(1, 1);
+	}
+
+	return <Table
+		size={props.size || "large"}
+		columns={columns}
+		dataSource={props.instances}
+		rowKey={instance => instance["id"]}
+		pagination={false}
+		onRow={(record, rowIndex) => ({
+			onClick: event => {
+				history.push(`/instances/${record.id}/view`);
+			},
+		})}
+	/>;
+}

packages/web_ui/src/components/InstanceStatusTag.jsx

@@ -0,0 +1,21 @@
+import { Tag } from "antd";
+import React from "react";
+
+export const statusColors = {
+	unknown: "#8c8c8c",
+	unassigned: "#eb2f96",
+	stopped: "#cf1322",
+	starting: "#ad8b00",
+	running: "#389e0d",
+	stopping: "#d48806",
+	creating_save: "#096dd9",
+	exporting_data: "#08979c",
+	deleted: "#262626",
+};
+
+
+export default function InstanceStatusTag(props) {
+	return <Tag color={statusColors[props.status]}>
+		{props.status.replace("_", " ")}
+	</Tag>;
+}

packages/web_ui/src/components/InstanceViewPage.jsx

@@ -1,6 +1,6 @@
 import React, { useContext, useState } from "react";
 import { useHistory, useParams } from "react-router-dom";
-import { Button, Descriptions, Popconfirm, Space, Spin, Typography } from "antd";
+import { Alert, Button, Descriptions, Popconfirm, Space, Spin, Typography } from "antd";
 import DeleteOutlined from "@ant-design/icons/DeleteOutlined";
 
 import { libLink } from "@clusterio/lib";
@@ -9,7 +9,7 @@ import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import PageLayout from "./PageLayout";
 import InstanceConfigTree from "./InstanceConfigTree";
-import InstanceConsole from "./InstanceConsole";
+import LogConsole from "./LogConsole";
 import InstanceRcon from "./InstanceRcon";
 import AssignInstanceModal from "./AssignInstanceModal";
 import StartStopInstanceButton from "./StartStopInstanceButton";
@@ -18,6 +18,7 @@ import SavesList from "./SavesList";
 import { notifyErrorHandler } from "../util/notify";
 import { useInstance } from "../model/instance";
 import { useSlave } from "../model/slave";
+import InstanceStatusTag from "./InstanceStatusTag";
 
 const { Title, Paragraph } = Typography;
 
@@ -40,10 +41,22 @@ export default function InstanceViewPage(props) {
 		return <PageLayout nav={nav}><Spin size="large" /></PageLayout>;
 	}
 
-	if (instance.missing) {
+	if (instance.missing || instance["status"] === "deleted") {
 		return <PageLayout nav={nav}>
-			<h2>Instance not found</h2>
-			<p>Instance with id {instanceId} was not found on the master server.</p>
+			<Alert
+				message={instance["status"] === "deleted" ? "Instance has been deleted" : "Instance not found" }
+				showIcon
+				description={<>Instance with id {instanceId} was not found on the master server.</>}
+				type="warning"
+				action={
+					<Button
+						type="text"
+						onClick={() => { history.push("/instances"); }}
+					>
+						Go back to instances list
+					</Button>
+				}
+			/>
 		</PageLayout>;
 	}
 
@@ -116,7 +129,7 @@ export default function InstanceViewPage(props) {
 					buttonContent={assigned ? "Reassign" : "Assign"}
 				/>}
 			</Descriptions.Item>
-			<Descriptions.Item label="Status">{instance["status"]}</Descriptions.Item>
+			<Descriptions.Item label="Status"><InstanceStatusTag status={instance["status"]} /></Descriptions.Item>
 		</Descriptions>
 
 		{
@@ -127,7 +140,7 @@ export default function InstanceViewPage(props) {
 			account.hasAnyPermission("core.log.follow", "core.instance.send_rcon")
 			&& <Title level={5} style={{ marginTop: 16 }}>Console</Title>
 		}
-		{account.hasPermission("core.log.follow") && <InstanceConsole id={instanceId} />}
+		{account.hasPermission("core.log.follow") && <LogConsole instances={[instanceId]} />}
 		{
 			account.hasPermission("core.instance.send_rcon")
 			&& <InstanceRcon id={instanceId} disabled={instance["status"] !== "running"} />

packages/web_ui/src/components/InstancesPage.jsx

@@ -1,6 +1,6 @@
 import React, { useContext, useState } from "react";
 import { useHistory } from "react-router-dom";
-import { Button, Form, Input, Modal, PageHeader, Table } from "antd";
+import { Button, Form, Input, Modal, PageHeader } from "antd";
 
 import { libConfig, libLink } from "@clusterio/lib";
 
@@ -8,11 +8,8 @@ import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import PageLayout from "./PageLayout";
 import { useInstanceList } from "../model/instance";
+import InstanceList from "./InstanceList";
 import { notifyErrorHandler } from "../util/notify";
-import { useSlaveList } from "../model/slave";
-
-const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare;
-
 
 function CreateInstanceButton(props) {
 	let control = useContext(ControlContext);
@@ -53,7 +50,7 @@ function CreateInstanceButton(props) {
 		>
 			<Form form={form}>
 				<Form.Item name="instanceName" label="Name">
-					<Input/>
+					<Input />
 				</Form.Item>
 			</Form>
 		</Modal>
@@ -61,54 +58,41 @@ function CreateInstanceButton(props) {
 }
 
 export default function InstancesPage() {
-	let history = useHistory();
+	let control = useContext(ControlContext);
 	let account = useAccount();
-	let [slaveList] = useSlaveList();
 	let [instanceList] = useInstanceList();
 
-	function slaveName(slaveId) {
-		let slave = slaveList.find(s => s.id === slaveId);
-		if (slave) {
-			return slave.name;
-		}
-		return String(slaveId);
-	}
-
 	return <PageLayout nav={[{ name: "Instances" }]}>
 		<PageHeader
 			className="site-page-header"
 			title="Instances"
-			extra={account.hasPermission("core.instance.create") && <CreateInstanceButton />}
+			extra={<>
+				{account.hasPermission("core.instance.create") && <CreateInstanceButton />}
+				{account.hasPermission("core.instance.start")
+					&& <Button onClick={e => instanceList.forEach(instance => {
+						if (instance.status === "stopped") {
+							libLink.messages.startInstance.send(
+								control, { instance_id: instance.id, save: null }
+							).catch(notifyErrorHandler("Error starting instance"));
+						}
+					})
+					}>
+						Start all
+					</Button>}
+				{account.hasPermission("core.instance.stop")
+					&& <Button onClick={e => instanceList.forEach(instance => {
+						if (["starting", "running"].includes(instance.status)) {
+							libLink.messages.stopInstance.send(
+								control, { instance_id: instance.id }
+							).catch(notifyErrorHandler("Error stopping instance"));
+						}
+					})
+					}>
+						Stop all
+					</Button>}
+			</>}
 		/>
 
-		<Table
-			columns={[
-				{
-					title: "Name",
-					dataIndex: "name",
-					defaultSortOrder: "ascend",
-					sorter: (a, b) => strcmp(a["name"], b["name"]),
-				},
-				{
-					title: "Assigned Slave",
-					key: "assigned_slave",
-					render: instance => slaveName(instance["assigned_slave"]),
-					sorter: (a, b) => strcmp(slaveName(a["assigned_slave"]), slaveName(b["assigned_slave"])),
-				},
-				{
-					title: "Status",
-					dataIndex: "status",
-					sorter: (a, b) => strcmp(a["status"], b["status"]),
-				},
-			]}
-			dataSource={instanceList}
-			rowKey={instance => instance["id"]}
-			pagination={false}
-			onRow={(record, rowIndex) => ({
-				onClick: event => {
-					history.push(`/instances/${record.id}/view`);
-				},
-			})}
-		/>
+		<InstanceList instances={instanceList} />
 	</PageLayout>;
 }

packages/web_ui/src/components/LogConsole.jsx

@@ -49,7 +49,7 @@ function formatLog(info, key) {
 	return <span key={key}>[{level}] {info.message}<br/></span>;
 }
 
-export default function InstanceConsole(props) {
+export default function LogConsole(props) {
 	let account = useAccount();
 	let control = useContext(ControlContext);
 	let anchor = useRef();
@@ -57,16 +57,21 @@ export default function InstanceConsole(props) {
 	let [lines, setLines] = useState([]);
 
 	useEffect(() => {
+		let logFilter = {
+			all: props.all || false,
+			master: props.master || false,
+			slave_ids: props.slaves || [],
+			instance_ids: props.instances || [],
+		};
+
 		// Scroll view to the anchor so it sticks to the bottom
 		let parent = anchor.current.parentElement;
 		parent.scrollTop = parent.scrollHeight - parent.clientHeight;
 
 		if (account.hasPermission("core.log.query")) {
+			setPastLines([<span key={0}>{"Loading past entries..."}<br/></span>]);
 			libLink.messages.queryLog.send(control, {
-				all: false,
-				master: false,
-				slave_ids: [],
-				instance_ids: [props.id],
+				...logFilter,
 				max_level: null,
 				limit: 400,
 				order: "desc",
@@ -85,11 +90,11 @@ export default function InstanceConsole(props) {
 			));
 		}
 
-		control.onInstanceLog(props.id, logHandler);
+		control.onLog(logFilter, logHandler);
 		return () => {
-			control.offInstanceLog(props.id, logHandler);
+			control.offLog(logFilter, logHandler);
 		};
-	}, [props.id]);
+	}, [props.all, props.master, (props.slaves || []).join(), (props.instances || []).join()]);
 
 	return <>
 		<Paragraph code className="instance-console">

packages/web_ui/src/components/MasterPage.jsx

@@ -1,15 +1,23 @@
 import React, { useContext } from "react";
+import { Typography } from "antd";
 
+import LogConsole from "./LogConsole";
 import { useAccount } from "../model/account";
 import MasterConfigTree from "./MasterConfigTree";
 import PageLayout from "./PageLayout";
 
+const { Title } = Typography;
+
 
 export default function SlavesPage() {
 	let account = useAccount();
 
 	return <PageLayout nav={[{ name: "Master" }]}>
 		<h2>Master</h2>
+		{account.hasPermission("core.log.follow") && <>
+			<Title level={5} style={{ marginTop: 16 }}>Console</Title>
+			<LogConsole master={true} />
+		</>}
 		{account.hasPermission("core.master.get_config") && <MasterConfigTree />}
 	</PageLayout>;
 };

packages/web_ui/src/components/SlaveViewPage.jsx

@@ -0,0 +1,64 @@
+import React, { useContext, useState } from "react";
+import { Descriptions, PageHeader, Spin, Tag, Typography } from "antd";
+import { useParams } from "react-router-dom";
+
+import InstanceList from "./InstanceList";
+import LogConsole from "./LogConsole";
+import { useAccount } from "../model/account";
+import { useInstanceList } from "../model/instance";
+import { useSlave } from "../model/slave";
+import PageLayout from "./PageLayout";
+
+const { Title } = Typography;
+
+
+export default function SlaveViewPage(props) {
+	let params = useParams();
+	let slaveId = Number(params.id);
+	let account = useAccount();
+	let [instanceList] = useInstanceList();
+	let [slave] = useSlave(slaveId);
+
+	instanceList = instanceList.filter(instance => instance["assigned_slave"] === slaveId);
+
+	let nav = [{ name: "Slaves", path: "/slaves" }, { name: slave.name || slaveId }];
+	if (slave.loading) {
+		return <PageLayout nav={nav}><Spin size="large" /></PageLayout>;
+	}
+
+	if (slave.missing) {
+		return <PageLayout nav={nav}>
+			<PageHeader
+				className="site-page-header"
+				title={slaveId}
+			/>
+			<p>Slave with id {slaveId} was not found on the master server.</p>
+		</PageLayout>;
+	}
+
+	return <PageLayout nav={nav}>
+		<PageHeader
+			className="site-page-header"
+			title={slave.name || slaveId}
+		/>
+
+		<Descriptions bordered size="small" column={{ xs: 1, sm: 2, xl: 4 }}>
+			<Descriptions.Item label="Name">{slave["name"]}</Descriptions.Item>
+			<Descriptions.Item label="Connected">
+				<Tag color={slave["connected"] ? "#389e0d" : "#cf1322"}>
+					{slave["connected"] ? "Connected" : "Disconnected"}
+				</Tag>
+			</Descriptions.Item>
+			<Descriptions.Item label="Agent">{slave["agent"]}</Descriptions.Item>
+			<Descriptions.Item label="Version">{slave["version"]}</Descriptions.Item>
+		</Descriptions>
+		{account.hasPermission("core.instance.list") && <>
+			<Title level={5} style={{ marginTop: 16 }}>Instances</Title>
+			<InstanceList instances={instanceList} size="small" hideAssignedSlave />
+		</>}
+		{account.hasPermission("core.log.follow") && <>
+			<Title level={5} style={{ marginTop: 16 }}>Console</Title>
+			<LogConsole slaves={[slaveId]} />
+		</>}
+	</PageLayout>;
+}

packages/web_ui/src/components/SlavesPage.jsx

@@ -1,5 +1,6 @@
 import React, { useContext, useRef, useState } from "react";
-import { Button, Form, Input, Modal, PageHeader, Table } from "antd";
+import { useHistory } from "react-router-dom";
+import { Button, Form, Input, Modal, PageHeader, Table, Tag } from "antd";
 
 import { libLink } from "@clusterio/lib";
 
@@ -75,6 +76,7 @@ function GenerateSlaveTokenButton(props) {
 
 export default function SlavesPage() {
 	let account = useAccount();
+	let history = useHistory();
 	let [slaveList] = useSlaveList();
 
 	return <PageLayout nav={[{ name: "Slaves" }]}>
@@ -102,16 +104,30 @@ export default function SlavesPage() {
 					dataIndex: "version",
 					sorter: (a, b) => strcmp(a["version"], b["version"]),
 				},
+				{
+					title: "Public address",
+					dataIndex: "public_address",
+					sorter: (a, b) => strcmp(a["public_address"], b["public_address"]),
+				},
 				{
 					title: "Connected",
 					key: "connected",
-					render: slave => slave["connected"] && "Yes",
+					render: slave => <Tag
+						color={slave["connected"] ? "#389e0d" : "#cf1322"}
+					>
+						{slave["connected"] ? "Connected" : "Disconnected"}
+					</Tag>,
 					sorter: (a, b) => a["connected"] - b["connected"],
 				},
 			]}
 			dataSource={slaveList}
 			rowKey={slave => slave["id"]}
 			pagination={false}
+			onRow={(record, rowIndex) => ({
+				onClick: event => {
+					history.push(`/slaves/${record.id}/view`);
+				},
+			})}
 		/>
 	</PageLayout>;
 };

packages/web_ui/src/index.jsx

@@ -5,6 +5,7 @@ export * from "./model/instance.jsx";
 export * from "./model/slave.jsx";
 export * from "./model/locale.jsx";
 export * from "./model/item_metadata.jsx";
+export * from "./model/account.jsx";
 
 
 export { default as ControlContext } from "./components/ControlContext";
@@ -13,9 +14,14 @@ export { default as BaseConfigTree } from "./components/BaseConfigTree";
 export { default as InstanceConfigTree } from "./components/InstanceConfigTree";
 export { default as MasterConfigTree } from "./components/MasterConfigTree";
 
+export { default as LogConsole } from "./components/LogConsole";
+
 export { default as AssignInstanceModal } from "./components/AssignInstanceModal";
-export { default as InstanceConsole } from "./components/InstanceConsole";
+export { default as InstanceList } from "./components/InstanceList";
 export { default as InstanceRcon } from "./components/InstanceRcon";
+export { default as InstanceStatusTag } from "./components/InstanceStatusTag";
+export { default as StartStopInstanceButton } from "./components/StartStopInstanceButton";
+export { statusColors } from "./components/InstanceStatusTag";
 
 export { default as PageLayout } from "./components/PageLayout";
 export { default as SavesList } from "./components/SavesList";

packages/web_ui/src/model/saves.jsx

@@ -14,10 +14,15 @@ export function useSaves(instanceId) {
 			setSaves(result.list);
 		}).catch(err => {
 			logger.error(`Failed to list instance saves: ${err}`);
+			setSaves([]);
 		});
 	}
 
 	useEffect(() => {
+		if (!Number.isInteger(instanceId)) {
+			setSaves([]);
+			return undefined;
+		}
 		updateSaves();
 
 		function updateHandler(data) {

packages/web_ui/src/pages.jsx

@@ -2,6 +2,7 @@ import React from "react";
 
 import MasterPage from "./components/MasterPage";
 import SlavesPage from "./components/SlavesPage";
+import SlaveViewPage from "./components/SlaveViewPage";
 import InstancesPage from "./components/InstancesPage";
 import InstanceViewPage from "./components/InstanceViewPage";
 import UsersPage from "./components/UsersPage";
@@ -24,6 +25,11 @@ export const pages = [
 		permission: "core.slave.list",
 		content: <SlavesPage />,
 	},
+	{
+		path: "/slaves/:id/view",
+		sidebarPath: "/slaves",
+		content: <SlaveViewPage />,
+	},
 	{
 		path: "/instances",
 		sidebarName: "Instances",

packages/web_ui/src/util/websocket.jsx

@@ -27,6 +27,22 @@ export class ControlConnector extends libLink.WebSocketClientConnector {
 	}
 }
 
+function logFilter({ all, master, slave_ids, instance_ids }, info) {
+	if (all) {
+		return true;
+	}
+	if (master && info.slave_id === undefined) {
+		return true;
+	}
+	if (info.slave_id && slave_ids.includes(info.slave_id)) {
+		return true;
+	}
+	if (info.instance_id && instance_ids.includes(info.instance_id)) {
+		return true;
+	}
+	return false;
+}
+
 /**
  * Handles running the control
  *
@@ -70,7 +86,7 @@ export class Control extends libLink.Link {
 		this.slaveUpdateHandlers = new Map();
 		this.instanceUpdateHandlers = new Map();
 		this.saveListUpdateHandlers = new Map();
-		this.instanceLogHandlers = new Map();
+		this.logHandlers = new Map();
 
 		this.connector.on("connect", data => {
 			this.accountName = data.account.name;
@@ -302,23 +318,20 @@ export class Control extends libLink.Link {
 	async logMessageEventHandler(message) {
 		let info = message.data.info;
 
-		if (info.instance_id !== undefined) {
-			let instanceHandlers = this.instanceLogHandlers.get(info.instance_id);
-			for (let handler of instanceHandlers || []) {
-				handler(info);
+		for (let [filter, handlers] of this.logHandlers) {
+			if (logFilter(filter, info)) {
+				for (let handler of handlers) {
+					handler(info);
+				}
 			}
 		}
 	}
 
-	async onInstanceLog(id, handler) {
-		if (!Number.isInteger(id)) {
-			throw new Error("Invalid instance id");
-		}
-
-		let handlers = this.instanceLogHandlers.get(id);
+	async onLog(filter, handler) {
+		let handlers = this.logHandlers.get(filter);
 		if (!handlers) {
 			handlers = [];
-			this.instanceLogHandlers.set(id, handlers);
+			this.logHandlers.set(filter, handlers);
 		}
 
 		handlers.push(handler);
@@ -328,15 +341,15 @@ export class Control extends libLink.Link {
 		}
 	}
 
-	async offInstanceLog(id, handler) {
-		let handlers = this.instanceLogHandlers.get(id);
+	async offLog(filter, handler) {
+		let handlers = this.logHandlers.get(filter);
 		if (!handlers || !handlers.length) {
-			throw new Error(`No handlers for instance ${id} exist`);
+			throw new Error("No handlers for the given filter exists");
 		}
 
 		let index = handlers.lastIndexOf(handler);
 		if (index === -1) {
-			throw new Error(`Given handler is not registered for instance ${id}`);
+			throw new Error("Given handler is not registered for the filter");
 		}
 
 		handlers.splice(index, 1);
@@ -350,13 +363,29 @@ export class Control extends libLink.Link {
 			return;
 		}
 
-		await libLink.messages.setLogSubscriptions.send(this, {
+		let combinedFilter = {
 			all: false,
 			master: false,
 			slave_ids: [],
-			instance_ids: [...this.instanceLogHandlers.keys()],
-			max_level: null,
-		});
+			instance_ids: [],
+		};
+
+		for (let filter of this.logHandlers.keys()) {
+			if (filter.all) { combinedFilter.all = true; }
+			if (filter.master) { combinedFilter.master = true; }
+			for (let slaveId of filter.slave_ids || []) {
+				if (!combinedFilter.slave_ids.includes(slaveId)) {
+					combinedFilter.slave_ids.push(slaveId);
+				}
+			}
+			for (let instanceId of filter.instance_ids || []) {
+				if (!combinedFilter.instance_ids.includes(instanceId)) {
+					combinedFilter.instance_ids.push(instanceId);
+				}
+			}
+		}
+
+		await libLink.messages.setLogSubscriptions.send(this, { ...combinedFilter, max_level: null });
 	}
 
 	async debugWsMessageEventHandler(message) {

plugins/global_chat/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-global_chat",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "description": "Clusterio plugin forwarding between Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/clusterio#readme",
@@ -20,8 +20,8 @@
     "@clusterio/lib": "^2.0.0-alpha.0"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.10",
-    "@clusterio/web_ui": "^2.0.0-alpha.10",
+    "@clusterio/lib": "^2.0.0-alpha.11",
+    "@clusterio/web_ui": "^2.0.0-alpha.11",
     "webpack": "^5.36.2",
     "webpack-cli": "^4.7.0",
     "webpack-merge": "^5.2.0"

plugins/inventory_sync/master.js

@@ -2,6 +2,7 @@
 const fs = require("fs-extra");
 const path = require("path");
 
+const libFileOps = require("@clusterio/lib/file_ops");
 const libPlugin = require("@clusterio/lib/plugin");
 const libErrors = require("@clusterio/lib/errors");
 
@@ -26,7 +27,7 @@ async function saveDatabase(masterConfig, playerDatastore, logger) {
 		let file = path.resolve(masterConfig.get("master.database_directory"), "inventories.json");
 		logger.verbose(`writing ${file}`);
 		let content = JSON.stringify(Array.from(playerDatastore));
-		await fs.outputFile(file, content);
+		await libFileOps.safeOutputFile(file, content);
 	}
 }
 

plugins/inventory_sync/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-inventory_sync",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "description": "Clusterio plugin forwarding between Factorio servers",
   "author": "Danielv <danielv@danielv.no>",
   "homepage": "https://github.com/clusterio/factorioClusterio#readme",
@@ -23,8 +23,8 @@
     "fs-extra": "^8.1.0"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.10",
-    "@clusterio/web_ui": "^2.0.0-alpha.10",
+    "@clusterio/lib": "^2.0.0-alpha.11",
+    "@clusterio/web_ui": "^2.0.0-alpha.11",
     "antd": "^4.16.0",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",

plugins/player_auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-player_auth",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "description": "Clusterio plugin authenticating logged in players to the web interface",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/clusterio#readme",
@@ -24,8 +24,8 @@
     "jsonwebtoken": "^8.2.1"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.10",
-    "@clusterio/web_ui": "^2.0.0-alpha.10",
+    "@clusterio/lib": "^2.0.0-alpha.11",
+    "@clusterio/web_ui": "^2.0.0-alpha.11",
     "antd": "^4.16.0",
     "phin": "^3.5.1",
     "react": "^16.13.1",

plugins/research_sync/master.js

@@ -2,6 +2,7 @@
 const fs = require("fs-extra");
 const path = require("path");
 
+const libFileOps = require("@clusterio/lib/file_ops");
 const libPlugin = require("@clusterio/lib/plugin");
 const RateLimiter = require("@clusterio/lib/RateLimiter");
 
@@ -25,7 +26,7 @@ async function loadTechnologies(masterConfig, logger) {
 async function saveTechnologies(masterConfig, technologies, logger) {
 	let filePath = path.join(masterConfig.get("master.database_directory"), "technologies.json");
 	logger.verbose(`writing ${filePath}`);
-	await fs.outputFile(filePath, JSON.stringify([...technologies.entries()], null, 4));
+	await libFileOps.safeOutputFile(filePath, JSON.stringify([...technologies.entries()], null, 4));
 }
 
 class MasterPlugin extends libPlugin.BaseMasterPlugin {

plugins/research_sync/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-research_sync",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "description": "Clusterio plugin for synchronizing research between Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/clusterio#readme",
@@ -23,8 +23,8 @@
     "fs-extra": "^8.1.0"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.10",
-    "@clusterio/web_ui": "^2.0.0-alpha.10",
+    "@clusterio/lib": "^2.0.0-alpha.11",
+    "@clusterio/web_ui": "^2.0.0-alpha.11",
     "webpack": "^5.36.2",
     "webpack-cli": "^4.7.0",
     "webpack-merge": "^5.2.0"

plugins/statistics_exporter/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-statistics_exporter",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "description": "Clusterio plugin for gathering statistics from Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/clusterio#readme",
@@ -20,8 +20,8 @@
     "@clusterio/lib": "^2.0.0-alpha.0"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.10",
-    "@clusterio/web_ui": "^2.0.0-alpha.10",
+    "@clusterio/lib": "^2.0.0-alpha.11",
+    "@clusterio/web_ui": "^2.0.0-alpha.11",
     "webpack": "^5.36.2",
     "webpack-cli": "^4.7.0",
     "webpack-merge": "^5.2.0"

plugins/subspace_storage/master.js

@@ -3,6 +3,7 @@ const fs = require("fs-extra");
 const path = require("path");
 
 const libDatabase = require("@clusterio/lib/database");
+const libFileOps = require("@clusterio/lib/file_ops");
 const libPlugin = require("@clusterio/lib/plugin");
 const { Counter, Gauge } = require("@clusterio/lib/prometheus");
 const RateLimiter = require("@clusterio/lib/RateLimiter");
@@ -49,7 +50,7 @@ async function saveDatabase(masterConfig, items, logger) {
 		let file = path.resolve(masterConfig.get("master.database_directory"), "items.json");
 		logger.verbose(`writing ${file}`);
 		let content = JSON.stringify(items.serialize());
-		await fs.outputFile(file, content);
+		await libFileOps.safeOutputFile(file, content);
 	} else if (items) {
 		logger.error(`Item database too large, not saving (${items.size})`);
 	}

plugins/subspace_storage/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@clusterio/plugin-subspace_storage",
-  "version": "2.0.0-alpha.10",
+  "version": "2.0.0-alpha.11",
   "description": "Clusterio plugin for sharing storage between Factorio servers",
   "author": "Hornwitser <github@hornwitser.no>",
   "homepage": "https://github.com/clusterio/clusterio#readme",
@@ -23,8 +23,8 @@
     "fs-extra": "^8.1.0"
   },
   "devDependencies": {
-    "@clusterio/lib": "^2.0.0-alpha.10",
-    "@clusterio/web_ui": "^2.0.0-alpha.10",
+    "@clusterio/lib": "^2.0.0-alpha.11",
+    "@clusterio/web_ui": "^2.0.0-alpha.11",
     "antd": "^4.16.0",
     "react": "^16.13.1",
     "react-dom": "^16.13.1",

test/file/log/excluded.txt

@@ -0,0 +1 @@
+This file should be excluded by queries and the index

test/file/log/test-2021-12-10.log

@@ -0,0 +1,9 @@
+{"timestamp":"2021-12-10T10:00:00.000Z","level":"verbose","message":"Master start"}
+{"timestamp":"2021-12-10T10:00:01.000Z","level":"verbose","message":"Loading"}
+{"timestamp":"2021-12-10T10:00:03.000Z","level":"info","message":"Ready"}
+{"timestamp":"2021-12-10T10:00:10.000Z","slave_id":1,"level":"verbose","message":"Slave connected"}
+{"timestamp":"2021-12-10T10:00:11.000Z","slave_id":1,"level":"verbose","message":"Found instance 10"}
+{"timestamp":"2021-12-10T10:00:13.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Starting up"}
+{"timestamp":"2021-12-10T10:00:14.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Loading map"}
+{"timestamp":"2021-12-10T10:00:15.000Z","slave_id":1,"instance_id":10,"level":"server","message":"Game ready"}
+{"timestamp":"2021-12-10T10:01:00.000Z","slave_id":1,"instance_id":10,"level":"error","message":"Unexpected crash"}

test/file/log/test-2021-12-11.log

@@ -0,0 +1,3 @@
+{"timestamp":"2021-12-11T10:00:00.000Z","level":"verbose","message":"Master start"}
+{"timestamp":"2021-12-11T10:00:01.000Z","level":"verbose","message":"Loading"}
+{"timestamp":"2021-12-11T10:00:03.000Z","level":"fatal","message":"An error occured"}

test/file/log/test-2021-12-12.log

@@ -0,0 +1 @@
+{"timestamp":"2021-12-12T10:00:00.000Z","leve

test/integration/index.js

@@ -9,7 +9,7 @@ const util = require("util");
 const events = require("events");
 
 const libLink = require("@clusterio/lib/link");
-const server = require("@clusterio/lib/factorio/server");
+const { LineSplitter } = require("@clusterio/lib/stream");
 const { ConsoleTransport, logger } = require("@clusterio/lib/logging");
 const libLoggingUtils = require("@clusterio/lib/logging_utils");
 
@@ -133,18 +133,18 @@ function spawn(name, cmd, waitFor) {
 		console.log(cmd);
 		let parts = cmd.split(" ");
 		let process = child_process.spawn(parts[0], parts.slice(1), { cwd: path.join("temp", "test") });
-		let stdout = new server._LineSplitter((line) => {
+		let stdout = new LineSplitter({ readableObjectMode: true });
+		stdout.on("data", line => {
 			line = line.toString("utf8");
 			if (waitFor.test(line)) {
 				resolve(process);
 			}
 			console.log(name, line);
 		});
-		let stderr = new server._LineSplitter((line) => { console.log(name, line.toString("utf8")); });
-		process.stdout.on("data", chunk => { stdout.data(chunk); });
-		process.stdout.on("close", () => { stdout.end(); });
-		process.stderr.on("data", chunk => { stderr.data(chunk); });
-		process.stderr.on("close", () => { stderr.end(); });
+		let stderr = new LineSplitter({ readableObjectMode: true });
+		stderr.on("data", line => { console.log(name, line.toString("utf8")); });
+		process.stdout.pipe(stdout);
+		process.stderr.pipe(stderr);
 	});
 }
 

test/lib/factorio/server.js

@@ -119,43 +119,6 @@ describe("lib/factorio/server", function() {
 		});
 	});
 
-	describe("class LineSplitter", function() {
-		function createSplitter(lines) {
-			return new libFactorio._LineSplitter(line => lines.push(line.toString("utf-8")));
-		}
-
-		it("should split three lines", function() {
-			let lines = [];
-			let ls = createSplitter(lines);
-			ls.data(Buffer.from("line 1\nline 2\nline 3\n"));
-			ls.end();
-			assert.deepEqual(lines, ["line 1", "line 2", "line 3"]);
-		});
-		it("should split three Windows line endings lines", function() {
-			let lines = [];
-			let ls = createSplitter(lines);
-			ls.data(Buffer.from("line 1\r\nline 2\r\nline 3\r\n"));
-			assert.deepEqual(lines, ["line 1", "line 2", "line 3"]);
-		});
-		it("should give the last non-terminated line on .end()", function() {
-			let lines = [];
-			let ls = createSplitter(lines);
-			ls.data(Buffer.from("line a\nline b"));
-			assert.deepEqual(lines, ["line a"]);
-			ls.end();
-			assert.deepEqual(lines, ["line a", "line b"]);
-		});
-		it("should handled partial lines", function() {
-			let lines = [];
-			let ls = createSplitter(lines);
-			ls.data(Buffer.from("part 1"));
-			ls.data(Buffer.from(" part 2 "));
-			ls.data(Buffer.from("part 3\n"));
-			ls.end();
-			assert.deepEqual(lines, ["part 1 part 2 part 3"]);
-		});
-	});
-
 	describe("parseOutput()", function() {
 		it("should parse the test lines", function() {
 			for (let [line, reference] of testLines) {

test/lib/file_ops.js

@@ -10,6 +10,8 @@ describe("lib/file_ops", function() {
 	async function setupTestingEnv() {
 		await fs.ensureDir(path.join(baseDir, "test", "folder"));
 		await fs.ensureDir(path.join(baseDir, "test", "another folder"));
+		await fs.remove(path.join(baseDir, "safe"));
+		await fs.ensureDir(path.join(baseDir, "safe"));
 
 		await fs.outputFile(path.join(baseDir, "test", "file.txt"), "contents");
 		await fs.outputFile(path.join(baseDir, "test", "another file.txt"), "more contents");
@@ -66,4 +68,20 @@ describe("lib/file_ops", function() {
 			}
 		});
 	});
+
+	describe("safeOutputFile()", function() {
+		it("should write new target file", async function() {
+			let target = path.join(baseDir, "safe", "simple.txt");
+			await libFileOps.safeOutputFile(target, "a text file", "utf8");
+			assert(!await fs.pathExists(`${target}.tmp`), "temporary was left behind");
+			assert.equal(await fs.readFile(target, "utf8"), "a text file");
+		});
+		it("should overwrite existing target file", async function() {
+			let target = path.join(baseDir, "safe", "exists.txt");
+			await fs.outputFile(target, "previous", "utf8");
+			await libFileOps.safeOutputFile(target, "current", "utf8");
+			assert(!await fs.pathExists(`${target}.tmp`), "temporary was left behind");
+			assert.equal(await fs.readFile(target, "utf8"), "current");
+		});
+	});
 });

test/lib/logging_utils.js

@@ -1,8 +1,13 @@
 "use strict";
 const assert = require("assert").strict;
+const fs = require("fs-extra");
+const path = require("path");
+const util = require("util");
+
 
 const { testLines } = require("./factorio/lines");
 const libLoggingUtils = require("@clusterio/lib/logging_utils");
+const libStream = require("@clusterio/lib/stream");
 
 
 describe("lib/logging_utils.js", function() {
@@ -16,4 +21,110 @@ describe("lib/logging_utils.js", function() {
 			}
 		});
 	});
+
+	let logDir = path.join("test", "file", "log");
+	describe("class LogIndex", function() {
+		it("should create a new empty index if index.json does not exist", async function() {
+			let logIndex = await libLoggingUtils.LogIndex.load(logDir);
+			assert.deepEqual(logIndex.index, new Map());
+		});
+		it("should build index of only past log files", async function() {
+			let logIndex = await libLoggingUtils.LogIndex.load(logDir);
+			await logIndex.buildIndex();
+			assert.deepEqual(
+				new Set(logIndex.index.keys()),
+				new Set(["test-2021-12-10.log", "test-2021-12-11.log", "test-2021-12-12.log"])
+			);
+		});
+		it("should roundtrip index on save load cycle", async function() {
+			let logIndex = await libLoggingUtils.LogIndex.load(logDir);
+			await logIndex.buildIndex();
+			let tempDir = path.join("temp", "test", "log");
+			logIndex.logDirectory = tempDir;
+			await logIndex.save();
+			let loadedIndex = await libLoggingUtils.LogIndex.load(tempDir);
+			assert.deepEqual(
+				loadedIndex,
+				logIndex
+			);
+		});
+		it("should handle loading broken index", async function() {
+			let tempDir = path.join("temp", "test", "log");
+			await fs.outputFile(path.join(tempDir, "index.json"), "Broken JSON");
+			let logIndex = await libLoggingUtils.LogIndex.load(tempDir);
+			assert.deepEqual(logIndex.index, new Map());
+		});
+	});
+
+	let allLines = [];
+	let logIndex = null;
+	before(async function() {
+		for (let file of (await fs.readdir(logDir)).sort()) {
+			if (file === "excluded.txt") {
+				continue;
+			}
+			let fileStream = fs.createReadStream(path.join(logDir, file));
+			let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
+			fileStream.pipe(lineStream);
+			for await (let line of lineStream) {
+				try {
+					allLines.push(JSON.parse(line));
+				} catch (err) {
+					allLines.push({ "level": "info", "message": line.toString() });
+				}
+			}
+		}
+
+		logIndex = await libLoggingUtils.LogIndex.load(logDir);
+		await logIndex.buildIndex();
+	});
+
+	describe("queryLog", function() {
+		for (let index of [() => undefined, () => logIndex]) {
+			describe(index() === undefined ? "without index" : "with index", function() {
+				it("returns nothing with an empty filter", async function() {
+					let log = await libLoggingUtils.queryLog(logDir, {}, index());
+					assert.deepEqual(log, []);
+				});
+				it("should return the whole log when querying all", async function() {
+					let log = await libLoggingUtils.queryLog(logDir, { all: true }, index());
+					assert.deepEqual(log, allLines);
+				});
+				it("should return the whole log in reverse when querying all desc", async function() {
+					let log = await libLoggingUtils.queryLog(logDir, { all: true, order: "desc" }, index());
+					assert.deepEqual(log, [...allLines].reverse());
+				});
+				it("should limit by limit", async function() {
+					let log = await libLoggingUtils.queryLog(logDir, { limit: 2, all: true }, index());
+					assert.deepEqual(log, allLines.slice(0, 2));
+				});
+				it("should filter by max_level", async function() {
+					let log = await libLoggingUtils.queryLog(logDir, { max_level: "info", all: true }, index());
+					assert.deepEqual(log, allLines.filter(
+						info => ["fatal", "error", "warn", "audit", "info"].includes(info.level)
+					));
+					log = await libLoggingUtils.queryLog(logDir, { max_level: "fatal", all: true }, index());
+					assert.deepEqual(log, allLines.filter(
+						info => ["fatal"].includes(info.level)
+					));
+				});
+				it("should filter by master", async function() {
+					let log = await libLoggingUtils.queryLog(logDir, { master: true }, index());
+					assert.deepEqual(log, allLines.filter(
+						info => info.slave_id === undefined && info.instance_id === undefined
+					));
+				});
+				it("should filter by slave_ids", async function() {
+					let log = await libLoggingUtils.queryLog(logDir, { slave_ids: [1, 2] }, index());
+					assert.deepEqual(log, allLines.filter(
+						info => [1, 2].includes(info.slave_id) && info.instance_id === undefined
+					));
+				});
+				it("should filter by instance_ids", async function() {
+					let log = await libLoggingUtils.queryLog(logDir, { instance_ids: [10, 11] }, index());
+					assert.deepEqual(log, allLines.filter(info => [10, 11].includes(info.instance_id)));
+				});
+			});
+		}
+	});
 });

test/lib/stream.js

@@ -0,0 +1,151 @@
+"use strict";
+const assert = require("assert").strict;
+const events = require("events");
+const fs = require("fs-extra");
+const path = require("path");
+const stream = require("stream");
+const util = require("util");
+
+const libStream = require("@clusterio/lib/stream");
+
+const finished = util.promisify(stream.finished);
+
+
+describe("lib/stream", function() {
+	describe("class LineSplitter", function() {
+		function createSplitter(lines) {
+			let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
+			lineStream.on("data", line => lines.push(line.toString("utf-8")));
+			return lineStream;
+		}
+
+		it("should split three lines", function() {
+			let lines = [];
+			let ls = createSplitter(lines);
+			ls.write(Buffer.from("line 1\nline 2\nline 3\n"));
+			ls.end();
+			assert.deepEqual(lines, ["line 1", "line 2", "line 3"]);
+		});
+		it("should split three Windows line endings lines", function() {
+			let lines = [];
+			let ls = createSplitter(lines);
+			ls.write(Buffer.from("line 1\r\nline 2\r\nline 3\r\n"));
+			ls.end();
+			assert.deepEqual(lines, ["line 1", "line 2", "line 3"]);
+		});
+		it("should handle mixed line endings", function() {
+			let lines = [];
+			let ls = createSplitter(lines);
+			ls.write(Buffer.from("1\n\n2\r\n\r\n3\n4"));
+			ls.end();
+			assert.deepEqual(lines, ["1", "", "2", "", "3", "4"]);
+		});
+		it("should give the last non-terminated line on .end()", function() {
+			let lines = [];
+			let ls = createSplitter(lines);
+			ls.write(Buffer.from("line a\nline b"));
+			assert.deepEqual(lines, ["line a"]);
+			ls.end();
+			assert.deepEqual(lines, ["line a", "line b"]);
+		});
+		it("should handled partial lines", function() {
+			let lines = [];
+			let ls = createSplitter(lines);
+			ls.write(Buffer.from("part 1"));
+			ls.write(Buffer.from(" part 2 "));
+			ls.write(Buffer.from("part 3\n"));
+			ls.end();
+			assert.deepEqual(lines, ["part 1 part 2 part 3"]);
+		});
+	});
+
+	describe("class ReverseLineSplitter", function() {
+		function createReverseSplitter(lines) {
+			let lineStream = new libStream.ReverseLineSplitter({ readableObjectMode: true });
+			lineStream.on("data", line => lines.push(line.toString("utf-8")));
+			return lineStream;
+		}
+
+		it("should split three lines", function() {
+			let lines = [];
+			let ls = createReverseSplitter(lines);
+			ls.write(Buffer.from("line 1\nline 2\nline 3\n"));
+			ls.end();
+			assert.deepEqual(lines, ["line 3", "line 2", "line 1"]);
+		});
+		it("should split three Windows line endings lines", function() {
+			let lines = [];
+			let ls = createReverseSplitter(lines);
+			ls.write(Buffer.from("line 1\r\nline 2\r\nline 3\r\n"));
+			ls.end();
+			assert.deepEqual(lines, ["line 3", "line 2", "line 1"]);
+		});
+		it("should handle mixed line endings", function() {
+			let lines = [];
+			let ls = createReverseSplitter(lines);
+			ls.write(Buffer.from("1\n\n2\r\n\r\n3\n4"));
+			ls.end();
+			assert.deepEqual(lines, ["4", "3", "", "2", "", "1"]);
+		});
+		it("should give the first line on .end()", function() {
+			let lines = [];
+			let ls = createReverseSplitter(lines);
+			ls.write(Buffer.from("line a\nline b"));
+			assert.deepEqual(lines, ["line b"]);
+			ls.end();
+			assert.deepEqual(lines, ["line b", "line a"]);
+		});
+		it("should handled partial lines", function() {
+			let lines = [];
+			let ls = createReverseSplitter(lines);
+			ls.write(Buffer.from("part 3\n"));
+			ls.write(Buffer.from(" part 2 "));
+			ls.write(Buffer.from("part 1"));
+			ls.end();
+			assert.deepEqual(lines, ["part 1 part 2 part 3"]);
+		});
+	});
+
+	describe("createReverseReadStream()", function() {
+		it("should read chunks in a file in reverse", async function() {
+			let content = "";
+			for (let i = 0; i < 10; i++) { content += String(i).repeat(10); }
+			await fs.outputFile(path.join("temp", "test", "reverse.txt"), content);
+			let reverseStream = await libStream.createReverseReadStream(
+				path.join("temp", "test", "reverse.txt"),
+				{ encoding: "utf8", highWaterMark: 10 }
+			);
+			let index = 9;
+			reverseStream.on("data", data => {
+				assert.equal(data, String(index).repeat(10));
+				index -= 1;
+			});
+			await finished(reverseStream);
+		});
+
+		it("should reverse the lines of a file", async function() {
+			let ws = fs.createWriteStream(path.join("temp", "test", "reverse.txt"));
+			for (let i = 1; i < 100000; i++) {
+				ws.write(Buffer.from(`${i}\n`));
+			}
+			ws.end();
+			await finished(ws);
+
+			let lineStream = new libStream.ReverseLineSplitter({ readableObjectMode: true });
+			let index = 99999;
+			let rs = await libStream.createReverseReadStream(path.join("temp", "test", "reverse.txt"));
+			lineStream.on("data", line => {
+				let n = Number(line.toString());
+				if (index !== n) {
+					rs.destroy();
+					assert.equal(n, index);
+				}
+				index -= 1;
+			});
+			rs.pipe(lineStream);
+			await finished(lineStream);
+
+			assert.equal(index, 0, "line check did not reach the end");
+		});
+	});
+});