Add upload save feature

Add /api/upload-save endpoint that streams the data via the slave
assigned to the target instance to the instance saves directory, along
with a ctl command and web interface button to do the upload.

test/integration/clusterio.js

@@ -358,6 +358,25 @@ describe("Integration of Clusterio", function() {
 			});
 		});
 
+		describe("instance upload-save", function() {
+			it("should upload a zip file", async function() {
+				await fs.outputFile(path.join("temp", "test", "upload.zip"), "a test");
+				await execCtl("instance upload-save 44 upload.zip");
+				assert(
+					await fs.pathExists(path.join("temp", "test", "instances", "test", "saves", "upload.zip")),
+					"file not uploaded to saves directory"
+				);
+			});
+			it("should reject non-zip files", async function() {
+				await fs.outputFile(path.join("temp", "test", "invalid"), "a test");
+				await assert.rejects(execCtl("instance upload-save 44 invalid"));
+			});
+			it("should reject path traversal attacks", async function() {
+				await fs.outputFile(path.join("temp", "test", "upload.zip"), "a test");
+				await assert.rejects(execCtl("instance upload-save 44 upload.zip --name ../traversal.zip"));
+			});
+		});
+
 		describe("instance delete", function() {
 			it("deletes the instance", async function() {
 				slowTest(this);

test/master.js

@@ -1,9 +0,0 @@
-"use strict";
-const assert = require("assert").strict;
-
-const mock = require("./mock");
-const master = require("@clusterio/master/master");
-
-
-describe("Master testing", function() {
-});

test/master/routes.js

@@ -0,0 +1,290 @@
+"use strict";
+const assert = require("assert").strict;
+const events = require("events");
+const FormData = require("form-data");
+const jwt = require("jsonwebtoken");
+const http = require("http");
+const phin = require("phin");
+
+const routes = require("@clusterio/master/src/routes");
+const mock = require("../mock");
+
+
+describe("master/src/routes", function() {
+	let master;
+	let server;
+	let port;
+	beforeEach(async function() {
+		master = new mock.MockMaster();
+		routes.addRouteHandlers(master.app);
+		server = http.createServer(master.app);
+		server.listen(0, "localhost");
+		await events.once(server, "listening");
+		port = server.address().port;
+	});
+	afterEach(function() {
+		server.close();
+	});
+	describe("/api/stream/:id", function() {
+		it("should respond with 404 if stream doesn't exist", async function() {
+			let response = await phin(`http://localhost:${port}/api/stream/01020304`);
+			assert.equal(response.statusCode, 404);
+		});
+		it("should respond with 500 if stream times out", async function() {
+			let stream = await routes.createProxyStream(master.app);
+			let response = await phin(`http://localhost:${port}/api/stream/${stream.id}`);
+			assert.equal(response.statusCode, 500);
+		});
+	});
+	describe("/api/upload-save", function() {
+		it("should respond with 401 with invalid token", async function() {
+			let response;
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: { "Content-Type": "application/zip" },
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 401);
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"Content-Type": "application/zip",
+					"X-Access-Token": "invalid",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 401);
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"Content-Type": "application/zip",
+					"X-Access-Token": jwt.sign({ aud: "user", user: "invalid" }, "TestSecretDoNotUse"),
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 401);
+			master.userManager.users.get("test").tokenValidAfter = Math.floor((Date.now() + 60e3) / 1000);
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"Content-Type": "application/zip",
+					"X-Access-Token": jwt.sign({ aud: "user", user: "test" }, "TestSecretDoNotUse"),
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 401);
+		});
+
+		it("should respond with 403 when user has insufficient permission", async function() {
+			let response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"Content-Type": "application/zip",
+					"X-Access-Token": jwt.sign({ aud: "user", user: "player" }, "TestSecretDoNotUse"),
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 403);
+		});
+
+		let testToken = jwt.sign({ aud: "user", user: "test" }, "TestSecretDoNotUse");
+		it("should respond with 415 if content type is missing or invalid", async function() {
+			let response;
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 415);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "invalid",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 415);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=file.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "text/plain",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 415);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid Content-Type"] });
+		});
+
+		it("should respond with 400 if invalid multipart request", async function() {
+			let response;
+			let form;
+			form = new FormData();
+			form.append("instance_id", "123");
+			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "text/plain" });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					...form.getHeaders(),
+				},
+				data: form.getBuffer(),
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["invalid file Content-Type"] });
+			form = new FormData();
+			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" });
+			form.append("instance_id", "123");
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					...form.getHeaders(),
+				},
+				data: form.getBuffer(),
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(
+				JSON.parse(response.body), { request_errors: ["instance_id must come before files uploaded"] }
+			);
+			form = new FormData();
+			form.append("instance_id", "123");
+			form.append("file", "totally a zip file", { filename: "test.txt", contentType: "application/zip" });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					...form.getHeaders(),
+				},
+				data: form.getBuffer(),
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(
+				JSON.parse(response.body), { request_errors: ["filename must end with .zip"] }
+			);
+			form = new FormData();
+			form.append("instance_id", "invalid");
+			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					...form.getHeaders(),
+				},
+				data: form.getBuffer(),
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(
+				JSON.parse(response.body), { request_errors: ["invalid instance_id"] }
+			);
+		});
+
+		it("should respond with 400 if invalid file request", async function() {
+			let response;
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["Missing or invalid filename parameter"] });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.txt`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(JSON.parse(response.body), { request_errors: ["filename must end with .zip"] });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?filename=test.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 400);
+			assert.deepEqual(
+				JSON.parse(response.body), { request_errors: ["Missing or invalid instance_id parameter"] }
+			);
+		});
+
+		it("should respond with 500 if the transfer failed", async function() {
+			master.forwardRequestToInstance = async (data, request) => {
+				throw new Error("Something went wrong");
+			};
+			let response;
+			let form;
+			form = new FormData();
+			form.append("instance_id", "123");
+			form.append("file", "totally a zip file", { filename: "test.zip", contentType: "application/zip" });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					...form.getHeaders(),
+				},
+				data: form.getBuffer(),
+			});
+			assert.equal(response.statusCode, 500);
+			assert.deepEqual(JSON.parse(response.body), { errors: ["Something went wrong"], request_errors: [] });
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 500);
+			assert.deepEqual(JSON.parse(response.body), { errors: ["Something went wrong"], request_errors: [] });
+			master.forwardRequestToInstance = async (data, request) => {
+				await new Promise(() => {});
+			};
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.equal(response.statusCode, 500);
+			assert.deepEqual(
+				JSON.parse(response.body), { errors: ["Timed out establishing stream to slave"], request_errors: [] }
+			);
+		});
+
+		it("should complete a valid transfer", async function() {
+			master.forwardRequestToInstance = async (request, data) => {
+				let stream = await phin({
+					url: `http://localhost:${port}/api/stream/${data.stream_id}`,
+				});
+				assert.equal(stream.body.toString(), "totally a zip file");
+				return { save: data.filename };
+			};
+			let response;
+			response = await phin({
+				url: `http://localhost:${port}/api/upload-save?instance_id=123&filename=test.zip`, method: "POST",
+				headers: {
+					"X-Access-Token": testToken,
+					"Content-Type": "application/zip",
+				},
+				data: "totally a zip file",
+			});
+			assert.deepEqual(JSON.parse(response.body), { saves: ["test.zip"] });
+			assert.equal(response.statusCode, 200);
+		});
+	});
+});

test/mock.js

@@ -150,9 +150,12 @@ class MockControl extends libLink.Link {
 class MockMaster {
 	constructor() {
 		this.app = express();
+		this.app.locals.master = this;
+		this.app.locals.streams = new Map();
 		this.mockConfigEntries = new Map([
 			["master.external_address", "test"],
 			["master.auth_secret", "TestSecretDoNotUse"],
+			["master.proxy_stream_timeout", 1],
 		]);
 		this.config = {
 			get: (name) => {
@@ -227,6 +230,7 @@ module.exports = {
 	MockInstance,
 	MockSlave,
 	MockControl,
+	MockMaster,
 
 	createMasterPlugin,
 	createInstancePlugin,

test/package.json

@@ -10,6 +10,7 @@
     "@clusterio/master": "^2.0.0-alpha.6",
     "@clusterio/slave": "^2.0.0-alpha.6",
     "express": "^4.16.2",
+    "form-data": "^4.0.0",
     "fs-extra": "^8.1.0",
     "jsonwebtoken": "^8.2.1",
     "jszip": "^3.2.2",