From 38c249ec703fd2c8519b5e0b94dc262213da02e3 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 28 Mar 2022 20:37:35 +0200 Subject: [PATCH] Implement save transfer Add ability to move or copy saves between instances. For instances on the same slave the saves are transferred with filesystem operations, for instances on two different slaves the proxy stream interface is used to send the save via the master server. --- CHANGELOG.md | 2 +- packages/ctl/ctl.js | 28 ++++ packages/lib/link/messages.js | 16 +++ packages/lib/users.js | 5 + packages/master/src/ControlConnection.js | 80 +++++++++++ packages/slave/slave.js | 39 +++++ packages/web_ui/src/components/SavesList.jsx | 93 +++++++++++- test/integration/clusterio.js | 144 +++++++++++++++++++ 8 files changed, 405 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db398cca..39386312 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ - Packaged and uploaded to npm again, greatly simplifying distribution and installation of both Clusterio and plugins. - Added centralized logging for the cluster. All logs from both Clusterio and Factorio is stored in a shared log which can be inspected and queried. -- Added remote save management with the ability to list, create, rename, copy, upload, download, delete and load saves for instances. +- Added remote save management with the ability to list, create, rename, copy, upload, download, transfer, delete and load saves for instances. ### Features diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 0f6dbedf..41454c3d 100755 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -659,6 +659,34 @@ instanceSaveCommands.add(new libCommand.Command({ }, })); +instanceSaveCommands.add(new libCommand.Command({ + definition: [ + "transfer [target-save]", + "Transfer a save between instances", + (yargs) => { + yargs.positional("source-instance", { describe: "Instance to transfer save from", type: "string" }); + yargs.positional("source-save", { describe: "Save to transfer.", type: "string" }); + yargs.positional("target-instance", { describe: "Instance to transfer to", type: "string" }); + yargs.positional("target-save", { describe: "Name to give transferred save.", type: "string" }); + yargs.options({ + "copy": { describe: "Copy instead of moving the save", type: "boolean", default: false }, + }); + }, + ], + handler: async function(args, control) { + let sourceInstanceId = await libCommand.resolveInstance(control, args.sourceInstance); + let targetInstanceId = await libCommand.resolveInstance(control, args.targetInstance); + let result = await libLink.messages.transferSave.send(control, { + instance_id: sourceInstanceId, + source_save: args.sourceSave, + target_instance_id: targetInstanceId, + target_save: args.targetSave || args.sourceSave, + copy: args.copy, + }); + print(`Transferred as ${result.save} to ${args.targetInstance}.`); + }, +})); + instanceSaveCommands.add(new libCommand.Command({ definition: ["download ", "Download a save from an instance", (yargs) => { yargs.positional("instance", { describe: "Instance to download save from", type: "string" }); diff --git a/packages/lib/link/messages.js b/packages/lib/link/messages.js index 13f5180a..dd187266 100644 --- a/packages/lib/link/messages.js +++ b/packages/lib/link/messages.js @@ -595,6 +595,22 @@ messages.downloadSave = new Request({ }, }); +messages.transferSave = new Request({ + type: "transfer_save", + links: ["control-master", "master-slave"], + permission: "core.instance.save.transfer", + requestProperties: { + "instance_id": { type: "integer" }, + "source_save": { type: "string" }, + "target_instance_id": { type: "integer" }, + "target_save": { type: "string" }, + "copy": { type: "boolean" }, + }, + responseProperties: { + "save": { type: "string" }, + }, +}); + messages.pullSave = new Request({ type: "pull_save", links: ["master-slave"], diff --git a/packages/lib/users.js b/packages/lib/users.js index a5539b7c..8432191b 100644 --- a/packages/lib/users.js +++ b/packages/lib/users.js @@ -412,6 +412,11 @@ definePermission({ title: "Upload save", description: "Upload savegames to instances.", }); +definePermission({ + name: "core.instance.save.transfer", + title: "Transfer save", + description: "Transfer savegames between instances.", +}); definePermission({ name: "core.instance.save.download", title: "Download save", diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index 5a10a780..05a31320 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -1,5 +1,6 @@ "use strict"; const jwt = require("jsonwebtoken"); +const events = require("events"); const libConfig = require("@clusterio/lib/config"); const libErrors = require("@clusterio/lib/errors"); @@ -373,6 +374,85 @@ class ControlConnection extends BaseConnection { return { stream_id: stream.id }; } + async transferSaveRequestHandler(message, request) { + let { source_save, target_save, copy, instance_id, target_instance_id } = message.data; + if (copy) { + this.user.checkPermission("core.instance.save.copy"); + } else if (source_save !== target_save) { + this.user.checkPermission("core.instance.save.rename"); + } + + if (instance_id === target_instance_id) { + throw new libErrors.RequestError("Source and target instance may not be the same"); + } + let sourceInstance = this._master.getRequestInstance(instance_id); + let targetInstance = this._master.getRequestInstance(target_instance_id); + let sourceSlaveId = sourceInstance.config.get("instance.assigned_slave"); + let targetSlaveId = targetInstance.config.get("instance.assigned_slave"); + if (sourceSlaveId === null) { + throw new libErrors.RequestError("Source instance is not assigned a slave"); + } + if (targetSlaveId === null) { + throw new libErrors.RequestError("Target instance is not assigned a slave"); + } + + // Let slave handle request if source and target is on the same slave. + if (sourceSlaveId === targetSlaveId) { + return await this.forwardRequestToInstance(message, request); + } + + // Check connectivity + let sourceSlaveConnection = this._master.wsServer.slaveConnections.get(sourceSlaveId); + if (!sourceSlaveConnection || sourceSlaveConnection.closing) { + throw new libErrors.RequestError("Source slave is not connected to the master server"); + } + + let targetSlaveConnection = this._master.wsServer.slaveConnections.get(targetSlaveId); + if (!targetSlaveConnection || targetSlaveConnection.closing) { + throw new libErrors.RequestError("Target slave is not connected to the master server"); + } + + // Create stream to proxy from target to source + let stream = await routes.createProxyStream(this._master.app); + stream.events.on("timeout", () => { + if (stream.source) { + stream.source.destroy(); + } + stream.events.emit("error", new libErrors.RequestError("Timed out establishing transfer stream")); + }); + + // Ignore errors if not listening for them to avoid crash. + stream.events.on("error", () => { /* ignore */ }); + + // Establish push from source slave to stream, this is done first to + // ensure the file size is known prior to the target slave pull. + await Promise.all([ + this._master.forwardRequestToInstance(libLink.messages.pushSave, { + instance_id, + stream_id: stream.id, + save: source_save, + }), + events.once(stream.events, "source"), + ]); + + // Establish pull from target slave to stream and wait for completion. + let result = await this._master.forwardRequestToInstance(libLink.messages.pullSave, { + instance_id: target_instance_id, + stream_id: stream.id, + filename: target_save, + }); + + // Delete source save if this is not a copy + if (!copy) { + await this._master.forwardRequestToInstance(libLink.messages.deleteSave, { + instance_id, + save: source_save, + }); + } + + return { save: result.save }; + } + async setLogSubscriptionsRequestHandler(message) { this.logSubscriptions = message.data; this.updateLogSubscriptions(); diff --git a/packages/slave/slave.js b/packages/slave/slave.js index 61e4f110..db3d0281 100755 --- a/packages/slave/slave.js +++ b/packages/slave/slave.js @@ -1489,6 +1489,45 @@ class Slave extends libLink.Link { await this.sendSaveListUpdate(instance_id, path.join(instanceInfo.path, "saves")); } + async transferSaveRequestHandler(message) { + let { source_save, target_save, copy, instance_id, target_instance_id } = message.data; + checkRequestSaveName(source_save); + checkRequestSaveName(target_save); + let sourceInstanceInfo = this.getRequestInstanceInfo(instance_id); + let targetInstanceInfo = this.getRequestInstanceInfo(target_instance_id); + + // For consistency with remote transfer initiated through pullSave the + // target is renamed if it already exists. + target_save = await libFileOps.findUnusedName( + path.join(targetInstanceInfo.path, "saves"), target_save, ".zip" + ); + + try { + if (copy) { + await fs.copy( + path.join(sourceInstanceInfo.path, "saves", source_save), + path.join(targetInstanceInfo.path, "saves", target_save), + { overwrite: true }, + ); + } else { + await fs.move( + path.join(sourceInstanceInfo.path, "saves", source_save), + path.join(targetInstanceInfo.path, "saves", target_save), + { overwrite: true }, + ); + } + } catch (err) { + if (err.code === "ENOENT") { + throw new libErrors.RequestError(`${source_save} does not exist`); + } + throw err; + } + await this.sendSaveListUpdate(instance_id, path.join(sourceInstanceInfo.path, "saves")); + await this.sendSaveListUpdate(target_instance_id, path.join(targetInstanceInfo.path, "saves")); + + return { save: target_save }; + } + async sendSaveListUpdate(instance_id, savesDir) { let instanceConnection = this.instanceConnections.get(instance_id); let saveList; diff --git a/packages/web_ui/src/components/SavesList.jsx b/packages/web_ui/src/components/SavesList.jsx index e63524fa..133cffac 100644 --- a/packages/web_ui/src/components/SavesList.jsx +++ b/packages/web_ui/src/components/SavesList.jsx @@ -1,5 +1,8 @@ import React, { useContext, useState } from "react"; -import { Button, Form, Input, List, Modal, Popconfirm, Progress, Space, Table, Tooltip, Upload } from "antd"; +import { + message, Button, Checkbox, Form, Input, List, Modal, + Popconfirm, Progress, Select, Space, Table, Tooltip, Upload, +} from "antd"; import CaretLeftOutlined from "@ant-design/icons/CaretLeftOutlined"; import LeftOutlined from "@ant-design/icons/LeftOutlined"; @@ -9,6 +12,7 @@ import { useAccount } from "../model/account"; import ControlContext from "./ControlContext"; import CreateSaveModal from "./CreateSaveModal"; import SectionHeader from "./SectionHeader"; +import { useInstanceList } from "../model/instance"; import { useSaves } from "../model/saves"; import { notifyErrorHandler } from "../util/notify"; @@ -91,6 +95,90 @@ function CopyModal(props) { ; } +function TransferModal(props) { + let account = useAccount(); + let control = useContext(ControlContext); + let [visible, setVisible] = useState(false); + let [form] = Form.useForm(); + let [instanceList] = useInstanceList(); + + return <> + + form.submit()} + onCancel={() => setVisible(false)} + destroyOnClose + > +
{ + setVisible(false); + let hide = message.loading("Transferring save...", 0); + libLink.messages.transferSave.send(control, { + instance_id: props.instanceId, + source_save: props.save.name, + target_instance_id: values.targetInstanceId, + target_save: values.transferredName || props.save.name, + copy: values.copy, + }).then(() => { + message.success("Transfer complete"); + form.resetFields(); + }).catch( + notifyErrorHandler("Error transfering save") + ).finally(() => hide()); + }} + > + + + + + + + + + +
+ + ; +} + function formatBytes(bytes) { if (bytes === 0) { return "0 Bytes"; @@ -179,6 +267,9 @@ export default function SavesList(props) { ); }} >Download} + {account.hasPermission("core.instance.save.transfer") && } {account.hasPermission("core.instance.save.delete") &&