Merge pull request #297 from Hornwitser/automate-export

Automate export process

LICENSE.md

@@ -19,8 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
-
-The icons for angels modded items located in /static/pictures/ are licensed
-by https://mods.factorio.com/user/Arch666Angel under CC BY-NC-ND 4.0
-https://creativecommons.org/licenses/by-nc-nd/4.0/
-

lib/factorio/export.js

@@ -306,6 +306,7 @@ async function exportLocale(server, modVersions, modOrder, languageCode) {
  *
  * @param {module:lib/factorio.FactorioServer} server -
  *     The server to export the data from.
+ * @returns {JSZip} zip file with exported data.
  * @memberof module:lib/factorio
  */
 async function exportData(server) {
@@ -347,8 +348,7 @@ async function exportData(server) {
 	zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
 	zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
 	zip.file("export/locale.json", JSON.stringify([...locale.entries()]));
-	let content = await zip.generateAsync({ type: "nodebuffer" });
-	await fs.outputFile("temp/export.zip", content);
+	return zip;
 }
 
 

master.js

@@ -27,6 +27,7 @@ const setBlocking = require("set-blocking");
 const events = require("events");
 const yargs = require("yargs");
 const WebSocket = require("ws");
+const JSZip = require("jszip");
 const version = require("./package").version;
 
 // ugly globals
@@ -174,6 +175,45 @@ async function getMetrics(req, res, next) {
 }
 app.get('/metrics', (req, res, next) => getMetrics(req, res, next).catch(next));
 
+// Handle an uploaded export package.
+async function uploadExport(req, res, next) {
+	endpointHitCounter.labels(req.route.path).inc();
+	if (req.get("Content-Type") !== "application/zip") {
+		res.sendStatus(415);
+		return;
+	}
+
+	let data = [];
+	for await (let chunk of req) {
+		data.push(chunk);
+	}
+	data = Buffer.concat(data);
+	let zip = await JSZip.loadAsync(data);
+	delete data;
+
+	// This is hardcoded to prevent path expansion attacks
+	let exportFiles = [
+		"export/item-spritesheet.png",
+		"export/item-metadata.json",
+		"export/locale.json",
+	];
+
+	for (let filePath of exportFiles) {
+		let file = zip.file(filePath);
+		if (!file) {
+			continue;
+		}
+
+		let name = path.posix.basename(filePath);
+		await fs.outputFile(path.join("static", "export", name), await file.async("nodebuffer"));
+	}
+
+	res.sendStatus(200);
+}
+app.put("/api/upload-export",
+	authenticate.middleware,
+	(req, res, next) => uploadExport(req, res, next).catch(next)
+);
 
 const masterConnectedClientsCount = new prometheus.Gauge(
 	'clusterio_master_connected_clients_count', "How many clients are currently connected to this master server",

package.json

@@ -55,6 +55,7 @@
     "needle": "^2.4.0",
     "node-factorio-api": "^0.3.8",
     "node-forge": "^0.7.1",
+    "phin": "^3.4.1",
     "pidusage": "^2.0.18",
     "rcon-client": "^4.2.0",
     "request": "^2.83.0",

slave.js

@@ -16,6 +16,7 @@ const yargs = require("yargs");
 const events = require("events");
 const pidusage = require("pidusage");
 const setBlocking = require("set-blocking");
+const phin = require("phin");
 const version = require("./package").version;
 
 // internal libraries
@@ -58,9 +59,10 @@ const instanceFactorioAutosaveSize = new prometheus.Gauge(
  * Keeps track of the runtime parameters of an instance
  */
 class Instance extends link.Link{
-	constructor(connector, dir, factorioDir, instanceConfig) {
+	constructor(slave, connector, dir, factorioDir, instanceConfig) {
 		super('instance', 'slave', connector);
 		link.attachAllMessages(this);
+		this._slave = slave;
 		this._dir = dir;
 
 		this.plugins = new Map();
@@ -141,21 +143,21 @@ class Instance extends link.Link{
 		console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`);
 	}
 
-	async init(pluginInfos, slave) {
+	async init(pluginInfos) {
 		await this.server.init();
 
 		// load plugins
 		for (let pluginInfo of pluginInfos) {
 			if (
 				!pluginInfo.instanceEntrypoint
-				|| !slave.serverPlugins.has(pluginInfo.name)
+				|| !this._slave.serverPlugins.has(pluginInfo.name)
 				|| !this.config.group(pluginInfo.name).get("enabled")
 			) {
 				continue;
 			}
 
 			try {
-				await this._loadPlugin(pluginInfo, slave);
+				await this._loadPlugin(pluginInfo, this._slave);
 			} catch (err) {
 				this.notifyExit();
 				throw err;
@@ -445,7 +447,23 @@ class Instance extends link.Link{
 
 			console.log("Exporting data .....");
 			await symlinkMods(this, "sharedMods", console);
-			await factorio.exportData(this.server);
+			let zip = await factorio.exportData(this.server);
+
+			let content = await zip.generateAsync({ type: "nodebuffer" });
+			let url = new URL(this._slave.config.get("slave.master_url"));
+			url.pathname += "api/upload-export";
+			let response = await phin({
+				url, method: "PUT",
+				data: content,
+				core: { rejectUnauthorized: false },
+				headers: {
+					"Content-Type": "application/zip",
+					"x-access-token": this._slave.config.get("slave.master_token"),
+				},
+			});
+			if (response.statusCode !== 200) {
+				throw Error(`Upload failed: ${response.statusCode} ${response.statusMessage}: ${response.body}`)
+			}
 
 		} finally {
 			this.notifyExit();
@@ -801,9 +819,9 @@ class Slave extends link.Link {
 		let [connectionClient, connectionServer] = link.VirtualConnector.makePair();
 		let instanceConnection = new InstanceConnection(connectionServer, this, instanceId);
 		let instance = new Instance(
-			connectionClient, instanceInfo.path, this.config.get("slave.factorio_directory"), instanceInfo.config
+			this, connectionClient, instanceInfo.path, this.config.get("slave.factorio_directory"), instanceInfo.config
 		);
-		await instance.init(this.pluginInfos, this);
+		await instance.init(this.pluginInfos);
 
 		// XXX: race condition on multiple simultanious calls
 		this.instanceConnections.set(instanceId, instanceConnection);

test/integration/clusterio.js

@@ -62,7 +62,7 @@ describe("Integration of Clusterio", function() {
 			it("exports the data", async function() {
 				slowTest(this);
 				await exec(`node clusterctl --config ${controlConfigPath} export-data --instance test`);
-				assert(await fs.exists(path.join("temp", "export.zip")), "Export was not created");
+				assert(await fs.exists(path.join("static", "export", "locale.json")), "Export was not created");
 			});
 		});
 

test/slave.js

@@ -17,7 +17,7 @@ describe("Slave testing", function() {
 			let instanceConfig = new config.InstanceConfig();
 			await instanceConfig.init();
 			instanceConfig.set("instance.name", "foo");
-			instance = new slave._Instance(new link.VirtualConnector(), "dir", "factorioDir", instanceConfig);
+			instance = new slave._Instance({}, new link.VirtualConnector(), "dir", "factorioDir", instanceConfig);
 		});
 
 		describe(".name", function() {
@@ -109,7 +109,7 @@ describe("Slave testing", function() {
 			await instanceConfig.init();
 			instanceConfig.set("instance.name", "test");
 			instance = new slave._Instance(
-				new link.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig
+				{}, new link.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig
 			);
 			await fs.outputFile(instance.path("mods", "mod_i.zip"), "i");
 		});