From f8597f500c105f31da5e89ee53aa076271c1681b Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 26 May 2020 13:17:58 +0200 Subject: [PATCH 1/3] Remove license note for angels icons Remove notice about license for the now removed modded angels icons. --- LICENSE.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/LICENSE.md b/LICENSE.md index f8b7533f..604ded16 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -19,8 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -The icons for angels modded items located in /static/pictures/ are licensed -by https://mods.factorio.com/user/Arch666Angel under CC BY-NC-ND 4.0 -https://creativecommons.org/licenses/by-nc-nd/4.0/ - From f3d4cfdcf5f89ebc0cf6fbff1a692db3e7291423 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 26 May 2020 14:04:25 +0200 Subject: [PATCH 2/3] Add reference to Slave in Instance Add a reference to the Slave that's running the given instance in order for it to be able to access the slave config. --- slave.js | 13 +++++++------ test/slave.js | 4 ++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/slave.js b/slave.js index 13b07ef8..ca2301ac 100644 --- a/slave.js +++ b/slave.js @@ -58,9 +58,10 @@ const instanceFactorioAutosaveSize = new prometheus.Gauge( * Keeps track of the runtime parameters of an instance */ class Instance extends link.Link{ - constructor(connector, dir, factorioDir, instanceConfig) { + constructor(slave, connector, dir, factorioDir, instanceConfig) { super('instance', 'slave', connector); link.attachAllMessages(this); + this._slave = slave; this._dir = dir; this.plugins = new Map(); @@ -141,21 +142,21 @@ class Instance extends link.Link{ console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`); } - async init(pluginInfos, slave) { + async init(pluginInfos) { await this.server.init(); // load plugins for (let pluginInfo of pluginInfos) { if ( !pluginInfo.instanceEntrypoint - || !slave.serverPlugins.has(pluginInfo.name) + || !this._slave.serverPlugins.has(pluginInfo.name) || !this.config.group(pluginInfo.name).get("enabled") ) { continue; } try { - await this._loadPlugin(pluginInfo, slave); + await this._loadPlugin(pluginInfo, this._slave); } catch (err) { this.notifyExit(); throw err; @@ -801,9 +802,9 @@ class Slave extends link.Link { let [connectionClient, connectionServer] = link.VirtualConnector.makePair(); let instanceConnection = new InstanceConnection(connectionServer, this, instanceId); let instance = new Instance( - connectionClient, instanceInfo.path, this.config.get("slave.factorio_directory"), instanceInfo.config + this, connectionClient, instanceInfo.path, this.config.get("slave.factorio_directory"), instanceInfo.config ); - await instance.init(this.pluginInfos, this); + await instance.init(this.pluginInfos); // XXX: race condition on multiple simultanious calls this.instanceConnections.set(instanceId, instanceConnection); diff --git a/test/slave.js b/test/slave.js index 47a146b9..d9309753 100644 --- a/test/slave.js +++ b/test/slave.js @@ -17,7 +17,7 @@ describe("Slave testing", function() { let instanceConfig = new config.InstanceConfig(); await instanceConfig.init(); instanceConfig.set("instance.name", "foo"); - instance = new slave._Instance(new link.VirtualConnector(), "dir", "factorioDir", instanceConfig); + instance = new slave._Instance({}, new link.VirtualConnector(), "dir", "factorioDir", instanceConfig); }); describe(".name", function() { @@ -109,7 +109,7 @@ describe("Slave testing", function() { await instanceConfig.init(); instanceConfig.set("instance.name", "test"); instance = new slave._Instance( - new link.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig + {}, new link.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig ); await fs.outputFile(instance.path("mods", "mod_i.zip"), "i"); }); From 36cf14074ea0aeb1feda05d77678aa235bc1df96 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 26 May 2020 15:31:42 +0200 Subject: [PATCH 3/3] Automatically upload export to master server Upload the created export data zip file to a new /api/upload-export endpoint on the master server which unpacks it into the appropriate directory on the master. --- lib/factorio/export.js | 4 ++-- master.js | 40 +++++++++++++++++++++++++++++++++++ package.json | 1 + slave.js | 19 ++++++++++++++++- test/integration/clusterio.js | 2 +- 5 files changed, 62 insertions(+), 4 deletions(-) diff --git a/lib/factorio/export.js b/lib/factorio/export.js index 31f81b0f..4c2d08cb 100644 --- a/lib/factorio/export.js +++ b/lib/factorio/export.js @@ -306,6 +306,7 @@ async function exportLocale(server, modVersions, modOrder, languageCode) { * * @param {module:lib/factorio.FactorioServer} server - * The server to export the data from. + * @returns {JSZip} zip file with exported data. * @memberof module:lib/factorio */ async function exportData(server) { @@ -347,8 +348,7 @@ async function exportData(server) { zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG)); zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()])); zip.file("export/locale.json", JSON.stringify([...locale.entries()])); - let content = await zip.generateAsync({ type: "nodebuffer" }); - await fs.outputFile("temp/export.zip", content); + return zip; } diff --git a/master.js b/master.js index 365063bc..f0abf712 100644 --- a/master.js +++ b/master.js @@ -27,6 +27,7 @@ const setBlocking = require("set-blocking"); const events = require("events"); const yargs = require("yargs"); const WebSocket = require("ws"); +const JSZip = require("jszip"); const version = require("./package").version; // ugly globals @@ -174,6 +175,45 @@ async function getMetrics(req, res, next) { } app.get('/metrics', (req, res, next) => getMetrics(req, res, next).catch(next)); +// Handle an uploaded export package. +async function uploadExport(req, res, next) { + endpointHitCounter.labels(req.route.path).inc(); + if (req.get("Content-Type") !== "application/zip") { + res.sendStatus(415); + return; + } + + let data = []; + for await (let chunk of req) { + data.push(chunk); + } + data = Buffer.concat(data); + let zip = await JSZip.loadAsync(data); + delete data; + + // This is hardcoded to prevent path expansion attacks + let exportFiles = [ + "export/item-spritesheet.png", + "export/item-metadata.json", + "export/locale.json", + ]; + + for (let filePath of exportFiles) { + let file = zip.file(filePath); + if (!file) { + continue; + } + + let name = path.posix.basename(filePath); + await fs.outputFile(path.join("static", "export", name), await file.async("nodebuffer")); + } + + res.sendStatus(200); +} +app.put("/api/upload-export", + authenticate.middleware, + (req, res, next) => uploadExport(req, res, next).catch(next) +); const masterConnectedClientsCount = new prometheus.Gauge( 'clusterio_master_connected_clients_count', "How many clients are currently connected to this master server", diff --git a/package.json b/package.json index 831572ca..fc2842fb 100644 --- a/package.json +++ b/package.json @@ -55,6 +55,7 @@ "needle": "^2.4.0", "node-factorio-api": "^0.3.8", "node-forge": "^0.7.1", + "phin": "^3.4.1", "pidusage": "^2.0.18", "rcon-client": "^4.2.0", "request": "^2.83.0", diff --git a/slave.js b/slave.js index ca2301ac..4001d10b 100644 --- a/slave.js +++ b/slave.js @@ -16,6 +16,7 @@ const yargs = require("yargs"); const events = require("events"); const pidusage = require("pidusage"); const setBlocking = require("set-blocking"); +const phin = require("phin"); const version = require("./package").version; // internal libraries @@ -446,7 +447,23 @@ class Instance extends link.Link{ console.log("Exporting data ....."); await symlinkMods(this, "sharedMods", console); - await factorio.exportData(this.server); + let zip = await factorio.exportData(this.server); + + let content = await zip.generateAsync({ type: "nodebuffer" }); + let url = new URL(this._slave.config.get("slave.master_url")); + url.pathname += "api/upload-export"; + let response = await phin({ + url, method: "PUT", + data: content, + core: { rejectUnauthorized: false }, + headers: { + "Content-Type": "application/zip", + "x-access-token": this._slave.config.get("slave.master_token"), + }, + }); + if (response.statusCode !== 200) { + throw Error(`Upload failed: ${response.statusCode} ${response.statusMessage}: ${response.body}`) + } } finally { this.notifyExit(); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 2d5bd828..f48353b4 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -62,7 +62,7 @@ describe("Integration of Clusterio", function() { it("exports the data", async function() { slowTest(this); await exec(`node clusterctl --config ${controlConfigPath} export-data --instance test`); - assert(await fs.exists(path.join("temp", "export.zip")), "Export was not created"); + assert(await fs.exists(path.join("static", "export", "locale.json")), "Export was not created"); }); });