Merge pull request #297 from Hornwitser/automate-export

Automate export process
This commit is contained in:
Hornwitser authored and GitHub committed 2020-05-26 18:13:17 +02:00
commit 01ea301a8c
7 files changed
+71 -17

No files matched your search

-5
View File
@@ -19,8 +19,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE. SOFTWARE.
The icons for angels modded items located in /static/pictures/ are licensed
by https://mods.factorio.com/user/Arch666Angel under CC BY-NC-ND 4.0
https://creativecommons.org/licenses/by-nc-nd/4.0/
+2 -2
View File
@@ -306,6 +306,7 @@ async function exportLocale(server, modVersions, modOrder, languageCode) {
* *
* @param {module:lib/factorio.FactorioServer} server - * @param {module:lib/factorio.FactorioServer} server -
* The server to export the data from. * The server to export the data from.
* @returns {JSZip} zip file with exported data.
* @memberof module:lib/factorio * @memberof module:lib/factorio
*/ */
async function exportData(server) { async function exportData(server) {
@@ -347,8 +348,7 @@ async function exportData(server) {
zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG)); zip.file("export/item-spritesheet.png", await iconSheet.getBufferAsync(Jimp.MIME_PNG));
zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()])); zip.file("export/item-metadata.json", JSON.stringify([...itemData.entries()]));
zip.file("export/locale.json", JSON.stringify([...locale.entries()])); zip.file("export/locale.json", JSON.stringify([...locale.entries()]));
let content = await zip.generateAsync({ type: "nodebuffer" }); return zip;
await fs.outputFile("temp/export.zip", content);
} }
+40
View File
@@ -27,6 +27,7 @@ const setBlocking = require("set-blocking");
const events = require("events"); const events = require("events");
const yargs = require("yargs"); const yargs = require("yargs");
const WebSocket = require("ws"); const WebSocket = require("ws");
const JSZip = require("jszip");
const version = require("./package").version; const version = require("./package").version;
// ugly globals // ugly globals
@@ -174,6 +175,45 @@ async function getMetrics(req, res, next) {
} }
app.get('/metrics', (req, res, next) => getMetrics(req, res, next).catch(next)); app.get('/metrics', (req, res, next) => getMetrics(req, res, next).catch(next));
// Handle an uploaded export package.
async function uploadExport(req, res, next) {
endpointHitCounter.labels(req.route.path).inc();
if (req.get("Content-Type") !== "application/zip") {
res.sendStatus(415);
return;
}
let data = [];
for await (let chunk of req) {
data.push(chunk);
}
data = Buffer.concat(data);
let zip = await JSZip.loadAsync(data);
delete data;
// This is hardcoded to prevent path expansion attacks
let exportFiles = [
"export/item-spritesheet.png",
"export/item-metadata.json",
"export/locale.json",
];
for (let filePath of exportFiles) {
let file = zip.file(filePath);
if (!file) {
continue;
}
let name = path.posix.basename(filePath);
await fs.outputFile(path.join("static", "export", name), await file.async("nodebuffer"));
}
res.sendStatus(200);
}
app.put("/api/upload-export",
authenticate.middleware,
(req, res, next) => uploadExport(req, res, next).catch(next)
);
const masterConnectedClientsCount = new prometheus.Gauge( const masterConnectedClientsCount = new prometheus.Gauge(
'clusterio_master_connected_clients_count', "How many clients are currently connected to this master server", 'clusterio_master_connected_clients_count', "How many clients are currently connected to this master server",
+1
View File
@@ -55,6 +55,7 @@
"needle": "^2.4.0", "needle": "^2.4.0",
"node-factorio-api": "^0.3.8", "node-factorio-api": "^0.3.8",
"node-forge": "^0.7.1", "node-forge": "^0.7.1",
"phin": "^3.4.1",
"pidusage": "^2.0.18", "pidusage": "^2.0.18",
"rcon-client": "^4.2.0", "rcon-client": "^4.2.0",
"request": "^2.83.0", "request": "^2.83.0",
+25 -7
View File
@@ -16,6 +16,7 @@ const yargs = require("yargs");
const events = require("events"); const events = require("events");
const pidusage = require("pidusage"); const pidusage = require("pidusage");
const setBlocking = require("set-blocking"); const setBlocking = require("set-blocking");
const phin = require("phin");
const version = require("./package").version; const version = require("./package").version;
// internal libraries // internal libraries
@@ -58,9 +59,10 @@ const instanceFactorioAutosaveSize = new prometheus.Gauge(
* Keeps track of the runtime parameters of an instance * Keeps track of the runtime parameters of an instance
*/ */
class Instance extends link.Link{ class Instance extends link.Link{
constructor(connector, dir, factorioDir, instanceConfig) { constructor(slave, connector, dir, factorioDir, instanceConfig) {
super('instance', 'slave', connector); super('instance', 'slave', connector);
link.attachAllMessages(this); link.attachAllMessages(this);
this._slave = slave;
this._dir = dir; this._dir = dir;
this.plugins = new Map(); this.plugins = new Map();
@@ -141,21 +143,21 @@ class Instance extends link.Link{
console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`); console.log(`Clusterio | Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStarted}ms`);
} }
async init(pluginInfos, slave) { async init(pluginInfos) {
await this.server.init(); await this.server.init();
// load plugins // load plugins
for (let pluginInfo of pluginInfos) { for (let pluginInfo of pluginInfos) {
if ( if (
!pluginInfo.instanceEntrypoint !pluginInfo.instanceEntrypoint
|| !slave.serverPlugins.has(pluginInfo.name) || !this._slave.serverPlugins.has(pluginInfo.name)
|| !this.config.group(pluginInfo.name).get("enabled") || !this.config.group(pluginInfo.name).get("enabled")
) { ) {
continue; continue;
} }
try { try {
await this._loadPlugin(pluginInfo, slave); await this._loadPlugin(pluginInfo, this._slave);
} catch (err) { } catch (err) {
this.notifyExit(); this.notifyExit();
throw err; throw err;
@@ -445,7 +447,23 @@ class Instance extends link.Link{
console.log("Exporting data ....."); console.log("Exporting data .....");
await symlinkMods(this, "sharedMods", console); await symlinkMods(this, "sharedMods", console);
await factorio.exportData(this.server); let zip = await factorio.exportData(this.server);
let content = await zip.generateAsync({ type: "nodebuffer" });
let url = new URL(this._slave.config.get("slave.master_url"));
url.pathname += "api/upload-export";
let response = await phin({
url, method: "PUT",
data: content,
core: { rejectUnauthorized: false },
headers: {
"Content-Type": "application/zip",
"x-access-token": this._slave.config.get("slave.master_token"),
},
});
if (response.statusCode !== 200) {
throw Error(`Upload failed: ${response.statusCode} ${response.statusMessage}: ${response.body}`)
}
} finally { } finally {
this.notifyExit(); this.notifyExit();
@@ -801,9 +819,9 @@ class Slave extends link.Link {
let [connectionClient, connectionServer] = link.VirtualConnector.makePair(); let [connectionClient, connectionServer] = link.VirtualConnector.makePair();
let instanceConnection = new InstanceConnection(connectionServer, this, instanceId); let instanceConnection = new InstanceConnection(connectionServer, this, instanceId);
let instance = new Instance( let instance = new Instance(
connectionClient, instanceInfo.path, this.config.get("slave.factorio_directory"), instanceInfo.config this, connectionClient, instanceInfo.path, this.config.get("slave.factorio_directory"), instanceInfo.config
); );
await instance.init(this.pluginInfos, this); await instance.init(this.pluginInfos);
// XXX: race condition on multiple simultanious calls // XXX: race condition on multiple simultanious calls
this.instanceConnections.set(instanceId, instanceConnection); this.instanceConnections.set(instanceId, instanceConnection);
+1 -1
View File
@@ -62,7 +62,7 @@ describe("Integration of Clusterio", function() {
it("exports the data", async function() { it("exports the data", async function() {
slowTest(this); slowTest(this);
await exec(`node clusterctl --config ${controlConfigPath} export-data --instance test`); await exec(`node clusterctl --config ${controlConfigPath} export-data --instance test`);
assert(await fs.exists(path.join("temp", "export.zip")), "Export was not created"); assert(await fs.exists(path.join("static", "export", "locale.json")), "Export was not created");
}); });
}); });
+2 -2
View File
@@ -17,7 +17,7 @@ describe("Slave testing", function() {
let instanceConfig = new config.InstanceConfig(); let instanceConfig = new config.InstanceConfig();
await instanceConfig.init(); await instanceConfig.init();
instanceConfig.set("instance.name", "foo"); instanceConfig.set("instance.name", "foo");
instance = new slave._Instance(new link.VirtualConnector(), "dir", "factorioDir", instanceConfig); instance = new slave._Instance({}, new link.VirtualConnector(), "dir", "factorioDir", instanceConfig);
}); });
describe(".name", function() { describe(".name", function() {
@@ -109,7 +109,7 @@ describe("Slave testing", function() {
await instanceConfig.init(); await instanceConfig.init();
instanceConfig.set("instance.name", "test"); instanceConfig.set("instance.name", "test");
instance = new slave._Instance( instance = new slave._Instance(
new link.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig {}, new link.VirtualConnector(), path.join(testDir, "instance"), "factorioDir", instanceConfig
); );
await fs.outputFile(instance.path("mods", "mod_i.zip"), "i"); await fs.outputFile(instance.path("mods", "mod_i.zip"), "i");
}); });