Add shared mods management

Add the ability to manage a set of mods to be shared by the master
server to the rest of the cluster.  This implements storage, uploads,
downloads, deletetion, and listing details of mods but does not actually
do anything with the stored mods.
This commit is contained in:
Hornwitser committed 2022-09-13 19:15:46 +02:00
1 parent 2ab915cc00
commit 3b57c1e0ae
23 files changed
+1311 -5

No files matched your search

+163 -1
View File
@@ -2,11 +2,14 @@
const Busboy = require("busboy");
const crypto = require("crypto");
const events = require("events");
const fs = require("fs-extra");
const JSZip = require("jszip");
const jwt = require("jsonwebtoken");
const path = require("path");
const nodeStream = require("stream");
const util = require("util");
const libData = require("@clusterio/lib/data");
const libErrors = require("@clusterio/lib/errors");
const libFileOps = require("@clusterio/lib/file_ops");
const libHash = require("@clusterio/lib/hash");
@@ -18,6 +21,8 @@ const { logger } = require("@clusterio/lib/logging");
const { endpointHitCounter } = require("./metrics");
const finished = util.promisify(nodeStream.finished);
// Merges samples from sourceResult to destinationResult
function mergeSamples(destinationResult, sourceResult) {
@@ -306,6 +311,8 @@ const zipMimes = [
"application/x-zip-compressed",
];
const contentTypeRegExp = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/;
async function uploadSave(req, res) {
try {
res.locals.user.checkPermission("core.instance.save.upload");
@@ -313,9 +320,10 @@ async function uploadSave(req, res) {
res.status(403).json({ request_errors: [err.message] });
return;
}
endpointHitCounter.labels(req.route.path).inc();
let contentType = req.get("Content-Type");
let match = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/.exec(contentType);
let match = contentTypeRegExp.exec(contentType);
if (!match) {
res.status(415).json({ request_errors: ["invalid Content-Type"] });
return;
@@ -436,6 +444,155 @@ async function uploadSave(req, res) {
res.json({ saves });
}
function checkModName(name) {
try {
libFileOps.checkFilename(name);
} catch (err) {
throw new libErrors.RequestError(`Mod name ${err.message}`);
}
}
async function uploadMod(req, res) {
try {
res.locals.user.checkPermission("core.mod.upload");
} catch (err) {
res.status(403).json({ request_errors: [err.message] });
return;
}
endpointHitCounter.labels(req.route.path).inc();
let contentType = req.get("Content-Type");
let match = contentTypeRegExp.exec(contentType);
if (!match) {
res.status(415).json({ request_errors: ["invalid Content-Type"] });
return;
}
let mimeType = match[1].toLowerCase();
let tasks = [];
let errors = [];
let requestErrors = [];
let mods = [];
async function handleFile(stream, filename) {
try {
checkModName(filename);
} catch (err) {
logger.error(`Error uploading mod: ${err.message}`);
errors.push(err.message);
stream.resume();
return;
}
const modsDirectory = req.app.locals.master.config.get("master.mods_directory");
let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
try {
let writeStream;
while (true) {
try {
writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" });
await events.once(writeStream, "open");
break;
} catch (err) {
if (err.code === "EEXIST") {
tempFilename = await libFileOps.findUnusedName(modsDirectory, tempFilename, ".tmp.zip");
} else {
throw err;
}
}
}
stream.pipe(writeStream);
await finished(writeStream);
const modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, tempFilename));
modInfo.filename = filename;
await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, filename));
req.app.locals.master.mods.set(`${modInfo.name}_${modInfo.version}`, modInfo);
req.app.locals.master.modUpdated(modInfo);
mods.push(modInfo.toJSON());
} catch (err) {
logger.error(`Error uploading mod: ${err.message}`);
errors.push(err.message);
stream.resume();
// Attempt to clean up.
writeStream.destroy();
try {
await fs.unlink(path.join(modsDirectory, tempFilename));
} catch (unlinkErr) {
if (unlinkErr.code !== "ENOENT") {
logger.error(`Error removing ${tempFilename}: ${err.message}`);
}
}
}
}
if (mimeType === "multipart/form-data") {
await new Promise(resolve => {
let busboy = new Busboy({ headers: req.headers });
busboy.on("file", (fieldname, stream, filename, transferEncoding, fileMime) => {
if (!zipMimes.includes(fileMime)) {
requestErrors.push("invalid file Content-Type");
}
if (!filename.endsWith(".zip")) {
requestErrors.push("filename must end with .zip");
}
if (errors.length || requestErrors.length) {
stream.resume();
return;
}
tasks.push(handleFile(stream, filename));
});
busboy.on("finish", resolve);
busboy.on("error", (err) => {
logger.error(`Error parsing multipart request in upload-mod:\n${err.stack}`);
errors.push(err.message);
});
req.pipe(busboy);
});
} else if (zipMimes.includes(mimeType)) {
let filename = req.query.filename;
if (typeof filename !== "string") {
requestErrors.push("Missing or invalid filename parameter");
} else if (!filename.endsWith(".zip")) {
requestErrors.push("filename must end with .zip");
}
if (errors.length || requestErrors.length) {
req.resume();
} else {
tasks.push(handleFile(req, filename));
}
} else {
res.status(415).json({ request_errors: ["invalid Content-Type"] });
return;
}
await Promise.all(tasks);
if (errors.length) {
res.status(500);
res.json({ errors, request_errors: requestErrors });
return;
}
if (requestErrors.length) {
res.status(400);
res.json({ request_errors: requestErrors });
return;
}
res.json({ mods });
}
function addRouteHandlers(app) {
app.get("/metrics", (req, res, next) => getMetrics(req, res, next).catch(next));
app.get("/api/plugins", getPlugins);
@@ -450,6 +607,10 @@ function addRouteHandlers(app) {
validateUserToken,
(req, res, next) => uploadSave(req, res).catch(next)
);
app.post("/api/upload-mod",
validateUserToken,
(req, res, next) => uploadMod(req, res).catch(next)
);
}
// Routes used in the web interface and served by the master server
@@ -460,6 +621,7 @@ const webRoutes = [
"/slaves/:id/view",
"/instances",
"/instances/:id/view",
"/mods",
"/users",
"/users/:id/view",
"/roles",