Add shared mods management

Add the ability to manage a set of mods to be shared by the master
server to the rest of the cluster.  This implements storage, uploads,
downloads, deletetion, and listing details of mods but does not actually
do anything with the stored mods.
This commit is contained in:
Hornwitser committed 2022-09-13 19:15:46 +02:00
1 parent 2ab915cc00
commit 3b57c1e0ae
23 files changed
+1311 -5

No files matched your search

+59 -1
View File
@@ -1,6 +1,8 @@
"use strict";
const jwt = require("jsonwebtoken");
const events = require("events");
const fs = require("fs-extra");
const jwt = require("jsonwebtoken");
const path = require("path");
const libConfig = require("@clusterio/lib/config");
const libErrors = require("@clusterio/lib/errors");
@@ -53,6 +55,11 @@ class ControlConnection extends BaseConnection {
instance_ids: [],
};
this.modSubscriptions = {
all: false,
mod_names: [],
};
this.userSubscriptions = {
all: false,
names: [],
@@ -458,11 +465,62 @@ class ControlConnection extends BaseConnection {
return { save: result.save };
}
async getModRequestHandler(message) {
let { name, version } = message.data;
let key = `${name}_${version}`;
let mod = this._master.mods.get(key);
if (!mod) {
throw new libErrors.RequestError(`Mod ${key} does not exist`);
}
return {
mod: mod.toJSON(),
};
}
async listModsRequestHandler(message) {
return { list: [...this._master.mods.values()].map(mod => mod.toJSON()) };
}
async setModSubscriptionsRequestHandler(message) {
this.modSubscriptions = message.data;
}
async downloadModRequestHandler(message) {
let { name, version } = message.data;
let key = `${name}_${version}`;
let mod = this._master.mods.get(key);
if (!mod) {
throw new libErrors.RequestError(`Mod ${key} does not exist`);
}
let modPath = path.join(this._master.config.get("master.mods_directory"), mod.filename);
let stream = await routes.createProxyStream(this._master.app);
stream.filename = mod.filename;
stream.source = fs.createReadStream(modPath);
stream.mime = "application/zip";
stream.size = mod.size;
return { stream_id: stream.id };
}
async deleteModRequestHandler(message) {
await this._master.deleteMod(message.data.name, message.data.version);
}
async setLogSubscriptionsRequestHandler(message) {
this.logSubscriptions = message.data;
this.updateLogSubscriptions();
}
modUpdated(mod) {
if (
this.modSubscriptions.all
|| this.modSubscriptions.mod_names.includes(mod.name)
) {
libLink.messages.modUpdate.send(this, { mod: mod.toJSON() });
}
}
updateLogSubscriptions() {
let { all, master, slave_ids, instance_ids } = this.logSubscriptions;
if (all || master || slave_ids.length || instance_ids.length) {
+71
View File
@@ -9,6 +9,7 @@ const path = require("path");
const libConfig = require("@clusterio/lib/config");
const libData = require("@clusterio/lib/data");
const libErrors = require("@clusterio/lib/errors");
const libFileOps = require("@clusterio/lib/file_ops");
const libLink = require("@clusterio/lib/link");
@@ -65,6 +66,12 @@ class Master {
*/
this.instances = null;
/**
* Mapping of mod names to mod infos
* @type {Map<string, module:lib/data/ModInfo>}
*/
this.mods = null;
/**
* User and roles manager for the cluster
* @type {module:master/src/UserManager}
@@ -165,6 +172,10 @@ class Master {
await this.userManager.load(path.join(databaseDirectory, "users.json"));
this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json"));
let modsDirectory = this.config.get("master.mods_directory");
await fs.ensureDir(modsDirectory);
this.mods = await Master.loadModInfos(modsDirectory);
this.config.on("fieldChanged", (group, field, prev) => {
libPlugin.invokeHook(this.plugins, "onMasterConfigFieldChanged", group, field, prev);
});
@@ -360,6 +371,40 @@ class Master {
await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4));
}
static async loadModInfos(modsDirectory) {
let mods = new Map();
for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) {
if (entry.isDirectory()) {
logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`);
continue;
}
if (entry.name.toLowerCase().endsWith("tmp.zip")) {
continue;
}
logger.info(`Loading info for Mod ${entry.name}`);
let modInfo;
try {
modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, entry.name));
} catch (err) {
logger.error(`Error loading mod ${entry.name}: ${err.message}`);
continue;
}
let key = `${modInfo.name}_${modInfo.version}`;
if (mods.has(key)) {
logger.error(
`Ignoring Mod ${entry.name} as it contains the same name and ` +
`version as ${mods.get(key).filename}`
);
continue;
}
mods.set(key, modInfo);
}
return mods;
}
static async loadJsonObject(filePath, throwOnMissing = false) {
let manifest = {};
try {
@@ -380,6 +425,20 @@ class Master {
this.exportManifest = newManifest;
}
async deleteMod(name, version) {
let key = `${name}_${version}`;
let mod = this.mods.get(key);
if (!mod) {
throw new Error(`Mod ${key} does not exist`);
}
let modsDirectory = this.config.get("master.mods_directory");
await fs.unlink(path.join(modsDirectory, mod.filename));
this.mods.delete(key);
mod.isDeleted = true;
this.modUpdated(mod);
}
/**
* Query master log
*
@@ -503,6 +562,18 @@ class Master {
}
}
modUpdated(mod) {
for (let controlConnection of this.wsServer.controlConnections) {
if (controlConnection.connector.closing) {
continue;
}
controlConnection.modUpdated(mod);
}
libPlugin.invokeHook(this.plugins, "onModUpdated", mod);
}
userUpdated(user) {
for (let controlConnection of this.wsServer.controlConnections) {
if (controlConnection.connector.closing) {
+163 -1
View File
@@ -2,11 +2,14 @@
const Busboy = require("busboy");
const crypto = require("crypto");
const events = require("events");
const fs = require("fs-extra");
const JSZip = require("jszip");
const jwt = require("jsonwebtoken");
const path = require("path");
const nodeStream = require("stream");
const util = require("util");
const libData = require("@clusterio/lib/data");
const libErrors = require("@clusterio/lib/errors");
const libFileOps = require("@clusterio/lib/file_ops");
const libHash = require("@clusterio/lib/hash");
@@ -18,6 +21,8 @@ const { logger } = require("@clusterio/lib/logging");
const { endpointHitCounter } = require("./metrics");
const finished = util.promisify(nodeStream.finished);
// Merges samples from sourceResult to destinationResult
function mergeSamples(destinationResult, sourceResult) {
@@ -306,6 +311,8 @@ const zipMimes = [
"application/x-zip-compressed",
];
const contentTypeRegExp = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/;
async function uploadSave(req, res) {
try {
res.locals.user.checkPermission("core.instance.save.upload");
@@ -313,9 +320,10 @@ async function uploadSave(req, res) {
res.status(403).json({ request_errors: [err.message] });
return;
}
endpointHitCounter.labels(req.route.path).inc();
let contentType = req.get("Content-Type");
let match = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/.exec(contentType);
let match = contentTypeRegExp.exec(contentType);
if (!match) {
res.status(415).json({ request_errors: ["invalid Content-Type"] });
return;
@@ -436,6 +444,155 @@ async function uploadSave(req, res) {
res.json({ saves });
}
function checkModName(name) {
try {
libFileOps.checkFilename(name);
} catch (err) {
throw new libErrors.RequestError(`Mod name ${err.message}`);
}
}
async function uploadMod(req, res) {
try {
res.locals.user.checkPermission("core.mod.upload");
} catch (err) {
res.status(403).json({ request_errors: [err.message] });
return;
}
endpointHitCounter.labels(req.route.path).inc();
let contentType = req.get("Content-Type");
let match = contentTypeRegExp.exec(contentType);
if (!match) {
res.status(415).json({ request_errors: ["invalid Content-Type"] });
return;
}
let mimeType = match[1].toLowerCase();
let tasks = [];
let errors = [];
let requestErrors = [];
let mods = [];
async function handleFile(stream, filename) {
try {
checkModName(filename);
} catch (err) {
logger.error(`Error uploading mod: ${err.message}`);
errors.push(err.message);
stream.resume();
return;
}
const modsDirectory = req.app.locals.master.config.get("master.mods_directory");
let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
try {
let writeStream;
while (true) {
try {
writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" });
await events.once(writeStream, "open");
break;
} catch (err) {
if (err.code === "EEXIST") {
tempFilename = await libFileOps.findUnusedName(modsDirectory, tempFilename, ".tmp.zip");
} else {
throw err;
}
}
}
stream.pipe(writeStream);
await finished(writeStream);
const modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, tempFilename));
modInfo.filename = filename;
await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, filename));
req.app.locals.master.mods.set(`${modInfo.name}_${modInfo.version}`, modInfo);
req.app.locals.master.modUpdated(modInfo);
mods.push(modInfo.toJSON());
} catch (err) {
logger.error(`Error uploading mod: ${err.message}`);
errors.push(err.message);
stream.resume();
// Attempt to clean up.
writeStream.destroy();
try {
await fs.unlink(path.join(modsDirectory, tempFilename));
} catch (unlinkErr) {
if (unlinkErr.code !== "ENOENT") {
logger.error(`Error removing ${tempFilename}: ${err.message}`);
}
}
}
}
if (mimeType === "multipart/form-data") {
await new Promise(resolve => {
let busboy = new Busboy({ headers: req.headers });
busboy.on("file", (fieldname, stream, filename, transferEncoding, fileMime) => {
if (!zipMimes.includes(fileMime)) {
requestErrors.push("invalid file Content-Type");
}
if (!filename.endsWith(".zip")) {
requestErrors.push("filename must end with .zip");
}
if (errors.length || requestErrors.length) {
stream.resume();
return;
}
tasks.push(handleFile(stream, filename));
});
busboy.on("finish", resolve);
busboy.on("error", (err) => {
logger.error(`Error parsing multipart request in upload-mod:\n${err.stack}`);
errors.push(err.message);
});
req.pipe(busboy);
});
} else if (zipMimes.includes(mimeType)) {
let filename = req.query.filename;
if (typeof filename !== "string") {
requestErrors.push("Missing or invalid filename parameter");
} else if (!filename.endsWith(".zip")) {
requestErrors.push("filename must end with .zip");
}
if (errors.length || requestErrors.length) {
req.resume();
} else {
tasks.push(handleFile(req, filename));
}
} else {
res.status(415).json({ request_errors: ["invalid Content-Type"] });
return;
}
await Promise.all(tasks);
if (errors.length) {
res.status(500);
res.json({ errors, request_errors: requestErrors });
return;
}
if (requestErrors.length) {
res.status(400);
res.json({ request_errors: requestErrors });
return;
}
res.json({ mods });
}
function addRouteHandlers(app) {
app.get("/metrics", (req, res, next) => getMetrics(req, res, next).catch(next));
app.get("/api/plugins", getPlugins);
@@ -450,6 +607,10 @@ function addRouteHandlers(app) {
validateUserToken,
(req, res, next) => uploadSave(req, res).catch(next)
);
app.post("/api/upload-mod",
validateUserToken,
(req, res, next) => uploadMod(req, res).catch(next)
);
}
// Routes used in the web interface and served by the master server
@@ -460,6 +621,7 @@ const webRoutes = [
"/slaves/:id/view",
"/instances",
"/instances/:id/view",
"/mods",
"/users",
"/users/:id/view",
"/roles",