diff --git a/.gitignore b/.gitignore index 84213197..a769fb6c 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ Factorio* /factorio /instances /sharedMods +/mods /database /plugin-list.json /config-master.json diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 9211543c..d054f89b 100755 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -89,6 +89,8 @@ class Control extends libLink.Link { async saveListUpdateEventHandler() { } + async modUpdateEventHandler() { } + async userUpdateEventHandler() { } async setLogSubscriptions({ diff --git a/packages/ctl/src/commands.js b/packages/ctl/src/commands.js index ba5df6bd..2d09982c 100644 --- a/packages/ctl/src/commands.js +++ b/packages/ctl/src/commands.js @@ -860,6 +860,151 @@ instanceCommands.add(new libCommand.Command({ }, })); +const modCommands = new libCommand.CommandTree({ name: "mod", description: "Manage uploaded mods" }); +modCommands.add(new libCommand.Command({ + definition: ["show ", "Show details for a mod stored in the cluster", (yargs) => { + yargs.positional("name", { describe: "Mod name to show details for", type: "string" }); + yargs.positional("mod-version", { describe: "Version of the mod", type: "string" }); + }], + handler: async function(args, control) { + let response = await libLink.messages.getMod.send(control, { name: args.name, version: args.modVersion }); + for (let [field, value] of Object.entries(response.mod)) { + if (value instanceof Array) { + print(`${field}:`); + for (let entry of value) { + print(` ${entry}`); + } + } else { + print(`${field}: ${value}`); + } + } + }, +})); + +modCommands.add(new libCommand.Command({ + definition: [["list", "l"], "List mods stored in the cluster", (yargs) => { + yargs.options({ + "fields": { + describe: "Fields", + array: true, + type: "string", + default: ["name", "version", "title", "factorio_version"], + }, + }); + }], + handler: async function(args, control) { + let response = await libLink.messages.listMods.send(control); + if (!args.fields.includes("all")) { + for (let entry of response.list) { + for (let field of Object.keys(entry)) { + if (!args.fields.includes(field)) { + delete entry[field]; + } + } + } + } + print(asTable(response.list)); + }, +})); + +modCommands.add(new libCommand.Command({ + definition: ["upload ", "Upload mod to the cluster", (yargs) => { + yargs.positional("file", { describe: "File to upload", type: "string" }); + }], + handler: async function(args, control) { + let filename = path.basename(args.file); + if (!filename.endsWith(".zip")) { + throw new libErrors.CommandError("Mod filename must end with .zip"); + } + // phin doesn't support streaming requests :( + let content = await fs.readFile(args.file); + + let url = new URL(control.config.get("control.master_url")); + url.pathname += "api/upload-mod"; + url.searchParams.append("filename", filename); + + let result = await phin({ + url, method: "POST", + headers: { + "X-Access-Token": control.config.get("control.master_token"), + "Content-Type": "application/zip", + }, + core: { ca: control.tlsCa }, + data: content, + parse: "json", + }); + + for (let error of result.body.errors || []) { + logger.error(error); + } + + for (let requestError of result.body.request_errors || []) { + logger.error(requestError); + } + + if (result.body.mods && result.body.mods.length) { + logger.info(`Successfully uploaded ${result.body.mods[0].filename}`); + } + + if ((result.body.errors || []).length || (result.body.request_errors || []).length) { + throw new libErrors.CommandError("Uploading mod failed"); + } + }, +})); + +modCommands.add(new libCommand.Command({ + definition: ["download ", "Download a mod from the cluster", (yargs) => { + yargs.positional("name", { describe: "Internal name of mod to download", type: "string" }); + yargs.positional("mod-version", { describe: "Version of mod to download", type: "string" }); + }], + handler: async function(args, control) { + let result = await libLink.messages.downloadMod.send(control, { + name: args.name, + version: args.modVersion, + }); + + let url = new URL(control.config.get("control.master_url")); + url.pathname += `api/stream/${result.stream_id}`; + let response = await phin({ + url, method: "GET", + core: { ca: control.tlsCa }, + stream: true, + }); + + let writeStream; + let filename = `${args.name}_${args.modVersion}.zip`; + let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); + while (true) { + try { + writeStream = fs.createWriteStream(tempFilename, { flags: "wx" }); + await events.once(writeStream, "open"); + break; + } catch (err) { + if (err.code === "EEXIST") { + tempFilename = await libFileOps.findUnusedName(".", tempFilename, ".tmp.zip"); + } else { + throw err; + } + } + } + response.pipe(writeStream); + await finished(writeStream); + await fs.rename(tempFilename, filename); + + logger.info(`Downloaded ${filename}`); + }, +})); + +modCommands.add(new libCommand.Command({ + definition: ["delete ", "Delete a mod stored in the cluster", (yargs) => { + yargs.positional("name", { describe: "Name of mod to delete", type: "string" }); + yargs.positional("mod-version", { describe: "Version of mod to delete", type: "string" }); + }], + handler: async function(args, control) { + await libLink.messages.deleteMod.send(control, { name: args.name, version: args.modVersion }); + }, +})); + const permissionCommands = new libCommand.CommandTree({ name: "permission", description: "Permission inspection" }); permissionCommands.add(new libCommand.Command({ definition: [["list", "l"], "List permissions in the cluster"], @@ -1194,6 +1339,7 @@ async function registerCommands(controlPlugins, yargs) { rootCommands.add(masterCommands); rootCommands.add(slaveCommands); rootCommands.add(instanceCommands); + rootCommands.add(modCommands); rootCommands.add(permissionCommands); rootCommands.add(roleCommands); rootCommands.add(userCommands); diff --git a/packages/lib/browser.js b/packages/lib/browser.js index 23c9850e..20866a72 100644 --- a/packages/lib/browser.js +++ b/packages/lib/browser.js @@ -1,4 +1,5 @@ export { default as libConfig } from "./config"; +export { default as libData } from "./data"; export { default as libErrors } from "./errors"; export { default as libFactorio } from "./factorio/exchange_string"; export { default as libHelpers } from "./helpers"; diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index e713f48f..01d74f78 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -19,6 +19,13 @@ MasterGroup.define({ type: "string", initial_value: "Your Cluster", }); +MasterGroup.define({ + name: "mods_directory", + title: "Mods Directory", + description: "Path to directory where mods shared with the cluster are stored.", + type: "string", + initial_value: "mods", +}); MasterGroup.define({ name: "database_directory", title: "Database directory", diff --git a/packages/lib/data/ModInfo.js b/packages/lib/data/ModInfo.js new file mode 100644 index 00000000..e1949d0c --- /dev/null +++ b/packages/lib/data/ModInfo.js @@ -0,0 +1,230 @@ +"use strict"; +const fs = require("fs-extra"); +const JSZip = require("jszip"); +const path = require("path"); + +const libHash = require("../hash"); +const libSchema = require("../schema"); +const { findRoot } = require("../zip_ops"); + + +/** + * Info about a mod available on the master server. + * + * See https://wiki.factorio.com/Tutorial:Mod_structure#info.json for + * details of the invidiual fields sourced from info.json. + * @alias module:lib/data/ModInfo + */ +class ModInfo { + /** + * Internal name of this mod. + * This is the name of the zip file as well as the name that appears in + * info.json. + * @type {string} + */ + name = ""; + + /** + * Version of the mod. + * Sourced from info.json. + * @type {string} + */ + version = ""; + + /** + * Integer representation of the version + * @type {number} + */ + get integerVersion() { + const [major, minor, sub] = this.version.split(".").map(n => Number.parseInt(n, 10)); + return major * 0x100000000 + minor * 0x10000 + sub; // Can't use bitwise here because this is 48-bits. + } + + /** + * Display name of the mod. + * Sourced from info.json. + * @type {string} + */ + title = ""; + + /** + * Author of the mod. + * Sourced from info.json. + * @type {string} + */ + author = ""; + + /** + * Contact field for mod. + * Sourced from info.json. + * @type {string} + */ + contact = ""; + + /** + * Homepage of the mod. + * Sourced from info.json. + * @type {string} + */ + homepage = ""; + + /** + * Description of the mod. + * Sourced from info.json. + * @type {string} + */ + description = ""; + + /** + * Major version of Factorio this mod supports. + * Sourced from info.json. + * @type {string} + */ + factorioVersion = "0.12"; + + /** + * Integer representation of the factorioVersion + * @type {number} + */ + get integerFactorioVersion() { + const [major, minor] = this.factorioVersion.split(".").map(n => Number.parseInt(n, 10)); + return major * 0x100000000 + minor * 0x10000; // Can't use bitwise here because this is 48-bits. + } + + /** + * Dependiences for this mod. + * Sourced from info.json. + * @type {Array} + */ + dependencies = ["base"]; + + /** + * Name of the file containing this mod. + * @type {number} + */ + filename = ""; + + /** + * Size of the mod in bytes + * @type {number} + */ + size = 0; + + /** + * Hash of this mod + * @type {string=} + */ + hash; + + /** + * True if this mod has been deleted + * @type {boolean} + */ + isDeleted = false; + + // Content of info.json found in mod files + static infoJsonSchema = { + type: "object", + required: ["name", "version", "title", "author"], + properties: { + "name": { type: "string" }, + "version": { type: "string" }, + "title": { type: "string" }, + "author": { type: "string" }, + "contact": { type: "string" }, + "homepage": { type: "string" }, + "description": { type: "string" }, + "factorio_version": { type: "string" }, + "dependencies": { type: "array", items: { type: "string" } }, + }, + }; + + static validateInfo = libSchema.compile(this.infoJsonSchema); + + static jsonSchema = { + type: "object", + properties: { + ...this.infoJsonSchema.properties, + "filename": { type: "string" }, + "size": { type: "integer" }, + "hash": { type: "string" }, + "is_deleted": { type: "boolean" }, + }, + }; + + static validate = libSchema.compile(this.jsonSchema); + + constructor(json = {}) { + // info.json fields + if (json.name) { this.name = json.name; } + if (json.version) { this.version = json.version; } + if (json.title) { this.title = json.title; } + if (json.author) { this.author = json.author; } + if (json.contact) { this.contact = json.contact; } + if (json.homepage) { this.homepage = json.homepage; } + if (json.description) { this.description = json.description; } + if (json.factorio_version) { this.factorioVersion = json.factorio_version; } + if (json.dependencies) { this.dependencies = json.dependencies; } + + // Additional data + if (json.filename) { this.filename = json.filename; } + if (json.size) { this.size = json.size; } + if (json.hash) { this.hash = json.hash; } + if (json.is_deleted) { this.isDeleted = json.is_deleted; } + } + + toJSON() { + let json = { + name: this.name, + version: this.version, + title: this.title, + author: this.author, + }; + if (this.contact) { json.contact = this.contact; } + if (this.homepage) { json.homepage = this.homepage; } + if (this.description) { json.description = this.description; } + if (this.factorio_version !== "0.12") { json.factorio_version = this.factorioVersion; } + if (this.dependencies.length !== 1 || this.dependencies[0] !== "base") { + json.dependencies = this.dependencies; + } + if (this.filename) { json.filename = this.filename; } + if (this.size) { json.size = this.size; } + if (this.hash) { json.hash = this.hash; } + if (this.isDeleted) { json.is_deleted = this.isDeleted; } + return json; + } + + static async fromModFile(modPath) { + let modInfo; + { + // XXX: JSZip needs the whole archive loaded in memory to work. + // This is clearly untenable and will be replaced later. + let zip = await JSZip.loadAsync(await fs.readFile(modPath)); + let root = zip.folder(findRoot(zip)); + + let infoFile = root.file("info.json"); + if (!infoFile) { + throw new Error("Mod contains no info.json file"); + } + + modInfo = JSON.parse(await infoFile.async("string")); + } + + let valid = this.validateInfo(modInfo); + if (!valid) { + throw new Error("Mod's info.json is not valid"); + } + + let hash = `sha1:${await libHash.hashFile(modPath)}`; + return new this({ + ...modInfo, + + filename: path.basename(modPath), + size: (await fs.stat(modPath)).size, + hash, + is_deleted: false, + }); + } +} + +module.exports = ModInfo; diff --git a/packages/lib/data/index.js b/packages/lib/data/index.js new file mode 100644 index 00000000..17402e53 --- /dev/null +++ b/packages/lib/data/index.js @@ -0,0 +1,12 @@ +/** + * Shared data types used in Clusterio + * @module lib/data + * @author Hornwitser + */ +"use strict"; +const ModInfo = require("./ModInfo"); + +module.exports = { + ModInfo, +}; + diff --git a/packages/lib/index.js b/packages/lib/index.js index 37777950..a642c629 100644 --- a/packages/lib/index.js +++ b/packages/lib/index.js @@ -2,6 +2,7 @@ const libBuildMod = require("./build_mod"); const libCommand = require("./command"); const libConfig = require("./config"); +const libData = require("./data"); const libDatabase = require("./database"); const libErrors = require("./errors"); const libFactorio = require("./factorio"); @@ -28,6 +29,7 @@ module.exports = { libBuildMod, libCommand, libConfig, + libData, libDatabase, libErrors, libFactorio, diff --git a/packages/lib/link/messages.js b/packages/lib/link/messages.js index 5b5361da..c066a0cb 100644 --- a/packages/lib/link/messages.js +++ b/packages/lib/link/messages.js @@ -1,8 +1,9 @@ // Message definitions for links "use strict"; -const libSchema = require("../schema"); +const libData = require("../data"); const libErrors = require("../errors"); const { logger } = require("../logging"); +const libSchema = require("../schema"); const PlayerStats = require("../PlayerStats"); class MissingLinkHandlerError extends Error { @@ -709,6 +710,67 @@ messages.sendRcon = new Request({ }, }); +messages.getMod = new Request({ + type: "get_mod", + links: ["control-master"], + permission: "core.mod.get", + requestProperties: { + "name": { type: "string" }, + "version": { type: "string" }, + }, + responseProperties: { + "mod": libData.ModInfo.jsonSchema, + }, +}); + +messages.listMods = new Request({ + type: "list_mods", + links: ["control-master"], + permission: "core.mod.list", + responseProperties: { + "list": { + type: "array", + items: libData.ModInfo.jsonSchema, + }, + }, +}); + +messages.setModSubscriptions = new Request({ + type: "set_mod_subscriptions", + links: ["control-master"], + permission: "core.mod.subscribe", + requestProperties: { + "all": { type: "boolean" }, + "mod_names": { + type: "array", + items: { type: "string" }, + }, + }, +}); + +messages.downloadMod = new Request({ + type: "download_mod", + links: ["control-master"], + permission: "core.mod.download", + requestProperties: { + "name": { type: "string" }, + "version": { type: "string" }, + }, + responseProperties: { + "stream_id": { type: "string" }, + }, +}); + +messages.deleteMod = new Request({ + type: "delete_mod", + links: ["control-master"], + permission: "core.mod.delete", + requestProperties: { + "name": { type: "string" }, + "version": { type: "string" }, + }, +}); + messages.listPermissions = new Request({ type: "list_permissions", links: ["control-master"], @@ -1263,6 +1325,14 @@ messages.saveListUpdate = new Event({ }, }); +messages.modUpdate = new Event({ + type: "mod_update", + links: ["master-control"], + eventProperties: { + "mod": libData.ModInfo.jsonSchema, + }, +}); + messages.userUpdate = new Event({ type: "user_update", links: ["master-control"], diff --git a/packages/lib/plugin.js b/packages/lib/plugin.js index 9a1cc25f..22fb96e4 100644 --- a/packages/lib/plugin.js +++ b/packages/lib/plugin.js @@ -479,7 +479,6 @@ class BaseMasterPlugin { */ onControlConnectionEvent(connection, event) { } - /** * Called when a slave is preparing to disconnect from the master * @@ -494,6 +493,18 @@ class BaseMasterPlugin { */ async onPrepareSlaveDisconnect(connection) { } + /** + * Called when a mod stored on the master server is updated + * + * Invoked when a mod has been added, updated or deleted from the pool + * of shared mods stored on the cluster. + * + * If a mod has been deleted its `.isDeleted` property will be true. + * + * @param {module:lib/data/ModInfo} mod - Mod that updated. + */ + async onModUpdated(mod) { } + /** * Called when a player joins or leaves an instance * diff --git a/packages/lib/users.js b/packages/lib/users.js index a212c519..300393d3 100644 --- a/packages/lib/users.js +++ b/packages/lib/users.js @@ -511,6 +511,41 @@ definePermission({ description: "Send arbitrary RCON commands to instances.", }); +definePermission({ + name: "core.mod.get", + title: "Get mods", + description: "Get the details of a mod stored in the cluster.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.list", + title: "List mods", + description: "Get the full list of mods stored on the master", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.subscribe", + title: "Subscribe to mod updates", + description: "Subscribe to be notified on updates on the details of mods.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.upload", + title: "Upload mod", + description: "Upload mods to the master.", +}); +definePermission({ + name: "core.mod.download", + title: "Download mod", + description: "Download mods stored on the master.", + grantByDefault: true, +}); +definePermission({ + name: "core.mod.delete", + title: "Delete mod", + description: "Delete mods stored on the master.", +}); + definePermission({ name: "core.permission.list", title: "List permissions", diff --git a/packages/master/src/ControlConnection.js b/packages/master/src/ControlConnection.js index e60f1fd5..a090712b 100644 --- a/packages/master/src/ControlConnection.js +++ b/packages/master/src/ControlConnection.js @@ -1,6 +1,8 @@ "use strict"; -const jwt = require("jsonwebtoken"); const events = require("events"); +const fs = require("fs-extra"); +const jwt = require("jsonwebtoken"); +const path = require("path"); const libConfig = require("@clusterio/lib/config"); const libErrors = require("@clusterio/lib/errors"); @@ -53,6 +55,11 @@ class ControlConnection extends BaseConnection { instance_ids: [], }; + this.modSubscriptions = { + all: false, + mod_names: [], + }; + this.userSubscriptions = { all: false, names: [], @@ -458,11 +465,62 @@ class ControlConnection extends BaseConnection { return { save: result.save }; } + async getModRequestHandler(message) { + let { name, version } = message.data; + let key = `${name}_${version}`; + let mod = this._master.mods.get(key); + if (!mod) { + throw new libErrors.RequestError(`Mod ${key} does not exist`); + } + return { + mod: mod.toJSON(), + }; + } + + async listModsRequestHandler(message) { + return { list: [...this._master.mods.values()].map(mod => mod.toJSON()) }; + } + + async setModSubscriptionsRequestHandler(message) { + this.modSubscriptions = message.data; + } + + async downloadModRequestHandler(message) { + let { name, version } = message.data; + let key = `${name}_${version}`; + let mod = this._master.mods.get(key); + if (!mod) { + throw new libErrors.RequestError(`Mod ${key} does not exist`); + } + let modPath = path.join(this._master.config.get("master.mods_directory"), mod.filename); + + let stream = await routes.createProxyStream(this._master.app); + stream.filename = mod.filename; + stream.source = fs.createReadStream(modPath); + stream.mime = "application/zip"; + stream.size = mod.size; + + return { stream_id: stream.id }; + } + + async deleteModRequestHandler(message) { + await this._master.deleteMod(message.data.name, message.data.version); + } + async setLogSubscriptionsRequestHandler(message) { this.logSubscriptions = message.data; this.updateLogSubscriptions(); } + modUpdated(mod) { + if ( + this.modSubscriptions.all + || this.modSubscriptions.mod_names.includes(mod.name) + ) { + libLink.messages.modUpdate.send(this, { mod: mod.toJSON() }); + } + } + updateLogSubscriptions() { let { all, master, slave_ids, instance_ids } = this.logSubscriptions; if (all || master || slave_ids.length || instance_ids.length) { diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index 8268021c..994916c1 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -9,6 +9,7 @@ const path = require("path"); const libConfig = require("@clusterio/lib/config"); +const libData = require("@clusterio/lib/data"); const libErrors = require("@clusterio/lib/errors"); const libFileOps = require("@clusterio/lib/file_ops"); const libLink = require("@clusterio/lib/link"); @@ -65,6 +66,12 @@ class Master { */ this.instances = null; + /** + * Mapping of mod names to mod infos + * @type {Map} + */ + this.mods = null; + /** * User and roles manager for the cluster * @type {module:master/src/UserManager} @@ -165,6 +172,10 @@ class Master { await this.userManager.load(path.join(databaseDirectory, "users.json")); this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json")); + let modsDirectory = this.config.get("master.mods_directory"); + await fs.ensureDir(modsDirectory); + this.mods = await Master.loadModInfos(modsDirectory); + this.config.on("fieldChanged", (group, field, prev) => { libPlugin.invokeHook(this.plugins, "onMasterConfigFieldChanged", group, field, prev); }); @@ -360,6 +371,40 @@ class Master { await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4)); } + static async loadModInfos(modsDirectory) { + let mods = new Map(); + for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) { + if (entry.isDirectory()) { + logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`); + continue; + } + + if (entry.name.toLowerCase().endsWith("tmp.zip")) { + continue; + } + + logger.info(`Loading info for Mod ${entry.name}`); + let modInfo; + try { + modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, entry.name)); + } catch (err) { + logger.error(`Error loading mod ${entry.name}: ${err.message}`); + continue; + } + + let key = `${modInfo.name}_${modInfo.version}`; + if (mods.has(key)) { + logger.error( + `Ignoring Mod ${entry.name} as it contains the same name and ` + + `version as ${mods.get(key).filename}` + ); + continue; + } + mods.set(key, modInfo); + } + return mods; + } + static async loadJsonObject(filePath, throwOnMissing = false) { let manifest = {}; try { @@ -380,6 +425,20 @@ class Master { this.exportManifest = newManifest; } + async deleteMod(name, version) { + let key = `${name}_${version}`; + let mod = this.mods.get(key); + if (!mod) { + throw new Error(`Mod ${key} does not exist`); + } + + let modsDirectory = this.config.get("master.mods_directory"); + await fs.unlink(path.join(modsDirectory, mod.filename)); + this.mods.delete(key); + mod.isDeleted = true; + this.modUpdated(mod); + } + /** * Query master log * @@ -503,6 +562,18 @@ class Master { } } + modUpdated(mod) { + for (let controlConnection of this.wsServer.controlConnections) { + if (controlConnection.connector.closing) { + continue; + } + + controlConnection.modUpdated(mod); + } + + libPlugin.invokeHook(this.plugins, "onModUpdated", mod); + } + userUpdated(user) { for (let controlConnection of this.wsServer.controlConnections) { if (controlConnection.connector.closing) { diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index 4052554a..a34a2259 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -2,11 +2,14 @@ const Busboy = require("busboy"); const crypto = require("crypto"); const events = require("events"); +const fs = require("fs-extra"); const JSZip = require("jszip"); const jwt = require("jsonwebtoken"); const path = require("path"); +const nodeStream = require("stream"); const util = require("util"); +const libData = require("@clusterio/lib/data"); const libErrors = require("@clusterio/lib/errors"); const libFileOps = require("@clusterio/lib/file_ops"); const libHash = require("@clusterio/lib/hash"); @@ -18,6 +21,8 @@ const { logger } = require("@clusterio/lib/logging"); const { endpointHitCounter } = require("./metrics"); +const finished = util.promisify(nodeStream.finished); + // Merges samples from sourceResult to destinationResult function mergeSamples(destinationResult, sourceResult) { @@ -306,6 +311,8 @@ const zipMimes = [ "application/x-zip-compressed", ]; +const contentTypeRegExp = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/; + async function uploadSave(req, res) { try { res.locals.user.checkPermission("core.instance.save.upload"); @@ -313,9 +320,10 @@ async function uploadSave(req, res) { res.status(403).json({ request_errors: [err.message] }); return; } + endpointHitCounter.labels(req.route.path).inc(); let contentType = req.get("Content-Type"); - let match = /^([!#$%&'*+\-.^_`|~0-9A-Za-z]+\/[!#$%&'*+\-.^_`|~0-9A-Za-z]+)/.exec(contentType); + let match = contentTypeRegExp.exec(contentType); if (!match) { res.status(415).json({ request_errors: ["invalid Content-Type"] }); return; @@ -436,6 +444,155 @@ async function uploadSave(req, res) { res.json({ saves }); } +function checkModName(name) { + try { + libFileOps.checkFilename(name); + } catch (err) { + throw new libErrors.RequestError(`Mod name ${err.message}`); + } +} + +async function uploadMod(req, res) { + try { + res.locals.user.checkPermission("core.mod.upload"); + } catch (err) { + res.status(403).json({ request_errors: [err.message] }); + return; + } + endpointHitCounter.labels(req.route.path).inc(); + + let contentType = req.get("Content-Type"); + let match = contentTypeRegExp.exec(contentType); + if (!match) { + res.status(415).json({ request_errors: ["invalid Content-Type"] }); + return; + } + let mimeType = match[1].toLowerCase(); + + let tasks = []; + let errors = []; + let requestErrors = []; + let mods = []; + + async function handleFile(stream, filename) { + try { + checkModName(filename); + + } catch (err) { + logger.error(`Error uploading mod: ${err.message}`); + errors.push(err.message); + stream.resume(); + return; + } + + const modsDirectory = req.app.locals.master.config.get("master.mods_directory"); + let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); + try { + + let writeStream; + while (true) { + try { + writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" }); + await events.once(writeStream, "open"); + break; + } catch (err) { + if (err.code === "EEXIST") { + tempFilename = await libFileOps.findUnusedName(modsDirectory, tempFilename, ".tmp.zip"); + } else { + throw err; + } + } + } + stream.pipe(writeStream); + await finished(writeStream); + + const modInfo = await libData.ModInfo.fromModFile(path.join(modsDirectory, tempFilename)); + modInfo.filename = filename; + await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, filename)); + req.app.locals.master.mods.set(`${modInfo.name}_${modInfo.version}`, modInfo); + req.app.locals.master.modUpdated(modInfo); + mods.push(modInfo.toJSON()); + + } catch (err) { + logger.error(`Error uploading mod: ${err.message}`); + errors.push(err.message); + stream.resume(); + + // Attempt to clean up. + writeStream.destroy(); + try { + await fs.unlink(path.join(modsDirectory, tempFilename)); + } catch (unlinkErr) { + if (unlinkErr.code !== "ENOENT") { + logger.error(`Error removing ${tempFilename}: ${err.message}`); + } + } + } + } + + if (mimeType === "multipart/form-data") { + await new Promise(resolve => { + let busboy = new Busboy({ headers: req.headers }); + busboy.on("file", (fieldname, stream, filename, transferEncoding, fileMime) => { + if (!zipMimes.includes(fileMime)) { + requestErrors.push("invalid file Content-Type"); + } + + if (!filename.endsWith(".zip")) { + requestErrors.push("filename must end with .zip"); + } + + if (errors.length || requestErrors.length) { + stream.resume(); + return; + } + + tasks.push(handleFile(stream, filename)); + }); + busboy.on("finish", resolve); + busboy.on("error", (err) => { + logger.error(`Error parsing multipart request in upload-mod:\n${err.stack}`); + errors.push(err.message); + }); + req.pipe(busboy); + }); + + } else if (zipMimes.includes(mimeType)) { + let filename = req.query.filename; + if (typeof filename !== "string") { + requestErrors.push("Missing or invalid filename parameter"); + } else if (!filename.endsWith(".zip")) { + requestErrors.push("filename must end with .zip"); + } + + if (errors.length || requestErrors.length) { + req.resume(); + } else { + tasks.push(handleFile(req, filename)); + } + + } else { + res.status(415).json({ request_errors: ["invalid Content-Type"] }); + return; + } + + await Promise.all(tasks); + + if (errors.length) { + res.status(500); + res.json({ errors, request_errors: requestErrors }); + return; + } + if (requestErrors.length) { + res.status(400); + res.json({ request_errors: requestErrors }); + return; + } + + res.json({ mods }); +} + + function addRouteHandlers(app) { app.get("/metrics", (req, res, next) => getMetrics(req, res, next).catch(next)); app.get("/api/plugins", getPlugins); @@ -450,6 +607,10 @@ function addRouteHandlers(app) { validateUserToken, (req, res, next) => uploadSave(req, res).catch(next) ); + app.post("/api/upload-mod", + validateUserToken, + (req, res, next) => uploadMod(req, res).catch(next) + ); } // Routes used in the web interface and served by the master server @@ -460,6 +621,7 @@ const webRoutes = [ "/slaves/:id/view", "/instances", "/instances/:id/view", + "/mods", "/users", "/users/:id/view", "/roles", diff --git a/packages/web_ui/src/components/ModsPage.jsx b/packages/web_ui/src/components/ModsPage.jsx new file mode 100644 index 00000000..41625b36 --- /dev/null +++ b/packages/web_ui/src/components/ModsPage.jsx @@ -0,0 +1,165 @@ +import React, { Fragment, useContext } from "react"; +import { Button, Descriptions, Grid, PageHeader, Popconfirm, Space, Table, Typography, Upload } from "antd"; +import UploadOutlined from "@ant-design/icons/UploadOutlined"; + +import { libHelpers, libLink } from "@clusterio/lib"; + +import { useAccount } from "../model/account"; +import { useModList } from "../model/mods"; +import { notifyErrorHandler } from "../util/notify"; +import ControlContext from "./ControlContext"; +import PageLayout from "./PageLayout"; +import PluginExtra from "./PluginExtra"; + +const { useBreakpoint } = Grid; +const strcmp = new Intl.Collator(undefined, { numerice: "true", sensitivity: "base" }).compare; + + +export default function ModsPage() { + let account = useAccount(); + let control = useContext(ControlContext); + let [modList] = useModList(); + let screens = useBreakpoint(); + + function actions(mod) { + return + {account.hasPermission("core.mod.download") + && { + libLink.messages.downloadMod.send( + control, { name: mod.name, version: mod.version } + ).then(response => { + let url = new URL(webRoot, document.location); + url.pathname += `api/stream/${response.stream_id}`; + document.location = url; + }).catch( + notifyErrorHandler("Error downloading save") + ); + }} + >download + } + {account.hasPermission("core.mod.delete") + && { + libLink.messages.deleteMod.send( + control, { name: mod.name, version: mod.version } + ).catch(notifyErrorHandler("Error deleting mod")); + }} + okText="Delete" + okButtonProps={{ danger: true }} + > + delete + + } + ; + } + + let uploadButton; + if (account.hasPermission("core.mod.upload")) { + uploadButton = + + ; + } + + return + + ( + strcmp(a.name, b.name) || a.integerVersion - b.integerVersion + ), + }, + { + title: "Version", + key: "version", + align: "right", + render: mod => <> + {`${mod.version} `} + {`/ ${mod.factorioVersion}`} + , + }, + { + title: "Filename", + dataIndex: "filename", + responsive: ["xl"], + sorter: (a, b) => strcmp(a.filename, b.filename), + }, + { + title: "Size", + key: "size", + responsive: ["lg"], + render: mod => libHelpers.formatBytes(mod.size), + align: "right", + sorter: (a, b) => a.size - b.size, + }, + { + title: "Action", + key: "action", + responsive: ["lg"], + render: mod => actions(mod), + }, + ]} + expandable={{ + expandedRowRender: mod => <> + + {!screens.lg + && {actions(mod)} + } + {mod.title} + {mod.version} + {mod.description + && {mod.description} + } + {mod.author} + + + {`https://mods.factorio.com/mod/${mod.name}`} + + + {mod.contact + && {mod.contact} + } + {mod.homepage + && {mod.homepage} + } + {mod.name} + {mod.factorioVersion} + {mod.filename} + {libHelpers.formatBytes(mod.size)} + {mod.hash} + {mod.dependencies.length + && + {mod.dependencies + .filter(e => !e.startsWith("(?)")) + .map((e, i) => {e}
) + } +
+ } +
+ , + expandedRowClassName: () => "no-expanded-padding", + }} + dataSource={modList} + pagination={false} + rowKey={mod => `${mod.name}_${mod.version}`} + /> + + ; +} diff --git a/packages/web_ui/src/index.css b/packages/web_ui/src/index.css index 37f3889b..9fd4b010 100644 --- a/packages/web_ui/src/index.css +++ b/packages/web_ui/src/index.css @@ -314,3 +314,12 @@ body { top: 8px; right: 8px; } + + +.ant-descriptions-bordered.borderless > .ant-descriptions-view { + border: none; +} + +tr.no-expanded-padding > td { + padding: 0; +} diff --git a/packages/web_ui/src/model/mods.jsx b/packages/web_ui/src/model/mods.jsx new file mode 100644 index 00000000..2ef9971c --- /dev/null +++ b/packages/web_ui/src/model/mods.jsx @@ -0,0 +1,50 @@ +import { useContext, useEffect, useState } from "react"; + +import { libData, libLink, libLogging } from "@clusterio/lib"; + +import ControlContext from "../components/ControlContext"; + +const { logger } = libLogging; + + +export function useModList() { + let control = useContext(ControlContext); + let [modList, setModList] = useState([]); + + function updateModList() { + libLink.messages.listMods.send(control).then(result => { + setModList(result.list.map(mod => new libData.ModInfo(mod))); + }).catch(err => { + logger.error(`Failed to list mods:\n${err}`); + }); + } + + useEffect(() => { + updateModList(); + + function updateHandler(newMod) { + setModList(oldList => { + let newList = oldList.concat(); + let index = newList.findIndex(mod => mod.name === newMod.name && mod.version === newMod.version); + if (!newMod.isDeleted) { + if (index !== -1) { + newList[index] = newMod; + } else { + newList.push(newMod); + } + } else if (index !== -1) { + newList.splice(index, 1); + } + return newList; + }); + } + + control.onModUpdate(null, updateHandler); + return () => { + control.offModUpdate(null, updateHandler); + }; + }, []); + + return [modList]; +} + diff --git a/packages/web_ui/src/pages.jsx b/packages/web_ui/src/pages.jsx index cec0de30..7f190564 100644 --- a/packages/web_ui/src/pages.jsx +++ b/packages/web_ui/src/pages.jsx @@ -5,6 +5,7 @@ import SlavesPage from "./components/SlavesPage"; import SlaveViewPage from "./components/SlaveViewPage"; import InstancesPage from "./components/InstancesPage"; import InstanceViewPage from "./components/InstanceViewPage"; +import ModsPage from "./components/ModsPage"; import UsersPage from "./components/UsersPage"; import UserViewPage from "./components/UserViewPage"; import RolesPage from "./components/RolesPage"; @@ -41,6 +42,12 @@ export const pages = [ sidebarPath: "/instances", content: , }, + { + path: "/mods", + sidebarName: "Mods", + permission: "core.mod.list", + content: , + }, { path: "/users", sidebarName: "Users", diff --git a/packages/web_ui/src/util/websocket.jsx b/packages/web_ui/src/util/websocket.jsx index 12d8eacd..7ac65b0f 100644 --- a/packages/web_ui/src/util/websocket.jsx +++ b/packages/web_ui/src/util/websocket.jsx @@ -1,4 +1,4 @@ -import { libErrors, libLink, libLogging, libPlugin } from "@clusterio/lib"; +import { libData, libErrors, libLink, libLogging, libPlugin } from "@clusterio/lib"; const { logger } = libLogging; import packageJson from "../../package.json"; @@ -86,6 +86,7 @@ export class Control extends libLink.Link { this.slaveUpdateHandlers = new Map(); this.instanceUpdateHandlers = new Map(); this.saveListUpdateHandlers = new Map(); + this.modUpdateHandlers = new Map(); this.userUpdateHandlers = new Map(); this.logHandlers = new Map(); @@ -102,6 +103,9 @@ export class Control extends libLink.Link { this.updateSaveListSubscriptions().catch(err => logger.error( `Unexpected error updating save list subscriptions:\n${err.stack}` )); + this.updateModSubscriptions().catch(err => logger.error( + `Unexpected error updating mod subscriptions:\n${err.stack}` + )); this.updateUserSubscriptions().catch(err => logger.error( `Unexpected error updating user subscriptions:\n${err.stack}` )); @@ -319,6 +323,64 @@ export class Control extends libLink.Link { }); } + async modUpdateEventHandler(message) { + let mod = new libData.ModInfo(message.data.mod); + let handlers = [].concat( + this.modUpdateHandlers.get(null) || [], + this.modUpdateHandlers.get(mod.name) || [] + ); + for (let handler of handlers) { + handler(mod); + } + } + + async onModUpdate(name, handler) { + if (name !== null && typeof name !== "string") { + throw new Error("Invalid mod name"); + } + + let handlers = this.modUpdateHandlers.get(name); + if (!handlers) { + handlers = []; + this.modUpdateHandlers.set(name, handlers); + } + + handlers.push(handler); + + if (handlers.length === 1) { + await this.updateModSubscriptions(); + } + } + + async offModUpdate(name, handler) { + let handlers = this.modUpdateHandlers.get(name); + if (!handlers || !handlers.length) { + throw new Error(`No handlers for mod pack ${name} exist`); + } + + let index = handlers.lastIndexOf(handler); + if (index === -1) { + throw new Error(`Given handler is not registered for mod pack ${name}`); + } + + handlers.splice(index, 1); + if (!handlers.length) { + this.modUpdateHandlers.delete(name); + await this.updateModSubscriptions(); + } + } + + async updateModSubscriptions() { + if (!this.connector.connected) { + return; + } + + await libLink.messages.setModSubscriptions.send(this, { + all: this.modUpdateHandlers.has(null), + mod_names: [...this.modUpdateHandlers.keys()].filter(k => k !== null), + }); + } + async userUpdateEventHandler(message) { let handlers = [].concat( this.userUpdateHandlers.get(null) || [], diff --git a/test/file/empty_mod/info.json b/test/file/empty_mod/info.json new file mode 100644 index 00000000..0537aa9a --- /dev/null +++ b/test/file/empty_mod/info.json @@ -0,0 +1,9 @@ +{ + "name": "empty_mod", + "version": "1.0.0", + "title": "An Empty Mod", + "author": "Me", + "description": "An empty mod for testing", + "factorio_version": "1.1", + "dependencies": [] +} diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index cb91a631..3c9a58f6 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -5,6 +5,8 @@ const path = require("path"); const events = require("events"); const phin = require("phin"); +const libBuildMod = require("@clusterio/lib/build_mod"); +const libHash = require("@clusterio/lib/hash"); const libLink = require("@clusterio/lib/link"); const libUsers = require("@clusterio/lib/users"); const { wait } = require("@clusterio/lib/helpers"); @@ -818,6 +820,91 @@ describe("Integration of Clusterio", function() { }); }); + describe("mod upload", function() { + it("uploads a mod", async function() { + await libBuildMod.build({ + build: true, + pack: true, + sourceDir: path.join("test", "file", "empty_mod"), + outputDir: path.join("temp", "test"), + }); + await execCtl("mod upload empty_mod_1.0.0.zip"); + assert( + await fs.pathExists(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")), + "mod not present in mods directory" + ); + }); + }); + + describe("mod show", function() { + it("gives details of a mod", async function() { + let result = await execCtl("mod show empty_mod 1.0.0"); + let hash = await libHash.hashFile(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")); + let stat = await fs.stat(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")); + assert.equal( + result.stdout, + "name: empty_mod\n" + + "version: 1.0.0\n" + + "title: An Empty Mod\n" + + "author: Me\n" + + "description: An empty mod for testing\n" + + "factorio_version: 1.1\n" + + "dependencies:\n" + + "filename: empty_mod_1.0.0.zip\n" + + `size: ${stat.size}\n` + + `hash: sha1:${hash}\n`, + ); + }); + }); + + describe("mod list", function() { + it("shows the list of mods", async function() { + let result = await execCtl("mod list"); + assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the list"); + }); + }); + + describe("mod download", function() { + it("downloads a mod", async function() { + await fs.unlink(path.join("temp", "test", "empty_mod_1.0.0.zip")); + await execCtl("mod download empty_mod 1.0.0"); + assert( + await fs.pathExists(path.join("temp", "test", "empty_mod_1.0.0.zip")), + "mod not downloaded to cwd" + ); + }); + }); + + describe("mod delete", function() { + it("deletes a mod", async function() { + await execCtl("mod delete empty_mod 1.0.0"); + assert( + !await fs.pathExists(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip")), + "mod still present in mods dir" + ); + }); + }); + + describe("modUpdateEventHandler()", function() { + it("should have triggered for the previous mod updates", function() { + let eventsToCheck = new Set(["updated", "deleted"]); + let eventsNotSeen = new Set(eventsToCheck); + + for (let modUpdate of getControl().modUpdates) { + if (modUpdate.name !== "empty_mod" || modUpdate.version !== "1.0.0") { + continue; + } + if (modUpdate.is_deleted) { + eventsNotSeen.delete("deleted"); + } else { + eventsNotSeen.delete("updated"); + } + } + + assert(eventsNotSeen.size === 0, `Did not see the events ${[...eventsNotSeen]}`); + }); + }); + describe("permission list", function() { it("runs", async function() { await execCtl("permission list"); diff --git a/test/integration/index.js b/test/integration/index.js index 1084570e..0a4485af 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -28,6 +28,7 @@ class TestControl extends libLink.Link { this.slaveUpdates = []; this.instanceUpdates = []; this.saveListUpdates = []; + this.modUpdates = []; this.userUpdates = []; this.connector.on("connect", () => { @@ -40,6 +41,9 @@ class TestControl extends libLink.Link { libLink.messages.setSaveListSubscriptions.send( this, { all: true, instance_ids: [] } ).catch(err => logger.error(`Error setting save list subscriptions:\n${err.stack}`)); + libLink.messages.setModSubscriptions.send( + this, { all: true, mod_names: [] } + ).catch(err => logger.error(`Error setting mod subscriptions:\n${err.stack}`)); libLink.messages.setUserSubscriptions.send( this, { all: true, names: [] } ).catch(err => logger.error(`Error setting user subscriptions:\n${err.stack}`)); @@ -67,6 +71,10 @@ class TestControl extends libLink.Link { this.saveListUpdates.push(message.data); } + async modUpdateEventHandler(message) { + this.modUpdates.push(message.data.mod); + } + async userUpdateEventHandler(message) { this.userUpdates.push(message.data); } diff --git a/test/lib/data/ModInfo.js b/test/lib/data/ModInfo.js new file mode 100644 index 00000000..f3de1794 --- /dev/null +++ b/test/lib/data/ModInfo.js @@ -0,0 +1,101 @@ +"use strict"; +const assert = require("assert").strict; +const fs = require("fs-extra"); +const path = require("path"); + +const libSchema = require("@clusterio/lib/schema"); +const libHash = require("@clusterio/lib/hash"); +const libBuildMod = require("@clusterio/lib/build_mod"); +const ModInfo = require("@clusterio/lib/data/ModInfo"); + + +describe("lib/data/ModInfo", function() { + describe("class ModInfo", function() { + it("should round trip serialize", function() { + const validate = libSchema.compile(ModInfo.jsonSchema); + function check(pack) { + const json = JSON.parse(JSON.stringify(pack)); + if (!validate(json)) { + throw validate.errors; + } + assert.deepEqual(new ModInfo(json), pack); + } + + check(new ModInfo()); + check(new ModInfo({ name: "MyMod" })); + check(new ModInfo({ version: "1.0.0" })); + check(new ModInfo({ title: "My Extravagent Mod" })); + check(new ModInfo({ author: "Me" })); + check(new ModInfo({ contact: "Use my email" })); + check(new ModInfo({ homepage: "It's over there" })); + check(new ModInfo({ description: "My Description" })); + check(new ModInfo({ factorio_version: "2.0" })); + check(new ModInfo({ dependencies: [] })); + check(new ModInfo({ dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"] })); + check(new ModInfo({ filename: "MyMod_1.0.0.zip" })); + check(new ModInfo({ size: 1024 })); + check(new ModInfo({ hash: "sha5:verified-as-MyMod" })); + check(new ModInfo({ is_deleted: true })); + + // All at once + check(new ModInfo({ + name: "MyMod", + version: "1.0.0", + title: "My Extravagent Mod", + author: "Me", + contact: "Use my email", + homepage: "It's over there", + description: "My Description", + factorio_version: "2.0", + dependencies: ["UltraMod", "SuperLib >= 1.00", "! bad-mod"], + filename: "MyMod_1.0.0.zip", + size: 1024, + hash: "sha5:verified-as-MyMod", + is_deleted: true, + })); + }); + + it("should sort integer mod versions lexicographically", function() { + let unsortedVersions = ["1.0.0", "1.1.0", "0.1.0", "3.0.0", "1.2.0", "0.3.1", "0.3.3", "2.1.1", "0.0.1"]; + let sortedVersions = ["0.0.1", "0.1.0", "0.3.1", "0.3.3", "1.0.0", "1.1.0", "1.2.0", "2.1.1", "3.0.0"]; + let mods = unsortedVersions.map(v => new ModInfo({ version: v })); + mods.sort((a, b) => a.integerVersion - b.integerVersion); + assert.deepEqual(mods.map(mod => mod.version), sortedVersions); + }); + it("should sort integer factorio versions lexicographically", function() { + let unsortedVersions = ["1.0", "1.1", "0.1", "3.0", "1.2", "0.3", "2.1"]; + let sortedVersions = ["0.1", "0.3", "1.0", "1.1", "1.2", "2.1", "3.0"]; + let factorioMods = unsortedVersions.map(v => new ModInfo({ factorio_version: v })); + factorioMods.sort((a, b) => a.integerFactorioVersion - b.integerFactorioVersion); + assert.deepEqual(factorioMods.map(mod => mod.factorioVersion), sortedVersions); + }); + + it("should load from a mod zip file", async function() { + await libBuildMod.build({ + build: true, + pack: true, + sourceDir: path.join("test", "file", "empty_mod"), + outputDir: path.join("temp", "test"), + }); + const filePath = path.join("temp", "test", "empty_mod_1.0.0.zip"); + const hash = await libHash.hashFile(filePath); + const stat = await fs.stat(filePath); + const mod = await ModInfo.fromModFile(filePath); + assert.deepEqual( + mod, + new ModInfo({ + name: "empty_mod", + version: "1.0.0", + title: "An Empty Mod", + author: "Me", + description: "An empty mod for testing", + factorio_version: "1.1", + dependencies: [], + filename: "empty_mod_1.0.0.zip", + size: stat.size, + hash: `sha1:${hash}`, + }) + ); + }); + }); +});