diff --git a/.gitignore b/.gitignore index a2891657..56c8fbd4 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ Factorio* /instances /sharedMods /database +/plugin-list.json /config-master.json /config-control.json /config-slave.json @@ -44,16 +45,6 @@ static/export saves /temp/ -# weird pattern to ignore sharedPlugins except not ignore some core modules -!sharedPlugins/ -sharedPlugins/* -!sharedPlugins/inventoryImports/ -!sharedPlugins/researchSync/ -!sharedPlugins/clusterioMod/ - -# File used for marking plugins as disabled -sharedPlugins/*/DISABLED - # IDE .idea/ .vs/ diff --git a/README.md b/README.md index ae3e136e..04f0a500 100644 --- a/README.md +++ b/README.md @@ -33,11 +33,11 @@ the [1.2.x branch][1.2.x] for instructions on how to install the stable version. * [Introduction & methodology](#introduction) * [Ubuntu setup](#ubuntu-setup) * [Windows setup](#windows-setup) +* [Installing Plugins](#installing-plugins) * [Running Clusterio](#running-clusterio) * [Master Server](#master-server) * [Slaves](#slaves) * [Instances](#instances) -* [Optional plugins](#Plugins) * [Common problems](#Common-problems) ## Introduction @@ -176,6 +176,29 @@ up, v11.13.0+ and v10.16.0+. the Factorio folder into the factorio folder created in step 1. +## Installing Plugins + +Installing plugins to make them work with Clusterio consists of two +steps. First install the package via npm, for example + + npm install @clusterio/plugin-subspace_storage + +Then tell clusterio that this plugin exists by adding it as a plugin. + + npx clusteriomaster plugin add @clusterio/plugin-subspace_storage + +This adds it to the default `plugin-list.json` file which in the shared +folder setup is loaded by master, slave and ctl. If you have slaves or +ctl installed on separate computers (or directories) then you need to +repeat the plugin install process for all of them. The clusteriomaster, +clusterioslave and clusterioctl commands has the plugin sub-command +so you do not need to install clusteriomaster to add plugins. + +For development purposes the `plugin add` command supports adding +plugins by the absolute path to them, or a relative path that must start +with either . or .. (which will then be resolved to an absolute path). + + ## Running Clusterio After following the installation instructions you can use the following @@ -257,12 +280,6 @@ There are many more commands available with clusterioctl. See `npx clusterioctl --help` for a full list of them. -## Plugins -Here are the known Clusterio plugins in the wild: -1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta) -2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook. -3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers. - ## Common problems ### EACCESS [...] LISTEN 443 diff --git a/package.json b/package.json index efdf2697..74f67bfc 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "root", "private": true, "scripts": { - "test": "mocha --exit --check-leaks -R spec --recursive test plugins/*/test", + "test": "mocha --exit --check-leaks -R spec --recursive test", "fast-test": "FAST_TEST=y npm test", "cover": "nyc npm test", "fast-cover": "FAST_TEST=y nyc npm test", diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 42a56469..a44d6820 100755 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -18,6 +18,7 @@ const errors = require("@clusterio/lib/errors"); const config = require("@clusterio/lib/config"); const plugin = require("@clusterio/lib/plugin"); const command = require("@clusterio/lib/command"); +const sharedCommands = require("@clusterio/lib/shared_commands"); /** @@ -586,11 +587,38 @@ async function startControl() { defaultDescription: "auto", type: "string", }) + .option("plugin-list", { + nargs: 1, + describe: "File containing list of plugins available with their install path", + default: "plugin-list.json", + type: "string", + }) + .command("plugin", "Manage available plugins", sharedCommands.pluginCommand) .command("control-config", "Manage Control config", config.configCommand) .wrap(yargs.terminalWidth()) - .strict() + .help(false) // Disable help to avoid triggering it on the first parse. ; + // Parse the args first to get the configured plugin list. + let args = yargs.argv; + + console.log(`Loading available plugins from ${args.pluginList}`); + let pluginList = new Map(); + try { + pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList))); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + // If the command is plugin management we don't try to load plugins + if (args._[0] === "plugin") { + await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList); + return; + } + + // Add all cluster management commands including ones from plugins const rootCommands = new command.CommandTree({ name: "clusterioctl", description: "Manage cluster" }); rootCommands.add(slaveCommands); rootCommands.add(instanceCommands); @@ -600,7 +628,7 @@ async function startControl() { rootCommands.add(debugCommands); console.log("Loading Plugin info"); - let pluginInfos = await plugin.loadPluginInfos("plugins"); + let pluginInfos = await plugin.loadPluginInfos(pluginList); config.registerPluginConfigGroups(pluginInfos); config.finalizeConfigs(); @@ -610,9 +638,7 @@ async function startControl() { continue; } - // XXX Does not work on Windows - let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name)); - let { ControlPlugin } = require(path.join(pluginPath, pluginInfo.controlEntrypoint)); + let { ControlPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.controlEntrypoint)); let controlPlugin = new ControlPlugin(pluginInfo); controlPlugins.set(pluginInfo.name, controlPlugin); await controlPlugin.init(); @@ -625,7 +651,12 @@ async function startControl() { } } - const args = yargs.argv; + // Reparse after commands have been added with help and strict checking. + args = yargs + .help() + .strict() + .argv + ; console.log(`Loading config from ${args.config}`); let controlConfig = new config.ControlConfig(); diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index 45e86672..7958db7e 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -394,6 +394,7 @@ function configCommand(yargs) { .command("show ", "Show value of the given config field") .command("list", "List all configuration fields and their values") .demandCommand(1, "You need to specify a command to run") + .help() .strict() ; } diff --git a/packages/lib/plugin.js b/packages/lib/plugin.js index d4a1a6f5..2bd00b00 100644 --- a/packages/lib/plugin.js +++ b/packages/lib/plugin.js @@ -3,7 +3,6 @@ * @module lib/plugin */ "use strict"; -const fs = require("fs-extra"); const path = require("path"); const errors = require("@clusterio/lib/errors"); @@ -390,39 +389,34 @@ class BaseControlPlugin { /** * Load plugin information * - * Searches through the plugin directory and loads each plugin's info - * module. Once loaded the info modules will not be reloaded should this - * function be called again. + * Loads plugin info modules for the paths to the given plugins. Once + * loaded the info modules will not be reloaded should this function be + * called again. * - * @param {string} baseDir - posix relative path to plugin directory. - * @returns {Map} mapping of plugin name to info module. + * @param {Map} pluginList - + * Mapping of plugin name to require path for the plugins to load. + * @returns {Array} Array of plugin info modules. * @static */ -async function loadPluginInfos(baseDir) { +async function loadPluginInfos(pluginList) { let plugins = []; - for (let pluginDir of await fs.readdir(baseDir)) { + for (let [pluginName, pluginPath] of pluginList) { let pluginInfo; try { - // Note: Require path is relative to this module - // XXX Does not work on Windows - let pluginPath = path.relative(__dirname, path.join(baseDir, pluginDir)); - pluginInfo = require(path.join(pluginPath, "info")); + pluginInfo = require(path.posix.join(pluginPath, "info")); } catch (err) { - if (err.code === "MODULE_NOT_FOUND") { - continue; - } else { - throw new errors.PluginError(pluginDir, err); - } + throw new errors.PluginError(pluginName, err); } - if (pluginInfo.name !== pluginDir) { + if (pluginInfo.name !== pluginName) { throw new errors.EnvironmentError( - `Plugin dir ${baseDir}/${pluginDir} does not match the name of the plugin (${pluginInfo.name})` + `Expected plugin at ${pluginPath} to be named ${pluginName} but got ${pluginInfo.name}` ); } + pluginInfo.requirePath = pluginPath; plugins.push(pluginInfo); } return plugins; diff --git a/packages/lib/shared_commands.js b/packages/lib/shared_commands.js new file mode 100644 index 00000000..3d16bd06 --- /dev/null +++ b/packages/lib/shared_commands.js @@ -0,0 +1,86 @@ +/** + * Implementation of commands shared between master/slave/ctl. + * @module lib/shared_commands + */ +"use strict"; +const fs = require("fs-extra"); +const path = require("path"); + +/** + * Yargs plugin command + * + * Can be passed to yargs.command to implement a plugin list management + * command. Use handlePluginCommand to do the requested action. + * + * @param {Object} yargs - yargs command builder. + */ +function pluginCommand(yargs) { + yargs + .command("add ", "Add plugin by require path") + .command("remove ", "Remove plugin by name") + .command("list", "List all plugins and their path") + .demandCommand(1, "You need to specify a command to run") + .help() + .strict() + ; +} + +/** + * Handle yargs plugin command + * + * Handle the actions that are made available by pluginCommand. + * + * @param {Object} args - yargs args object. + * @param {Map} pluginList - Current list of plugins. + * @param {string} pluginListPath - Path to plugin list config file. + */ +async function handlePluginCommand(args, pluginList, pluginListPath) { + let command = args._[1]; + + if (command === "add") { + let pluginPath = args.path; + if (/\.\.?[\/\\]/.test(pluginPath)) { + pluginPath = path.resolve(pluginPath); + } + + let pluginInfo; + try { + pluginInfo = require(path.posix.join(pluginPath, "info")); + } catch (err) { + console.error(`Unable to import plugin info from ${args.path}`); + console.error(err); + process.exitCode = 1; + return; + } + + if (pluginList.has(pluginInfo.name)) { + console.error(`Plugin with the same ${pluginInfo.name} already exists`); + process.exitCode = 1; + return; + } + + pluginList.set(pluginInfo.name, args.path); + await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4)); + console.log(`Added ${pluginInfo.name}`); + + } else if (command === "remove") { + if (!pluginList.delete(args.name)) { + console.error(`Plugin with name ${args.name} does not exist`); + process.exitCode = 1; + return; + } + + await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4)); + console.log(`Removed ${args.name}`); + + } else if (command === "list") { + for (let [pluginName, pluginPath] of pluginList) { + console.log(`${pluginName} - ${pluginPath}`); + } + } +} + +module.exports = { + pluginCommand, + handlePluginCommand, +}; diff --git a/packages/master/master.js b/packages/master/master.js index 70179711..a27e308d 100755 --- a/packages/master/master.js +++ b/packages/master/master.js @@ -34,6 +34,8 @@ const util = require("util"); // ugly globals let masterConfig; let masterConfigPath; +let masterPlugins = new Map(); +let pluginInfos = new Map(); let stopAcceptingNewSessions = false; let debugEvents = new events.EventEmitter(); let pluginList = {}; @@ -50,6 +52,7 @@ const plugin = require("@clusterio/lib/plugin"); const prometheus = require("@clusterio/lib/prometheus"); const config = require("@clusterio/lib/config"); const users = require("@clusterio/lib/users"); +const sharedCommands = require("@clusterio/lib/shared_commands"); const express = require("express"); const compression = require("compression"); @@ -1441,7 +1444,6 @@ async function handleHandshake(message, socket, req, attachHandler) { } // handle plugins on the master -var masterPlugins = new Map(); async function pluginManagement(pluginInfos) { let startPluginLoad = Date.now(); masterPlugins = await loadPlugins(pluginInfos); @@ -1461,9 +1463,7 @@ async function loadPlugins(pluginInfos) { let MasterPlugin = plugin.BaseMasterPlugin; try { if (pluginInfo.masterEntrypoint) { - // XXX Does not work on Windows - let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name)); - ({ MasterPlugin } = require(path.join(pluginPath, pluginInfo.masterEntrypoint))); + ({ MasterPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.masterEntrypoint))); } let masterPlugin = new MasterPlugin( @@ -1555,6 +1555,13 @@ async function startServer() { default: "config-master.json", type: "string", }) + .option("plugin-list", { + nargs: 1, + describe: "File containing list of plugins available with their install path", + default: "plugin-list.json", + type: "string", + }) + .command("plugin", "Manage available plugins", sharedCommands.pluginCommand) .command("config", "Manage Master config", config.configCommand) .command("bootstrap", "Bootstrap access to cluster", yargs => { yargs @@ -1575,8 +1582,25 @@ async function startServer() { .argv ; + console.log(`Loading available plugins from ${args.pluginList}`); + let pluginList = new Map(); + try { + pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList))); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + // If the command is plugin management we don't try to load plugins + let command = args._[0]; + if (command === "plugin") { + await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList); + return; + } + console.log("Loading Plugin info"); - let pluginInfos = await plugin.loadPluginInfos("plugins"); + pluginInfos = await plugin.loadPluginInfos(pluginList); config.registerPluginConfigGroups(pluginInfos); config.finalizeConfigs(); @@ -1604,7 +1628,6 @@ async function startServer() { await fs.outputFile(masterConfigPath, JSON.stringify(masterConfig.serialize(), null, 4)); } - let command = args._[0]; if (command === "config") { await config.handleConfigCommand(args, masterConfig, masterConfigPath); return; diff --git a/packages/slave/slave.js b/packages/slave/slave.js index b2947b08..026785ed 100755 --- a/packages/slave/slave.js +++ b/packages/slave/slave.js @@ -32,6 +32,7 @@ const errors = require("@clusterio/lib/errors"); const prometheus = require("@clusterio/lib/prometheus"); const luaTools = require("@clusterio/lib/luaTools"); const config = require("@clusterio/lib/config"); +const sharedCommands = require("@clusterio/lib/shared_commands"); const instanceRconCommandsCounter = new prometheus.Counter( @@ -206,9 +207,7 @@ class Instance extends link.Link{ async _loadPlugin(pluginInfo, slave) { let pluginLoadStarted = Date.now(); - // XXX Does not work on Windows - let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name)); - let { InstancePlugin } = require(path.join(pluginPath, pluginInfo.instanceEntrypoint)); + let { InstancePlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.instanceEntrypoint)); let instancePlugin = new InstancePlugin(pluginInfo, this, slave); this.plugins.set(pluginInfo.name, instancePlugin); await instancePlugin.init(); @@ -1385,6 +1384,13 @@ async function startSlave() { default: "config-slave.json", type: "string", }) + .option("plugin-list", { + nargs: 1, + describe: "File containing list of plugins available with their install path", + default: "plugin-list.json", + type: "string", + }) + .command("plugin", "Manage available plugins", sharedCommands.pluginCommand) .command("config", "Manage Slave config", config.configCommand) .command("run", "Run slave") .demandCommand(1, "You need to specify a command to run") @@ -1392,8 +1398,25 @@ async function startSlave() { .argv ; + console.log(`Loading available plugins from ${args.pluginList}`); + let pluginList = new Map(); + try { + pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList))); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + // If the command is plugin management we don't try to load plugins + let command = args._[0]; + if (command === "plugin") { + await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList); + return; + } + console.log("Loading Plugin info"); - let pluginInfos = await plugin.loadPluginInfos("plugins"); + let pluginInfos = await plugin.loadPluginInfos(pluginList); config.registerPluginConfigGroups(pluginInfos); config.finalizeConfigs(); @@ -1412,7 +1435,6 @@ async function startSlave() { } } - let command = args._[0]; if (command === "config") { await config.handleConfigCommand(args, slaveConfig, args.config); return; diff --git a/test/lib/plugin.js b/test/lib/plugin.js index 879e225f..445ab317 100644 --- a/test/lib/plugin.js +++ b/test/lib/plugin.js @@ -44,56 +44,48 @@ describe("lib/plugin", function() { describe("loadPluginInfos()", function() { let baseDir = path.join("temp", "test", "plugin"); - let emptyDir = path.join(baseDir, "emptyDir"); - let emptyPlugin = path.join(baseDir, "emptyPlugin"); - let testPlugin = path.join(baseDir, "testPlugin"); - let brokenPlugin = path.join(baseDir, "brokenPlugin"); - let invalidPlugin = path.join(baseDir, "invalidPlugin"); + let missingPlugin = path.join(baseDir, "missing_plugin"); + let testPlugin = path.join(baseDir, "test_plugin"); + let brokenPlugin = path.join(baseDir, "broken_plugin"); + let invalidPlugin = path.join(baseDir, "invalid_plugin"); before(async function() { - await fs.ensureDir(emptyDir); - await fs.ensureDir(path.join(emptyPlugin, "empty")); - - async function writePlugin(pluginPath, name, infoName = name) { + async function writePlugin(pluginPath, infoName) { await fs.outputFile( - path.join(pluginPath, name, "info.js"), + path.join(pluginPath, "info.js"), `module.exports = { name: "${infoName}" };` ); } await writePlugin(testPlugin, "test"); await writePlugin(brokenPlugin, "broken"); - await fs.outputFile(path.join(brokenPlugin, "broken", "info.js"), "Syntax Error"); - await writePlugin(invalidPlugin, "invalid", "wrong"); + await fs.outputFile(path.join(brokenPlugin, "info.js"), "Syntax Error"); + await writePlugin(invalidPlugin, "wrong"); }); - it("should return an empty array for an empty directory", async function() { - assert.deepEqual(await plugin.loadPluginInfos(emptyDir), []); + it("should throw on missing plugin", async function() { + await assert.rejects( + plugin.loadPluginInfos(new Map([["missing", missingPlugin]]), []), + new RegExp(`^Error: PluginError: Cannot find module '${missingPlugin}/info'`) + ); }); - it("should ignore plugin dirs without info module", async function() { - assert.deepEqual(await plugin.loadPluginInfos(emptyPlugin), []); - }); - it("should discover test plugin", async function() { + it("should load test plugin", async function() { assert.deepEqual( - await plugin.loadPluginInfos(testPlugin), - [{ name: "test" }] + await plugin.loadPluginInfos(new Map([["test", path.resolve(testPlugin)]])), + [{ name: "test", requirePath: path.resolve(testPlugin) }] ); }); it("should reject on broken plugin", async function() { await assert.rejects( - plugin.loadPluginInfos(brokenPlugin), + plugin.loadPluginInfos(new Map([["broken", path.resolve(brokenPlugin)]])), { message: "PluginError: Unexpected identifier" } ); }); it("should reject on invalid plugin", async function() { await assert.rejects( - plugin.loadPluginInfos(invalidPlugin), - { message: `Plugin dir ${invalidPlugin}/invalid does not match the name of the plugin (wrong)` } + plugin.loadPluginInfos(new Map([["invalid", path.resolve(invalidPlugin)]])), + { message: `Expected plugin at ${path.resolve(invalidPlugin)} to be named invalid but got wrong` } ); }); - - after(async function() { - await fs.unlink(path.join(brokenPlugin, "broken", "info.js")); - }); }); describe("attachPluginMessages()", function() {