Allow plugins to be installed as npm packages

Add a separate plugin-list.json file for storing which plugins are
available and the path to pass to require in order to load it.  This
allows plugins to be published and installed as npm packages.  And also
replaces the old method of installing plugins by putting them in the
plugins folder.

.gitignore

@@ -35,6 +35,7 @@ Factorio*
 /instances
 /sharedMods
 /database
+/plugin-list.json
 /config-master.json
 /config-control.json
 /config-slave.json
@@ -44,16 +45,6 @@ static/export
 saves
 /temp/
 
-# weird pattern to ignore sharedPlugins except not ignore some core modules
-!sharedPlugins/
-sharedPlugins/*
-!sharedPlugins/inventoryImports/
-!sharedPlugins/researchSync/
-!sharedPlugins/clusterioMod/
-
-# File used for marking plugins as disabled
-sharedPlugins/*/DISABLED
-
 # IDE
 .idea/
 .vs/

README.md

@@ -33,11 +33,11 @@ the [1.2.x branch][1.2.x] for instructions on how to install the stable version.
 * [Introduction & methodology](#introduction)
 * [Ubuntu setup](#ubuntu-setup)
 * [Windows setup](#windows-setup)
+* [Installing Plugins](#installing-plugins)
 * [Running Clusterio](#running-clusterio)
   * [Master Server](#master-server)
   * [Slaves](#slaves)
   * [Instances](#instances)
-* [Optional plugins](#Plugins)
 * [Common problems](#Common-problems)
 
 ## Introduction
@@ -176,6 +176,29 @@ up, v11.13.0+ and v10.16.0+.
             the Factorio folder into the factorio folder created in step 1.
 
 
+## Installing Plugins
+
+Installing plugins to make them work with Clusterio consists of two
+steps.  First install the package via npm, for example
+
+    npm install @clusterio/plugin-subspace_storage
+
+Then tell clusterio that this plugin exists by adding it as a plugin.
+
+    npx clusteriomaster plugin add @clusterio/plugin-subspace_storage
+
+This adds it to the default `plugin-list.json` file which in the shared
+folder setup is loaded by master, slave and ctl.  If you have slaves or
+ctl installed on separate computers (or directories) then you need to
+repeat the plugin install process for all of them.  The clusteriomaster,
+clusterioslave and clusterioctl commands has the plugin sub-command
+so you do not need to install clusteriomaster to add plugins.
+
+For development purposes the `plugin add` command supports adding
+plugins by the absolute path to them, or a relative path that must start
+with either . or .. (which will then be resolved to an absolute path).
+
+
 ## Running Clusterio
 
 After following the installation instructions you can use the following
@@ -257,12 +280,6 @@ There are many more commands available with clusterioctl.  See
 `npx clusterioctl --help` for a full list of them.
 
 
-## Plugins
-Here are the known Clusterio plugins in the wild:
-1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta)
-2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook.
-3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers.
-
 ## Common problems
 
 ### EACCESS [...] LISTEN 443

package.json

@@ -2,7 +2,7 @@
   "name": "root",
   "private": true,
   "scripts": {
-    "test": "mocha --exit --check-leaks -R spec --recursive test plugins/*/test",
+    "test": "mocha --exit --check-leaks -R spec --recursive test",
     "fast-test": "FAST_TEST=y npm test",
     "cover": "nyc npm test",
     "fast-cover": "FAST_TEST=y nyc npm test",

packages/ctl/ctl.js

@@ -18,6 +18,7 @@ const errors = require("@clusterio/lib/errors");
 const config = require("@clusterio/lib/config");
 const plugin = require("@clusterio/lib/plugin");
 const command = require("@clusterio/lib/command");
+const sharedCommands = require("@clusterio/lib/shared_commands");
 
 
 /**
@@ -586,11 +587,38 @@ async function startControl() {
 			defaultDescription: "auto",
 			type: "string",
 		})
+		.option("plugin-list", {
+			nargs: 1,
+			describe: "File containing list of plugins available with their install path",
+			default: "plugin-list.json",
+			type: "string",
+		})
+		.command("plugin", "Manage available plugins", sharedCommands.pluginCommand)
 		.command("control-config", "Manage Control config", config.configCommand)
 		.wrap(yargs.terminalWidth())
-		.strict()
+		.help(false) // Disable help to avoid triggering it on the first parse.
 	;
 
+	// Parse the args first to get the configured plugin list.
+	let args = yargs.argv;
+
+	console.log(`Loading available plugins from ${args.pluginList}`);
+	let pluginList = new Map();
+	try {
+		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
+	} catch (err) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+	}
+
+	// If the command is plugin management we don't try to load plugins
+	if (args._[0] === "plugin") {
+		await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList);
+		return;
+	}
+
+	// Add all cluster management commands including ones from plugins
 	const rootCommands = new command.CommandTree({ name: "clusterioctl", description: "Manage cluster" });
 	rootCommands.add(slaveCommands);
 	rootCommands.add(instanceCommands);
@@ -600,7 +628,7 @@ async function startControl() {
 	rootCommands.add(debugCommands);
 
 	console.log("Loading Plugin info");
-	let pluginInfos = await plugin.loadPluginInfos("plugins");
+	let pluginInfos = await plugin.loadPluginInfos(pluginList);
 	config.registerPluginConfigGroups(pluginInfos);
 	config.finalizeConfigs();
 
@@ -610,9 +638,7 @@ async function startControl() {
 			continue;
 		}
 
-		// XXX Does not work on Windows
-		let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name));
-		let { ControlPlugin } = require(path.join(pluginPath, pluginInfo.controlEntrypoint));
+		let { ControlPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.controlEntrypoint));
 		let controlPlugin = new ControlPlugin(pluginInfo);
 		controlPlugins.set(pluginInfo.name, controlPlugin);
 		await controlPlugin.init();
@@ -625,7 +651,12 @@ async function startControl() {
 		}
 	}
 
-	const args = yargs.argv;
+	// Reparse after commands have been added with help and strict checking.
+	args = yargs
+		.help()
+		.strict()
+		.argv
+	;
 
 	console.log(`Loading config from ${args.config}`);
 	let controlConfig = new config.ControlConfig();

packages/lib/config/definitions.js

@@ -394,6 +394,7 @@ function configCommand(yargs) {
 		.command("show <field>", "Show value of the given config field")
 		.command("list", "List all configuration fields and their values")
 		.demandCommand(1, "You need to specify a command to run")
+		.help()
 		.strict()
 	;
 }

packages/lib/plugin.js

@@ -3,7 +3,6 @@
  * @module lib/plugin
  */
 "use strict";
-const fs = require("fs-extra");
 const path = require("path");
 
 const errors = require("@clusterio/lib/errors");
@@ -390,39 +389,34 @@ class BaseControlPlugin {
 /**
  * Load plugin information
  *
- * Searches through the plugin directory and loads each plugin's info
- * module.  Once loaded the info modules will not be reloaded should this
- * function be called again.
+ * Loads plugin info modules for the paths to the given plugins.  Once
+ * loaded the info modules will not be reloaded should this function be
+ * called again.
  *
- * @param {string} baseDir - posix relative path to plugin directory.
- * @returns {Map<string, Object>} mapping of plugin name to info module.
+ * @param {Map<string, string>} pluginList -
+ *     Mapping of plugin name to require path for the plugins to load.
+ * @returns {Array<Object>} Array of plugin info modules.
  * @static
  */
-async function loadPluginInfos(baseDir) {
+async function loadPluginInfos(pluginList) {
 	let plugins = [];
-	for (let pluginDir of await fs.readdir(baseDir)) {
+	for (let [pluginName, pluginPath] of pluginList) {
 		let pluginInfo;
 
 		try {
-			// Note: Require path is relative to this module
-			// XXX Does not work on Windows
-			let pluginPath = path.relative(__dirname, path.join(baseDir, pluginDir));
-			pluginInfo = require(path.join(pluginPath, "info"));
+			pluginInfo = require(path.posix.join(pluginPath, "info"));
 
 		} catch (err) {
-			if (err.code === "MODULE_NOT_FOUND") {
-				continue;
-			} else {
-				throw new errors.PluginError(pluginDir, err);
-			}
+			throw new errors.PluginError(pluginName, err);
 		}
 
-		if (pluginInfo.name !== pluginDir) {
+		if (pluginInfo.name !== pluginName) {
 			throw new errors.EnvironmentError(
-				`Plugin dir ${baseDir}/${pluginDir} does not match the name of the plugin (${pluginInfo.name})`
+				`Expected plugin at ${pluginPath} to be named ${pluginName} but got ${pluginInfo.name}`
 			);
 		}
 
+		pluginInfo.requirePath = pluginPath;
 		plugins.push(pluginInfo);
 	}
 	return plugins;

packages/lib/shared_commands.js

@@ -0,0 +1,86 @@
+/**
+ * Implementation of commands shared between master/slave/ctl.
+ * @module lib/shared_commands
+ */
+"use strict";
+const fs = require("fs-extra");
+const path = require("path");
+
+/**
+ * Yargs plugin command
+ *
+ * Can be passed to yargs.command to implement a plugin list management
+ * command.  Use handlePluginCommand to do the requested action.
+ *
+ * @param {Object} yargs - yargs command builder.
+ */
+function pluginCommand(yargs) {
+	yargs
+		.command("add <path>", "Add plugin by require path")
+		.command("remove <name>", "Remove plugin by name")
+		.command("list", "List all plugins and their path")
+		.demandCommand(1, "You need to specify a command to run")
+		.help()
+		.strict()
+	;
+}
+
+/**
+ * Handle yargs plugin command
+ *
+ * Handle the actions that are made available by pluginCommand.
+ *
+ * @param {Object} args - yargs args object.
+ * @param {Map<string, string>} pluginList - Current list of plugins.
+ * @param {string} pluginListPath - Path to plugin list config file.
+ */
+async function handlePluginCommand(args, pluginList, pluginListPath) {
+	let command = args._[1];
+
+	if (command === "add") {
+		let pluginPath = args.path;
+		if (/\.\.?[\/\\]/.test(pluginPath)) {
+			pluginPath = path.resolve(pluginPath);
+		}
+
+		let pluginInfo;
+		try {
+			pluginInfo = require(path.posix.join(pluginPath, "info"));
+		} catch (err) {
+			console.error(`Unable to import plugin info from ${args.path}`);
+			console.error(err);
+			process.exitCode = 1;
+			return;
+		}
+
+		if (pluginList.has(pluginInfo.name)) {
+			console.error(`Plugin with the same ${pluginInfo.name} already exists`);
+			process.exitCode = 1;
+			return;
+		}
+
+		pluginList.set(pluginInfo.name, args.path);
+		await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
+		console.log(`Added ${pluginInfo.name}`);
+
+	} else if (command === "remove") {
+		if (!pluginList.delete(args.name)) {
+			console.error(`Plugin with name ${args.name} does not exist`);
+			process.exitCode = 1;
+			return;
+		}
+
+		await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
+		console.log(`Removed ${args.name}`);
+
+	} else if (command === "list") {
+		for (let [pluginName, pluginPath] of pluginList) {
+			console.log(`${pluginName} - ${pluginPath}`);
+		}
+	}
+}
+
+module.exports = {
+	pluginCommand,
+	handlePluginCommand,
+};

packages/master/master.js

@@ -34,6 +34,8 @@ const util = require("util");
 // ugly globals
 let masterConfig;
 let masterConfigPath;
+let masterPlugins = new Map();
+let pluginInfos = new Map();
 let stopAcceptingNewSessions = false;
 let debugEvents = new events.EventEmitter();
 let pluginList = {};
@@ -50,6 +52,7 @@ const plugin = require("@clusterio/lib/plugin");
 const prometheus = require("@clusterio/lib/prometheus");
 const config = require("@clusterio/lib/config");
 const users = require("@clusterio/lib/users");
+const sharedCommands = require("@clusterio/lib/shared_commands");
 
 const express = require("express");
 const compression = require("compression");
@@ -1441,7 +1444,6 @@ async function handleHandshake(message, socket, req, attachHandler) {
 }
 
 // handle plugins on the master
-var masterPlugins = new Map();
 async function pluginManagement(pluginInfos) {
 	let startPluginLoad = Date.now();
 	masterPlugins = await loadPlugins(pluginInfos);
@@ -1461,9 +1463,7 @@ async function loadPlugins(pluginInfos) {
 		let MasterPlugin = plugin.BaseMasterPlugin;
 		try {
 			if (pluginInfo.masterEntrypoint) {
-				// XXX Does not work on Windows
-				let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name));
-				({ MasterPlugin } = require(path.join(pluginPath, pluginInfo.masterEntrypoint)));
+				({ MasterPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.masterEntrypoint)));
 			}
 
 			let masterPlugin = new MasterPlugin(
@@ -1555,6 +1555,13 @@ async function startServer() {
 			default: "config-master.json",
 			type: "string",
 		})
+		.option("plugin-list", {
+			nargs: 1,
+			describe: "File containing list of plugins available with their install path",
+			default: "plugin-list.json",
+			type: "string",
+		})
+		.command("plugin", "Manage available plugins", sharedCommands.pluginCommand)
 		.command("config", "Manage Master config", config.configCommand)
 		.command("bootstrap", "Bootstrap access to cluster", yargs => {
 			yargs
@@ -1575,8 +1582,25 @@ async function startServer() {
 		.argv
 	;
 
+	console.log(`Loading available plugins from ${args.pluginList}`);
+	let pluginList = new Map();
+	try {
+		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
+	} catch (err) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+	}
+
+	// If the command is plugin management we don't try to load plugins
+	let command = args._[0];
+	if (command === "plugin") {
+		await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList);
+		return;
+	}
+
 	console.log("Loading Plugin info");
-	let pluginInfos = await plugin.loadPluginInfos("plugins");
+	pluginInfos = await plugin.loadPluginInfos(pluginList);
 	config.registerPluginConfigGroups(pluginInfos);
 	config.finalizeConfigs();
 
@@ -1604,7 +1628,6 @@ async function startServer() {
 		await fs.outputFile(masterConfigPath, JSON.stringify(masterConfig.serialize(), null, 4));
 	}
 
-	let command = args._[0];
 	if (command === "config") {
 		await config.handleConfigCommand(args, masterConfig, masterConfigPath);
 		return;

packages/slave/slave.js

@@ -32,6 +32,7 @@ const errors = require("@clusterio/lib/errors");
 const prometheus = require("@clusterio/lib/prometheus");
 const luaTools = require("@clusterio/lib/luaTools");
 const config = require("@clusterio/lib/config");
+const sharedCommands = require("@clusterio/lib/shared_commands");
 
 
 const instanceRconCommandsCounter = new prometheus.Counter(
@@ -206,9 +207,7 @@ class Instance extends link.Link{
 
 	async _loadPlugin(pluginInfo, slave) {
 		let pluginLoadStarted = Date.now();
-		// XXX Does not work on Windows
-		let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name));
-		let { InstancePlugin } = require(path.join(pluginPath, pluginInfo.instanceEntrypoint));
+		let { InstancePlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.instanceEntrypoint));
 		let instancePlugin = new InstancePlugin(pluginInfo, this, slave);
 		this.plugins.set(pluginInfo.name, instancePlugin);
 		await instancePlugin.init();
@@ -1385,6 +1384,13 @@ async function startSlave() {
 			default: "config-slave.json",
 			type: "string",
 		})
+		.option("plugin-list", {
+			nargs: 1,
+			describe: "File containing list of plugins available with their install path",
+			default: "plugin-list.json",
+			type: "string",
+		})
+		.command("plugin", "Manage available plugins", sharedCommands.pluginCommand)
 		.command("config", "Manage Slave config", config.configCommand)
 		.command("run", "Run slave")
 		.demandCommand(1, "You need to specify a command to run")
@@ -1392,8 +1398,25 @@ async function startSlave() {
 		.argv
 	;
 
+	console.log(`Loading available plugins from ${args.pluginList}`);
+	let pluginList = new Map();
+	try {
+		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
+	} catch (err) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+	}
+
+	// If the command is plugin management we don't try to load plugins
+	let command = args._[0];
+	if (command === "plugin") {
+		await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList);
+		return;
+	}
+
 	console.log("Loading Plugin info");
-	let pluginInfos = await plugin.loadPluginInfos("plugins");
+	let pluginInfos = await plugin.loadPluginInfos(pluginList);
 	config.registerPluginConfigGroups(pluginInfos);
 	config.finalizeConfigs();
 
@@ -1412,7 +1435,6 @@ async function startSlave() {
 		}
 	}
 
-	let command = args._[0];
 	if (command === "config") {
 		await config.handleConfigCommand(args, slaveConfig, args.config);
 		return;

test/lib/plugin.js

@@ -44,56 +44,48 @@ describe("lib/plugin", function() {
 
 	describe("loadPluginInfos()", function() {
 		let baseDir = path.join("temp", "test", "plugin");
-		let emptyDir = path.join(baseDir, "emptyDir");
-		let emptyPlugin = path.join(baseDir, "emptyPlugin");
-		let testPlugin = path.join(baseDir, "testPlugin");
-		let brokenPlugin = path.join(baseDir, "brokenPlugin");
-		let invalidPlugin = path.join(baseDir, "invalidPlugin");
+		let missingPlugin = path.join(baseDir, "missing_plugin");
+		let testPlugin = path.join(baseDir, "test_plugin");
+		let brokenPlugin = path.join(baseDir, "broken_plugin");
+		let invalidPlugin = path.join(baseDir, "invalid_plugin");
 		before(async function() {
-			await fs.ensureDir(emptyDir);
-			await fs.ensureDir(path.join(emptyPlugin, "empty"));
-
-			async function writePlugin(pluginPath, name, infoName = name) {
+			async function writePlugin(pluginPath, infoName) {
 				await fs.outputFile(
-					path.join(pluginPath, name, "info.js"),
+					path.join(pluginPath, "info.js"),
 					`module.exports = { name: "${infoName}" };`
 				);
 			}
 
 			await writePlugin(testPlugin, "test");
 			await writePlugin(brokenPlugin, "broken");
-			await fs.outputFile(path.join(brokenPlugin, "broken", "info.js"), "Syntax Error");
-			await writePlugin(invalidPlugin, "invalid", "wrong");
+			await fs.outputFile(path.join(brokenPlugin, "info.js"), "Syntax Error");
+			await writePlugin(invalidPlugin, "wrong");
 		});
 
-		it("should return an empty array for an empty directory", async function() {
-			assert.deepEqual(await plugin.loadPluginInfos(emptyDir), []);
+		it("should throw on missing plugin", async function() {
+			await assert.rejects(
+				plugin.loadPluginInfos(new Map([["missing", missingPlugin]]), []),
+				new RegExp(`^Error: PluginError: Cannot find module '${missingPlugin}/info'`)
+			);
 		});
-		it("should ignore plugin dirs without info module", async function() {
-			assert.deepEqual(await plugin.loadPluginInfos(emptyPlugin), []);
-		});
-		it("should discover test plugin", async function() {
+		it("should load test plugin", async function() {
 			assert.deepEqual(
-				await plugin.loadPluginInfos(testPlugin),
-				[{ name: "test" }]
+				await plugin.loadPluginInfos(new Map([["test", path.resolve(testPlugin)]])),
+				[{ name: "test", requirePath: path.resolve(testPlugin) }]
 			);
 		});
 		it("should reject on broken plugin", async function() {
 			await assert.rejects(
-				plugin.loadPluginInfos(brokenPlugin),
+				plugin.loadPluginInfos(new Map([["broken", path.resolve(brokenPlugin)]])),
 				{ message: "PluginError: Unexpected identifier" }
 			);
 		});
 		it("should reject on invalid plugin", async function() {
 			await assert.rejects(
-				plugin.loadPluginInfos(invalidPlugin),
-				{ message: `Plugin dir ${invalidPlugin}/invalid does not match the name of the plugin (wrong)` }
+				plugin.loadPluginInfos(new Map([["invalid", path.resolve(invalidPlugin)]])),
+				{ message: `Expected plugin at ${path.resolve(invalidPlugin)} to be named invalid but got wrong` }
 			);
 		});
-
-		after(async function() {
-			await fs.unlink(path.join(brokenPlugin, "broken", "info.js"));
-		});
 	});
 
 	describe("attachPluginMessages()", function() {