Merge pull request #323 from Hornwitser/plugin-restructure

Convert plugins to node packages

.gitignore

@@ -35,6 +35,7 @@ Factorio*
 /instances
 /sharedMods
 /database
+/plugin-list.json
 /config-master.json
 /config-control.json
 /config-slave.json
@@ -44,16 +45,6 @@ static/export
 saves
 /temp/
 
-# weird pattern to ignore sharedPlugins except not ignore some core modules
-!sharedPlugins/
-sharedPlugins/*
-!sharedPlugins/inventoryImports/
-!sharedPlugins/researchSync/
-!sharedPlugins/clusterioMod/
-
-# File used for marking plugins as disabled
-sharedPlugins/*/DISABLED
-
 # IDE
 .idea/
 .vs/

.travis.yml

@@ -7,8 +7,8 @@ before_install:
   - npm install -g codecov
 script:
   - npx lerna bootstrap --no-ci
-  - node packages/lib/build_mod --bump-patch --factorio-version 0.18 --output-dir sharedMods
-  - curl -o sharedMods/subspace_storage_1.99.1.zip -L https://github.com/clusterio/factorioClusterioMod/releases/download/1.99.1/subspace_storage_1.99.1.zip
+  - node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --bump-patch --factorio-version 0.18 --output-dir temp/test/sharedMods
+  - curl -o temp/test/sharedMods/subspace_storage_1.99.1.zip -L https://github.com/clusterio/factorioClusterioMod/releases/download/1.99.1/subspace_storage_1.99.1.zip
   - npm run-script ci-cover
   - cat temp/test/integration/log.txt
   - codecov -f coverage/lcov.info

README.md

@@ -33,11 +33,11 @@ the [1.2.x branch][1.2.x] for instructions on how to install the stable version.
 * [Introduction & methodology](#introduction)
 * [Ubuntu setup](#ubuntu-setup)
 * [Windows setup](#windows-setup)
+* [Installing Plugins](#installing-plugins)
 * [Running Clusterio](#running-clusterio)
   * [Master Server](#master-server)
   * [Slaves](#slaves)
   * [Instances](#instances)
-* [Optional plugins](#Plugins)
 * [Common problems](#Common-problems)
 
 ## Introduction
@@ -176,6 +176,29 @@ up, v11.13.0+ and v10.16.0+.
             the Factorio folder into the factorio folder created in step 1.
 
 
+## Installing Plugins
+
+Installing plugins to make them work with Clusterio consists of two
+steps.  First install the package via npm, for example
+
+    npm install @clusterio/plugin-subspace_storage
+
+Then tell clusterio that this plugin exists by adding it as a plugin.
+
+    npx clusteriomaster plugin add @clusterio/plugin-subspace_storage
+
+This adds it to the default `plugin-list.json` file which in the shared
+folder setup is loaded by master, slave and ctl.  If you have slaves or
+ctl installed on separate computers (or directories) then you need to
+repeat the plugin install process for all of them.  The clusteriomaster,
+clusterioslave and clusterioctl commands has the plugin sub-command
+so you do not need to install clusteriomaster to add plugins.
+
+For development purposes the `plugin add` command supports adding
+plugins by the absolute path to them, or a relative path that must start
+with either . or .. (which will then be resolved to an absolute path).
+
+
 ## Running Clusterio
 
 After following the installation instructions you can use the following
@@ -257,12 +280,6 @@ There are many more commands available with clusterioctl.  See
 `npx clusterioctl --help` for a full list of them.
 
 
-## Plugins
-Here are the known Clusterio plugins in the wild:
-1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta)
-2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook.
-3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers.
-
 ## Common problems
 
 ### EACCESS [...] LISTEN 443

docs/writing-plugins.md

@@ -39,8 +39,21 @@ The basic file structure of a plugin is the following.
          +- module.json
          +- plugin.lua
 
-The most important file is the `info.js` file.  Without it the plugin
-will not recognized by Clusterio.  Here's an example of it:
+Clusterio plugins are Node.js packages that can be published on npm for
+ease of installation and distribution.  The usual guides for creating
+such packages apply.  At minimum the `package.json` file must contain a
+version entry.
+
+A possible workflow for developing plugins is to place the plugin in a
+sub-directory of where clusterio has been installed, and rely on Node.js
+searching up the folder heirarchy for it to find `@clusterio/lib`.  To
+add it to `plugin-list.json` so that it gets loaded use the `plugin add
+<path>` sub-command to either clusteriomaster, clusterioslave or
+clusterioctl.  Note that it's important that the path starts with ./ or
+../ (use .\ or ..\ on Windows).
+
+For a plugin the most important file is the `info.js` file.  Without it
+the plugin will not recognized by Clusterio.  Here's an example of it:
 
     const link = require('@clusterio/lib/link'); // For messages
 
@@ -48,7 +61,6 @@ will not recognized by Clusterio.  Here's an example of it:
         name: "foo_frobber",
         title: "Foo Frobber",
         description: "Does advanced frobnication",
-        version: "0.8.1",
         instanceEntrypoint: "instance",
         masterEntrypoint: "master",
         messages: {
@@ -68,11 +80,6 @@ The following properties are recognized:
 **description**:
     Brief description of what the plugin does.  Currently not used.
 
-**version**:
-    Semver compatible version string of the plugin.  Currently only
-    servers as the default for the Clusterio module embedded in the
-    plugin, if any.
-
 **instanceEntrypoint**:
     Path to a Node.js module relative to the plugin directory which
     contains the InstancePlugin class definition for this plugin.  This

lerna.json

@@ -1,6 +1,7 @@
 {
   "packages": [
-    "packages/*"
+    "packages/*",
+    "plugins/*"
   ],
   "version": "2.0.0-alpha0"
 }

packages/ctl/ctl.js

@@ -18,6 +18,7 @@ const errors = require("@clusterio/lib/errors");
 const config = require("@clusterio/lib/config");
 const plugin = require("@clusterio/lib/plugin");
 const command = require("@clusterio/lib/command");
+const sharedCommands = require("@clusterio/lib/shared_commands");
 
 
 /**
@@ -586,11 +587,38 @@ async function startControl() {
 			defaultDescription: "auto",
 			type: "string",
 		})
+		.option("plugin-list", {
+			nargs: 1,
+			describe: "File containing list of plugins available with their install path",
+			default: "plugin-list.json",
+			type: "string",
+		})
+		.command("plugin", "Manage available plugins", sharedCommands.pluginCommand)
 		.command("control-config", "Manage Control config", config.configCommand)
 		.wrap(yargs.terminalWidth())
-		.strict()
+		.help(false) // Disable help to avoid triggering it on the first parse.
 	;
 
+	// Parse the args first to get the configured plugin list.
+	let args = yargs.argv;
+
+	console.log(`Loading available plugins from ${args.pluginList}`);
+	let pluginList = new Map();
+	try {
+		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
+	} catch (err) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+	}
+
+	// If the command is plugin management we don't try to load plugins
+	if (args._[0] === "plugin") {
+		await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList);
+		return;
+	}
+
+	// Add all cluster management commands including ones from plugins
 	const rootCommands = new command.CommandTree({ name: "clusterioctl", description: "Manage cluster" });
 	rootCommands.add(slaveCommands);
 	rootCommands.add(instanceCommands);
@@ -600,7 +628,7 @@ async function startControl() {
 	rootCommands.add(debugCommands);
 
 	console.log("Loading Plugin info");
-	let pluginInfos = await plugin.loadPluginInfos("plugins");
+	let pluginInfos = await plugin.loadPluginInfos(pluginList);
 	config.registerPluginConfigGroups(pluginInfos);
 	config.finalizeConfigs();
 
@@ -610,9 +638,7 @@ async function startControl() {
 			continue;
 		}
 
-		// XXX Does not work on Windows
-		let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name));
-		let { ControlPlugin } = require(path.join(pluginPath, pluginInfo.controlEntrypoint));
+		let { ControlPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.controlEntrypoint));
 		let controlPlugin = new ControlPlugin(pluginInfo);
 		controlPlugins.set(pluginInfo.name, controlPlugin);
 		await controlPlugin.init();
@@ -625,7 +651,12 @@ async function startControl() {
 		}
 	}
 
-	const args = yargs.argv;
+	// Reparse after commands have been added with help and strict checking.
+	args = yargs
+		.help()
+		.strict()
+		.argv
+	;
 
 	console.log(`Loading config from ${args.config}`);
 	let controlConfig = new config.ControlConfig();

packages/lib/build_mod.js

@@ -8,14 +8,13 @@ const yargs = require("yargs");
 
 
 async function main() {
-	const libPath = path.join("lua", "clusterio_lib");
 	const args = yargs
 		.scriptName("build")
 		.options({
 			"clean": { describe: "Remove previous builds", type: "boolean", default: false },
 			"build": { describe: "Build mod", type: "boolean", default: true },
 			"pack": { describe: "Pack into zip file", type: "boolean", default: true },
-			"source-dir": { describe: "Path to mod source files", nargs: 1, type: "string", default: libPath },
+			"source-dir": { describe: "Path to mod source files", nargs: 1, type: "string", demandOption: true },
 			"output-dir": { describe: "Path to output built mod", nargs: 1, type: "string", default: "dist" },
 			"bump-patch": { describe: "Increment patch number of build", type: "boolean", default: false },
 			"factorio-version": { describe: "Override factorio_version", type: "string" },
@@ -79,7 +78,10 @@ async function build(args) {
 			await events.once(walker, "end");
 
 			for (let pathParts of info.additional_files || []) {
-				let filePath = path.join(...pathParts);
+				let filePath = path.join(args.sourceDir, ...pathParts);
+				if (!await fs.pathExists(filePath)) {
+					throw new Error(`Additional file ${filePath} does not exist`);
+				}
 				zip.file(path.posix.join(modName, path.basename(filePath)), fs.createReadStream(filePath));
 			}
 			delete info.additional_files;

packages/lib/config/definitions.js

@@ -1,9 +1,6 @@
 // Core definitions for the configuration system
 "use strict";
 
-const util = require("util");
-const crypto = require("crypto");
-const path = require("path");
 const fs = require("fs-extra");
 
 const classes = require("./classes");
@@ -76,12 +73,7 @@ MasterGroup.define({
 		+"  Should be a long string of random letters and numbers."
 		+"  Do not share this.",
 	type: "string",
-	initial_value: async function() {
-		console.log("Generating new master authentication secret");
-		let asyncRandomBytes = util.promisify(crypto.randomBytes);
-		let bytes = await asyncRandomBytes(256);
-		return bytes.toString("base64");
-	},
+	optional: true,
 });
 MasterGroup.define({
 	name: "heartbeat_interval",
@@ -402,6 +394,7 @@ function configCommand(yargs) {
 		.command("show <field>", "Show value of the given config field")
 		.command("list", "List all configuration fields and their values")
 		.demandCommand(1, "You need to specify a command to run")
+		.help()
 		.strict()
 	;
 }

packages/lib/factorio/export.js

@@ -33,7 +33,8 @@ async function generateExportMod(server) {
 		clean: false,
 		build: true,
 		pack: true,
-		sourceDir: path.join("lua", "export"),
+		// XXX fix path when moving lib/factorio to slave
+		sourceDir: path.join(__dirname, "..", "..", "slave", "lua", "export"),
 		outputDir: server.writePath("mods"),
 		bumpPatch: false,
 		factorioVersion: server.version.replace(/\.\d+$/, ""),

packages/lib/plugin.js

@@ -3,7 +3,6 @@
  * @module lib/plugin
  */
 "use strict";
-const fs = require("fs-extra");
 const path = require("path");
 
 const errors = require("@clusterio/lib/errors");
@@ -390,39 +389,37 @@ class BaseControlPlugin {
 /**
  * Load plugin information
  *
- * Searches through the plugin directory and loads each plugin's info
- * module.  Once loaded the info modules will not be reloaded should this
- * function be called again.
+ * Loads plugin info modules for the paths to the given plugins.  Once
+ * loaded the info modules will not be reloaded should this function be
+ * called again.
  *
- * @param {string} baseDir - posix relative path to plugin directory.
- * @returns {Map<string, Object>} mapping of plugin name to info module.
+ * @param {Map<string, string>} pluginList -
+ *     Mapping of plugin name to require path for the plugins to load.
+ * @returns {Array<Object>} Array of plugin info modules.
  * @static
  */
-async function loadPluginInfos(baseDir) {
+async function loadPluginInfos(pluginList) {
 	let plugins = [];
-	for (let pluginDir of await fs.readdir(baseDir)) {
+	for (let [pluginName, pluginPath] of pluginList) {
 		let pluginInfo;
+		let pluginPackage;
 
 		try {
-			// Note: Require path is relative to this module
-			// XXX Does not work on Windows
-			let pluginPath = path.relative(__dirname, path.join(baseDir, pluginDir));
-			pluginInfo = require(path.join(pluginPath, "info"));
+			pluginInfo = require(path.posix.join(pluginPath, "info"));
+			pluginPackage = require(path.posix.join(pluginPath, "package.json"));
 
 		} catch (err) {
-			if (err.code === "MODULE_NOT_FOUND") {
-				continue;
-			} else {
-				throw new errors.PluginError(pluginDir, err);
-			}
+			throw new errors.PluginError(pluginName, err);
 		}
 
-		if (pluginInfo.name !== pluginDir) {
+		if (pluginInfo.name !== pluginName) {
 			throw new errors.EnvironmentError(
-				`Plugin dir ${baseDir}/${pluginDir} does not match the name of the plugin (${pluginInfo.name})`
+				`Expected plugin at ${pluginPath} to be named ${pluginName} but got ${pluginInfo.name}`
 			);
 		}
 
+		pluginInfo.requirePath = pluginPath;
+		pluginInfo.version = pluginPackage.version;
 		plugins.push(pluginInfo);
 	}
 	return plugins;

packages/lib/shared_commands.js

@@ -0,0 +1,86 @@
+/**
+ * Implementation of commands shared between master/slave/ctl.
+ * @module lib/shared_commands
+ */
+"use strict";
+const fs = require("fs-extra");
+const path = require("path");
+
+/**
+ * Yargs plugin command
+ *
+ * Can be passed to yargs.command to implement a plugin list management
+ * command.  Use handlePluginCommand to do the requested action.
+ *
+ * @param {Object} yargs - yargs command builder.
+ */
+function pluginCommand(yargs) {
+	yargs
+		.command("add <path>", "Add plugin by require path")
+		.command("remove <name>", "Remove plugin by name")
+		.command("list", "List all plugins and their path")
+		.demandCommand(1, "You need to specify a command to run")
+		.help()
+		.strict()
+	;
+}
+
+/**
+ * Handle yargs plugin command
+ *
+ * Handle the actions that are made available by pluginCommand.
+ *
+ * @param {Object} args - yargs args object.
+ * @param {Map<string, string>} pluginList - Current list of plugins.
+ * @param {string} pluginListPath - Path to plugin list config file.
+ */
+async function handlePluginCommand(args, pluginList, pluginListPath) {
+	let command = args._[1];
+
+	if (command === "add") {
+		let pluginPath = args.path;
+		if (/^\.\.?[\/\\]/.test(pluginPath)) {
+			pluginPath = path.resolve(pluginPath);
+		}
+
+		let pluginInfo;
+		try {
+			pluginInfo = require(path.posix.join(pluginPath, "info"));
+		} catch (err) {
+			console.error(`Unable to import plugin info from ${args.path}`);
+			console.error(err);
+			process.exitCode = 1;
+			return;
+		}
+
+		if (pluginList.has(pluginInfo.name)) {
+			console.error(`Plugin with the same ${pluginInfo.name} already exists`);
+			process.exitCode = 1;
+			return;
+		}
+
+		pluginList.set(pluginInfo.name, pluginPath);
+		await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
+		console.log(`Added ${pluginInfo.name}`);
+
+	} else if (command === "remove") {
+		if (!pluginList.delete(args.name)) {
+			console.error(`Plugin with name ${args.name} does not exist`);
+			process.exitCode = 1;
+			return;
+		}
+
+		await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
+		console.log(`Removed ${args.name}`);
+
+	} else if (command === "list") {
+		for (let [pluginName, pluginPath] of pluginList) {
+			console.log(`${pluginName} - ${pluginPath}`);
+		}
+	}
+}
+
+module.exports = {
+	pluginCommand,
+	handlePluginCommand,
+};

packages/master/master.js

@@ -29,10 +29,13 @@ const yargs = require("yargs");
 const WebSocket = require("ws");
 const JSZip = require("jszip");
 const version = require("./package").version;
+const util = require("util");
 
 // ugly globals
 let masterConfig;
 let masterConfigPath;
+let masterPlugins = new Map();
+let pluginInfos = new Map();
 let stopAcceptingNewSessions = false;
 let debugEvents = new events.EventEmitter();
 let pluginList = {};
@@ -49,6 +52,7 @@ const plugin = require("@clusterio/lib/plugin");
 const prometheus = require("@clusterio/lib/prometheus");
 const config = require("@clusterio/lib/config");
 const users = require("@clusterio/lib/users");
+const sharedCommands = require("@clusterio/lib/shared_commands");
 
 const express = require("express");
 const compression = require("compression");
@@ -1440,7 +1444,6 @@ async function handleHandshake(message, socket, req, attachHandler) {
 }
 
 // handle plugins on the master
-var masterPlugins = new Map();
 async function pluginManagement(pluginInfos) {
 	let startPluginLoad = Date.now();
 	masterPlugins = await loadPlugins(pluginInfos);
@@ -1460,9 +1463,7 @@ async function loadPlugins(pluginInfos) {
 		let MasterPlugin = plugin.BaseMasterPlugin;
 		try {
 			if (pluginInfo.masterEntrypoint) {
-				// XXX Does not work on Windows
-				let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name));
-				({ MasterPlugin } = require(path.join(pluginPath, pluginInfo.masterEntrypoint)));
+				({ MasterPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.masterEntrypoint)));
 			}
 
 			let masterPlugin = new MasterPlugin(
@@ -1554,6 +1555,13 @@ async function startServer() {
 			default: "config-master.json",
 			type: "string",
 		})
+		.option("plugin-list", {
+			nargs: 1,
+			describe: "File containing list of plugins available with their install path",
+			default: "plugin-list.json",
+			type: "string",
+		})
+		.command("plugin", "Manage available plugins", sharedCommands.pluginCommand)
 		.command("config", "Manage Master config", config.configCommand)
 		.command("bootstrap", "Bootstrap access to cluster", yargs => {
 			yargs
@@ -1574,8 +1582,25 @@ async function startServer() {
 		.argv
 	;
 
+	console.log(`Loading available plugins from ${args.pluginList}`);
+	let pluginList = new Map();
+	try {
+		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
+	} catch (err) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+	}
+
+	// If the command is plugin management we don't try to load plugins
+	let command = args._[0];
+	if (command === "plugin") {
+		await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList);
+		return;
+	}
+
 	console.log("Loading Plugin info");
-	let pluginInfos = await plugin.loadPluginInfos("plugins");
+	pluginInfos = await plugin.loadPluginInfos(pluginList);
 	config.registerPluginConfigGroups(pluginInfos);
 	config.finalizeConfigs();
 
@@ -1595,7 +1620,14 @@ async function startServer() {
 		}
 	}
 
-	let command = args._[0];
+	if (!masterConfig.get("master.auth_secret")) {
+		console.log("Generating new master authentication secret");
+		let asyncRandomBytes = util.promisify(crypto.randomBytes);
+		let bytes = await asyncRandomBytes(256);
+		masterConfig.set("master.auth_secret", bytes.toString("base64"));
+		await fs.outputFile(masterConfigPath, JSON.stringify(masterConfig.serialize(), null, 4));
+	}
+
 	if (command === "config") {
 		await config.handleConfigCommand(args, masterConfig, masterConfigPath);
 		return;
@@ -1642,8 +1674,6 @@ async function startServer() {
 			}
 		}
 
-		// Save config in case the auth_secret was generated during this invocation.
-		await fs.outputFile(masterConfigPath, JSON.stringify(masterConfig.serialize(), null, 4));
 		return;
 	}
 

packages/slave/lua/clusterio_lib/info.json

@@ -7,6 +7,6 @@
     "description": "Library for interfacing with Clusterio.  Warning: work in progress alpha.",
     "factorio_version": "0.17",
     "additional_files": [
-        ["modules", "clusterio", "serialize.lua"]
+        ["..", "..", "modules", "clusterio", "serialize.lua"]
     ]
 }

packages/slave/slave.js

@@ -32,6 +32,7 @@ const errors = require("@clusterio/lib/errors");
 const prometheus = require("@clusterio/lib/prometheus");
 const luaTools = require("@clusterio/lib/luaTools");
 const config = require("@clusterio/lib/config");
+const sharedCommands = require("@clusterio/lib/shared_commands");
 
 
 const instanceRconCommandsCounter = new prometheus.Counter(
@@ -206,9 +207,7 @@ class Instance extends link.Link{
 
 	async _loadPlugin(pluginInfo, slave) {
 		let pluginLoadStarted = Date.now();
-		// XXX Does not work on Windows
-		let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name));
-		let { InstancePlugin } = require(path.join(pluginPath, pluginInfo.instanceEntrypoint));
+		let { InstancePlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.instanceEntrypoint));
 		let instancePlugin = new InstancePlugin(pluginInfo, this, slave);
 		this.plugins.set(pluginInfo.name, instancePlugin);
 		await instancePlugin.init();
@@ -385,7 +384,8 @@ class Instance extends link.Link{
 		// Find plugin modules to patch in
 		let modules = new Map();
 		for (let [pluginName, plugin] of this.plugins) {
-			let modulePath = path.join("plugins", pluginName, "module");
+			let pluginPackagePath = require.resolve(path.posix.join(plugin.info.requirePath, "package.json"));
+			let modulePath = path.join(path.dirname(pluginPackagePath), "module");
 			if (!await fs.pathExists(modulePath)) {
 				continue;
 			}
@@ -412,14 +412,14 @@ class Instance extends link.Link{
 		}
 
 		// Find stand alone modules to load
-		// XXX for now it's assumed all available modules should be loaded.
-		for (let entry of await fs.readdir("modules", { withFileTypes: true })) {
+		// XXX for now only the included clusterio module is loaded
+		for (let entry of await fs.readdir(path.join(__dirname, "modules"), { withFileTypes: true })) {
 			if (entry.isDirectory()) {
 				if (modules.has(entry.name)) {
 					throw new Error(`Module with name ${entry.name} already exists in a plugin`);
 				}
 
-				let moduleJsonPath = path.join("modules", entry.name, "module.json");
+				let moduleJsonPath = path.join(__dirname, "modules", entry.name, "module.json");
 				if (!await fs.pathExists(moduleJsonPath)) {
 					throw new Error(`Module ${entry.name} is missing module.json`);
 				}
@@ -430,7 +430,7 @@ class Instance extends link.Link{
 				}
 
 				module = {
-					path: path.join("modules", entry.name),
+					path: path.join(__dirname, "modules", entry.name),
 					dependencies: { "clusterio": "*" },
 					load: [],
 					require: [],
@@ -1385,6 +1385,13 @@ async function startSlave() {
 			default: "config-slave.json",
 			type: "string",
 		})
+		.option("plugin-list", {
+			nargs: 1,
+			describe: "File containing list of plugins available with their install path",
+			default: "plugin-list.json",
+			type: "string",
+		})
+		.command("plugin", "Manage available plugins", sharedCommands.pluginCommand)
 		.command("config", "Manage Slave config", config.configCommand)
 		.command("run", "Run slave")
 		.demandCommand(1, "You need to specify a command to run")
@@ -1392,8 +1399,25 @@ async function startSlave() {
 		.argv
 	;
 
+	console.log(`Loading available plugins from ${args.pluginList}`);
+	let pluginList = new Map();
+	try {
+		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
+	} catch (err) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+	}
+
+	// If the command is plugin management we don't try to load plugins
+	let command = args._[0];
+	if (command === "plugin") {
+		await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList);
+		return;
+	}
+
 	console.log("Loading Plugin info");
-	let pluginInfos = await plugin.loadPluginInfos("plugins");
+	let pluginInfos = await plugin.loadPluginInfos(pluginList);
 	config.registerPluginConfigGroups(pluginInfos);
 	config.finalizeConfigs();
 
@@ -1412,7 +1436,6 @@ async function startSlave() {
 		}
 	}
 
-	let command = args._[0];
 	if (command === "config") {
 		await config.handleConfigCommand(args, slaveConfig, args.config);
 		return;

plugins/global_chat/info.js

@@ -6,7 +6,6 @@ module.exports = {
 	name: "global_chat",
 	title: "Global Chat",
 	description: "Forwards chat between instances.",
-	version: "2.0.0-alpha",
 	instanceEntrypoint: "instance",
 	controlEntrypoint: "control",
 

plugins/global_chat/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@clusterio/plugin-global_chat",
+  "version": "2.0.0-alpha0",
+  "description": "Clusterio plugin forwarding between Factorio servers",
+  "author": "Hornwitser <github@hornwitser.no>",
+  "homepage": "https://github.com/clusterio/factorioClusterio#readme",
+  "license": "MIT",
+  "repository": "clusterio/factorioClusterio",
+  "scripts": {
+    "test": "echo \"Error: run tests from root\" && exit 1"
+  },
+  "bugs": {
+    "url": "https://github.com/clusterio/factorioClusterio/issues"
+  },
+  "peerDependencies": {
+    "@clusterio/lib": "^2.0.0-alpha0"
+  }
+}

plugins/research_sync/info.js

@@ -22,7 +22,6 @@ module.exports = {
 	name: "research_sync",
 	title: "Research Sync",
 	description: "Synchronises technology research progress between instances.",
-	version: "2.0.0-alpha",
 	instanceEntrypoint: "instance",
 	masterEntrypoint: "master",
 

plugins/research_sync/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@clusterio/plugin-research_sync",
+  "version": "2.0.0-alpha0",
+  "description": "Clusterio plugin for synchronizing research between Factorio servers",
+  "author": "Hornwitser <github@hornwitser.no>",
+  "homepage": "https://github.com/clusterio/factorioClusterio#readme",
+  "license": "MIT",
+  "repository": "clusterio/factorioClusterio",
+  "scripts": {
+    "test": "echo \"Error: run tests from root\" && exit 1"
+  },
+  "bugs": {
+    "url": "https://github.com/clusterio/factorioClusterio/issues"
+  },
+  "peerDependencies": {
+    "@clusterio/lib": "^2.0.0-alpha0"
+  },
+  "ependencies": {
+    "fs-extra": "^8.1.0"
+  }
+}

plugins/statistics_exporter/info.js

@@ -5,6 +5,5 @@ module.exports = {
 	name: "statistics_exporter",
 	title: "Prometheus Statistics Export",
 	description: "Exports statistics to Prometheus",
-	version: "2.0.0-alpha",
 	instanceEntrypoint: "instance",
 };

plugins/statistics_exporter/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@clusterio/plugin-statistics_exporter",
+  "version": "2.0.0-alpha0",
+  "description": "Clusterio plugin for gathering statistics from Factorio servers",
+  "author": "Hornwitser <github@hornwitser.no>",
+  "homepage": "https://github.com/clusterio/factorioClusterio#readme",
+  "license": "MIT",
+  "repository": "clusterio/factorioClusterio",
+  "scripts": {
+    "test": "echo \"Error: run tests from root\" && exit 1"
+  },
+  "bugs": {
+    "url": "https://github.com/clusterio/factorioClusterio/issues"
+  },
+  "peerDependencies": {
+    "@clusterio/lib": "^2.0.0-alpha0"
+  }
+}

plugins/subspace_storage/info.js

@@ -57,7 +57,6 @@ module.exports = {
 	name: "subspace_storage",
 	title: "Subspace Storage",
 	description: "Provides shared storage across instances for the Subspace Storage mod",
-	version: "2.0.0-alpha",
 	instanceEntrypoint: "instance",
 	InstanceConfigGroup,
 

plugins/subspace_storage/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@clusterio/plugin-subspace_storage",
+  "version": "2.0.0-alpha0",
+  "description": "Clusterio plugin for sharing storage between Factorio servers",
+  "author": "Hornwitser <github@hornwitser.no>",
+  "homepage": "https://github.com/clusterio/clusterio#readme",
+  "license": "MIT",
+  "repository": "clusterio/factorioClusterio",
+  "scripts": {
+    "test": "echo \"Error: run tests from root\" && exit 1"
+  },
+  "bugs": {
+    "url": "https://github.com/clusterio/factorioClusterio/issues"
+  },
+  "peerDependencies": {
+    "@clusterio/lib": "^2.0.0-alpha0"
+  },
+  "dependencies": {
+    "fs-extra": "^8.1.0"
+  }
+}

plugins/subspace_storage/routes.js

@@ -1,5 +1,4 @@
 "use strict";
-const express = require("express");
 const path = require("path");
 
 const prometheus = require("@clusterio/lib/prometheus");

test/integration/clusterio.js

@@ -59,8 +59,10 @@ describe("Integration of Clusterio", function() {
 		describe("instance export-data", function() {
 			it("exports the data", async function() {
 				slowTest(this);
+				let exportPath = path.join("temp", "test", "static", "export", "locale.json");
+				await fs.remove(exportPath);
 				await execCtl("instance export-data test");
-				assert(await fs.exists(path.join("static", "export", "locale.json")), "Export was not created");
+				assert(await fs.exists(exportPath), "Export was not created");
 				await checkInstanceStatus(44, "stopped");
 			});
 		});

test/integration/index.js

@@ -63,18 +63,20 @@ let control;
 let url = "https://localhost:4443/";
 let controlToken = jwt.sign({ aud: "user", user: "test" }, "TestSecretDoNotUse");
 let instancesDir = path.join("temp", "test", "instances");
-let databaseDir = path.join("temp", "test", "databse");
-let masterConfigPath = path.join("temp", "test", "master-integration.json");
-let slaveConfigPath = path.join("temp", "test", "slave-integration.json");
-let controlConfigPath = path.join("temp", "test", "control-integration.json");
+let databaseDir = path.join("temp", "test", "database");
+let pluginListPath = path.join("temp", "test", "plugin-list.json");
+let masterConfigPath = path.join("temp", "test", "config-master.json");
+let slaveConfigPath = path.join("temp", "test", "config-slave.json");
+let controlConfigPath = path.join("temp", "test", "config-control.json");
 
-async function exec(...args) {
-	console.log(args[0]);
-	return await util.promisify(child_process.exec)(...args);
+async function exec(command, options = {}) {
+	console.log(command);
+	options = { cwd: path.join("temp", "test"), ...options };
+	return await util.promisify(child_process.exec)(command, options);
 }
 
 async function execCtl(...args) {
-	args[0] = `node packages/ctl --config ${controlConfigPath} ${args[0]}`;
+	args[0] = `node ../../packages/ctl ${args[0]}`;
 	return await exec(...args);
 }
 
@@ -91,7 +93,7 @@ function spawn(name, cmd, waitFor) {
 	return new Promise((resolve, reject) => {
 		console.log(cmd);
 		let parts = cmd.split(" ");
-		let process = child_process.spawn(parts[0], parts.slice(1));
+		let process = child_process.spawn(parts[0], parts.slice(1), { cwd: path.join("temp", "test") });
 		let stdout = new server._LineSplitter((line) => {
 			line = line.toString("utf8");
 			if (line.startsWith(waitFor)) {
@@ -113,30 +115,35 @@ before(async function() {
 	await fs.remove(databaseDir);
 	await fs.remove(instancesDir);
 
+	await fs.remove(pluginListPath);
 	await fs.remove(masterConfigPath);
 	await fs.remove(slaveConfigPath);
 	await fs.remove(controlConfigPath);
 
-	await exec(`node packages/master --config ${masterConfigPath} config set master.database_directory ${databaseDir}`);
-	await exec(`node packages/master --config ${masterConfigPath} config set master.auth_secret TestSecretDoNotUse`);
-	await exec(`node packages/master --config ${masterConfigPath} config set master.http_port 8880`);
-	await exec(`node packages/master --config ${masterConfigPath} config set master.https_port 4443`);
-	await exec(`node packages/master --config ${masterConfigPath} config set master.tls_certificate ${path.join(databaseDir, "cert.crt")}`);
-	await exec(`node packages/master --config ${masterConfigPath} config set master.tls_private_key ${path.join(databaseDir, "cert.key")}`);
-	await exec(`node packages/master --config ${masterConfigPath} config set master.tls_bits 1024`);
-	await exec(`node packages/master --config ${masterConfigPath} config set master.heartbeat_interval 0.25`);
-	await exec(`node packages/master --config ${masterConfigPath} config set master.connector_shutdown_timeout 2`);
+	await fs.ensureDir(path.join("temp", "test"));
 
-	await exec(`node packages/master --config ${masterConfigPath} bootstrap create-admin test`);
-	await exec(`node packages/master --config ${masterConfigPath} bootstrap create-ctl-config test --output ${controlConfigPath}`);
+	await exec("node ../../packages/master config set master.auth_secret TestSecretDoNotUse");
+	await exec("node ../../packages/master config set master.http_port 8880");
+	await exec("node ../../packages/master config set master.https_port 4443");
+	await exec("node ../../packages/master config set master.tls_bits 1024");
+	await exec("node ../../packages/master config set master.heartbeat_interval 0.25");
+	await exec("node ../../packages/master config set master.connector_shutdown_timeout 2");
 
-	masterProcess = await spawn("master:", `node packages/master --config ${masterConfigPath} run`, "All plugins loaded");
+	await exec("node ../../packages/ctl plugin add ../../plugins/global_chat");
+	await exec("node ../../packages/ctl plugin add ../../plugins/research_sync");
+	await exec("node ../../packages/ctl plugin add ../../plugins/statistics_exporter");
+	await exec("node ../../packages/ctl plugin add ../../plugins/subspace_storage");
 
-	let createArgs = `--id 4 --name slave --generate-token --output ${slaveConfigPath}`;
+	await exec("node ../../packages/master bootstrap create-admin test");
+	await exec("node ../../packages/master bootstrap create-ctl-config test");
+
+	masterProcess = await spawn("master:", "node ../../packages/master run", "All plugins loaded");
+
+	let createArgs = "--id 4 --name slave --generate-token";
 	await execCtl(`slave create-config ${createArgs}`);
-	await exec(`node packages/slave --config ${slaveConfigPath} config set slave.instances_directory ${instancesDir}`);
 
-	slaveProcess = await spawn("slave:", `node packages/slave --config ${slaveConfigPath} run`, "SOCKET | received ready from master");
+	await exec(`node ../../packages/slave config set slave.factorio_directory ${path.join("..", "..", "factorio")}`);
+	slaveProcess = await spawn("slave:", "node ../../packages/slave run", "SOCKET | received ready from master");
 
 	let controlConnector = new TestControlConnector(url, 2);
 	controlConnector.token = controlToken;

test/lib/plugin.js

@@ -44,56 +44,52 @@ describe("lib/plugin", function() {
 
 	describe("loadPluginInfos()", function() {
 		let baseDir = path.join("temp", "test", "plugin");
-		let emptyDir = path.join(baseDir, "emptyDir");
-		let emptyPlugin = path.join(baseDir, "emptyPlugin");
-		let testPlugin = path.join(baseDir, "testPlugin");
-		let brokenPlugin = path.join(baseDir, "brokenPlugin");
-		let invalidPlugin = path.join(baseDir, "invalidPlugin");
+		let missingPlugin = path.join(baseDir, "missing_plugin");
+		let testPlugin = path.join(baseDir, "test_plugin");
+		let brokenPlugin = path.join(baseDir, "broken_plugin");
+		let invalidPlugin = path.join(baseDir, "invalid_plugin");
 		before(async function() {
-			await fs.ensureDir(emptyDir);
-			await fs.ensureDir(path.join(emptyPlugin, "empty"));
-
-			async function writePlugin(pluginPath, name, infoName = name) {
+			async function writePlugin(pluginPath, infoName) {
 				await fs.outputFile(
-					path.join(pluginPath, name, "info.js"),
+					path.join(pluginPath, "info.js"),
 					`module.exports = { name: "${infoName}" };`
 				);
+				await fs.outputFile(
+					path.join(pluginPath, "package.json"),
+					'{ "version": "0.0.1" }'
+				);
 			}
 
 			await writePlugin(testPlugin, "test");
 			await writePlugin(brokenPlugin, "broken");
-			await fs.outputFile(path.join(brokenPlugin, "broken", "info.js"), "Syntax Error");
-			await writePlugin(invalidPlugin, "invalid", "wrong");
+			await fs.outputFile(path.join(brokenPlugin, "info.js"), "Syntax Error");
+			await writePlugin(invalidPlugin, "wrong");
 		});
 
-		it("should return an empty array for an empty directory", async function() {
-			assert.deepEqual(await plugin.loadPluginInfos(emptyDir), []);
+		it("should throw on missing plugin", async function() {
+			await assert.rejects(
+				plugin.loadPluginInfos(new Map([["missing", missingPlugin]]), []),
+				new RegExp(`^Error: PluginError: Cannot find module '${missingPlugin}/info'`)
+			);
 		});
-		it("should ignore plugin dirs without info module", async function() {
-			assert.deepEqual(await plugin.loadPluginInfos(emptyPlugin), []);
-		});
-		it("should discover test plugin", async function() {
+		it("should load test plugin", async function() {
 			assert.deepEqual(
-				await plugin.loadPluginInfos(testPlugin),
-				[{ name: "test" }]
+				await plugin.loadPluginInfos(new Map([["test", path.resolve(testPlugin)]])),
+				[{ name: "test", version: "0.0.1", requirePath: path.resolve(testPlugin) }]
 			);
 		});
 		it("should reject on broken plugin", async function() {
 			await assert.rejects(
-				plugin.loadPluginInfos(brokenPlugin),
+				plugin.loadPluginInfos(new Map([["broken", path.resolve(brokenPlugin)]])),
 				{ message: "PluginError: Unexpected identifier" }
 			);
 		});
 		it("should reject on invalid plugin", async function() {
 			await assert.rejects(
-				plugin.loadPluginInfos(invalidPlugin),
-				{ message: `Plugin dir ${invalidPlugin}/invalid does not match the name of the plugin (wrong)` }
+				plugin.loadPluginInfos(new Map([["invalid", path.resolve(invalidPlugin)]])),
+				{ message: `Expected plugin at ${path.resolve(invalidPlugin)} to be named invalid but got wrong` }
 			);
 		});
-
-		after(async function() {
-			await fs.unlink(path.join(brokenPlugin, "broken", "info.js"));
-		});
 	});
 
 	describe("attachPluginMessages()", function() {