Allow plugins to be installed as npm packages

Add a separate plugin-list.json file for storing which plugins are
available and the path to pass to require in order to load it.  This
allows plugins to be published and installed as npm packages.  And also
replaces the old method of installing plugins by putting them in the
plugins folder.
This commit is contained in:
Hornwitser committed 2020-10-21 19:35:24 +02:00
1 parent 2a7bc75929
commit 18e609f3fe
10 files changed
+238 -81

No files matched your search

+1 -10
View File
@@ -35,6 +35,7 @@ Factorio*
/instances /instances
/sharedMods /sharedMods
/database /database
/plugin-list.json
/config-master.json /config-master.json
/config-control.json /config-control.json
/config-slave.json /config-slave.json
@@ -44,16 +45,6 @@ static/export
saves saves
/temp/ /temp/
# weird pattern to ignore sharedPlugins except not ignore some core modules
!sharedPlugins/
sharedPlugins/*
!sharedPlugins/inventoryImports/
!sharedPlugins/researchSync/
!sharedPlugins/clusterioMod/
# File used for marking plugins as disabled
sharedPlugins/*/DISABLED
# IDE # IDE
.idea/ .idea/
.vs/ .vs/
+24 -7
View File
@@ -33,11 +33,11 @@ the [1.2.x branch][1.2.x] for instructions on how to install the stable version.
* [Introduction & methodology](#introduction) * [Introduction & methodology](#introduction)
* [Ubuntu setup](#ubuntu-setup) * [Ubuntu setup](#ubuntu-setup)
* [Windows setup](#windows-setup) * [Windows setup](#windows-setup)
* [Installing Plugins](#installing-plugins)
* [Running Clusterio](#running-clusterio) * [Running Clusterio](#running-clusterio)
* [Master Server](#master-server) * [Master Server](#master-server)
* [Slaves](#slaves) * [Slaves](#slaves)
* [Instances](#instances) * [Instances](#instances)
* [Optional plugins](#Plugins)
* [Common problems](#Common-problems) * [Common problems](#Common-problems)
## Introduction ## Introduction
@@ -176,6 +176,29 @@ up, v11.13.0+ and v10.16.0+.
the Factorio folder into the factorio folder created in step 1. the Factorio folder into the factorio folder created in step 1.
## Installing Plugins
Installing plugins to make them work with Clusterio consists of two
steps. First install the package via npm, for example
npm install @clusterio/plugin-subspace_storage
Then tell clusterio that this plugin exists by adding it as a plugin.
npx clusteriomaster plugin add @clusterio/plugin-subspace_storage
This adds it to the default `plugin-list.json` file which in the shared
folder setup is loaded by master, slave and ctl. If you have slaves or
ctl installed on separate computers (or directories) then you need to
repeat the plugin install process for all of them. The clusteriomaster,
clusterioslave and clusterioctl commands has the plugin sub-command
so you do not need to install clusteriomaster to add plugins.
For development purposes the `plugin add` command supports adding
plugins by the absolute path to them, or a relative path that must start
with either . or .. (which will then be resolved to an absolute path).
## Running Clusterio ## Running Clusterio
After following the installation instructions you can use the following After following the installation instructions you can use the following
@@ -257,12 +280,6 @@ There are many more commands available with clusterioctl. See
`npx clusterioctl --help` for a full list of them. `npx clusterioctl --help` for a full list of them.
## Plugins
Here are the known Clusterio plugins in the wild:
1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta)
2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook.
3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers.
## Common problems ## Common problems
### EACCESS [...] LISTEN 443 ### EACCESS [...] LISTEN 443
+1 -1
View File
@@ -2,7 +2,7 @@
"name": "root", "name": "root",
"private": true, "private": true,
"scripts": { "scripts": {
"test": "mocha --exit --check-leaks -R spec --recursive test plugins/*/test", "test": "mocha --exit --check-leaks -R spec --recursive test",
"fast-test": "FAST_TEST=y npm test", "fast-test": "FAST_TEST=y npm test",
"cover": "nyc npm test", "cover": "nyc npm test",
"fast-cover": "FAST_TEST=y nyc npm test", "fast-cover": "FAST_TEST=y nyc npm test",
+37 -6
View File
@@ -18,6 +18,7 @@ const errors = require("@clusterio/lib/errors");
const config = require("@clusterio/lib/config"); const config = require("@clusterio/lib/config");
const plugin = require("@clusterio/lib/plugin"); const plugin = require("@clusterio/lib/plugin");
const command = require("@clusterio/lib/command"); const command = require("@clusterio/lib/command");
const sharedCommands = require("@clusterio/lib/shared_commands");
/** /**
@@ -586,11 +587,38 @@ async function startControl() {
defaultDescription: "auto", defaultDescription: "auto",
type: "string", type: "string",
}) })
.option("plugin-list", {
nargs: 1,
describe: "File containing list of plugins available with their install path",
default: "plugin-list.json",
type: "string",
})
.command("plugin", "Manage available plugins", sharedCommands.pluginCommand)
.command("control-config", "Manage Control config", config.configCommand) .command("control-config", "Manage Control config", config.configCommand)
.wrap(yargs.terminalWidth()) .wrap(yargs.terminalWidth())
.strict() .help(false) // Disable help to avoid triggering it on the first parse.
; ;
// Parse the args first to get the configured plugin list.
let args = yargs.argv;
console.log(`Loading available plugins from ${args.pluginList}`);
let pluginList = new Map();
try {
pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
} catch (err) {
if (err.code !== "ENOENT") {
throw err;
}
}
// If the command is plugin management we don't try to load plugins
if (args._[0] === "plugin") {
await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList);
return;
}
// Add all cluster management commands including ones from plugins
const rootCommands = new command.CommandTree({ name: "clusterioctl", description: "Manage cluster" }); const rootCommands = new command.CommandTree({ name: "clusterioctl", description: "Manage cluster" });
rootCommands.add(slaveCommands); rootCommands.add(slaveCommands);
rootCommands.add(instanceCommands); rootCommands.add(instanceCommands);
@@ -600,7 +628,7 @@ async function startControl() {
rootCommands.add(debugCommands); rootCommands.add(debugCommands);
console.log("Loading Plugin info"); console.log("Loading Plugin info");
let pluginInfos = await plugin.loadPluginInfos("plugins"); let pluginInfos = await plugin.loadPluginInfos(pluginList);
config.registerPluginConfigGroups(pluginInfos); config.registerPluginConfigGroups(pluginInfos);
config.finalizeConfigs(); config.finalizeConfigs();
@@ -610,9 +638,7 @@ async function startControl() {
continue; continue;
} }
// XXX Does not work on Windows let { ControlPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.controlEntrypoint));
let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name));
let { ControlPlugin } = require(path.join(pluginPath, pluginInfo.controlEntrypoint));
let controlPlugin = new ControlPlugin(pluginInfo); let controlPlugin = new ControlPlugin(pluginInfo);
controlPlugins.set(pluginInfo.name, controlPlugin); controlPlugins.set(pluginInfo.name, controlPlugin);
await controlPlugin.init(); await controlPlugin.init();
@@ -625,7 +651,12 @@ async function startControl() {
} }
} }
const args = yargs.argv; // Reparse after commands have been added with help and strict checking.
args = yargs
.help()
.strict()
.argv
;
console.log(`Loading config from ${args.config}`); console.log(`Loading config from ${args.config}`);
let controlConfig = new config.ControlConfig(); let controlConfig = new config.ControlConfig();
+1
View File
@@ -394,6 +394,7 @@ function configCommand(yargs) {
.command("show <field>", "Show value of the given config field") .command("show <field>", "Show value of the given config field")
.command("list", "List all configuration fields and their values") .command("list", "List all configuration fields and their values")
.demandCommand(1, "You need to specify a command to run") .demandCommand(1, "You need to specify a command to run")
.help()
.strict() .strict()
; ;
} }
+13 -19
View File
@@ -3,7 +3,6 @@
* @module lib/plugin * @module lib/plugin
*/ */
"use strict"; "use strict";
const fs = require("fs-extra");
const path = require("path"); const path = require("path");
const errors = require("@clusterio/lib/errors"); const errors = require("@clusterio/lib/errors");
@@ -390,39 +389,34 @@ class BaseControlPlugin {
/** /**
* Load plugin information * Load plugin information
* *
* Searches through the plugin directory and loads each plugin's info * Loads plugin info modules for the paths to the given plugins. Once
* module. Once loaded the info modules will not be reloaded should this * loaded the info modules will not be reloaded should this function be
* function be called again. * called again.
* *
* @param {string} baseDir - posix relative path to plugin directory. * @param {Map<string, string>} pluginList -
* @returns {Map<string, Object>} mapping of plugin name to info module. * Mapping of plugin name to require path for the plugins to load.
* @returns {Array<Object>} Array of plugin info modules.
* @static * @static
*/ */
async function loadPluginInfos(baseDir) { async function loadPluginInfos(pluginList) {
let plugins = []; let plugins = [];
for (let pluginDir of await fs.readdir(baseDir)) { for (let [pluginName, pluginPath] of pluginList) {
let pluginInfo; let pluginInfo;
try { try {
// Note: Require path is relative to this module pluginInfo = require(path.posix.join(pluginPath, "info"));
// XXX Does not work on Windows
let pluginPath = path.relative(__dirname, path.join(baseDir, pluginDir));
pluginInfo = require(path.join(pluginPath, "info"));
} catch (err) { } catch (err) {
if (err.code === "MODULE_NOT_FOUND") { throw new errors.PluginError(pluginName, err);
continue;
} else {
throw new errors.PluginError(pluginDir, err);
}
} }
if (pluginInfo.name !== pluginDir) { if (pluginInfo.name !== pluginName) {
throw new errors.EnvironmentError( throw new errors.EnvironmentError(
`Plugin dir ${baseDir}/${pluginDir} does not match the name of the plugin (${pluginInfo.name})` `Expected plugin at ${pluginPath} to be named ${pluginName} but got ${pluginInfo.name}`
); );
} }
pluginInfo.requirePath = pluginPath;
plugins.push(pluginInfo); plugins.push(pluginInfo);
} }
return plugins; return plugins;
+86
View File
@@ -0,0 +1,86 @@
/**
* Implementation of commands shared between master/slave/ctl.
* @module lib/shared_commands
*/
"use strict";
const fs = require("fs-extra");
const path = require("path");
/**
* Yargs plugin command
*
* Can be passed to yargs.command to implement a plugin list management
* command. Use handlePluginCommand to do the requested action.
*
* @param {Object} yargs - yargs command builder.
*/
function pluginCommand(yargs) {
yargs
.command("add <path>", "Add plugin by require path")
.command("remove <name>", "Remove plugin by name")
.command("list", "List all plugins and their path")
.demandCommand(1, "You need to specify a command to run")
.help()
.strict()
;
}
/**
* Handle yargs plugin command
*
* Handle the actions that are made available by pluginCommand.
*
* @param {Object} args - yargs args object.
* @param {Map<string, string>} pluginList - Current list of plugins.
* @param {string} pluginListPath - Path to plugin list config file.
*/
async function handlePluginCommand(args, pluginList, pluginListPath) {
let command = args._[1];
if (command === "add") {
let pluginPath = args.path;
if (/\.\.?[\/\\]/.test(pluginPath)) {
pluginPath = path.resolve(pluginPath);
}
let pluginInfo;
try {
pluginInfo = require(path.posix.join(pluginPath, "info"));
} catch (err) {
console.error(`Unable to import plugin info from ${args.path}`);
console.error(err);
process.exitCode = 1;
return;
}
if (pluginList.has(pluginInfo.name)) {
console.error(`Plugin with the same ${pluginInfo.name} already exists`);
process.exitCode = 1;
return;
}
pluginList.set(pluginInfo.name, args.path);
await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
console.log(`Added ${pluginInfo.name}`);
} else if (command === "remove") {
if (!pluginList.delete(args.name)) {
console.error(`Plugin with name ${args.name} does not exist`);
process.exitCode = 1;
return;
}
await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4));
console.log(`Removed ${args.name}`);
} else if (command === "list") {
for (let [pluginName, pluginPath] of pluginList) {
console.log(`${pluginName} - ${pluginPath}`);
}
}
}
module.exports = {
pluginCommand,
handlePluginCommand,
};
+29 -6
View File
@@ -34,6 +34,8 @@ const util = require("util");
// ugly globals // ugly globals
let masterConfig; let masterConfig;
let masterConfigPath; let masterConfigPath;
let masterPlugins = new Map();
let pluginInfos = new Map();
let stopAcceptingNewSessions = false; let stopAcceptingNewSessions = false;
let debugEvents = new events.EventEmitter(); let debugEvents = new events.EventEmitter();
let pluginList = {}; let pluginList = {};
@@ -50,6 +52,7 @@ const plugin = require("@clusterio/lib/plugin");
const prometheus = require("@clusterio/lib/prometheus"); const prometheus = require("@clusterio/lib/prometheus");
const config = require("@clusterio/lib/config"); const config = require("@clusterio/lib/config");
const users = require("@clusterio/lib/users"); const users = require("@clusterio/lib/users");
const sharedCommands = require("@clusterio/lib/shared_commands");
const express = require("express"); const express = require("express");
const compression = require("compression"); const compression = require("compression");
@@ -1441,7 +1444,6 @@ async function handleHandshake(message, socket, req, attachHandler) {
} }
// handle plugins on the master // handle plugins on the master
var masterPlugins = new Map();
async function pluginManagement(pluginInfos) { async function pluginManagement(pluginInfos) {
let startPluginLoad = Date.now(); let startPluginLoad = Date.now();
masterPlugins = await loadPlugins(pluginInfos); masterPlugins = await loadPlugins(pluginInfos);
@@ -1461,9 +1463,7 @@ async function loadPlugins(pluginInfos) {
let MasterPlugin = plugin.BaseMasterPlugin; let MasterPlugin = plugin.BaseMasterPlugin;
try { try {
if (pluginInfo.masterEntrypoint) { if (pluginInfo.masterEntrypoint) {
// XXX Does not work on Windows ({ MasterPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.masterEntrypoint)));
let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name));
({ MasterPlugin } = require(path.join(pluginPath, pluginInfo.masterEntrypoint)));
} }
let masterPlugin = new MasterPlugin( let masterPlugin = new MasterPlugin(
@@ -1555,6 +1555,13 @@ async function startServer() {
default: "config-master.json", default: "config-master.json",
type: "string", type: "string",
}) })
.option("plugin-list", {
nargs: 1,
describe: "File containing list of plugins available with their install path",
default: "plugin-list.json",
type: "string",
})
.command("plugin", "Manage available plugins", sharedCommands.pluginCommand)
.command("config", "Manage Master config", config.configCommand) .command("config", "Manage Master config", config.configCommand)
.command("bootstrap", "Bootstrap access to cluster", yargs => { .command("bootstrap", "Bootstrap access to cluster", yargs => {
yargs yargs
@@ -1575,8 +1582,25 @@ async function startServer() {
.argv .argv
; ;
console.log(`Loading available plugins from ${args.pluginList}`);
let pluginList = new Map();
try {
pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
} catch (err) {
if (err.code !== "ENOENT") {
throw err;
}
}
// If the command is plugin management we don't try to load plugins
let command = args._[0];
if (command === "plugin") {
await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList);
return;
}
console.log("Loading Plugin info"); console.log("Loading Plugin info");
let pluginInfos = await plugin.loadPluginInfos("plugins"); pluginInfos = await plugin.loadPluginInfos(pluginList);
config.registerPluginConfigGroups(pluginInfos); config.registerPluginConfigGroups(pluginInfos);
config.finalizeConfigs(); config.finalizeConfigs();
@@ -1604,7 +1628,6 @@ async function startServer() {
await fs.outputFile(masterConfigPath, JSON.stringify(masterConfig.serialize(), null, 4)); await fs.outputFile(masterConfigPath, JSON.stringify(masterConfig.serialize(), null, 4));
} }
let command = args._[0];
if (command === "config") { if (command === "config") {
await config.handleConfigCommand(args, masterConfig, masterConfigPath); await config.handleConfigCommand(args, masterConfig, masterConfigPath);
return; return;
+27 -5
View File
@@ -32,6 +32,7 @@ const errors = require("@clusterio/lib/errors");
const prometheus = require("@clusterio/lib/prometheus"); const prometheus = require("@clusterio/lib/prometheus");
const luaTools = require("@clusterio/lib/luaTools"); const luaTools = require("@clusterio/lib/luaTools");
const config = require("@clusterio/lib/config"); const config = require("@clusterio/lib/config");
const sharedCommands = require("@clusterio/lib/shared_commands");
const instanceRconCommandsCounter = new prometheus.Counter( const instanceRconCommandsCounter = new prometheus.Counter(
@@ -206,9 +207,7 @@ class Instance extends link.Link{
async _loadPlugin(pluginInfo, slave) { async _loadPlugin(pluginInfo, slave) {
let pluginLoadStarted = Date.now(); let pluginLoadStarted = Date.now();
// XXX Does not work on Windows let { InstancePlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.instanceEntrypoint));
let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name));
let { InstancePlugin } = require(path.join(pluginPath, pluginInfo.instanceEntrypoint));
let instancePlugin = new InstancePlugin(pluginInfo, this, slave); let instancePlugin = new InstancePlugin(pluginInfo, this, slave);
this.plugins.set(pluginInfo.name, instancePlugin); this.plugins.set(pluginInfo.name, instancePlugin);
await instancePlugin.init(); await instancePlugin.init();
@@ -1385,6 +1384,13 @@ async function startSlave() {
default: "config-slave.json", default: "config-slave.json",
type: "string", type: "string",
}) })
.option("plugin-list", {
nargs: 1,
describe: "File containing list of plugins available with their install path",
default: "plugin-list.json",
type: "string",
})
.command("plugin", "Manage available plugins", sharedCommands.pluginCommand)
.command("config", "Manage Slave config", config.configCommand) .command("config", "Manage Slave config", config.configCommand)
.command("run", "Run slave") .command("run", "Run slave")
.demandCommand(1, "You need to specify a command to run") .demandCommand(1, "You need to specify a command to run")
@@ -1392,8 +1398,25 @@ async function startSlave() {
.argv .argv
; ;
console.log(`Loading available plugins from ${args.pluginList}`);
let pluginList = new Map();
try {
pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList)));
} catch (err) {
if (err.code !== "ENOENT") {
throw err;
}
}
// If the command is plugin management we don't try to load plugins
let command = args._[0];
if (command === "plugin") {
await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList);
return;
}
console.log("Loading Plugin info"); console.log("Loading Plugin info");
let pluginInfos = await plugin.loadPluginInfos("plugins"); let pluginInfos = await plugin.loadPluginInfos(pluginList);
config.registerPluginConfigGroups(pluginInfos); config.registerPluginConfigGroups(pluginInfos);
config.finalizeConfigs(); config.finalizeConfigs();
@@ -1412,7 +1435,6 @@ async function startSlave() {
} }
} }
let command = args._[0];
if (command === "config") { if (command === "config") {
await config.handleConfigCommand(args, slaveConfig, args.config); await config.handleConfigCommand(args, slaveConfig, args.config);
return; return;
+19 -27
View File
@@ -44,56 +44,48 @@ describe("lib/plugin", function() {
describe("loadPluginInfos()", function() { describe("loadPluginInfos()", function() {
let baseDir = path.join("temp", "test", "plugin"); let baseDir = path.join("temp", "test", "plugin");
let emptyDir = path.join(baseDir, "emptyDir"); let missingPlugin = path.join(baseDir, "missing_plugin");
let emptyPlugin = path.join(baseDir, "emptyPlugin"); let testPlugin = path.join(baseDir, "test_plugin");
let testPlugin = path.join(baseDir, "testPlugin"); let brokenPlugin = path.join(baseDir, "broken_plugin");
let brokenPlugin = path.join(baseDir, "brokenPlugin"); let invalidPlugin = path.join(baseDir, "invalid_plugin");
let invalidPlugin = path.join(baseDir, "invalidPlugin");
before(async function() { before(async function() {
await fs.ensureDir(emptyDir); async function writePlugin(pluginPath, infoName) {
await fs.ensureDir(path.join(emptyPlugin, "empty"));
async function writePlugin(pluginPath, name, infoName = name) {
await fs.outputFile( await fs.outputFile(
path.join(pluginPath, name, "info.js"), path.join(pluginPath, "info.js"),
`module.exports = { name: "${infoName}" };` `module.exports = { name: "${infoName}" };`
); );
} }
await writePlugin(testPlugin, "test"); await writePlugin(testPlugin, "test");
await writePlugin(brokenPlugin, "broken"); await writePlugin(brokenPlugin, "broken");
await fs.outputFile(path.join(brokenPlugin, "broken", "info.js"), "Syntax Error"); await fs.outputFile(path.join(brokenPlugin, "info.js"), "Syntax Error");
await writePlugin(invalidPlugin, "invalid", "wrong"); await writePlugin(invalidPlugin, "wrong");
}); });
it("should return an empty array for an empty directory", async function() { it("should throw on missing plugin", async function() {
assert.deepEqual(await plugin.loadPluginInfos(emptyDir), []); await assert.rejects(
plugin.loadPluginInfos(new Map([["missing", missingPlugin]]), []),
new RegExp(`^Error: PluginError: Cannot find module '${missingPlugin}/info'`)
);
}); });
it("should ignore plugin dirs without info module", async function() { it("should load test plugin", async function() {
assert.deepEqual(await plugin.loadPluginInfos(emptyPlugin), []);
});
it("should discover test plugin", async function() {
assert.deepEqual( assert.deepEqual(
await plugin.loadPluginInfos(testPlugin), await plugin.loadPluginInfos(new Map([["test", path.resolve(testPlugin)]])),
[{ name: "test" }] [{ name: "test", requirePath: path.resolve(testPlugin) }]
); );
}); });
it("should reject on broken plugin", async function() { it("should reject on broken plugin", async function() {
await assert.rejects( await assert.rejects(
plugin.loadPluginInfos(brokenPlugin), plugin.loadPluginInfos(new Map([["broken", path.resolve(brokenPlugin)]])),
{ message: "PluginError: Unexpected identifier" } { message: "PluginError: Unexpected identifier" }
); );
}); });
it("should reject on invalid plugin", async function() { it("should reject on invalid plugin", async function() {
await assert.rejects( await assert.rejects(
plugin.loadPluginInfos(invalidPlugin), plugin.loadPluginInfos(new Map([["invalid", path.resolve(invalidPlugin)]])),
{ message: `Plugin dir ${invalidPlugin}/invalid does not match the name of the plugin (wrong)` } { message: `Expected plugin at ${path.resolve(invalidPlugin)} to be named invalid but got wrong` }
); );
}); });
after(async function() {
await fs.unlink(path.join(brokenPlugin, "broken", "info.js"));
});
}); });
describe("attachPluginMessages()", function() { describe("attachPluginMessages()", function() {