diff --git a/.gitignore b/.gitignore index a2891657..56c8fbd4 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ Factorio* /instances /sharedMods /database +/plugin-list.json /config-master.json /config-control.json /config-slave.json @@ -44,16 +45,6 @@ static/export saves /temp/ -# weird pattern to ignore sharedPlugins except not ignore some core modules -!sharedPlugins/ -sharedPlugins/* -!sharedPlugins/inventoryImports/ -!sharedPlugins/researchSync/ -!sharedPlugins/clusterioMod/ - -# File used for marking plugins as disabled -sharedPlugins/*/DISABLED - # IDE .idea/ .vs/ diff --git a/.travis.yml b/.travis.yml index d46fbc66..52d21e37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ before_install: - npm install -g codecov script: - npx lerna bootstrap --no-ci - - node packages/lib/build_mod --bump-patch --factorio-version 0.18 --output-dir sharedMods - - curl -o sharedMods/subspace_storage_1.99.1.zip -L https://github.com/clusterio/factorioClusterioMod/releases/download/1.99.1/subspace_storage_1.99.1.zip + - node packages/lib/build_mod --source-dir packages/slave/lua/clusterio_lib --bump-patch --factorio-version 0.18 --output-dir temp/test/sharedMods + - curl -o temp/test/sharedMods/subspace_storage_1.99.1.zip -L https://github.com/clusterio/factorioClusterioMod/releases/download/1.99.1/subspace_storage_1.99.1.zip - npm run-script ci-cover - cat temp/test/integration/log.txt - codecov -f coverage/lcov.info diff --git a/README.md b/README.md index ae3e136e..04f0a500 100644 --- a/README.md +++ b/README.md @@ -33,11 +33,11 @@ the [1.2.x branch][1.2.x] for instructions on how to install the stable version. * [Introduction & methodology](#introduction) * [Ubuntu setup](#ubuntu-setup) * [Windows setup](#windows-setup) +* [Installing Plugins](#installing-plugins) * [Running Clusterio](#running-clusterio) * [Master Server](#master-server) * [Slaves](#slaves) * [Instances](#instances) -* [Optional plugins](#Plugins) * [Common problems](#Common-problems) ## Introduction @@ -176,6 +176,29 @@ up, v11.13.0+ and v10.16.0+. the Factorio folder into the factorio folder created in step 1. +## Installing Plugins + +Installing plugins to make them work with Clusterio consists of two +steps. First install the package via npm, for example + + npm install @clusterio/plugin-subspace_storage + +Then tell clusterio that this plugin exists by adding it as a plugin. + + npx clusteriomaster plugin add @clusterio/plugin-subspace_storage + +This adds it to the default `plugin-list.json` file which in the shared +folder setup is loaded by master, slave and ctl. If you have slaves or +ctl installed on separate computers (or directories) then you need to +repeat the plugin install process for all of them. The clusteriomaster, +clusterioslave and clusterioctl commands has the plugin sub-command +so you do not need to install clusteriomaster to add plugins. + +For development purposes the `plugin add` command supports adding +plugins by the absolute path to them, or a relative path that must start +with either . or .. (which will then be resolved to an absolute path). + + ## Running Clusterio After following the installation instructions you can use the following @@ -257,12 +280,6 @@ There are many more commands available with clusterioctl. See `npx clusterioctl --help` for a full list of them. -## Plugins -Here are the known Clusterio plugins in the wild: -1. [Player Manager](https://github.com/Danielv123/playerManager) - Adds player management to the Web UI and shared inventory handling (beta) -2. [DiscordChat](https://github.com/jakedraddy/ClusterioDiscordChat) - Logs in-game chat/joins/leave messages on all instances to a Discord webhook. -3. [TrainTeleports](https://github.com/Godmave/clusterioTrainTeleports) - Allows you to teleport cargotrains between servers. - ## Common problems ### EACCESS [...] LISTEN 443 diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index 60f6ecec..680b53a7 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -39,8 +39,21 @@ The basic file structure of a plugin is the following. +- module.json +- plugin.lua -The most important file is the `info.js` file. Without it the plugin -will not recognized by Clusterio. Here's an example of it: +Clusterio plugins are Node.js packages that can be published on npm for +ease of installation and distribution. The usual guides for creating +such packages apply. At minimum the `package.json` file must contain a +version entry. + +A possible workflow for developing plugins is to place the plugin in a +sub-directory of where clusterio has been installed, and rely on Node.js +searching up the folder heirarchy for it to find `@clusterio/lib`. To +add it to `plugin-list.json` so that it gets loaded use the `plugin add +` sub-command to either clusteriomaster, clusterioslave or +clusterioctl. Note that it's important that the path starts with ./ or +../ (use .\ or ..\ on Windows). + +For a plugin the most important file is the `info.js` file. Without it +the plugin will not recognized by Clusterio. Here's an example of it: const link = require('@clusterio/lib/link'); // For messages @@ -48,7 +61,6 @@ will not recognized by Clusterio. Here's an example of it: name: "foo_frobber", title: "Foo Frobber", description: "Does advanced frobnication", - version: "0.8.1", instanceEntrypoint: "instance", masterEntrypoint: "master", messages: { @@ -68,11 +80,6 @@ The following properties are recognized: **description**: Brief description of what the plugin does. Currently not used. -**version**: - Semver compatible version string of the plugin. Currently only - servers as the default for the Clusterio module embedded in the - plugin, if any. - **instanceEntrypoint**: Path to a Node.js module relative to the plugin directory which contains the InstancePlugin class definition for this plugin. This diff --git a/lerna.json b/lerna.json index 8732d9e9..fdf6300d 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,7 @@ { "packages": [ - "packages/*" + "packages/*", + "plugins/*" ], "version": "2.0.0-alpha0" } diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 42a56469..a44d6820 100755 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -18,6 +18,7 @@ const errors = require("@clusterio/lib/errors"); const config = require("@clusterio/lib/config"); const plugin = require("@clusterio/lib/plugin"); const command = require("@clusterio/lib/command"); +const sharedCommands = require("@clusterio/lib/shared_commands"); /** @@ -586,11 +587,38 @@ async function startControl() { defaultDescription: "auto", type: "string", }) + .option("plugin-list", { + nargs: 1, + describe: "File containing list of plugins available with their install path", + default: "plugin-list.json", + type: "string", + }) + .command("plugin", "Manage available plugins", sharedCommands.pluginCommand) .command("control-config", "Manage Control config", config.configCommand) .wrap(yargs.terminalWidth()) - .strict() + .help(false) // Disable help to avoid triggering it on the first parse. ; + // Parse the args first to get the configured plugin list. + let args = yargs.argv; + + console.log(`Loading available plugins from ${args.pluginList}`); + let pluginList = new Map(); + try { + pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList))); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + // If the command is plugin management we don't try to load plugins + if (args._[0] === "plugin") { + await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList); + return; + } + + // Add all cluster management commands including ones from plugins const rootCommands = new command.CommandTree({ name: "clusterioctl", description: "Manage cluster" }); rootCommands.add(slaveCommands); rootCommands.add(instanceCommands); @@ -600,7 +628,7 @@ async function startControl() { rootCommands.add(debugCommands); console.log("Loading Plugin info"); - let pluginInfos = await plugin.loadPluginInfos("plugins"); + let pluginInfos = await plugin.loadPluginInfos(pluginList); config.registerPluginConfigGroups(pluginInfos); config.finalizeConfigs(); @@ -610,9 +638,7 @@ async function startControl() { continue; } - // XXX Does not work on Windows - let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name)); - let { ControlPlugin } = require(path.join(pluginPath, pluginInfo.controlEntrypoint)); + let { ControlPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.controlEntrypoint)); let controlPlugin = new ControlPlugin(pluginInfo); controlPlugins.set(pluginInfo.name, controlPlugin); await controlPlugin.init(); @@ -625,7 +651,12 @@ async function startControl() { } } - const args = yargs.argv; + // Reparse after commands have been added with help and strict checking. + args = yargs + .help() + .strict() + .argv + ; console.log(`Loading config from ${args.config}`); let controlConfig = new config.ControlConfig(); diff --git a/packages/lib/build_mod.js b/packages/lib/build_mod.js index aef57612..ffe54f83 100644 --- a/packages/lib/build_mod.js +++ b/packages/lib/build_mod.js @@ -8,14 +8,13 @@ const yargs = require("yargs"); async function main() { - const libPath = path.join("lua", "clusterio_lib"); const args = yargs .scriptName("build") .options({ "clean": { describe: "Remove previous builds", type: "boolean", default: false }, "build": { describe: "Build mod", type: "boolean", default: true }, "pack": { describe: "Pack into zip file", type: "boolean", default: true }, - "source-dir": { describe: "Path to mod source files", nargs: 1, type: "string", default: libPath }, + "source-dir": { describe: "Path to mod source files", nargs: 1, type: "string", demandOption: true }, "output-dir": { describe: "Path to output built mod", nargs: 1, type: "string", default: "dist" }, "bump-patch": { describe: "Increment patch number of build", type: "boolean", default: false }, "factorio-version": { describe: "Override factorio_version", type: "string" }, @@ -79,7 +78,10 @@ async function build(args) { await events.once(walker, "end"); for (let pathParts of info.additional_files || []) { - let filePath = path.join(...pathParts); + let filePath = path.join(args.sourceDir, ...pathParts); + if (!await fs.pathExists(filePath)) { + throw new Error(`Additional file ${filePath} does not exist`); + } zip.file(path.posix.join(modName, path.basename(filePath)), fs.createReadStream(filePath)); } delete info.additional_files; diff --git a/packages/lib/config/definitions.js b/packages/lib/config/definitions.js index 7e4b25d6..7958db7e 100644 --- a/packages/lib/config/definitions.js +++ b/packages/lib/config/definitions.js @@ -1,9 +1,6 @@ // Core definitions for the configuration system "use strict"; -const util = require("util"); -const crypto = require("crypto"); -const path = require("path"); const fs = require("fs-extra"); const classes = require("./classes"); @@ -76,12 +73,7 @@ MasterGroup.define({ +" Should be a long string of random letters and numbers." +" Do not share this.", type: "string", - initial_value: async function() { - console.log("Generating new master authentication secret"); - let asyncRandomBytes = util.promisify(crypto.randomBytes); - let bytes = await asyncRandomBytes(256); - return bytes.toString("base64"); - }, + optional: true, }); MasterGroup.define({ name: "heartbeat_interval", @@ -402,6 +394,7 @@ function configCommand(yargs) { .command("show ", "Show value of the given config field") .command("list", "List all configuration fields and their values") .demandCommand(1, "You need to specify a command to run") + .help() .strict() ; } diff --git a/packages/lib/factorio/export.js b/packages/lib/factorio/export.js index 9a19617b..132ecada 100644 --- a/packages/lib/factorio/export.js +++ b/packages/lib/factorio/export.js @@ -33,7 +33,8 @@ async function generateExportMod(server) { clean: false, build: true, pack: true, - sourceDir: path.join("lua", "export"), + // XXX fix path when moving lib/factorio to slave + sourceDir: path.join(__dirname, "..", "..", "slave", "lua", "export"), outputDir: server.writePath("mods"), bumpPatch: false, factorioVersion: server.version.replace(/\.\d+$/, ""), diff --git a/packages/lib/plugin.js b/packages/lib/plugin.js index d4a1a6f5..f7ccbb6c 100644 --- a/packages/lib/plugin.js +++ b/packages/lib/plugin.js @@ -3,7 +3,6 @@ * @module lib/plugin */ "use strict"; -const fs = require("fs-extra"); const path = require("path"); const errors = require("@clusterio/lib/errors"); @@ -390,39 +389,37 @@ class BaseControlPlugin { /** * Load plugin information * - * Searches through the plugin directory and loads each plugin's info - * module. Once loaded the info modules will not be reloaded should this - * function be called again. + * Loads plugin info modules for the paths to the given plugins. Once + * loaded the info modules will not be reloaded should this function be + * called again. * - * @param {string} baseDir - posix relative path to plugin directory. - * @returns {Map} mapping of plugin name to info module. + * @param {Map} pluginList - + * Mapping of plugin name to require path for the plugins to load. + * @returns {Array} Array of plugin info modules. * @static */ -async function loadPluginInfos(baseDir) { +async function loadPluginInfos(pluginList) { let plugins = []; - for (let pluginDir of await fs.readdir(baseDir)) { + for (let [pluginName, pluginPath] of pluginList) { let pluginInfo; + let pluginPackage; try { - // Note: Require path is relative to this module - // XXX Does not work on Windows - let pluginPath = path.relative(__dirname, path.join(baseDir, pluginDir)); - pluginInfo = require(path.join(pluginPath, "info")); + pluginInfo = require(path.posix.join(pluginPath, "info")); + pluginPackage = require(path.posix.join(pluginPath, "package.json")); } catch (err) { - if (err.code === "MODULE_NOT_FOUND") { - continue; - } else { - throw new errors.PluginError(pluginDir, err); - } + throw new errors.PluginError(pluginName, err); } - if (pluginInfo.name !== pluginDir) { + if (pluginInfo.name !== pluginName) { throw new errors.EnvironmentError( - `Plugin dir ${baseDir}/${pluginDir} does not match the name of the plugin (${pluginInfo.name})` + `Expected plugin at ${pluginPath} to be named ${pluginName} but got ${pluginInfo.name}` ); } + pluginInfo.requirePath = pluginPath; + pluginInfo.version = pluginPackage.version; plugins.push(pluginInfo); } return plugins; diff --git a/packages/lib/shared_commands.js b/packages/lib/shared_commands.js new file mode 100644 index 00000000..29550cb6 --- /dev/null +++ b/packages/lib/shared_commands.js @@ -0,0 +1,86 @@ +/** + * Implementation of commands shared between master/slave/ctl. + * @module lib/shared_commands + */ +"use strict"; +const fs = require("fs-extra"); +const path = require("path"); + +/** + * Yargs plugin command + * + * Can be passed to yargs.command to implement a plugin list management + * command. Use handlePluginCommand to do the requested action. + * + * @param {Object} yargs - yargs command builder. + */ +function pluginCommand(yargs) { + yargs + .command("add ", "Add plugin by require path") + .command("remove ", "Remove plugin by name") + .command("list", "List all plugins and their path") + .demandCommand(1, "You need to specify a command to run") + .help() + .strict() + ; +} + +/** + * Handle yargs plugin command + * + * Handle the actions that are made available by pluginCommand. + * + * @param {Object} args - yargs args object. + * @param {Map} pluginList - Current list of plugins. + * @param {string} pluginListPath - Path to plugin list config file. + */ +async function handlePluginCommand(args, pluginList, pluginListPath) { + let command = args._[1]; + + if (command === "add") { + let pluginPath = args.path; + if (/^\.\.?[\/\\]/.test(pluginPath)) { + pluginPath = path.resolve(pluginPath); + } + + let pluginInfo; + try { + pluginInfo = require(path.posix.join(pluginPath, "info")); + } catch (err) { + console.error(`Unable to import plugin info from ${args.path}`); + console.error(err); + process.exitCode = 1; + return; + } + + if (pluginList.has(pluginInfo.name)) { + console.error(`Plugin with the same ${pluginInfo.name} already exists`); + process.exitCode = 1; + return; + } + + pluginList.set(pluginInfo.name, pluginPath); + await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4)); + console.log(`Added ${pluginInfo.name}`); + + } else if (command === "remove") { + if (!pluginList.delete(args.name)) { + console.error(`Plugin with name ${args.name} does not exist`); + process.exitCode = 1; + return; + } + + await fs.outputFile(pluginListPath, JSON.stringify([...pluginList], null, 4)); + console.log(`Removed ${args.name}`); + + } else if (command === "list") { + for (let [pluginName, pluginPath] of pluginList) { + console.log(`${pluginName} - ${pluginPath}`); + } + } +} + +module.exports = { + pluginCommand, + handlePluginCommand, +}; diff --git a/packages/master/master.js b/packages/master/master.js index e3832fba..a27e308d 100755 --- a/packages/master/master.js +++ b/packages/master/master.js @@ -29,10 +29,13 @@ const yargs = require("yargs"); const WebSocket = require("ws"); const JSZip = require("jszip"); const version = require("./package").version; +const util = require("util"); // ugly globals let masterConfig; let masterConfigPath; +let masterPlugins = new Map(); +let pluginInfos = new Map(); let stopAcceptingNewSessions = false; let debugEvents = new events.EventEmitter(); let pluginList = {}; @@ -49,6 +52,7 @@ const plugin = require("@clusterio/lib/plugin"); const prometheus = require("@clusterio/lib/prometheus"); const config = require("@clusterio/lib/config"); const users = require("@clusterio/lib/users"); +const sharedCommands = require("@clusterio/lib/shared_commands"); const express = require("express"); const compression = require("compression"); @@ -1440,7 +1444,6 @@ async function handleHandshake(message, socket, req, attachHandler) { } // handle plugins on the master -var masterPlugins = new Map(); async function pluginManagement(pluginInfos) { let startPluginLoad = Date.now(); masterPlugins = await loadPlugins(pluginInfos); @@ -1460,9 +1463,7 @@ async function loadPlugins(pluginInfos) { let MasterPlugin = plugin.BaseMasterPlugin; try { if (pluginInfo.masterEntrypoint) { - // XXX Does not work on Windows - let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name)); - ({ MasterPlugin } = require(path.join(pluginPath, pluginInfo.masterEntrypoint))); + ({ MasterPlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.masterEntrypoint))); } let masterPlugin = new MasterPlugin( @@ -1554,6 +1555,13 @@ async function startServer() { default: "config-master.json", type: "string", }) + .option("plugin-list", { + nargs: 1, + describe: "File containing list of plugins available with their install path", + default: "plugin-list.json", + type: "string", + }) + .command("plugin", "Manage available plugins", sharedCommands.pluginCommand) .command("config", "Manage Master config", config.configCommand) .command("bootstrap", "Bootstrap access to cluster", yargs => { yargs @@ -1574,8 +1582,25 @@ async function startServer() { .argv ; + console.log(`Loading available plugins from ${args.pluginList}`); + let pluginList = new Map(); + try { + pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList))); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + // If the command is plugin management we don't try to load plugins + let command = args._[0]; + if (command === "plugin") { + await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList); + return; + } + console.log("Loading Plugin info"); - let pluginInfos = await plugin.loadPluginInfos("plugins"); + pluginInfos = await plugin.loadPluginInfos(pluginList); config.registerPluginConfigGroups(pluginInfos); config.finalizeConfigs(); @@ -1595,7 +1620,14 @@ async function startServer() { } } - let command = args._[0]; + if (!masterConfig.get("master.auth_secret")) { + console.log("Generating new master authentication secret"); + let asyncRandomBytes = util.promisify(crypto.randomBytes); + let bytes = await asyncRandomBytes(256); + masterConfig.set("master.auth_secret", bytes.toString("base64")); + await fs.outputFile(masterConfigPath, JSON.stringify(masterConfig.serialize(), null, 4)); + } + if (command === "config") { await config.handleConfigCommand(args, masterConfig, masterConfigPath); return; @@ -1642,8 +1674,6 @@ async function startServer() { } } - // Save config in case the auth_secret was generated during this invocation. - await fs.outputFile(masterConfigPath, JSON.stringify(masterConfig.serialize(), null, 4)); return; } diff --git a/lua/clusterio_lib/api.lua b/packages/slave/lua/clusterio_lib/api.lua similarity index 100% rename from lua/clusterio_lib/api.lua rename to packages/slave/lua/clusterio_lib/api.lua diff --git a/lua/clusterio_lib/changelog.txt b/packages/slave/lua/clusterio_lib/changelog.txt similarity index 100% rename from lua/clusterio_lib/changelog.txt rename to packages/slave/lua/clusterio_lib/changelog.txt diff --git a/lua/clusterio_lib/info.json b/packages/slave/lua/clusterio_lib/info.json similarity index 85% rename from lua/clusterio_lib/info.json rename to packages/slave/lua/clusterio_lib/info.json index 43177950..b66e2a2e 100644 --- a/lua/clusterio_lib/info.json +++ b/packages/slave/lua/clusterio_lib/info.json @@ -7,6 +7,6 @@ "description": "Library for interfacing with Clusterio. Warning: work in progress alpha.", "factorio_version": "0.17", "additional_files": [ - ["modules", "clusterio", "serialize.lua"] + ["..", "..", "modules", "clusterio", "serialize.lua"] ] } diff --git a/lua/export/control.lua b/packages/slave/lua/export/control.lua similarity index 100% rename from lua/export/control.lua rename to packages/slave/lua/export/control.lua diff --git a/lua/export/data-final-fixes.lua b/packages/slave/lua/export/data-final-fixes.lua similarity index 100% rename from lua/export/data-final-fixes.lua rename to packages/slave/lua/export/data-final-fixes.lua diff --git a/lua/export/info.json b/packages/slave/lua/export/info.json similarity index 100% rename from lua/export/info.json rename to packages/slave/lua/export/info.json diff --git a/modules/clusterio/api.lua b/packages/slave/modules/clusterio/api.lua similarity index 100% rename from modules/clusterio/api.lua rename to packages/slave/modules/clusterio/api.lua diff --git a/modules/clusterio/impl.lua b/packages/slave/modules/clusterio/impl.lua similarity index 100% rename from modules/clusterio/impl.lua rename to packages/slave/modules/clusterio/impl.lua diff --git a/modules/clusterio/module.json b/packages/slave/modules/clusterio/module.json similarity index 100% rename from modules/clusterio/module.json rename to packages/slave/modules/clusterio/module.json diff --git a/modules/clusterio/serialize.lua b/packages/slave/modules/clusterio/serialize.lua similarity index 100% rename from modules/clusterio/serialize.lua rename to packages/slave/modules/clusterio/serialize.lua diff --git a/packages/slave/slave.js b/packages/slave/slave.js index b2947b08..17c2bc52 100755 --- a/packages/slave/slave.js +++ b/packages/slave/slave.js @@ -32,6 +32,7 @@ const errors = require("@clusterio/lib/errors"); const prometheus = require("@clusterio/lib/prometheus"); const luaTools = require("@clusterio/lib/luaTools"); const config = require("@clusterio/lib/config"); +const sharedCommands = require("@clusterio/lib/shared_commands"); const instanceRconCommandsCounter = new prometheus.Counter( @@ -206,9 +207,7 @@ class Instance extends link.Link{ async _loadPlugin(pluginInfo, slave) { let pluginLoadStarted = Date.now(); - // XXX Does not work on Windows - let pluginPath = path.relative(__dirname, path.join("plugins", pluginInfo.name)); - let { InstancePlugin } = require(path.join(pluginPath, pluginInfo.instanceEntrypoint)); + let { InstancePlugin } = require(path.posix.join(pluginInfo.requirePath, pluginInfo.instanceEntrypoint)); let instancePlugin = new InstancePlugin(pluginInfo, this, slave); this.plugins.set(pluginInfo.name, instancePlugin); await instancePlugin.init(); @@ -385,7 +384,8 @@ class Instance extends link.Link{ // Find plugin modules to patch in let modules = new Map(); for (let [pluginName, plugin] of this.plugins) { - let modulePath = path.join("plugins", pluginName, "module"); + let pluginPackagePath = require.resolve(path.posix.join(plugin.info.requirePath, "package.json")); + let modulePath = path.join(path.dirname(pluginPackagePath), "module"); if (!await fs.pathExists(modulePath)) { continue; } @@ -412,14 +412,14 @@ class Instance extends link.Link{ } // Find stand alone modules to load - // XXX for now it's assumed all available modules should be loaded. - for (let entry of await fs.readdir("modules", { withFileTypes: true })) { + // XXX for now only the included clusterio module is loaded + for (let entry of await fs.readdir(path.join(__dirname, "modules"), { withFileTypes: true })) { if (entry.isDirectory()) { if (modules.has(entry.name)) { throw new Error(`Module with name ${entry.name} already exists in a plugin`); } - let moduleJsonPath = path.join("modules", entry.name, "module.json"); + let moduleJsonPath = path.join(__dirname, "modules", entry.name, "module.json"); if (!await fs.pathExists(moduleJsonPath)) { throw new Error(`Module ${entry.name} is missing module.json`); } @@ -430,7 +430,7 @@ class Instance extends link.Link{ } module = { - path: path.join("modules", entry.name), + path: path.join(__dirname, "modules", entry.name), dependencies: { "clusterio": "*" }, load: [], require: [], @@ -1385,6 +1385,13 @@ async function startSlave() { default: "config-slave.json", type: "string", }) + .option("plugin-list", { + nargs: 1, + describe: "File containing list of plugins available with their install path", + default: "plugin-list.json", + type: "string", + }) + .command("plugin", "Manage available plugins", sharedCommands.pluginCommand) .command("config", "Manage Slave config", config.configCommand) .command("run", "Run slave") .demandCommand(1, "You need to specify a command to run") @@ -1392,8 +1399,25 @@ async function startSlave() { .argv ; + console.log(`Loading available plugins from ${args.pluginList}`); + let pluginList = new Map(); + try { + pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList))); + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + // If the command is plugin management we don't try to load plugins + let command = args._[0]; + if (command === "plugin") { + await sharedCommands.handlePluginCommand(args, pluginList, args.pluginList); + return; + } + console.log("Loading Plugin info"); - let pluginInfos = await plugin.loadPluginInfos("plugins"); + let pluginInfos = await plugin.loadPluginInfos(pluginList); config.registerPluginConfigGroups(pluginInfos); config.finalizeConfigs(); @@ -1412,7 +1436,6 @@ async function startSlave() { } } - let command = args._[0]; if (command === "config") { await config.handleConfigCommand(args, slaveConfig, args.config); return; diff --git a/plugins/global_chat/info.js b/plugins/global_chat/info.js index bb4dece1..ea36cf20 100644 --- a/plugins/global_chat/info.js +++ b/plugins/global_chat/info.js @@ -6,7 +6,6 @@ module.exports = { name: "global_chat", title: "Global Chat", description: "Forwards chat between instances.", - version: "2.0.0-alpha", instanceEntrypoint: "instance", controlEntrypoint: "control", diff --git a/plugins/global_chat/package.json b/plugins/global_chat/package.json new file mode 100644 index 00000000..89603685 --- /dev/null +++ b/plugins/global_chat/package.json @@ -0,0 +1,18 @@ +{ + "name": "@clusterio/plugin-global_chat", + "version": "2.0.0-alpha0", + "description": "Clusterio plugin forwarding between Factorio servers", + "author": "Hornwitser ", + "homepage": "https://github.com/clusterio/factorioClusterio#readme", + "license": "MIT", + "repository": "clusterio/factorioClusterio", + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1" + }, + "bugs": { + "url": "https://github.com/clusterio/factorioClusterio/issues" + }, + "peerDependencies": { + "@clusterio/lib": "^2.0.0-alpha0" + } +} diff --git a/plugins/research_sync/info.js b/plugins/research_sync/info.js index e28ef732..094f67e2 100644 --- a/plugins/research_sync/info.js +++ b/plugins/research_sync/info.js @@ -22,7 +22,6 @@ module.exports = { name: "research_sync", title: "Research Sync", description: "Synchronises technology research progress between instances.", - version: "2.0.0-alpha", instanceEntrypoint: "instance", masterEntrypoint: "master", diff --git a/plugins/research_sync/package.json b/plugins/research_sync/package.json new file mode 100644 index 00000000..b58518a0 --- /dev/null +++ b/plugins/research_sync/package.json @@ -0,0 +1,21 @@ +{ + "name": "@clusterio/plugin-research_sync", + "version": "2.0.0-alpha0", + "description": "Clusterio plugin for synchronizing research between Factorio servers", + "author": "Hornwitser ", + "homepage": "https://github.com/clusterio/factorioClusterio#readme", + "license": "MIT", + "repository": "clusterio/factorioClusterio", + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1" + }, + "bugs": { + "url": "https://github.com/clusterio/factorioClusterio/issues" + }, + "peerDependencies": { + "@clusterio/lib": "^2.0.0-alpha0" + }, + "ependencies": { + "fs-extra": "^8.1.0" + } +} diff --git a/plugins/statistics_exporter/info.js b/plugins/statistics_exporter/info.js index a0dabdd4..b54e56c4 100644 --- a/plugins/statistics_exporter/info.js +++ b/plugins/statistics_exporter/info.js @@ -5,6 +5,5 @@ module.exports = { name: "statistics_exporter", title: "Prometheus Statistics Export", description: "Exports statistics to Prometheus", - version: "2.0.0-alpha", instanceEntrypoint: "instance", }; diff --git a/plugins/statistics_exporter/package.json b/plugins/statistics_exporter/package.json new file mode 100644 index 00000000..e3ca5a71 --- /dev/null +++ b/plugins/statistics_exporter/package.json @@ -0,0 +1,18 @@ +{ + "name": "@clusterio/plugin-statistics_exporter", + "version": "2.0.0-alpha0", + "description": "Clusterio plugin for gathering statistics from Factorio servers", + "author": "Hornwitser ", + "homepage": "https://github.com/clusterio/factorioClusterio#readme", + "license": "MIT", + "repository": "clusterio/factorioClusterio", + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1" + }, + "bugs": { + "url": "https://github.com/clusterio/factorioClusterio/issues" + }, + "peerDependencies": { + "@clusterio/lib": "^2.0.0-alpha0" + } +} diff --git a/plugins/subspace_storage/info.js b/plugins/subspace_storage/info.js index cc0e9831..72298262 100644 --- a/plugins/subspace_storage/info.js +++ b/plugins/subspace_storage/info.js @@ -57,7 +57,6 @@ module.exports = { name: "subspace_storage", title: "Subspace Storage", description: "Provides shared storage across instances for the Subspace Storage mod", - version: "2.0.0-alpha", instanceEntrypoint: "instance", InstanceConfigGroup, diff --git a/plugins/subspace_storage/package.json b/plugins/subspace_storage/package.json new file mode 100644 index 00000000..d0e8f929 --- /dev/null +++ b/plugins/subspace_storage/package.json @@ -0,0 +1,21 @@ +{ + "name": "@clusterio/plugin-subspace_storage", + "version": "2.0.0-alpha0", + "description": "Clusterio plugin for sharing storage between Factorio servers", + "author": "Hornwitser ", + "homepage": "https://github.com/clusterio/clusterio#readme", + "license": "MIT", + "repository": "clusterio/factorioClusterio", + "scripts": { + "test": "echo \"Error: run tests from root\" && exit 1" + }, + "bugs": { + "url": "https://github.com/clusterio/factorioClusterio/issues" + }, + "peerDependencies": { + "@clusterio/lib": "^2.0.0-alpha0" + }, + "dependencies": { + "fs-extra": "^8.1.0" + } +} diff --git a/plugins/subspace_storage/routes.js b/plugins/subspace_storage/routes.js index 5247cd8b..f4d648a4 100644 --- a/plugins/subspace_storage/routes.js +++ b/plugins/subspace_storage/routes.js @@ -1,5 +1,4 @@ "use strict"; -const express = require("express"); const path = require("path"); const prometheus = require("@clusterio/lib/prometheus"); diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 3ec94b1e..5d173d8e 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -59,8 +59,10 @@ describe("Integration of Clusterio", function() { describe("instance export-data", function() { it("exports the data", async function() { slowTest(this); + let exportPath = path.join("temp", "test", "static", "export", "locale.json"); + await fs.remove(exportPath); await execCtl("instance export-data test"); - assert(await fs.exists(path.join("static", "export", "locale.json")), "Export was not created"); + assert(await fs.exists(exportPath), "Export was not created"); await checkInstanceStatus(44, "stopped"); }); }); diff --git a/test/integration/index.js b/test/integration/index.js index ed05ce5c..873dacc1 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -63,18 +63,20 @@ let control; let url = "https://localhost:4443/"; let controlToken = jwt.sign({ aud: "user", user: "test" }, "TestSecretDoNotUse"); let instancesDir = path.join("temp", "test", "instances"); -let databaseDir = path.join("temp", "test", "databse"); -let masterConfigPath = path.join("temp", "test", "master-integration.json"); -let slaveConfigPath = path.join("temp", "test", "slave-integration.json"); -let controlConfigPath = path.join("temp", "test", "control-integration.json"); +let databaseDir = path.join("temp", "test", "database"); +let pluginListPath = path.join("temp", "test", "plugin-list.json"); +let masterConfigPath = path.join("temp", "test", "config-master.json"); +let slaveConfigPath = path.join("temp", "test", "config-slave.json"); +let controlConfigPath = path.join("temp", "test", "config-control.json"); -async function exec(...args) { - console.log(args[0]); - return await util.promisify(child_process.exec)(...args); +async function exec(command, options = {}) { + console.log(command); + options = { cwd: path.join("temp", "test"), ...options }; + return await util.promisify(child_process.exec)(command, options); } async function execCtl(...args) { - args[0] = `node packages/ctl --config ${controlConfigPath} ${args[0]}`; + args[0] = `node ../../packages/ctl ${args[0]}`; return await exec(...args); } @@ -91,7 +93,7 @@ function spawn(name, cmd, waitFor) { return new Promise((resolve, reject) => { console.log(cmd); let parts = cmd.split(" "); - let process = child_process.spawn(parts[0], parts.slice(1)); + let process = child_process.spawn(parts[0], parts.slice(1), { cwd: path.join("temp", "test") }); let stdout = new server._LineSplitter((line) => { line = line.toString("utf8"); if (line.startsWith(waitFor)) { @@ -113,30 +115,35 @@ before(async function() { await fs.remove(databaseDir); await fs.remove(instancesDir); + await fs.remove(pluginListPath); await fs.remove(masterConfigPath); await fs.remove(slaveConfigPath); await fs.remove(controlConfigPath); - await exec(`node packages/master --config ${masterConfigPath} config set master.database_directory ${databaseDir}`); - await exec(`node packages/master --config ${masterConfigPath} config set master.auth_secret TestSecretDoNotUse`); - await exec(`node packages/master --config ${masterConfigPath} config set master.http_port 8880`); - await exec(`node packages/master --config ${masterConfigPath} config set master.https_port 4443`); - await exec(`node packages/master --config ${masterConfigPath} config set master.tls_certificate ${path.join(databaseDir, "cert.crt")}`); - await exec(`node packages/master --config ${masterConfigPath} config set master.tls_private_key ${path.join(databaseDir, "cert.key")}`); - await exec(`node packages/master --config ${masterConfigPath} config set master.tls_bits 1024`); - await exec(`node packages/master --config ${masterConfigPath} config set master.heartbeat_interval 0.25`); - await exec(`node packages/master --config ${masterConfigPath} config set master.connector_shutdown_timeout 2`); + await fs.ensureDir(path.join("temp", "test")); - await exec(`node packages/master --config ${masterConfigPath} bootstrap create-admin test`); - await exec(`node packages/master --config ${masterConfigPath} bootstrap create-ctl-config test --output ${controlConfigPath}`); + await exec("node ../../packages/master config set master.auth_secret TestSecretDoNotUse"); + await exec("node ../../packages/master config set master.http_port 8880"); + await exec("node ../../packages/master config set master.https_port 4443"); + await exec("node ../../packages/master config set master.tls_bits 1024"); + await exec("node ../../packages/master config set master.heartbeat_interval 0.25"); + await exec("node ../../packages/master config set master.connector_shutdown_timeout 2"); - masterProcess = await spawn("master:", `node packages/master --config ${masterConfigPath} run`, "All plugins loaded"); + await exec("node ../../packages/ctl plugin add ../../plugins/global_chat"); + await exec("node ../../packages/ctl plugin add ../../plugins/research_sync"); + await exec("node ../../packages/ctl plugin add ../../plugins/statistics_exporter"); + await exec("node ../../packages/ctl plugin add ../../plugins/subspace_storage"); - let createArgs = `--id 4 --name slave --generate-token --output ${slaveConfigPath}`; + await exec("node ../../packages/master bootstrap create-admin test"); + await exec("node ../../packages/master bootstrap create-ctl-config test"); + + masterProcess = await spawn("master:", "node ../../packages/master run", "All plugins loaded"); + + let createArgs = "--id 4 --name slave --generate-token"; await execCtl(`slave create-config ${createArgs}`); - await exec(`node packages/slave --config ${slaveConfigPath} config set slave.instances_directory ${instancesDir}`); - slaveProcess = await spawn("slave:", `node packages/slave --config ${slaveConfigPath} run`, "SOCKET | received ready from master"); + await exec(`node ../../packages/slave config set slave.factorio_directory ${path.join("..", "..", "factorio")}`); + slaveProcess = await spawn("slave:", "node ../../packages/slave run", "SOCKET | received ready from master"); let controlConnector = new TestControlConnector(url, 2); controlConnector.token = controlToken; diff --git a/test/lib/plugin.js b/test/lib/plugin.js index 879e225f..2c1e7602 100644 --- a/test/lib/plugin.js +++ b/test/lib/plugin.js @@ -44,56 +44,52 @@ describe("lib/plugin", function() { describe("loadPluginInfos()", function() { let baseDir = path.join("temp", "test", "plugin"); - let emptyDir = path.join(baseDir, "emptyDir"); - let emptyPlugin = path.join(baseDir, "emptyPlugin"); - let testPlugin = path.join(baseDir, "testPlugin"); - let brokenPlugin = path.join(baseDir, "brokenPlugin"); - let invalidPlugin = path.join(baseDir, "invalidPlugin"); + let missingPlugin = path.join(baseDir, "missing_plugin"); + let testPlugin = path.join(baseDir, "test_plugin"); + let brokenPlugin = path.join(baseDir, "broken_plugin"); + let invalidPlugin = path.join(baseDir, "invalid_plugin"); before(async function() { - await fs.ensureDir(emptyDir); - await fs.ensureDir(path.join(emptyPlugin, "empty")); - - async function writePlugin(pluginPath, name, infoName = name) { + async function writePlugin(pluginPath, infoName) { await fs.outputFile( - path.join(pluginPath, name, "info.js"), + path.join(pluginPath, "info.js"), `module.exports = { name: "${infoName}" };` ); + await fs.outputFile( + path.join(pluginPath, "package.json"), + '{ "version": "0.0.1" }' + ); } await writePlugin(testPlugin, "test"); await writePlugin(brokenPlugin, "broken"); - await fs.outputFile(path.join(brokenPlugin, "broken", "info.js"), "Syntax Error"); - await writePlugin(invalidPlugin, "invalid", "wrong"); + await fs.outputFile(path.join(brokenPlugin, "info.js"), "Syntax Error"); + await writePlugin(invalidPlugin, "wrong"); }); - it("should return an empty array for an empty directory", async function() { - assert.deepEqual(await plugin.loadPluginInfos(emptyDir), []); + it("should throw on missing plugin", async function() { + await assert.rejects( + plugin.loadPluginInfos(new Map([["missing", missingPlugin]]), []), + new RegExp(`^Error: PluginError: Cannot find module '${missingPlugin}/info'`) + ); }); - it("should ignore plugin dirs without info module", async function() { - assert.deepEqual(await plugin.loadPluginInfos(emptyPlugin), []); - }); - it("should discover test plugin", async function() { + it("should load test plugin", async function() { assert.deepEqual( - await plugin.loadPluginInfos(testPlugin), - [{ name: "test" }] + await plugin.loadPluginInfos(new Map([["test", path.resolve(testPlugin)]])), + [{ name: "test", version: "0.0.1", requirePath: path.resolve(testPlugin) }] ); }); it("should reject on broken plugin", async function() { await assert.rejects( - plugin.loadPluginInfos(brokenPlugin), + plugin.loadPluginInfos(new Map([["broken", path.resolve(brokenPlugin)]])), { message: "PluginError: Unexpected identifier" } ); }); it("should reject on invalid plugin", async function() { await assert.rejects( - plugin.loadPluginInfos(invalidPlugin), - { message: `Plugin dir ${invalidPlugin}/invalid does not match the name of the plugin (wrong)` } + plugin.loadPluginInfos(new Map([["invalid", path.resolve(invalidPlugin)]])), + { message: `Expected plugin at ${path.resolve(invalidPlugin)} to be named invalid but got wrong` } ); }); - - after(async function() { - await fs.unlink(path.join(brokenPlugin, "broken", "info.js")); - }); }); describe("attachPluginMessages()", function() {