From 5ce3fb6cc6102209dfcb19552bbc4bde28e3f2ba Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 11 Mar 2021 13:20:43 +0100 Subject: [PATCH 01/10] Add loading of web plugins in interface Add separate BaseWebPlugin class which plugins can extend and expose through a webEntrypoint in a similar fasion to the existing master, instance and control entrypoints. To simplify the logic for the rest of the web interface code the plugins context is changed to be a mapping of loaded WebPlugin instances. This in turn means the plugins and plugin view pages fetches the plugin list upon load in order to show plugins that failed to load (which incidentally also gives them an up to date list). --- packages/lib/plugin.js | 41 ++++++++++++++ packages/master/master.js | 3 +- packages/web_ui/src/bootstrap.jsx | 42 ++++++++------ .../web_ui/src/components/PluginViewPage.jsx | 55 ++++++++++++------- .../web_ui/src/components/PluginsPage.jsx | 34 +++++++++++- packages/web_ui/src/util/websocket.jsx | 10 ++-- 6 files changed, 140 insertions(+), 45 deletions(-) diff --git a/packages/lib/plugin.js b/packages/lib/plugin.js index 4dce8257..68687718 100644 --- a/packages/lib/plugin.js +++ b/packages/lib/plugin.js @@ -423,6 +423,46 @@ class BaseControlPlugin { async addCommands(rootCommand) { } } + +/** + * Base class for web interface plugins + * + * @static + */ +class BaseWebPlugin { + constructor(container, packageData, info, logger) { + /** + * Webpack container for this plugin + */ + this.container = container; + + /** + * Contents of the plugin's package.json file + * @type {Object} + */ + this.package = packageData; + + /** + * The plugin's own info module + */ + this.info = info; + + /** + * Logger for this plugin + * + * Instance of winston Logger for sending log messages from this + * plugin. Supported methods and their corresponding log levels are + * `error`, `warn`, `audit`, `info` and `verbose`. + */ + this.logger = logger.child({ plugin: this.info.name }); + } + + /** + * Called immediately after the class is instantiated. + */ + async init() { } +} + /** * Attach plugin messages * @@ -489,6 +529,7 @@ module.exports = { BaseInstancePlugin, BaseMasterPlugin, BaseControlPlugin, + BaseWebPlugin, attachPluginMessages, invokeHook, diff --git a/packages/master/master.js b/packages/master/master.js index b7af2e5d..7c1ce2a6 100755 --- a/packages/master/master.js +++ b/packages/master/master.js @@ -275,7 +275,8 @@ app.get("/api/plugins", (req, res) => { for (let pluginInfo of pluginInfos) { let name = pluginInfo.name; let loaded = masterPlugins.has(name); - plugins.push({ name, version: pluginInfo.version, loaded }); + let enabled = loaded && masterConfig.group(pluginInfo.name).get("load_plugin"); + plugins.push({ name, version: pluginInfo.version, enabled, loaded }); } res.send(plugins); }); diff --git a/packages/web_ui/src/bootstrap.jsx b/packages/web_ui/src/bootstrap.jsx index c84405b7..0db610cb 100644 --- a/packages/web_ui/src/bootstrap.jsx +++ b/packages/web_ui/src/bootstrap.jsx @@ -4,8 +4,10 @@ import React from "react"; import ReactDOM from "react-dom"; import libConfig from "@clusterio/lib/config"; +import libPlugin from "@clusterio/lib/plugin"; import { ConsoleTransport, WebConsoleFormat, logger } from "@clusterio/lib/logging"; +import basename from "./basename"; import App from "./components/App"; import { Control, ControlConnector } from "./util/websocket"; @@ -26,7 +28,7 @@ async function loadScript(url) { } async function loadPlugins() { - let response = await fetch(`${webRoot}api/plugins`); + let response = await fetch(`${basename}/api/plugins`); let pluginList; if (response.ok) { pluginList = await response.json(); @@ -36,30 +38,37 @@ async function loadPlugins() { pluginList = []; } - let plugins = []; + let plugins = new Map(); await __webpack_init_sharing__("default"); for (let meta of pluginList) { try { - await loadScript(`${webRoot}plugin/${meta.name}/remoteEntry.js`); + await loadScript(`${basename}/plugin/${meta.name}/remoteEntry.js`); let container = window[`plugin_${meta.name}`]; + if (!container) { + throw new Error(`Plugin did not expose its container via plugin_${meta.name}`); + } await container.init(__webpack_share_scopes__.default); let pluginInfo = (await container.get("./info"))(); let pluginPackage = (await container.get("./package.json"))(); - plugins.push({ - meta, - info: pluginInfo, - package: pluginPackage, - container, - }); + + let WebPluginClass = libPlugin.BaseWebPlugin; + if (meta.enabled && pluginInfo.webEntrypoint) { + let webModule = (await container.get(pluginInfo.webEntrypoint))(); + if (!webModule.WebPlugin) { + throw new Error("Plugin webEntrypoint does not export WebPlugin class"); + } + WebPluginClass = webModule.WebPlugin; + } + + let plugin = new WebPluginClass(container, pluginPackage, pluginInfo, logger); + await plugin.init(); + plugins.set(pluginInfo.name, plugin); } catch (err) { - logger.error(`Failed to load plugin info for ${meta.name}`); - if (err) { + logger.error(`Failed to load plugin ${meta.name}`); + if (err.stack) { logger.error(err.stack); } - plugins.push({ - meta, - }); } } return plugins; @@ -71,14 +80,15 @@ async function load() { format: new WebConsoleFormat(), })); let plugins = await loadPlugins(); - libConfig.registerPluginConfigGroups(plugins.filter(p => p.info).map(p => p.info)); + let pluginInfos = [...plugins.values()].map(p => p.info); + libConfig.registerPluginConfigGroups(pluginInfos); libConfig.finalizeConfigs(); let wsUrl = new URL(window.webRoot, document.location); wsUrl.protocol = wsUrl.protocol.replace("http", "ws"); let controlConnector = new ControlConnector(wsUrl, 10); - let control = new Control(controlConnector, []); + let control = new Control(controlConnector, plugins); ReactDOM.render(, document.getElementById("root")); } diff --git a/packages/web_ui/src/components/PluginViewPage.jsx b/packages/web_ui/src/components/PluginViewPage.jsx index ae847103..dbca70c8 100644 --- a/packages/web_ui/src/components/PluginViewPage.jsx +++ b/packages/web_ui/src/components/PluginViewPage.jsx @@ -1,7 +1,8 @@ import React, { useEffect, useContext, useState } from "react"; import { useHistory, useParams } from "react-router-dom"; -import { Alert, Descriptions } from "antd"; +import { Alert, Descriptions, Spin } from "antd"; +import basename from "../basename"; import PluginsContext from "./PluginsContext"; import PageLayout from "./PageLayout"; import { notifyErrorHandler } from "../util/notify"; @@ -10,27 +11,43 @@ import { notifyErrorHandler } from "../util/notify"; export default function PluginViewPage() { let params = useParams(); let plugins = useContext(PluginsContext); + let [pluginList, setPluginList] = useState(null); let pluginName = params.name; - let plugin = plugins.find(p => p.meta.name === pluginName); - let pluginTitle = plugin.info ? plugin.info.title : plugin.meta.name; + useEffect(() => { + (async () => { + let response = await fetch(`${basename}/api/plugins`); + if (response.ok) { + setPluginList(await response.json()); + } else { + notify("Failed to load plugin list"); + } + })(); + }, []); + + let plugin = plugins.get(pluginName); + let pluginTitle = plugin ? plugin.info.title : pluginName; let nav = [{ name: "Plugins", path: "/plugins" }, { name: pluginTitle }]; - if (!plugin) { + if (!pluginList) { + return + + + ; + } + + let pluginMeta = pluginList.find(p => p.name === pluginName); + if (!pluginMeta) { return

Plugin not found

Plugin with name {pluginName} was not found on the master server.

; } - if (!plugin.info) { + if (!plugin) { return - - {plugin.meta.version} - {plugin.meta.loaded ? "Yes" : "No"} + + {pluginMeta.version} + {pluginMeta.loaded ? "Yes" : "No"} - - {plugin.meta.version} - {plugin.meta.loaded ? "Yes" : "No"} + + {pluginMeta.version} + {pluginMeta.loaded ? "Yes" : "No"} {plugin.info.description} {plugin.package.homepage ? {plugin.package.homepage} @@ -66,14 +79,14 @@ export default function PluginViewPage() { : null} {plugin.package.license} - {plugin.meta.version !== plugin.package.version ? { + (async () => { + let response = await fetch(`${basename}/api/plugins`); + if (response.ok) { + setPluginList(await response.json()); + } else { + notify("Failed to load plugin list"); + } + })(); + }, []); + + let tableContents = []; + for (let meta of pluginList) { + if (plugins.has(meta.name)) { + let plugin = plugins.get(meta.name); + tableContents.push({ + meta, + info: plugin.info, + package: plugin.package, + }); + } else { + tableContents.push({ + meta, + }); + } + } return

Plugins

@@ -39,7 +69,7 @@ export default function PluginsPage() { responsive: ["sm"], }, ]} - dataSource={plugins} + dataSource={tableContents} rowKey={plugin => plugin.meta.name} pagination={false} onRow={plugin => ({ diff --git a/packages/web_ui/src/util/websocket.jsx b/packages/web_ui/src/util/websocket.jsx index f3de0e4d..13b78fef 100644 --- a/packages/web_ui/src/util/websocket.jsx +++ b/packages/web_ui/src/util/websocket.jsx @@ -36,17 +36,17 @@ export class ControlConnector extends libLink.WebSocketClientConnector { * @static */ export class Control extends libLink.Link { - constructor(connector, controlPlugins) { + constructor(connector, plugins) { super("control", "master", connector); libLink.attachAllMessages(this); /** * Mapping of plugin names to their instance for loaded plugins. - * @type {Map} + * @type {Map} */ - this.plugins = controlPlugins; - for (let controlPlugin of controlPlugins.values()) { - libPlugin.attachPluginMessages(this, controlPlugin.info, controlPlugin); + this.plugins = plugins; + for (let plugin of plugins.values()) { + libPlugin.attachPluginMessages(this, plugin); } this.instanceLogHandlers = new Map(); From 71804df3e00bf69613668888371289933969cc39 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 11 Mar 2021 13:32:52 +0100 Subject: [PATCH 02/10] Add --dev-plugin to compile plugins on the fly Add hidden development option to run the webpack compiller for the given plugin(s) similar to the --dev option for the built-in web interface. This requires publicPath to be set in the webpack config and an updated version of webpack-dev-middleware to function correctly. --- packages/master/master.js | 21 ++++++++++++++++--- packages/master/package.json | 2 +- packages/master/webpack.config.js | 1 + plugins/global_chat/webpack.config.js | 1 + plugins/research_sync/webpack.config.js | 1 + plugins/statistics_exporter/webpack.config.js | 1 + plugins/subspace_storage/webpack.config.js | 1 + 7 files changed, 24 insertions(+), 4 deletions(-) diff --git a/packages/master/master.js b/packages/master/master.js index 7c1ce2a6..a5044424 100755 --- a/packages/master/master.js +++ b/packages/master/master.js @@ -1862,6 +1862,7 @@ async function initialize() { }) .command("run", "Run master server", yargs => { yargs.option("dev", { hidden: true, type: "boolean", nargs: 0 }); + yargs.option("dev-plugin", { hidden: true, type: "array" }); }) .demandCommand(1, "You need to specify a command to run") .strict() @@ -1954,15 +1955,29 @@ async function initialize() { async function startServer(args) { // Start webpack development server if enabled - if (args.dev) { + if (args.dev || args.devPlugin) { logger.warn("Webpack development mode enabled"); /* eslint-disable global-require */ const webpack = require("webpack"); const webpackDevMiddleware = require("webpack-dev-middleware"); - const webpackConfig = require("./webpack.config"); + const webpackConfigs = []; + if (args.dev) { + webpackConfigs.push(require("./webpack.config")); + } + if (args.devPlugin) { + for (let name of args.devPlugin) { + let info = pluginInfos.find(i => i.name === name); + if (!info) { + throw new libErrors.StartupError(`No plugin named ${name}`); + } + webpackConfigs.push( + require(path.posix.join(info.requirePath, "webpack.config")) + ); + } + } /* eslint-enable global-require */ - const compiler = webpack(webpackConfig({})); + const compiler = webpack(webpackConfigs.map(config => config({}))); devMiddleware = webpackDevMiddleware(compiler, {}); app.use(devMiddleware); } diff --git a/packages/master/package.json b/packages/master/package.json index 937d5f0a..0b22b6bb 100644 --- a/packages/master/package.json +++ b/packages/master/package.json @@ -41,7 +41,7 @@ "react-dom": "^16.13.1", "react-router-dom": "^5.2.0", "webpack-cli": "^3.3.12", - "webpack-dev-middleware": "^3.7.2", + "webpack-dev-middleware": "^4.1.0", "webpack-merge": "^5.2.0" }, "publishConfig": { diff --git a/packages/master/webpack.config.js b/packages/master/webpack.config.js index 14350778..06fd9363 100644 --- a/packages/master/webpack.config.js +++ b/packages/master/webpack.config.js @@ -12,6 +12,7 @@ module.exports = (env = {}) => merge(common(env), { contentBase: "./dist/web", }, output: { + publicPath: "/", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/global_chat/webpack.config.js b/plugins/global_chat/webpack.config.js index 6efae364..87890343 100644 --- a/plugins/global_chat/webpack.config.js +++ b/plugins/global_chat/webpack.config.js @@ -9,6 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { + publicPath: "/plugin/global_chat/", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/research_sync/webpack.config.js b/plugins/research_sync/webpack.config.js index 78645a96..69ef00de 100644 --- a/plugins/research_sync/webpack.config.js +++ b/plugins/research_sync/webpack.config.js @@ -9,6 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { + publicPath: "/plugin/research_sync/", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/statistics_exporter/webpack.config.js b/plugins/statistics_exporter/webpack.config.js index 6ca9a1a4..3f1ee548 100644 --- a/plugins/statistics_exporter/webpack.config.js +++ b/plugins/statistics_exporter/webpack.config.js @@ -9,6 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { + publicPath: "/plugin/statistics_exporter/", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/subspace_storage/webpack.config.js b/plugins/subspace_storage/webpack.config.js index 65853b7f..cefb283b 100644 --- a/plugins/subspace_storage/webpack.config.js +++ b/plugins/subspace_storage/webpack.config.js @@ -9,6 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { + publicPath: "/plugin/subspace_storage/", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, From 78ce6fc3827d0e8099363ee594048e1a2e60a40f Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 11 Mar 2021 13:49:15 +0100 Subject: [PATCH 03/10] Move /plugin/name/ to /plugins/name/ The web files for plugins were hosted at /plugin/plugin_name/ while the view page for a plugin is hosted at /plugins/plugin_name/view. Reduce confusion by making both urls use the plural version. --- packages/master/master.js | 2 +- packages/web_ui/src/bootstrap.jsx | 2 +- plugins/global_chat/webpack.config.js | 2 +- plugins/research_sync/webpack.config.js | 2 +- plugins/statistics_exporter/webpack.config.js | 2 +- plugins/subspace_storage/webpack.config.js | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/master/master.js b/packages/master/master.js index a5044424..6b80e181 100755 --- a/packages/master/master.js +++ b/packages/master/master.js @@ -2049,7 +2049,7 @@ async function startServer(args) { let pluginPackagePath = require.resolve(path.posix.join(pluginInfo.requirePath, "package.json")); let webPath = path.join(path.dirname(pluginPackagePath), "dist", "web"); - app.use(`/plugin/${pluginInfo.name}`, express.static(webPath)); + app.use(`/plugins/${pluginInfo.name}`, express.static(webPath)); } // Load plugins diff --git a/packages/web_ui/src/bootstrap.jsx b/packages/web_ui/src/bootstrap.jsx index 0db610cb..ee217769 100644 --- a/packages/web_ui/src/bootstrap.jsx +++ b/packages/web_ui/src/bootstrap.jsx @@ -42,7 +42,7 @@ async function loadPlugins() { await __webpack_init_sharing__("default"); for (let meta of pluginList) { try { - await loadScript(`${basename}/plugin/${meta.name}/remoteEntry.js`); + await loadScript(`${basename}/plugins/${meta.name}/remoteEntry.js`); let container = window[`plugin_${meta.name}`]; if (!container) { throw new Error(`Plugin did not expose its container via plugin_${meta.name}`); diff --git a/plugins/global_chat/webpack.config.js b/plugins/global_chat/webpack.config.js index 87890343..ccb9b636 100644 --- a/plugins/global_chat/webpack.config.js +++ b/plugins/global_chat/webpack.config.js @@ -9,7 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "/plugin/global_chat/", + publicPath: "/plugins/global_chat/", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/research_sync/webpack.config.js b/plugins/research_sync/webpack.config.js index 69ef00de..b34f7439 100644 --- a/plugins/research_sync/webpack.config.js +++ b/plugins/research_sync/webpack.config.js @@ -9,7 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "/plugin/research_sync/", + publicPath: "/plugins/research_sync/", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/statistics_exporter/webpack.config.js b/plugins/statistics_exporter/webpack.config.js index 3f1ee548..a42b5c65 100644 --- a/plugins/statistics_exporter/webpack.config.js +++ b/plugins/statistics_exporter/webpack.config.js @@ -9,7 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "/plugin/statistics_exporter/", + publicPath: "/plugins/statistics_exporter/", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/subspace_storage/webpack.config.js b/plugins/subspace_storage/webpack.config.js index cefb283b..1de46598 100644 --- a/plugins/subspace_storage/webpack.config.js +++ b/plugins/subspace_storage/webpack.config.js @@ -9,7 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "/plugin/subspace_storage/", + publicPath: "/plugins/subspace_storage/", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, From a8e197148381f545ecd8fecac4e9ce94113959d2 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 11 Mar 2021 13:51:29 +0100 Subject: [PATCH 04/10] Fix data loss on bad name passed --dev-plugin If data from the database is not loaded but a shutdown is triggered then the master server will overwrite the database with empty data, causing unexpected data loss. This occurs for example when the --dev-plugin option throws. Fix by setting the database entries to null before they are loaded so that should an attempt be made to save them without having been loaded first it will throw an error instead. --- packages/master/master.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/packages/master/master.js b/packages/master/master.js index 6b80e181..27ac8869 100755 --- a/packages/master/master.js +++ b/packages/master/master.js @@ -41,8 +41,10 @@ let loadedPlugins = {}; let devMiddleware; let clusterLogger; let db = { - instances: new Map(), - slaves: new Map(), + instances: null, + slaves: null, + roles: null, + users: null, }; let slaveConnections = new Map(); let controlConnections = []; @@ -489,6 +491,14 @@ async function shutdown() { logger.info("Saving configs"); await fs.outputFile(masterConfigPath, JSON.stringify(masterConfig.serialize(), null, 4)); + if (devMiddleware) { + await new Promise((resolve, reject) => { devMiddleware.close(resolve); }); + } + + if (!db.slaves) { + return; + } + await saveMap(masterConfig.get("master.database_directory"), "slaves.json", db.slaves); await saveInstances(masterConfig.get("master.database_directory"), "instances.json", db.instances); await saveUsers(masterConfig.get("master.database_directory"), "users.json"); @@ -497,10 +507,6 @@ async function shutdown() { stopAcceptingNewSessions = true; - if (devMiddleware) { - await new Promise((resolve, reject) => { devMiddleware.close(resolve); }); - } - let disconnectTasks = []; for (let controlConnection of controlConnections) { controlConnection.connector.setTimeout(masterConfig.get("master.connector_shutdown_timeout")); From b70303c4370ceed9ea4593478f66800422adcb65 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 11 Mar 2021 14:13:40 +0100 Subject: [PATCH 05/10] Add error boundary to App Catch errors in the component stack that reaches the App component and display an error message for them instead of a black screen. --- packages/web_ui/src/components/App.jsx | 30 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/packages/web_ui/src/components/App.jsx b/packages/web_ui/src/components/App.jsx index 2c90130d..255f1aa1 100644 --- a/packages/web_ui/src/components/App.jsx +++ b/packages/web_ui/src/components/App.jsx @@ -5,13 +5,25 @@ import { BrowserRouter } from "react-router-dom"; import { logger } from "@clusterio/lib/logging"; import basename from "../basename"; +import ErrorBoundary from "./ErrorBoundary"; import SiteLayout from "./SiteLayout"; import ControlContext from "./ControlContext"; import PluginsContext from "./PluginsContext"; import LoginForm from "./LoginForm"; -import { Card, Spin } from "antd"; +import { Card, Spin, Typography } from "antd"; +const { Paragraph } = Typography; + + +function ErrorCard(props) { + return
+ +

An unexpected error occured

+ {props.error.stack} + +
; +} export default function App(props) { let [connected, setConnected] = useState(false); @@ -77,12 +89,14 @@ export default function App(props) { } return ( - - - - {page} - - - + + + + + {page} + + + + ); } From a904a8fb9d62c91d49903010b3e9d541b5e1598c Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 13 Mar 2021 19:30:59 +0100 Subject: [PATCH 06/10] Add hit counter for HTTP requests Add a complementary total hit counter for all HTTP requests. The endpointHitCounter doesn't cover all endpoints and fixing it to cover all endpoints without making it possible for it to create an infinite number of time seriers is not straight forward. --- packages/master/master.js | 43 ++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/packages/master/master.js b/packages/master/master.js index 27ac8869..fa65b2f4 100755 --- a/packages/master/master.js +++ b/packages/master/master.js @@ -80,18 +80,6 @@ let httpServerCloser; let httpsServer; let httpsServerCloser; -app.use(cookieParser()); -app.use(bodyParser.json({ - limit: "10mb", -})); -app.use(bodyParser.urlencoded({ - parameterLimit: 100000, - limit: "10mb", - extended: true, -})); -app.use(fileUpload()); -app.use(compression()); - // Servers the web interface with the root path set apropriately. function serveWeb(route) { @@ -112,10 +100,6 @@ function serveWeb(route) { }; } -// Set folder to serve static content from (the website) -app.use(express.static(path.join(__dirname, "dist", "web"))); -app.use(express.static("static")); // Used for data export files - const slaveMappingGauge = new libPrometheus.Gauge( "clusterio_master_slave_mapping", "Mapping of Slave ID to name", @@ -151,6 +135,11 @@ const instanceMappingGauge = new libPrometheus.Gauge( } ); +const hitCounter = new libPrometheus.Counter( + "clusterio_master_http_hits_total", + "How many HTTP requests in total have been received" +); + const endpointHitCounter = new libPrometheus.Counter( "clusterio_master_http_endpoint_hits_total", "How many requests a particular HTTP endpoint has gotten", @@ -183,6 +172,28 @@ const wsActiveSlavesGauge = new libPrometheus.Gauge( "How many slaves are currently connected to the master" ); + +app.use((req, res, next) => { + hitCounter.inc(); + next(); +}); +app.use(cookieParser()); +app.use(bodyParser.json({ + limit: "10mb", +})); +app.use(bodyParser.urlencoded({ + parameterLimit: 100000, + limit: "10mb", + extended: true, +})); +app.use(fileUpload()); +app.use(compression()); + +// Set folder to serve static content from (the website) +app.use(express.static(path.join(__dirname, "dist", "web"))); +app.use(express.static("static")); // Used for data export files + + // Merges samples from sourceResult to destinationResult function mergeSamples(destinationResult, sourceResult) { // Merge metrics received by multiple slaves From 579f6421e43ae4c4d44e4312e928bf9db723af08 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 13 Mar 2021 20:21:49 +0100 Subject: [PATCH 07/10] Export web_ui as a single module Due to the way the imports and exports works with module fedderation each require path needs it's own shared module in the plugin. To avoid having to specify every possible import in the module federation plugin config, export all useful components to plugins in the browser entrypoint for web_ui. Note that for some reason the module wouldn't exist for import by plugins unless some part of the build imported it. Which is the reason for the master index.jsx to import bootstrap via it. --- packages/master/web/index.jsx | 4 +++- packages/master/webpack.config.js | 1 - packages/web_ui/package.json | 1 + packages/web_ui/src/bootstrap.jsx | 4 +--- packages/web_ui/src/index.jsx | 20 ++++++++++++++++++++ 5 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 packages/web_ui/src/index.jsx diff --git a/packages/master/web/index.jsx b/packages/master/web/index.jsx index 30fdbf62..64be3a06 100644 --- a/packages/master/web/index.jsx +++ b/packages/master/web/index.jsx @@ -1 +1,3 @@ -import("@clusterio/web_ui/src/bootstrap"); +import("@clusterio/web_ui").then(webUi => { + return webUi.bootstrap(); +}).catch(console.error); diff --git a/packages/master/webpack.config.js b/packages/master/webpack.config.js index 06fd9363..16223c8b 100644 --- a/packages/master/webpack.config.js +++ b/packages/master/webpack.config.js @@ -29,7 +29,6 @@ module.exports = (env = {}) => merge(common(env), { "@clusterio/lib/schema": { singleton: true }, "@clusterio/lib/users": { singleton: true }, "@clusterio/web_ui": { singleton: true }, - "ajv": {}, "antd": { singleton: true }, "react": { singleton: true }, "react-dom": { singleton: true }, diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index 5a4fe02a..3adf1e6c 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -4,6 +4,7 @@ "author": "Hornwitser", "license": "MIT", "version": "2.0.0-alpha.4", + "browser": "src/index.jsx", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" }, diff --git a/packages/web_ui/src/bootstrap.jsx b/packages/web_ui/src/bootstrap.jsx index ee217769..0e37a489 100644 --- a/packages/web_ui/src/bootstrap.jsx +++ b/packages/web_ui/src/bootstrap.jsx @@ -74,7 +74,7 @@ async function loadPlugins() { return plugins; } -async function load() { +export default async function bootstrap() { logger.add(new ConsoleTransport({ level: "verbose", format: new WebConsoleFormat(), @@ -92,5 +92,3 @@ async function load() { ReactDOM.render(, document.getElementById("root")); } - -load().catch((err) => logger.fatal(err.stack)); diff --git a/packages/web_ui/src/index.jsx b/packages/web_ui/src/index.jsx new file mode 100644 index 00000000..dc3e9887 --- /dev/null +++ b/packages/web_ui/src/index.jsx @@ -0,0 +1,20 @@ +export { default as basename } from "./basename.jsx"; +export { default as bootstrap } from "./bootstrap.jsx"; +export { default as notify, notifyErrorHandler } from "./util/notify.jsx"; + +export * from "./model/instance.jsx"; +export * from "./model/slave.jsx"; + + +export { default as ControlContext } from "./components/ControlContext"; +export { default as PluginsContext } from "./components/PluginsContext"; + +export { default as BaseConfigTree } from "./components/BaseConfigTree"; +export { default as InstanceConfigTree } from "./components/InstanceConfigTree"; +export { default as MasterConfigTree } from "./components/MasterConfigTree"; + +export { default as AssignInstanceModal } from "./components/AssignInstanceModal"; +export { default as InstanceConsole } from "./components/InstanceConsole"; +export { default as InstanceRcon } from "./components/InstanceRcon"; + +export { default as PageLayout } from "./components/PageLayout"; From ceae2b25b59189bcc17af5df3e8f10f76094c7d7 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Sat, 13 Mar 2021 20:39:29 +0100 Subject: [PATCH 08/10] Make the login form pluggable Allow extending the login form with additional authentication methods by adding them to the web plugin's loginForms array. --- packages/lib/plugin.js | 22 +++++++++ packages/web_ui/src/components/App.jsx | 5 +- packages/web_ui/src/components/LoginForm.jsx | 52 ++++++++++++++++---- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/packages/lib/plugin.js b/packages/lib/plugin.js index 68687718..960150a0 100644 --- a/packages/lib/plugin.js +++ b/packages/lib/plugin.js @@ -424,6 +424,21 @@ class BaseControlPlugin { } +/** + * Plugin supplied login form + * @typedef {Object} module:lib/plugin~LoginForm + * @property {string} name - + * Internal name of the login form, this should start with the + * plugin name followed by a dot. + * @property {string} title - + * Name displayed above this form in the login window. + * @property {React.Component} Component - + * React component that's rendered for this login form. This is + * supplied the setToken function via its props which should be + * called when an authentication token is aquired via this + * form. + */ + /** * Base class for web interface plugins * @@ -455,6 +470,13 @@ class BaseWebPlugin { * `error`, `warn`, `audit`, `info` and `verbose`. */ this.logger = logger.child({ plugin: this.info.name }); + + /** + * List of login forms provided by this plugin + * + * @type {Array} + */ + this.loginForms = []; } /** diff --git a/packages/web_ui/src/components/App.jsx b/packages/web_ui/src/components/App.jsx index 255f1aa1..c326d014 100644 --- a/packages/web_ui/src/components/App.jsx +++ b/packages/web_ui/src/components/App.jsx @@ -81,10 +81,7 @@ export default function App(props) { } else { page =
- -

Clusterio

- - +
; } diff --git a/packages/web_ui/src/components/LoginForm.jsx b/packages/web_ui/src/components/LoginForm.jsx index 6a5a13cf..7e78b915 100644 --- a/packages/web_ui/src/components/LoginForm.jsx +++ b/packages/web_ui/src/components/LoginForm.jsx @@ -1,27 +1,61 @@ -import React from "react"; -import { Form, Input, Button } from "antd"; +import React, { Fragment, useContext } from "react"; +import { Button, Card, Divider, Form, Image, Input, List, Space, Spin, Typography } from "antd"; import LockOutlined from "@ant-design/icons/LockOutlined"; -export default function LoginForm(props) { - return ( +import logo from "../images/logo.png"; +import PluginsContext from "./PluginsContext"; + +const { Paragraph, Text } = Typography; + + +function TokenAuth(props) { + return <> + If you have generated an authentication token from the master server, use that here.
{ props.setToken(values.token); }} + layout="inline" > } + style={{ width: "14em" }} + prefix= type="password" placeholder="Authentication Token" />
- ); + ; +} + +export default function LoginForm(props) { + let plugins = useContext(PluginsContext); + + let loginForms = []; + for (let plugin of plugins.values()) { + loginForms.push(...plugin.loginForms); + } + + loginForms.push({ + name: "core.token", + title: "Token", + Component: TokenAuth, + }); + + return + +

Clusterio

+ + Log in to the cluster using one of the methods below. + + {loginForms.map(form => + {form.title} + + )} + ; } From bd533395321e4a306f51d0fc965ed5841e7b0af5 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 16 Mar 2021 21:40:47 +0100 Subject: [PATCH 09/10] Fix web root path confusion Remove basename in favor of making webRoot absolute. The distinction between the two were supposed to be that basename was absolute and did not have a trailing slash while webRoot was relative and had a traling slash. This was not how it ended up working in the code, which lead to double forward slashes in requests sent out when webRoot was not equal to './'. --- packages/master/web/index.html | 2 +- packages/web_ui/src/basename.jsx | 6 ------ packages/web_ui/src/bootstrap.jsx | 5 ++--- packages/web_ui/src/components/App.jsx | 3 +-- packages/web_ui/src/components/PluginViewPage.jsx | 3 +-- packages/web_ui/src/components/PluginsPage.jsx | 3 +-- packages/web_ui/src/index.jsx | 1 - 7 files changed, 6 insertions(+), 17 deletions(-) delete mode 100644 packages/web_ui/src/basename.jsx diff --git a/packages/master/web/index.html b/packages/master/web/index.html index 61d74ee8..372cf1fe 100644 --- a/packages/master/web/index.html +++ b/packages/master/web/index.html @@ -5,7 +5,7 @@ Clusterio diff --git a/packages/web_ui/src/basename.jsx b/packages/web_ui/src/basename.jsx deleted file mode 100644 index 0e7e3de4..00000000 --- a/packages/web_ui/src/basename.jsx +++ /dev/null @@ -1,6 +0,0 @@ -// Calculate basedir of application based on current location and -// the provided web_root from master server. -const location = new URL(window.webRoot, document.location); -const basename = location.pathname !== "/" ? location.pathname : ""; - -export default basename; diff --git a/packages/web_ui/src/bootstrap.jsx b/packages/web_ui/src/bootstrap.jsx index 0e37a489..6acef556 100644 --- a/packages/web_ui/src/bootstrap.jsx +++ b/packages/web_ui/src/bootstrap.jsx @@ -7,7 +7,6 @@ import libConfig from "@clusterio/lib/config"; import libPlugin from "@clusterio/lib/plugin"; import { ConsoleTransport, WebConsoleFormat, logger } from "@clusterio/lib/logging"; -import basename from "./basename"; import App from "./components/App"; import { Control, ControlConnector } from "./util/websocket"; @@ -28,7 +27,7 @@ async function loadScript(url) { } async function loadPlugins() { - let response = await fetch(`${basename}/api/plugins`); + let response = await fetch(`${webRoot}api/plugins`); let pluginList; if (response.ok) { pluginList = await response.json(); @@ -42,7 +41,7 @@ async function loadPlugins() { await __webpack_init_sharing__("default"); for (let meta of pluginList) { try { - await loadScript(`${basename}/plugins/${meta.name}/remoteEntry.js`); + await loadScript(`${webRoot}plugins/${meta.name}/remoteEntry.js`); let container = window[`plugin_${meta.name}`]; if (!container) { throw new Error(`Plugin did not expose its container via plugin_${meta.name}`); diff --git a/packages/web_ui/src/components/App.jsx b/packages/web_ui/src/components/App.jsx index c326d014..7532d7c8 100644 --- a/packages/web_ui/src/components/App.jsx +++ b/packages/web_ui/src/components/App.jsx @@ -4,7 +4,6 @@ import { BrowserRouter } from "react-router-dom"; import { logger } from "@clusterio/lib/logging"; -import basename from "../basename"; import ErrorBoundary from "./ErrorBoundary"; import SiteLayout from "./SiteLayout"; import ControlContext from "./ControlContext"; @@ -89,7 +88,7 @@ export default function App(props) { - + {page} diff --git a/packages/web_ui/src/components/PluginViewPage.jsx b/packages/web_ui/src/components/PluginViewPage.jsx index dbca70c8..4ab2955f 100644 --- a/packages/web_ui/src/components/PluginViewPage.jsx +++ b/packages/web_ui/src/components/PluginViewPage.jsx @@ -2,7 +2,6 @@ import React, { useEffect, useContext, useState } from "react"; import { useHistory, useParams } from "react-router-dom"; import { Alert, Descriptions, Spin } from "antd"; -import basename from "../basename"; import PluginsContext from "./PluginsContext"; import PageLayout from "./PageLayout"; import { notifyErrorHandler } from "../util/notify"; @@ -16,7 +15,7 @@ export default function PluginViewPage() { useEffect(() => { (async () => { - let response = await fetch(`${basename}/api/plugins`); + let response = await fetch(`${webRoot}api/plugins`); if (response.ok) { setPluginList(await response.json()); } else { diff --git a/packages/web_ui/src/components/PluginsPage.jsx b/packages/web_ui/src/components/PluginsPage.jsx index 42b4a7e4..6c9390f2 100644 --- a/packages/web_ui/src/components/PluginsPage.jsx +++ b/packages/web_ui/src/components/PluginsPage.jsx @@ -3,7 +3,6 @@ import { useHistory } from "react-router-dom"; import { Table } from "antd"; import notify from "../util/notify"; -import basename from "../basename"; import PluginsContext from "./PluginsContext"; import PageLayout from "./PageLayout"; @@ -15,7 +14,7 @@ export default function PluginsPage() { useEffect(() => { (async () => { - let response = await fetch(`${basename}/api/plugins`); + let response = await fetch(`${webRoot}api/plugins`); if (response.ok) { setPluginList(await response.json()); } else { diff --git a/packages/web_ui/src/index.jsx b/packages/web_ui/src/index.jsx index dc3e9887..9761a022 100644 --- a/packages/web_ui/src/index.jsx +++ b/packages/web_ui/src/index.jsx @@ -1,4 +1,3 @@ -export { default as basename } from "./basename.jsx"; export { default as bootstrap } from "./bootstrap.jsx"; export { default as notify, notifyErrorHandler } from "./util/notify.jsx"; From 941639666d48165629c61fdb731b23c82050c11f Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 16 Mar 2021 21:44:30 +0100 Subject: [PATCH 10/10] Use auto publicPath deduction Change to "auto" as the publicPath to have webpack deduce the correct path at runtime. This fixes an issue with plugins being unable to be loaded due to trying to load bundles from the wrong path when the web interface is hosted under a proxied sub-path. --- packages/master/webpack.config.js | 2 +- plugins/global_chat/webpack.config.js | 2 +- plugins/research_sync/webpack.config.js | 2 +- plugins/statistics_exporter/webpack.config.js | 2 +- plugins/subspace_storage/webpack.config.js | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/master/webpack.config.js b/packages/master/webpack.config.js index 16223c8b..d7570127 100644 --- a/packages/master/webpack.config.js +++ b/packages/master/webpack.config.js @@ -12,7 +12,7 @@ module.exports = (env = {}) => merge(common(env), { contentBase: "./dist/web", }, output: { - publicPath: "/", + publicPath: "auto", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/global_chat/webpack.config.js b/plugins/global_chat/webpack.config.js index ccb9b636..b1b77ec3 100644 --- a/plugins/global_chat/webpack.config.js +++ b/plugins/global_chat/webpack.config.js @@ -9,7 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "/plugins/global_chat/", + publicPath: "auto", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/research_sync/webpack.config.js b/plugins/research_sync/webpack.config.js index b34f7439..e41d52e5 100644 --- a/plugins/research_sync/webpack.config.js +++ b/plugins/research_sync/webpack.config.js @@ -9,7 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "/plugins/research_sync/", + publicPath: "auto", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/statistics_exporter/webpack.config.js b/plugins/statistics_exporter/webpack.config.js index a42b5c65..81f1a2ad 100644 --- a/plugins/statistics_exporter/webpack.config.js +++ b/plugins/statistics_exporter/webpack.config.js @@ -9,7 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "/plugins/statistics_exporter/", + publicPath: "auto", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, diff --git a/plugins/subspace_storage/webpack.config.js b/plugins/subspace_storage/webpack.config.js index 1de46598..8305920b 100644 --- a/plugins/subspace_storage/webpack.config.js +++ b/plugins/subspace_storage/webpack.config.js @@ -9,7 +9,7 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "/plugins/subspace_storage/", + publicPath: "auto", filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), },