From e32637503d79006ac7f348fe9deff1faa7fe9114 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 8 Jun 2022 12:37:03 +0200 Subject: [PATCH] Move all web assets to shared cached space Put all web assets including plugin assets under a shared static/ route in the web interface that's declared as immutable and cacheable for 1 year. This allows all JavaScript bundles to be loaded from disk instead of having to check with the web server each time and significantly speeds up the web interface load time. It also fixes the public path issue when using the --dev-plugin option behind a proxy. To prevent collisions and stale data all assets now include a hash in the filename. This means the entrypoint is now dynamic and specified in a manifest.json file created from the web build. For plugins this means the filename attribute needs to be removed from webpack.config.js. --- packages/master/src/Master.js | 56 ++++++++++++++++--- packages/master/src/routes.js | 18 +++++- packages/master/web/index.html | 3 +- packages/master/webpack.config.js | 2 - packages/web_ui/package.json | 3 +- packages/web_ui/src/bootstrap.jsx | 9 ++- .../web_ui/src/components/PluginViewPage.jsx | 2 +- packages/web_ui/webpack.common.js | 12 +++- plugins/global_chat/webpack.config.js | 3 - plugins/inventory_sync/webpack.config.js | 3 - plugins/player_auth/webpack.config.js | 3 - plugins/research_sync/webpack.config.js | 3 - plugins/statistics_exporter/webpack.config.js | 3 - plugins/subspace_storage/webpack.config.js | 3 - 14 files changed, 90 insertions(+), 33 deletions(-) diff --git a/packages/master/src/Master.js b/packages/master/src/Master.js index 863bf48a..c65f5676 100644 --- a/packages/master/src/Master.js +++ b/packages/master/src/Master.js @@ -137,20 +137,22 @@ class Master { webpackConfigs.push(require("../webpack.config")({})); } if (args.devPlugin) { + let devPlugins = new Map(); for (let name of args.devPlugin) { let info = this.pluginInfos.find(i => i.name === name); if (!info) { throw new libErrors.StartupError(`No plugin named ${name}`); } let config = require(path.posix.join(info.requirePath, "webpack.config"))({}); - config.output.publicPath = `/plugins/${name}/`; + devPlugins.set(name, webpackConfigs.length); webpackConfigs.push(config); } + this.app.locals.devPlugins = devPlugins; } /* eslint-enable node/global-require, node/no-unpublished-require */ const compiler = webpack(webpackConfigs); - this.devMiddleware = webpackDevMiddleware(compiler, {}); + this.devMiddleware = webpackDevMiddleware(compiler, { serverSideRender: true }); this.app.use(this.devMiddleware); } @@ -190,6 +192,14 @@ class Master { Master.addAppRoutes(this.app, this.pluginInfos); + if (!args.dev) { + let manifest = await Master.loadJsonObject(path.join(__dirname, "..", "dist", "web", "manifest.json")); + if (!manifest["main.js"]) { + logger.error("Missing main.js entry in dist/web/manifest.json"); + } + this.app.locals.mainBundle = manifest["main.js"] || "no_web_build"; + } + // Load plugins await this.loadPlugins(); @@ -349,6 +359,18 @@ class Master { await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4)); } + static async loadJsonObject(filePath, throwOnMissing = false) { + let manifest = {}; + try { + manifest = JSON.parse(await fs.readFile(filePath)); + } catch (err) { + if (!throwOnMissing && err.code !== "ENOENT") { + throw err; + } + } + return manifest; + } + /** * Query master log * @@ -384,8 +406,9 @@ class Master { app.use(compression()); // Set folder to serve static content from (the website) - app.use(express.static(path.join(__dirname, "..", "dist", "web"))); - app.use(express.static("static")); // Used for data export files + const staticOptions = { immutable: true, maxAge: 1000 * 86400 * 365 }; + app.use("/static", express.static(path.join(__dirname, "..", "dist", "web", "static"), staticOptions)); + app.use("/static", express.static("static", staticOptions)); // Used for data export files // Add API routes routes.addRouteHandlers(app); @@ -400,8 +423,8 @@ class Master { } let pluginPackagePath = require.resolve(path.posix.join(pluginInfo.requirePath, "package.json")); - let webPath = path.join(path.dirname(pluginPackagePath), "dist", "web"); - app.use(`/plugins/${pluginInfo.name}`, express.static(webPath)); + let webPath = path.join(path.dirname(pluginPackagePath), "dist", "web", "static"); + app.use("/static", express.static(webPath, staticOptions)); } } @@ -511,6 +534,13 @@ class Master { async loadPlugins() { for (let pluginInfo of this.pluginInfos) { + try { + let manifestPath = path.posix.join(pluginInfo.requirePath, "dist", "web", "manifest.json"); + pluginInfo.manifest = await Master.loadJsonObject(require.resolve(manifestPath), true); + } catch (err) { + logger.warn(`Unable to load dist/web/manifest.json for plugin ${pluginInfo.name}`); + } + if (!this.config.group(pluginInfo.name).get("load_plugin")) { continue; } @@ -600,9 +630,21 @@ class Master { return function(req, res, next) { let depth = routeDepth + (req.path.slice(-1) === "/"); let webRoot = "../".repeat(depth) || "./"; + let staticRoot = webRoot; + let mainBundle; + if (res.app.locals.mainBundle) { + mainBundle = res.app.locals.mainBundle; + } else { + let stats = res.locals.webpack.devMiddleware.stats.stats[0]; + mainBundle = stats.toJson().assetsByChunkName["main"]; + } fs.readFile(path.join(__dirname, "..", "web", "index.html"), "utf8").then((content) => { res.type("text/html"); - res.send(content.replace(/__WEB_ROOT__/g, webRoot)); + res.send(content + .replace(/__WEB_ROOT__/g, webRoot) + .replace(/__STATIC_ROOT__/g, staticRoot) + .replace(/__MAIN_BUNDLE__/g, mainBundle) + ); }).catch(err => { next(err); }); diff --git a/packages/master/src/routes.js b/packages/master/src/routes.js index c0217d20..36c39402 100644 --- a/packages/master/src/routes.js +++ b/packages/master/src/routes.js @@ -100,7 +100,23 @@ function getPlugins(req, res) { let name = pluginInfo.name; let loaded = req.app.locals.master.plugins.has(name); let enabled = loaded && req.app.locals.master.config.group(pluginInfo.name).get("load_plugin"); - plugins.push({ name, version: pluginInfo.version, enabled, loaded }); + let web = {}; + let devPlugins = req.app.locals.devPlugins; + if (devPlugins && devPlugins.has(name)) { + let stats = res.locals.webpack.devMiddleware.stats.stats[devPlugins.get(name)]; + web.main = stats.toJson().assetsByChunkName[name]; + } else if (pluginInfo.manifest) { + web.main = pluginInfo.manifest[`${pluginInfo.name}.js`]; + if (!web.main) { + web.error = `Missing ${pluginInfo.name}.js entry in manifest.json`; + } + } else { + web.error = "Missing dist/web/manifest.json"; + } + if (web.main === "remoteEntry.js") { + web.error = "Incompatible old remoteEntry.js entrypoint."; + } + plugins.push({ name, version: pluginInfo.version, enabled, loaded, web }); } res.send(plugins); } diff --git a/packages/master/web/index.html b/packages/master/web/index.html index ba9bbf02..71682ed7 100644 --- a/packages/master/web/index.html +++ b/packages/master/web/index.html @@ -7,11 +7,12 @@ Clusterio
- + diff --git a/packages/master/webpack.config.js b/packages/master/webpack.config.js index 4c45a621..e7e9a9fa 100644 --- a/packages/master/webpack.config.js +++ b/packages/master/webpack.config.js @@ -12,8 +12,6 @@ module.exports = (env = {}) => merge(common(env), { contentBase: "./dist/web", }, output: { - publicPath: "auto", - filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, plugins: [ diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index de2910ed..7c74d8e4 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -39,7 +39,8 @@ "style-loader": "^2.0.0", "util": "^0.12.3", "webpack": "^5.73.0", - "webpack-cli": "^4.9.2" + "webpack-cli": "^4.9.2", + "webpack-manifest-plugin": "^5.0.0" }, "publishConfig": { "access": "public" diff --git a/packages/web_ui/src/bootstrap.jsx b/packages/web_ui/src/bootstrap.jsx index 213715b4..e64be130 100644 --- a/packages/web_ui/src/bootstrap.jsx +++ b/packages/web_ui/src/bootstrap.jsx @@ -39,8 +39,12 @@ async function loadPlugins() { let plugins = new Map(); await __webpack_init_sharing__("default"); for (let meta of pluginList) { + if (meta.web.error) { + logger.error(`Failed to load plugin ${meta.name}: ${meta.web.error}`); + continue; + } try { - await loadScript(`${webRoot}plugins/${meta.name}/remoteEntry.js`); + await loadScript(`${webRoot}${meta.web.main}`); let container = window[`plugin_${meta.name}`]; if (!container) { throw new Error(`Plugin did not expose its container via plugin_${meta.name}`); @@ -63,6 +67,9 @@ async function loadPlugins() { plugins.set(pluginInfo.name, plugin); } catch (err) { + if (err.message) { + meta.web.error = err.message; + } logger.error(`Failed to load plugin ${meta.name}`); if (err.stack) { logger.error(err.stack); diff --git a/packages/web_ui/src/components/PluginViewPage.jsx b/packages/web_ui/src/components/PluginViewPage.jsx index 4e9eebc1..1c72b08c 100644 --- a/packages/web_ui/src/components/PluginViewPage.jsx +++ b/packages/web_ui/src/components/PluginViewPage.jsx @@ -51,7 +51,7 @@ export default function PluginViewPage() { style={{ marginTop: "1em", }} - message="Error loading web module" + message={pluginMeta.web.error || "Error loading web module"} description={ "The web interface was unable to load the webpack module for this plugin. This is "+ "usually due to an incorrect or missing webpack build for the plugin. Due to the "+ diff --git a/packages/web_ui/webpack.common.js b/packages/web_ui/webpack.common.js index 6a850827..14f0696f 100644 --- a/packages/web_ui/webpack.common.js +++ b/packages/web_ui/webpack.common.js @@ -1,6 +1,7 @@ "use strict"; const webpack = require("webpack"); const { CleanWebpackPlugin } = require("clean-webpack-plugin"); +const { WebpackManifestPlugin } = require("webpack-manifest-plugin"); module.exports = (env = {}) => ({ mode: env.production ? "production" : "development", @@ -9,8 +10,14 @@ module.exports = (env = {}) => ({ maxAssetSize: 2**21, maxEntrypointSize: 2**21, }, + output: { + publicPath: "auto", + assetModuleFilename: "static/[hash][ext][query]", + filename: "static/[name].[contenthash].js", + }, plugins: [ new CleanWebpackPlugin(), + new WebpackManifestPlugin({ publicPath: "" }), // required for winston new webpack.ProvidePlugin({ @@ -81,7 +88,7 @@ module.exports = (env = {}) => ({ }, { test: /\.png$/, - use: require.resolve("file-loader"), + type: "asset/resource", }, ], }, @@ -101,4 +108,7 @@ module.exports = (env = {}) => ({ "zlib": require.resolve("browserify-zlib"), }, }, + optimization: { + moduleIds: "deterministic", + }, }); diff --git a/plugins/global_chat/webpack.config.js b/plugins/global_chat/webpack.config.js index d77230ab..ecba532e 100644 --- a/plugins/global_chat/webpack.config.js +++ b/plugins/global_chat/webpack.config.js @@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "auto", - filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, plugins: [ new webpack.container.ModuleFederationPlugin({ name: "global_chat", library: { type: "var", name: "plugin_global_chat" }, - filename: "remoteEntry.js", exposes: { "./info": "./info.js", "./package.json": "./package.json", diff --git a/plugins/inventory_sync/webpack.config.js b/plugins/inventory_sync/webpack.config.js index 1807a85a..c2d7703f 100644 --- a/plugins/inventory_sync/webpack.config.js +++ b/plugins/inventory_sync/webpack.config.js @@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "auto", - filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, plugins: [ new webpack.container.ModuleFederationPlugin({ name: "inventory_sync", library: { type: "var", name: "plugin_inventory_sync" }, - filename: "remoteEntry.js", exposes: { "./info": "./info.js", "./package.json": "./package.json", diff --git a/plugins/player_auth/webpack.config.js b/plugins/player_auth/webpack.config.js index d282ac14..bae8ede2 100644 --- a/plugins/player_auth/webpack.config.js +++ b/plugins/player_auth/webpack.config.js @@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "auto", - filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, plugins: [ new webpack.container.ModuleFederationPlugin({ name: "player_auth", library: { type: "var", name: "plugin_player_auth" }, - filename: "remoteEntry.js", exposes: { "./info": "./info.js", "./package.json": "./package.json", diff --git a/plugins/research_sync/webpack.config.js b/plugins/research_sync/webpack.config.js index 2afe61c6..0f2f0fba 100644 --- a/plugins/research_sync/webpack.config.js +++ b/plugins/research_sync/webpack.config.js @@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "auto", - filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, plugins: [ new webpack.container.ModuleFederationPlugin({ name: "research_sync", library: { type: "var", name: "plugin_research_sync" }, - filename: "remoteEntry.js", exposes: { "./info": "./info.js", "./package.json": "./package.json", diff --git a/plugins/statistics_exporter/webpack.config.js b/plugins/statistics_exporter/webpack.config.js index 6170fc62..d62c818f 100644 --- a/plugins/statistics_exporter/webpack.config.js +++ b/plugins/statistics_exporter/webpack.config.js @@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "auto", - filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, plugins: [ new webpack.container.ModuleFederationPlugin({ name: "statistics_exporter", library: { type: "var", name: "plugin_statistics_exporter" }, - filename: "remoteEntry.js", exposes: { "./info": "./info.js", "./package.json": "./package.json", diff --git a/plugins/subspace_storage/webpack.config.js b/plugins/subspace_storage/webpack.config.js index a4ae1db3..23a0b41f 100644 --- a/plugins/subspace_storage/webpack.config.js +++ b/plugins/subspace_storage/webpack.config.js @@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), { context: __dirname, entry: "./web/index.jsx", output: { - publicPath: "auto", - filename: "bundle.js", path: path.resolve(__dirname, "dist", "web"), }, plugins: [ new webpack.container.ModuleFederationPlugin({ name: "subspace_storage", library: { type: "var", name: "plugin_subspace_storage" }, - filename: "remoteEntry.js", exposes: { "./info": "./info.js", "./package.json": "./package.json",