Move all web assets to shared cached space

Put all web assets including plugin assets under a shared static/ route
in the web interface that's declared as immutable and cacheable for 1
year.  This allows all JavaScript bundles to be loaded from disk instead
of having to check with the web server each time and significantly
speeds up the web interface load time.  It also fixes the public path
issue when using the --dev-plugin option behind a proxy.

To prevent collisions and stale data all assets now include a hash in
the filename.  This means the entrypoint is now dynamic and specified in
a manifest.json file created from the web build.  For plugins this means
the filename attribute needs to be removed from webpack.config.js.

packages/master/src/Master.js

@@ -137,20 +137,22 @@ class Master {
 				webpackConfigs.push(require("../webpack.config")({}));
 			}
 			if (args.devPlugin) {
+				let devPlugins = new Map();
 				for (let name of args.devPlugin) {
 					let info = this.pluginInfos.find(i => i.name === name);
 					if (!info) {
 						throw new libErrors.StartupError(`No plugin named ${name}`);
 					}
 					let config = require(path.posix.join(info.requirePath, "webpack.config"))({});
-					config.output.publicPath = `/plugins/${name}/`;
+					devPlugins.set(name, webpackConfigs.length);
 					webpackConfigs.push(config);
 				}
+				this.app.locals.devPlugins = devPlugins;
 			}
 			/* eslint-enable node/global-require, node/no-unpublished-require */
 
 			const compiler = webpack(webpackConfigs);
-			this.devMiddleware = webpackDevMiddleware(compiler, {});
+			this.devMiddleware = webpackDevMiddleware(compiler, { serverSideRender: true });
 			this.app.use(this.devMiddleware);
 		}
 
@@ -190,6 +192,14 @@ class Master {
 
 		Master.addAppRoutes(this.app, this.pluginInfos);
 
+		if (!args.dev) {
+			let manifest = await Master.loadJsonObject(path.join(__dirname, "..", "dist", "web", "manifest.json"));
+			if (!manifest["main.js"]) {
+				logger.error("Missing main.js entry in dist/web/manifest.json");
+			}
+			this.app.locals.mainBundle = manifest["main.js"] || "no_web_build";
+		}
+
 		// Load plugins
 		await this.loadPlugins();
 
@@ -349,6 +359,18 @@ class Master {
 		await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4));
 	}
 
+	static async loadJsonObject(filePath, throwOnMissing = false) {
+		let manifest = {};
+		try {
+			manifest = JSON.parse(await fs.readFile(filePath));
+		} catch (err) {
+			if (!throwOnMissing && err.code !== "ENOENT") {
+				throw err;
+			}
+		}
+		return manifest;
+	}
+
 	/**
 	 * Query master log
 	 *
@@ -384,8 +406,9 @@ class Master {
 		app.use(compression());
 
 		// Set folder to serve static content from (the website)
-		app.use(express.static(path.join(__dirname, "..", "dist", "web")));
-		app.use(express.static("static")); // Used for data export files
+		const staticOptions = { immutable: true, maxAge: 1000 * 86400 * 365 };
+		app.use("/static", express.static(path.join(__dirname, "..", "dist", "web", "static"), staticOptions));
+		app.use("/static", express.static("static", staticOptions)); // Used for data export files
 
 		// Add API routes
 		routes.addRouteHandlers(app);
@@ -400,8 +423,8 @@ class Master {
 			}
 
 			let pluginPackagePath = require.resolve(path.posix.join(pluginInfo.requirePath, "package.json"));
-			let webPath = path.join(path.dirname(pluginPackagePath), "dist", "web");
-			app.use(`/plugins/${pluginInfo.name}`, express.static(webPath));
+			let webPath = path.join(path.dirname(pluginPackagePath), "dist", "web", "static");
+			app.use("/static", express.static(webPath, staticOptions));
 		}
 	}
 
@@ -511,6 +534,13 @@ class Master {
 
 	async loadPlugins() {
 		for (let pluginInfo of this.pluginInfos) {
+			try {
+				let manifestPath = path.posix.join(pluginInfo.requirePath, "dist", "web", "manifest.json");
+				pluginInfo.manifest = await Master.loadJsonObject(require.resolve(manifestPath), true);
+			} catch (err) {
+				logger.warn(`Unable to load dist/web/manifest.json for plugin ${pluginInfo.name}`);
+			}
+
 			if (!this.config.group(pluginInfo.name).get("load_plugin")) {
 				continue;
 			}
@@ -600,9 +630,21 @@ class Master {
 		return function(req, res, next) {
 			let depth = routeDepth + (req.path.slice(-1) === "/");
 			let webRoot = "../".repeat(depth) || "./";
+			let staticRoot = webRoot;
+			let mainBundle;
+			if (res.app.locals.mainBundle) {
+				mainBundle = res.app.locals.mainBundle;
+			} else {
+				let stats = res.locals.webpack.devMiddleware.stats.stats[0];
+				mainBundle = stats.toJson().assetsByChunkName["main"];
+			}
 			fs.readFile(path.join(__dirname, "..", "web", "index.html"), "utf8").then((content) => {
 				res.type("text/html");
-				res.send(content.replace(/__WEB_ROOT__/g, webRoot));
+				res.send(content
+					.replace(/__WEB_ROOT__/g, webRoot)
+					.replace(/__STATIC_ROOT__/g, staticRoot)
+					.replace(/__MAIN_BUNDLE__/g, mainBundle)
+				);
 			}).catch(err => {
 				next(err);
 			});

packages/master/src/routes.js

@@ -100,7 +100,23 @@ function getPlugins(req, res) {
 		let name = pluginInfo.name;
 		let loaded = req.app.locals.master.plugins.has(name);
 		let enabled = loaded && req.app.locals.master.config.group(pluginInfo.name).get("load_plugin");
-		plugins.push({ name, version: pluginInfo.version, enabled, loaded });
+		let web = {};
+		let devPlugins = req.app.locals.devPlugins;
+		if (devPlugins && devPlugins.has(name)) {
+			let stats = res.locals.webpack.devMiddleware.stats.stats[devPlugins.get(name)];
+			web.main = stats.toJson().assetsByChunkName[name];
+		} else if (pluginInfo.manifest) {
+			web.main = pluginInfo.manifest[`${pluginInfo.name}.js`];
+			if (!web.main) {
+				web.error = `Missing ${pluginInfo.name}.js entry in manifest.json`;
+			}
+		} else {
+			web.error = "Missing dist/web/manifest.json";
+		}
+		if (web.main === "remoteEntry.js") {
+			web.error = "Incompatible old remoteEntry.js entrypoint.";
+		}
+		plugins.push({ name, version: pluginInfo.version, enabled, loaded, web });
 	}
 	res.send(plugins);
 }

packages/master/web/index.html

@@ -7,11 +7,12 @@
 		<title>Clusterio</title>
 		<script>
 				window.webRoot = new URL("__WEB_ROOT__", document.location).pathname;
+				window.staticRoot = new URL("__STATIC_ROOT__", document.location).pathname;
 		</script>
 	</head>
 	<body>
 		<noscript>You need to enable JavaScript to run this app.</noscript>
 		<div id="root"></div>
-		<script src="__WEB_ROOT__bundle.js"></script>
+		<script src="__STATIC_ROOT____MAIN_BUNDLE__"></script>
 	</body>
 </html>

packages/master/webpack.config.js

@@ -12,8 +12,6 @@ module.exports = (env = {}) => merge(common(env), {
 		contentBase: "./dist/web",
 	},
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [

packages/web_ui/package.json

@@ -39,7 +39,8 @@
     "style-loader": "^2.0.0",
     "util": "^0.12.3",
     "webpack": "^5.73.0",
-    "webpack-cli": "^4.9.2"
+    "webpack-cli": "^4.9.2",
+    "webpack-manifest-plugin": "^5.0.0"
   },
   "publishConfig": {
     "access": "public"

packages/web_ui/src/bootstrap.jsx

@@ -39,8 +39,12 @@ async function loadPlugins() {
 	let plugins = new Map();
 	await __webpack_init_sharing__("default");
 	for (let meta of pluginList) {
+		if (meta.web.error) {
+			logger.error(`Failed to load plugin ${meta.name}: ${meta.web.error}`);
+			continue;
+		}
 		try {
-			await loadScript(`${webRoot}plugins/${meta.name}/remoteEntry.js`);
+			await loadScript(`${webRoot}${meta.web.main}`);
 			let container = window[`plugin_${meta.name}`];
 			if (!container) {
 				throw new Error(`Plugin did not expose its container via plugin_${meta.name}`);
@@ -63,6 +67,9 @@ async function loadPlugins() {
 			plugins.set(pluginInfo.name, plugin);
 
 		} catch (err) {
+			if (err.message) {
+				meta.web.error = err.message;
+			}
 			logger.error(`Failed to load plugin ${meta.name}`);
 			if (err.stack) {
 				logger.error(err.stack);

packages/web_ui/src/components/PluginViewPage.jsx

@@ -51,7 +51,7 @@ export default function PluginViewPage() {
 				style={{
 					marginTop: "1em",
 				}}
-				message="Error loading web module"
+				message={pluginMeta.web.error || "Error loading web module"}
 				description={
 					"The web interface was unable to load the webpack module for this plugin. This is "+
 					"usually due to an incorrect or missing webpack build for the plugin. Due to the "+

packages/web_ui/webpack.common.js

@@ -1,6 +1,7 @@
 "use strict";
 const webpack = require("webpack");
 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
+const { WebpackManifestPlugin } = require("webpack-manifest-plugin");
 
 module.exports = (env = {}) => ({
 	mode: env.production ? "production" : "development",
@@ -9,8 +10,14 @@ module.exports = (env = {}) => ({
 		maxAssetSize: 2**21,
 		maxEntrypointSize: 2**21,
 	},
+	output: {
+		publicPath: "auto",
+		assetModuleFilename: "static/[hash][ext][query]",
+		filename: "static/[name].[contenthash].js",
+	},
 	plugins: [
 		new CleanWebpackPlugin(),
+		new WebpackManifestPlugin({ publicPath: "" }),
 
 		// required for winston
 		new webpack.ProvidePlugin({
@@ -81,7 +88,7 @@ module.exports = (env = {}) => ({
 			},
 			{
 				test: /\.png$/,
-				use: require.resolve("file-loader"),
+				type: "asset/resource",
 			},
 		],
 	},
@@ -101,4 +108,7 @@ module.exports = (env = {}) => ({
 			"zlib": require.resolve("browserify-zlib"),
 		},
 	},
+	optimization: {
+		moduleIds: "deterministic",
+	},
 });

plugins/global_chat/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "global_chat",
 			library: { type: "var", name: "plugin_global_chat" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

plugins/inventory_sync/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "inventory_sync",
 			library: { type: "var", name: "plugin_inventory_sync" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

plugins/player_auth/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "player_auth",
 			library: { type: "var", name: "plugin_player_auth" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

plugins/research_sync/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "research_sync",
 			library: { type: "var", name: "plugin_research_sync" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

plugins/statistics_exporter/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "statistics_exporter",
 			library: { type: "var", name: "plugin_statistics_exporter" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

plugins/subspace_storage/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "subspace_storage",
 			library: { type: "var", name: "plugin_subspace_storage" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",