Merge pull request #460 from Hornwitser/cache-web

Cache static web files

.gitignore

@@ -40,7 +40,7 @@ Factorio*
 /config-slave.json
 pnpm-lock.yaml
 mod_cache.json
-static/export
+/static/
 saves
 /temp/
 

packages/master/src/Master.js

@@ -137,20 +137,22 @@ class Master {
 				webpackConfigs.push(require("../webpack.config")({}));
 			}
 			if (args.devPlugin) {
+				let devPlugins = new Map();
 				for (let name of args.devPlugin) {
 					let info = this.pluginInfos.find(i => i.name === name);
 					if (!info) {
 						throw new libErrors.StartupError(`No plugin named ${name}`);
 					}
 					let config = require(path.posix.join(info.requirePath, "webpack.config"))({});
-					config.output.publicPath = `/plugins/${name}/`;
+					devPlugins.set(name, webpackConfigs.length);
 					webpackConfigs.push(config);
 				}
+				this.app.locals.devPlugins = devPlugins;
 			}
 			/* eslint-enable node/global-require, node/no-unpublished-require */
 
 			const compiler = webpack(webpackConfigs);
-			this.devMiddleware = webpackDevMiddleware(compiler, {});
+			this.devMiddleware = webpackDevMiddleware(compiler, { serverSideRender: true });
 			this.app.use(this.devMiddleware);
 		}
 
@@ -161,6 +163,7 @@ class Master {
 		this.instances = await Master.loadInstances(path.join(databaseDirectory, "instances.json"));
 		this.userManager = new UserManager(this.config);
 		await this.userManager.load(path.join(databaseDirectory, "users.json"));
+		this.exportManifest = await Master.loadJsonObject(path.join(databaseDirectory, "export_manifest.json"));
 
 		this.config.on("fieldChanged", (group, field, prev) => {
 			libPlugin.invokeHook(this.plugins, "onMasterConfigFieldChanged", group, field, prev);
@@ -190,6 +193,14 @@ class Master {
 
 		Master.addAppRoutes(this.app, this.pluginInfos);
 
+		if (!args.dev) {
+			let manifest = await Master.loadJsonObject(path.join(__dirname, "..", "dist", "web", "manifest.json"));
+			if (!manifest["main.js"]) {
+				logger.error("Missing main.js entry in dist/web/manifest.json");
+			}
+			this.app.locals.mainBundle = manifest["main.js"] || "no_web_build";
+		}
+
 		// Load plugins
 		await this.loadPlugins();
 
@@ -349,6 +360,26 @@ class Master {
 		await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4));
 	}
 
+	static async loadJsonObject(filePath, throwOnMissing = false) {
+		let manifest = {};
+		try {
+			manifest = JSON.parse(await fs.readFile(filePath));
+		} catch (err) {
+			if (!throwOnMissing && err.code !== "ENOENT") {
+				throw err;
+			}
+		}
+		return manifest;
+	}
+
+	async updateExportManifest(newManifest) {
+		let databaseDirectory = this.config.get("master.database_directory");
+		await libFileOps.safeOutputFile(
+			path.join(databaseDirectory, "export_manifest.json"), JSON.stringify(newManifest, null, 4)
+		);
+		this.exportManifest = newManifest;
+	}
+
 	/**
 	 * Query master log
 	 *
@@ -384,8 +415,9 @@ class Master {
 		app.use(compression());
 
 		// Set folder to serve static content from (the website)
-		app.use(express.static(path.join(__dirname, "..", "dist", "web")));
-		app.use(express.static("static")); // Used for data export files
+		const staticOptions = { immutable: true, maxAge: 1000 * 86400 * 365 };
+		app.use("/static", express.static(path.join(__dirname, "..", "dist", "web", "static"), staticOptions));
+		app.use("/static", express.static("static", staticOptions)); // Used for data export files
 
 		// Add API routes
 		routes.addRouteHandlers(app);
@@ -400,8 +432,8 @@ class Master {
 			}
 
 			let pluginPackagePath = require.resolve(path.posix.join(pluginInfo.requirePath, "package.json"));
-			let webPath = path.join(path.dirname(pluginPackagePath), "dist", "web");
-			app.use(`/plugins/${pluginInfo.name}`, express.static(webPath));
+			let webPath = path.join(path.dirname(pluginPackagePath), "dist", "web", "static");
+			app.use("/static", express.static(webPath, staticOptions));
 		}
 	}
 
@@ -511,6 +543,13 @@ class Master {
 
 	async loadPlugins() {
 		for (let pluginInfo of this.pluginInfos) {
+			try {
+				let manifestPath = path.posix.join(pluginInfo.requirePath, "dist", "web", "manifest.json");
+				pluginInfo.manifest = await Master.loadJsonObject(require.resolve(manifestPath), true);
+			} catch (err) {
+				logger.warn(`Unable to load dist/web/manifest.json for plugin ${pluginInfo.name}`);
+			}
+
 			if (!this.config.group(pluginInfo.name).get("load_plugin")) {
 				continue;
 			}
@@ -600,9 +639,22 @@ class Master {
 		return function(req, res, next) {
 			let depth = routeDepth + (req.path.slice(-1) === "/");
 			let webRoot = "../".repeat(depth) || "./";
+			let staticRoot = webRoot;
+			let mainBundle;
+			if (res.app.locals.mainBundle) {
+				mainBundle = res.app.locals.mainBundle;
+			} else {
+				let stats = res.locals.webpack.devMiddleware.stats.stats[0];
+				mainBundle = stats.toJson().assetsByChunkName["main"];
+			}
 			fs.readFile(path.join(__dirname, "..", "web", "index.html"), "utf8").then((content) => {
 				res.type("text/html");
-				res.send(content.replace(/__WEB_ROOT__/g, webRoot));
+				res.send(content
+					.replace(/__CLUSTER_NAME__/g, res.app.locals.master.config.get("master.name"))
+					.replace(/__WEB_ROOT__/g, webRoot)
+					.replace(/__STATIC_ROOT__/g, staticRoot)
+					.replace(/__MAIN_BUNDLE__/g, mainBundle)
+				);
 			}).catch(err => {
 				next(err);
 			});

packages/master/src/routes.js

@@ -9,6 +9,7 @@ const util = require("util");
 
 const libErrors = require("@clusterio/lib/errors");
 const libFileOps = require("@clusterio/lib/file_ops");
+const libHash = require("@clusterio/lib/hash");
 const libHelpers = require("@clusterio/lib/helpers");
 const libLink = require("@clusterio/lib/link");
 const libPlugin = require("@clusterio/lib/plugin");
@@ -100,7 +101,23 @@ function getPlugins(req, res) {
 		let name = pluginInfo.name;
 		let loaded = req.app.locals.master.plugins.has(name);
 		let enabled = loaded && req.app.locals.master.config.group(pluginInfo.name).get("load_plugin");
-		plugins.push({ name, version: pluginInfo.version, enabled, loaded });
+		let web = {};
+		let devPlugins = req.app.locals.devPlugins;
+		if (devPlugins && devPlugins.has(name)) {
+			let stats = res.locals.webpack.devMiddleware.stats.stats[devPlugins.get(name)];
+			web.main = stats.toJson().assetsByChunkName[name];
+		} else if (pluginInfo.manifest) {
+			web.main = pluginInfo.manifest[`${pluginInfo.name}.js`];
+			if (!web.main) {
+				web.error = `Missing ${pluginInfo.name}.js entry in manifest.json`;
+			}
+		} else {
+			web.error = "Missing dist/web/manifest.json";
+		}
+		if (web.main === "remoteEntry.js") {
+			web.error = "Incompatible old remoteEntry.js entrypoint.";
+		}
+		plugins.push({ name, version: pluginInfo.version, enabled, loaded, web });
 	}
 	res.send(plugins);
 }
@@ -180,19 +197,28 @@ async function uploadExport(req, res) {
 		"export/locale.json",
 	];
 
+	let manifest = {};
 	for (let filePath of exportFiles) {
 		let file = zip.file(filePath);
 		if (!file) {
 			continue;
 		}
 
-		let name = path.posix.basename(filePath);
-		await libFileOps.safeOutputFile(path.join("static", "export", name), await file.async("nodebuffer"));
+		let { name, ext } = path.posix.parse(filePath);
+		let hash = await libHash.hashStream(file.nodeStream());
+		manifest[name] = `static/${name}.${hash}${ext}`;
+		await libFileOps.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
 	}
+	await res.app.locals.master.updateExportManifest(manifest);
 
 	res.sendStatus(200);
 }
 
+async function getExportManifest(req, res) {
+	endpointHitCounter.labels(req.route.path).inc();
+	res.json(res.app.locals.master.exportManifest);
+}
+
 async function createProxyStream(app) {
 	let asyncRandomBytes = util.promisify(crypto.randomBytes);
 	let id = (await asyncRandomBytes(8)).toString("hex");
@@ -417,6 +443,7 @@ function addRouteHandlers(app) {
 		validateSlaveToken,
 		(req, res, next) => uploadExport(req, res).catch(next)
 	);
+	app.get("/api/export-manifest", (req, res, next) => getExportManifest(req, res, next).catch(next));
 	app.put("/api/stream/:id", (req, res, next) => putStream(req, res).catch(next));
 	app.get("/api/stream/:id", (req, res, next) => getStream(req, res).catch(next));
 	app.post("/api/upload-save",

packages/master/web/index.html

@@ -4,14 +4,35 @@
 		<meta charset="utf-8" />
 		<meta name="viewport" content="width=device-width, initial-scale=1" />
 		<meta name="theme-color" content="#2a2a2a" />
-		<title>Clusterio</title>
+		<title>__CLUSTER_NAME__</title>
+		<style>
+			html {
+				color: #fff;
+				background-color: #000;
+			}
+
+			#bootstrap-msg {
+				font: 20px serif;
+				color: #aaa;
+				margin: auto;
+				margin-top: 1em;
+				width: fit-content;
+			}
+		</style>
 		<script>
-				window.webRoot = new URL("__WEB_ROOT__", document.location).pathname;
+			window.webRoot = new URL("__WEB_ROOT__", document.location).pathname;
+			window.staticRoot = new URL("__STATIC_ROOT__", document.location).pathname;
+			function scriptError() {
+				let bootstrapMsg = document.getElementById("bootstrap-msg");
+				bootstrapMsg.textContent = "Error loading main bundle, see web console for details.";
+			}
 		</script>
 	</head>
 	<body>
 		<noscript>You need to enable JavaScript to run this app.</noscript>
-		<div id="root"></div>
-		<script src="__WEB_ROOT__bundle.js"></script>
+		<div id="root">
+			<div id="bootstrap-msg">Loading web interface...</div>
+		</div>
+		<script src="__STATIC_ROOT____MAIN_BUNDLE__" defer onerror="scriptError()"></script>
 	</body>
 </html>

packages/master/webpack.config.js

@@ -12,8 +12,6 @@ module.exports = (env = {}) => merge(common(env), {
 		contentBase: "./dist/web",
 	},
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [

packages/web_ui/package.json

@@ -39,7 +39,8 @@
     "style-loader": "^2.0.0",
     "util": "^0.12.3",
     "webpack": "^5.73.0",
-    "webpack-cli": "^4.9.2"
+    "webpack-cli": "^4.9.2",
+    "webpack-manifest-plugin": "^5.0.0"
   },
   "publishConfig": {
     "access": "public"

packages/web_ui/src/bootstrap.jsx

@@ -39,8 +39,12 @@ async function loadPlugins() {
 	let plugins = new Map();
 	await __webpack_init_sharing__("default");
 	for (let meta of pluginList) {
+		if (meta.web.error) {
+			logger.error(`Failed to load plugin ${meta.name}: ${meta.web.error}`);
+			continue;
+		}
 		try {
-			await loadScript(`${webRoot}plugins/${meta.name}/remoteEntry.js`);
+			await loadScript(`${webRoot}${meta.web.main}`);
 			let container = window[`plugin_${meta.name}`];
 			if (!container) {
 				throw new Error(`Plugin did not expose its container via plugin_${meta.name}`);
@@ -63,6 +67,9 @@ async function loadPlugins() {
 			plugins.set(pluginInfo.name, plugin);
 
 		} catch (err) {
+			if (err.message) {
+				meta.web.error = err.message;
+			}
 			logger.error(`Failed to load plugin ${meta.name}`);
 			if (err.stack) {
 				logger.error(err.stack);

packages/web_ui/src/components/PluginViewPage.jsx

@@ -51,7 +51,7 @@ export default function PluginViewPage() {
 				style={{
 					marginTop: "1em",
 				}}
-				message="Error loading web module"
+				message={pluginMeta.web.error || "Error loading web module"}
 				description={
 					"The web interface was unable to load the webpack module for this plugin. This is "+
 					"usually due to an incorrect or missing webpack build for the plugin. Due to the "+

packages/web_ui/src/model/export_manifest.jsx

@@ -0,0 +1,23 @@
+import { useEffect, useState } from "react";
+
+
+let exportManifestCache = null;
+let emptyCache = {};
+export function useExportManifest() {
+	let [exportManifest, setExportManifest] = useState(exportManifestCache || emptyCache);
+	useEffect(() => {
+		async function load() {
+			let response = await fetch(`${webRoot}api/export-manifest`);
+			if (response.ok) {
+				exportManifestCache = await response.json();
+				setExportManifest(exportManifestCache);
+			}
+		}
+
+		if (!exportManifestCache) {
+			load();
+		}
+	}, []);
+
+	return exportManifest;
+}

packages/web_ui/src/model/item_metadata.jsx

@@ -1,12 +1,18 @@
 import { useEffect, useState } from "react";
 
+import { useExportManifest } from "./export_manifest";
+
 
 let itemMetadataCache = null;
 export function useItemMetadata() {
+	let exportManifest = useExportManifest();
 	let [itemMetadata, setItemMetadata] = useState(itemMetadataCache || new Map());
 	useEffect(() => {
 		async function load() {
-			let response = await fetch(`${webRoot}export/item-metadata.json`);
+			if (!exportManifest["item-metadata"] || !exportManifest["item-spritesheet"]) {
+				return;
+			}
+			let response = await fetch(`${staticRoot}${exportManifest["item-metadata"]}`);
 			if (response.ok) {
 				let data = await response.json();
 				itemMetadataCache = new Map(data);
@@ -16,7 +22,7 @@ export function useItemMetadata() {
 				for (let [name, meta] of itemMetadataCache) {
 					style.sheet.insertRule(
 						`.item-${CSS.escape(name)} {
-	background-image: url("${webRoot}export/item-spritesheet.png");
+	background-image: url("${staticRoot}${exportManifest["item-spritesheet"]}");
 	background-repeat: no-repeat;
 	background-position: -${meta.x}px -${meta.y}px;
 	height: ${meta.size}px;
@@ -31,7 +37,7 @@ export function useItemMetadata() {
 		if (!itemMetadataCache) {
 			load();
 		}
-	}, []);
+	}, [exportManifest]);
 
 	return itemMetadata;
 }

packages/web_ui/src/model/locale.jsx

@@ -1,12 +1,18 @@
 import React, { useEffect, useState } from "react";
 
+import { useExportManifest } from "./export_manifest";
+
 
 let localeCache = null;
 export function useLocale() {
+	let exportManifest = useExportManifest();
 	let [locale, setLocale] = useState(localeCache || new Map());
 	useEffect(() => {
 		async function load() {
-			let response = await fetch(`${webRoot}export/locale.json`);
+			if (!exportManifest["locale"]) {
+				return;
+			}
+			let response = await fetch(`${staticRoot}${exportManifest["locale"]}`);
 			if (response.ok) {
 				let data = await response.json();
 				localeCache = new Map(data);
@@ -17,7 +23,7 @@ export function useLocale() {
 		if (!localeCache) {
 			load();
 		}
-	}, []);
+	}, [exportManifest]);
 
 	return locale;
 }

packages/web_ui/webpack.common.js

@@ -1,6 +1,7 @@
 "use strict";
 const webpack = require("webpack");
 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
+const { WebpackManifestPlugin } = require("webpack-manifest-plugin");
 
 module.exports = (env = {}) => ({
 	mode: env.production ? "production" : "development",
@@ -9,8 +10,14 @@ module.exports = (env = {}) => ({
 		maxAssetSize: 2**21,
 		maxEntrypointSize: 2**21,
 	},
+	output: {
+		publicPath: "auto",
+		assetModuleFilename: "static/[hash][ext][query]",
+		filename: "static/[name].[contenthash].js",
+	},
 	plugins: [
 		new CleanWebpackPlugin(),
+		new WebpackManifestPlugin({ publicPath: "" }),
 
 		// required for winston
 		new webpack.ProvidePlugin({
@@ -81,7 +88,7 @@ module.exports = (env = {}) => ({
 			},
 			{
 				test: /\.png$/,
-				use: require.resolve("file-loader"),
+				type: "asset/resource",
 			},
 		],
 	},
@@ -101,4 +108,7 @@ module.exports = (env = {}) => ({
 			"zlib": require.resolve("browserify-zlib"),
 		},
 	},
+	optimization: {
+		moduleIds: "deterministic",
+	},
 });

plugins/global_chat/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "global_chat",
 			library: { type: "var", name: "plugin_global_chat" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

plugins/inventory_sync/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "inventory_sync",
 			library: { type: "var", name: "plugin_inventory_sync" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

plugins/player_auth/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "player_auth",
 			library: { type: "var", name: "plugin_player_auth" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

plugins/research_sync/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "research_sync",
 			library: { type: "var", name: "plugin_research_sync" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

plugins/statistics_exporter/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "statistics_exporter",
 			library: { type: "var", name: "plugin_statistics_exporter" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

plugins/subspace_storage/webpack.config.js

@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
 	context: __dirname,
 	entry: "./web/index.jsx",
 	output: {
-		publicPath: "auto",
-		filename: "bundle.js",
 		path: path.resolve(__dirname, "dist", "web"),
 	},
 	plugins: [
 		new webpack.container.ModuleFederationPlugin({
 			name: "subspace_storage",
 			library: { type: "var", name: "plugin_subspace_storage" },
-			filename: "remoteEntry.js",
 			exposes: {
 				"./info": "./info.js",
 				"./package.json": "./package.json",

test/integration/clusterio.js

@@ -300,10 +300,16 @@ describe("Integration of Clusterio", function() {
 		describe("instance export-data", function() {
 			it("exports the data", async function() {
 				slowTest(this);
-				let exportPath = path.join("temp", "test", "static", "export", "locale.json");
+				let exportPath = path.join("temp", "test", "static");
 				await fs.remove(exportPath);
 				await execCtl("instance export-data test");
-				assert(await fs.exists(exportPath), "Export was not created");
+				let response = await get("/api/export-manifest");
+				let manifest = JSON.parse(response.body);
+				assert(Object.keys(manifest).length > 1, "Export manifest is empty");
+				assert(
+					await fs.exists(path.join(exportPath, "..", manifest["item-metadata"])),
+					"Export was not written to filesystem"
+				);
 				await checkInstanceStatus(44, "stopped");
 			});
 		});