Move all web assets to shared cached space

Put all web assets including plugin assets under a shared static/ route
in the web interface that's declared as immutable and cacheable for 1
year.  This allows all JavaScript bundles to be loaded from disk instead
of having to check with the web server each time and significantly
speeds up the web interface load time.  It also fixes the public path
issue when using the --dev-plugin option behind a proxy.

To prevent collisions and stale data all assets now include a hash in
the filename.  This means the entrypoint is now dynamic and specified in
a manifest.json file created from the web build.  For plugins this means
the filename attribute needs to be removed from webpack.config.js.
This commit is contained in:
Hornwitser committed 2022-06-08 21:01:36 +02:00
1 parent 2401fe25fa
commit e32637503d
14 files changed
+90 -33

No files matched your search

+49 -7
View File
@@ -137,20 +137,22 @@ class Master {
webpackConfigs.push(require("../webpack.config")({})); webpackConfigs.push(require("../webpack.config")({}));
} }
if (args.devPlugin) { if (args.devPlugin) {
let devPlugins = new Map();
for (let name of args.devPlugin) { for (let name of args.devPlugin) {
let info = this.pluginInfos.find(i => i.name === name); let info = this.pluginInfos.find(i => i.name === name);
if (!info) { if (!info) {
throw new libErrors.StartupError(`No plugin named ${name}`); throw new libErrors.StartupError(`No plugin named ${name}`);
} }
let config = require(path.posix.join(info.requirePath, "webpack.config"))({}); let config = require(path.posix.join(info.requirePath, "webpack.config"))({});
config.output.publicPath = `/plugins/${name}/`; devPlugins.set(name, webpackConfigs.length);
webpackConfigs.push(config); webpackConfigs.push(config);
} }
this.app.locals.devPlugins = devPlugins;
} }
/* eslint-enable node/global-require, node/no-unpublished-require */ /* eslint-enable node/global-require, node/no-unpublished-require */
const compiler = webpack(webpackConfigs); const compiler = webpack(webpackConfigs);
this.devMiddleware = webpackDevMiddleware(compiler, {}); this.devMiddleware = webpackDevMiddleware(compiler, { serverSideRender: true });
this.app.use(this.devMiddleware); this.app.use(this.devMiddleware);
} }
@@ -190,6 +192,14 @@ class Master {
Master.addAppRoutes(this.app, this.pluginInfos); Master.addAppRoutes(this.app, this.pluginInfos);
if (!args.dev) {
let manifest = await Master.loadJsonObject(path.join(__dirname, "..", "dist", "web", "manifest.json"));
if (!manifest["main.js"]) {
logger.error("Missing main.js entry in dist/web/manifest.json");
}
this.app.locals.mainBundle = manifest["main.js"] || "no_web_build";
}
// Load plugins // Load plugins
await this.loadPlugins(); await this.loadPlugins();
@@ -349,6 +359,18 @@ class Master {
await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4)); await libFileOps.safeOutputFile(filePath, JSON.stringify(serialized, null, 4));
} }
static async loadJsonObject(filePath, throwOnMissing = false) {
let manifest = {};
try {
manifest = JSON.parse(await fs.readFile(filePath));
} catch (err) {
if (!throwOnMissing && err.code !== "ENOENT") {
throw err;
}
}
return manifest;
}
/** /**
* Query master log * Query master log
* *
@@ -384,8 +406,9 @@ class Master {
app.use(compression()); app.use(compression());
// Set folder to serve static content from (the website) // Set folder to serve static content from (the website)
app.use(express.static(path.join(__dirname, "..", "dist", "web"))); const staticOptions = { immutable: true, maxAge: 1000 * 86400 * 365 };
app.use(express.static("static")); // Used for data export files app.use("/static", express.static(path.join(__dirname, "..", "dist", "web", "static"), staticOptions));
app.use("/static", express.static("static", staticOptions)); // Used for data export files
// Add API routes // Add API routes
routes.addRouteHandlers(app); routes.addRouteHandlers(app);
@@ -400,8 +423,8 @@ class Master {
} }
let pluginPackagePath = require.resolve(path.posix.join(pluginInfo.requirePath, "package.json")); let pluginPackagePath = require.resolve(path.posix.join(pluginInfo.requirePath, "package.json"));
let webPath = path.join(path.dirname(pluginPackagePath), "dist", "web"); let webPath = path.join(path.dirname(pluginPackagePath), "dist", "web", "static");
app.use(`/plugins/${pluginInfo.name}`, express.static(webPath)); app.use("/static", express.static(webPath, staticOptions));
} }
} }
@@ -511,6 +534,13 @@ class Master {
async loadPlugins() { async loadPlugins() {
for (let pluginInfo of this.pluginInfos) { for (let pluginInfo of this.pluginInfos) {
try {
let manifestPath = path.posix.join(pluginInfo.requirePath, "dist", "web", "manifest.json");
pluginInfo.manifest = await Master.loadJsonObject(require.resolve(manifestPath), true);
} catch (err) {
logger.warn(`Unable to load dist/web/manifest.json for plugin ${pluginInfo.name}`);
}
if (!this.config.group(pluginInfo.name).get("load_plugin")) { if (!this.config.group(pluginInfo.name).get("load_plugin")) {
continue; continue;
} }
@@ -600,9 +630,21 @@ class Master {
return function(req, res, next) { return function(req, res, next) {
let depth = routeDepth + (req.path.slice(-1) === "/"); let depth = routeDepth + (req.path.slice(-1) === "/");
let webRoot = "../".repeat(depth) || "./"; let webRoot = "../".repeat(depth) || "./";
let staticRoot = webRoot;
let mainBundle;
if (res.app.locals.mainBundle) {
mainBundle = res.app.locals.mainBundle;
} else {
let stats = res.locals.webpack.devMiddleware.stats.stats[0];
mainBundle = stats.toJson().assetsByChunkName["main"];
}
fs.readFile(path.join(__dirname, "..", "web", "index.html"), "utf8").then((content) => { fs.readFile(path.join(__dirname, "..", "web", "index.html"), "utf8").then((content) => {
res.type("text/html"); res.type("text/html");
res.send(content.replace(/__WEB_ROOT__/g, webRoot)); res.send(content
.replace(/__WEB_ROOT__/g, webRoot)
.replace(/__STATIC_ROOT__/g, staticRoot)
.replace(/__MAIN_BUNDLE__/g, mainBundle)
);
}).catch(err => { }).catch(err => {
next(err); next(err);
}); });
+17 -1
View File
@@ -100,7 +100,23 @@ function getPlugins(req, res) {
let name = pluginInfo.name; let name = pluginInfo.name;
let loaded = req.app.locals.master.plugins.has(name); let loaded = req.app.locals.master.plugins.has(name);
let enabled = loaded && req.app.locals.master.config.group(pluginInfo.name).get("load_plugin"); let enabled = loaded && req.app.locals.master.config.group(pluginInfo.name).get("load_plugin");
plugins.push({ name, version: pluginInfo.version, enabled, loaded }); let web = {};
let devPlugins = req.app.locals.devPlugins;
if (devPlugins && devPlugins.has(name)) {
let stats = res.locals.webpack.devMiddleware.stats.stats[devPlugins.get(name)];
web.main = stats.toJson().assetsByChunkName[name];
} else if (pluginInfo.manifest) {
web.main = pluginInfo.manifest[`${pluginInfo.name}.js`];
if (!web.main) {
web.error = `Missing ${pluginInfo.name}.js entry in manifest.json`;
}
} else {
web.error = "Missing dist/web/manifest.json";
}
if (web.main === "remoteEntry.js") {
web.error = "Incompatible old remoteEntry.js entrypoint.";
}
plugins.push({ name, version: pluginInfo.version, enabled, loaded, web });
} }
res.send(plugins); res.send(plugins);
} }
+2 -1
View File
@@ -7,11 +7,12 @@
<title>Clusterio</title> <title>Clusterio</title>
<script> <script>
window.webRoot = new URL("__WEB_ROOT__", document.location).pathname; window.webRoot = new URL("__WEB_ROOT__", document.location).pathname;
window.staticRoot = new URL("__STATIC_ROOT__", document.location).pathname;
</script> </script>
</head> </head>
<body> <body>
<noscript>You need to enable JavaScript to run this app.</noscript> <noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div> <div id="root"></div>
<script src="__WEB_ROOT__bundle.js"></script> <script src="__STATIC_ROOT____MAIN_BUNDLE__"></script>
</body> </body>
</html> </html>
-2
View File
@@ -12,8 +12,6 @@ module.exports = (env = {}) => merge(common(env), {
contentBase: "./dist/web", contentBase: "./dist/web",
}, },
output: { output: {
publicPath: "auto",
filename: "bundle.js",
path: path.resolve(__dirname, "dist", "web"), path: path.resolve(__dirname, "dist", "web"),
}, },
plugins: [ plugins: [
+2 -1
View File
@@ -39,7 +39,8 @@
"style-loader": "^2.0.0", "style-loader": "^2.0.0",
"util": "^0.12.3", "util": "^0.12.3",
"webpack": "^5.73.0", "webpack": "^5.73.0",
"webpack-cli": "^4.9.2" "webpack-cli": "^4.9.2",
"webpack-manifest-plugin": "^5.0.0"
}, },
"publishConfig": { "publishConfig": {
"access": "public" "access": "public"
+8 -1
View File
@@ -39,8 +39,12 @@ async function loadPlugins() {
let plugins = new Map(); let plugins = new Map();
await __webpack_init_sharing__("default"); await __webpack_init_sharing__("default");
for (let meta of pluginList) { for (let meta of pluginList) {
if (meta.web.error) {
logger.error(`Failed to load plugin ${meta.name}: ${meta.web.error}`);
continue;
}
try { try {
await loadScript(`${webRoot}plugins/${meta.name}/remoteEntry.js`); await loadScript(`${webRoot}${meta.web.main}`);
let container = window[`plugin_${meta.name}`]; let container = window[`plugin_${meta.name}`];
if (!container) { if (!container) {
throw new Error(`Plugin did not expose its container via plugin_${meta.name}`); throw new Error(`Plugin did not expose its container via plugin_${meta.name}`);
@@ -63,6 +67,9 @@ async function loadPlugins() {
plugins.set(pluginInfo.name, plugin); plugins.set(pluginInfo.name, plugin);
} catch (err) { } catch (err) {
if (err.message) {
meta.web.error = err.message;
}
logger.error(`Failed to load plugin ${meta.name}`); logger.error(`Failed to load plugin ${meta.name}`);
if (err.stack) { if (err.stack) {
logger.error(err.stack); logger.error(err.stack);
@@ -51,7 +51,7 @@ export default function PluginViewPage() {
style={{ style={{
marginTop: "1em", marginTop: "1em",
}} }}
message="Error loading web module" message={pluginMeta.web.error || "Error loading web module"}
description={ description={
"The web interface was unable to load the webpack module for this plugin. This is "+ "The web interface was unable to load the webpack module for this plugin. This is "+
"usually due to an incorrect or missing webpack build for the plugin. Due to the "+ "usually due to an incorrect or missing webpack build for the plugin. Due to the "+
+11 -1
View File
@@ -1,6 +1,7 @@
"use strict"; "use strict";
const webpack = require("webpack"); const webpack = require("webpack");
const { CleanWebpackPlugin } = require("clean-webpack-plugin"); const { CleanWebpackPlugin } = require("clean-webpack-plugin");
const { WebpackManifestPlugin } = require("webpack-manifest-plugin");
module.exports = (env = {}) => ({ module.exports = (env = {}) => ({
mode: env.production ? "production" : "development", mode: env.production ? "production" : "development",
@@ -9,8 +10,14 @@ module.exports = (env = {}) => ({
maxAssetSize: 2**21, maxAssetSize: 2**21,
maxEntrypointSize: 2**21, maxEntrypointSize: 2**21,
}, },
output: {
publicPath: "auto",
assetModuleFilename: "static/[hash][ext][query]",
filename: "static/[name].[contenthash].js",
},
plugins: [ plugins: [
new CleanWebpackPlugin(), new CleanWebpackPlugin(),
new WebpackManifestPlugin({ publicPath: "" }),
// required for winston // required for winston
new webpack.ProvidePlugin({ new webpack.ProvidePlugin({
@@ -81,7 +88,7 @@ module.exports = (env = {}) => ({
}, },
{ {
test: /\.png$/, test: /\.png$/,
use: require.resolve("file-loader"), type: "asset/resource",
}, },
], ],
}, },
@@ -101,4 +108,7 @@ module.exports = (env = {}) => ({
"zlib": require.resolve("browserify-zlib"), "zlib": require.resolve("browserify-zlib"),
}, },
}, },
optimization: {
moduleIds: "deterministic",
},
}); });
-3
View File
@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
context: __dirname, context: __dirname,
entry: "./web/index.jsx", entry: "./web/index.jsx",
output: { output: {
publicPath: "auto",
filename: "bundle.js",
path: path.resolve(__dirname, "dist", "web"), path: path.resolve(__dirname, "dist", "web"),
}, },
plugins: [ plugins: [
new webpack.container.ModuleFederationPlugin({ new webpack.container.ModuleFederationPlugin({
name: "global_chat", name: "global_chat",
library: { type: "var", name: "plugin_global_chat" }, library: { type: "var", name: "plugin_global_chat" },
filename: "remoteEntry.js",
exposes: { exposes: {
"./info": "./info.js", "./info": "./info.js",
"./package.json": "./package.json", "./package.json": "./package.json",
-3
View File
@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
context: __dirname, context: __dirname,
entry: "./web/index.jsx", entry: "./web/index.jsx",
output: { output: {
publicPath: "auto",
filename: "bundle.js",
path: path.resolve(__dirname, "dist", "web"), path: path.resolve(__dirname, "dist", "web"),
}, },
plugins: [ plugins: [
new webpack.container.ModuleFederationPlugin({ new webpack.container.ModuleFederationPlugin({
name: "inventory_sync", name: "inventory_sync",
library: { type: "var", name: "plugin_inventory_sync" }, library: { type: "var", name: "plugin_inventory_sync" },
filename: "remoteEntry.js",
exposes: { exposes: {
"./info": "./info.js", "./info": "./info.js",
"./package.json": "./package.json", "./package.json": "./package.json",
-3
View File
@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
context: __dirname, context: __dirname,
entry: "./web/index.jsx", entry: "./web/index.jsx",
output: { output: {
publicPath: "auto",
filename: "bundle.js",
path: path.resolve(__dirname, "dist", "web"), path: path.resolve(__dirname, "dist", "web"),
}, },
plugins: [ plugins: [
new webpack.container.ModuleFederationPlugin({ new webpack.container.ModuleFederationPlugin({
name: "player_auth", name: "player_auth",
library: { type: "var", name: "plugin_player_auth" }, library: { type: "var", name: "plugin_player_auth" },
filename: "remoteEntry.js",
exposes: { exposes: {
"./info": "./info.js", "./info": "./info.js",
"./package.json": "./package.json", "./package.json": "./package.json",
-3
View File
@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
context: __dirname, context: __dirname,
entry: "./web/index.jsx", entry: "./web/index.jsx",
output: { output: {
publicPath: "auto",
filename: "bundle.js",
path: path.resolve(__dirname, "dist", "web"), path: path.resolve(__dirname, "dist", "web"),
}, },
plugins: [ plugins: [
new webpack.container.ModuleFederationPlugin({ new webpack.container.ModuleFederationPlugin({
name: "research_sync", name: "research_sync",
library: { type: "var", name: "plugin_research_sync" }, library: { type: "var", name: "plugin_research_sync" },
filename: "remoteEntry.js",
exposes: { exposes: {
"./info": "./info.js", "./info": "./info.js",
"./package.json": "./package.json", "./package.json": "./package.json",
@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
context: __dirname, context: __dirname,
entry: "./web/index.jsx", entry: "./web/index.jsx",
output: { output: {
publicPath: "auto",
filename: "bundle.js",
path: path.resolve(__dirname, "dist", "web"), path: path.resolve(__dirname, "dist", "web"),
}, },
plugins: [ plugins: [
new webpack.container.ModuleFederationPlugin({ new webpack.container.ModuleFederationPlugin({
name: "statistics_exporter", name: "statistics_exporter",
library: { type: "var", name: "plugin_statistics_exporter" }, library: { type: "var", name: "plugin_statistics_exporter" },
filename: "remoteEntry.js",
exposes: { exposes: {
"./info": "./info.js", "./info": "./info.js",
"./package.json": "./package.json", "./package.json": "./package.json",
@@ -9,15 +9,12 @@ module.exports = (env = {}) => merge(common(env), {
context: __dirname, context: __dirname,
entry: "./web/index.jsx", entry: "./web/index.jsx",
output: { output: {
publicPath: "auto",
filename: "bundle.js",
path: path.resolve(__dirname, "dist", "web"), path: path.resolve(__dirname, "dist", "web"),
}, },
plugins: [ plugins: [
new webpack.container.ModuleFederationPlugin({ new webpack.container.ModuleFederationPlugin({
name: "subspace_storage", name: "subspace_storage",
library: { type: "var", name: "plugin_subspace_storage" }, library: { type: "var", name: "plugin_subspace_storage" },
filename: "remoteEntry.js",
exposes: { exposes: {
"./info": "./info.js", "./info": "./info.js",
"./package.json": "./package.json", "./package.json": "./package.json",