Merge pull request #983 from Hornwitser/configurable-static-url

Configurable static assets URL

docs/configuration.md

@@ -60,6 +60,17 @@ This should be a full URL ending with a /.
 Defaults to null meaning assume localhost.
 
 
+### controller.static_url
+
+Used to override where static assets such as the web bundle is loaded from.
+This can be used if you want to put the static assets on a separate cdn url like `https://cdn.example.com/static/` while keeping the API hosted directly by clusterio.
+Due to a limitation on how the webpack assets are built the pathname component of the url have to be `/static/`.
+
+See also the `clusteriocontroller copy-static` command for how to obtain the static asset files that need to be hosted.
+
+Defaults to null meaning use the internally hosted assets.
+
+
 ### controller.tls_certificate
 
 Path to TLS certificate to use for the HTTPS server when controller.https_port is configured.

packages/controller/README.md

@@ -104,6 +104,16 @@ This can be used both in the web interface and as the `controller_token` to conn
 Creates a clusterioctl config for the given user with url and token set up for connecting to the cluster.
 
 
+### `copy-static [target]`
+
+Copies the static web assets to the given target folder (which defaults to `./static/`).
+Useful in case you want to host the assets on a CDN or separate web server and want provide the files directly instead of fetching them from the controller.
+
+Note that mod pack exports will create additional static files that also have to be hosted, these are written to the `./static/` folder in the clusterio installation directory.
+
+See also the `controller.public_url` config option for setting where static assets are loaded from.
+
+
 ### `run`
 
 Runs the controller.

packages/controller/controller.ts

@@ -211,6 +211,40 @@ async function handleBootstrapCommand(
 	}
 }
 
+async function handleCopyStaticCommand(
+	args: any,
+	pluginInfos: lib.PluginNodeEnvInfo[],
+) {
+	await fs.cp(
+		path.join(__dirname, "..", "web", "static"),
+		args.target,
+		{
+			recursive: true,
+			mode: fs.constants.COPYFILE_FICLONE,
+		},
+	);
+	logger.info("Copied main web interface files");
+	for (const pluginInfo of pluginInfos) {
+		try {
+			await fs.cp(
+				pluginInfo.webStaticPath,
+				args.target,
+				{
+					recursive: true,
+					mode: fs.constants.COPYFILE_FICLONE,
+				},
+			);
+		} catch (err: any) {
+			if (err.code === "ENOENT") {
+				logger.info(`Skipped web interface files for ${pluginInfo.name}: ${err.message}`);
+			} else {
+				logger.error(`Unexpected error copying web interface files for ${pluginInfo.name}:\n${err.stack}`);
+				process.exitCode = 1;
+			}
+		}
+		logger.info(`Copied web interface files for ${pluginInfo.name}`);
+	}
+}
 
 export interface ControllerArgs {
 	[x: string]: unknown;
@@ -284,6 +318,13 @@ async function initialize(): Promise<InitializeParameters> {
 				})
 				.demandCommand(1, "You need to specify a command to run");
 		})
+		.command("copy-static [target]", "Copy static Web UI files to a folder", yargs => {
+			yargs.positional("target", {
+				describe: "Target directory to copy files to",
+				type: "string",
+				default: "static",
+			});
+		})
 		.command("run", "Run controller", yargs => {
 			yargs.option("can-restart", {
 				type: "boolean", nargs: 0, default: false,
@@ -399,6 +440,8 @@ async function initialize(): Promise<InitializeParameters> {
 		await lib.handleConfigCommand(args, controllerConfig, controllerConfigLock);
 	} else if (command === "bootstrap") {
 		await handleBootstrapCommand(args, controllerConfig, controllerConfigLock);
+	} else if (command === "copy-static") {
+		await handleCopyStaticCommand(args, pluginInfos);
 	} else if (shouldRun) {
 		await controllerConfigLock.acquire(); // Hold the lock until process exit
 	}

packages/controller/src/Controller.ts

@@ -355,6 +355,8 @@ export default class Controller {
 				this.onSystemMetricsIntervalChanged();
 			} else if (field === "controller.trusted_proxies") {
 				this.trustedProxies = this.parseTrustedProxies();
+			} else if (field === "controller.static_url") {
+				this.app.locals.staticRoot = this.config.get("controller.static_url");
 			}
 			lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", field, curr, prev);
 		});
@@ -389,6 +391,7 @@ export default class Controller {
 			}
 			this.app.locals.mainBundle = manifest["main.js"] || "no_web_build";
 		}
+		this.app.locals.staticRoot = this.config.get("controller.static_url");
 
 		// Load plugins
 		await this.loadPlugins();
@@ -866,7 +869,7 @@ export default class Controller {
 		);
 	}
 
-	static addAppRoutes(app: Application, pluginInfos: any[]) {
+	static addAppRoutes(app: Application, pluginInfos: lib.PluginNodeEnvInfo[]) {
 		app.use((req: Request, res: Response, next) => {
 			const startNs = process.hrtime.bigint();
 			stream.finished(res, () => {
@@ -915,9 +918,7 @@ export default class Controller {
 				app.get(route, Controller.serveWeb(route));
 			}
 
-			let pluginPackagePath = require.resolve(path.posix.join(pluginInfo.requirePath, "package.json"));
-			let webPath = path.join(path.dirname(pluginPackagePath), "dist", "web", "static");
-			app.use("/static", express.static(webPath, staticOptions));
+			app.use("/static", express.static(pluginInfo.webStaticPath, staticOptions));
 		}
 	}
 
@@ -1231,7 +1232,7 @@ export default class Controller {
 		return function(req: Request, res: Response, next: NextFunction) {
 			let depth = routeDepth + Number(req.path.slice(-1) === "/");
 			let webRoot = "../".repeat(depth) || "./";
-			let staticRoot = `${webRoot}static/`;
+			let staticRoot = res.app.locals.staticRoot ?? `${webRoot}static/`;
 			let mainBundle: string = "";
 			if (res.app.locals.mainBundle) {
 				mainBundle = res.app.locals.mainBundle;

packages/controller/src/routes.ts

@@ -22,6 +22,7 @@ declare global {
 		export interface Locals {
 			controller: Controller,
 			mainBundle: string,
+			staticRoot: string | null,
 			devPlugins: Map<string, number>,
 			streams: Map<string, ProxyStream>
 		}

packages/controller/web/index.html

@@ -29,7 +29,7 @@
 		<script id="script-bundle" src="__STATIC_ROOT____MAIN_BUNDLE__" defer></script>
 		<script>
 			webRoot = new URL("__WEB_ROOT__", document.location).pathname;
-			staticRoot = new URL("__STATIC_ROOT__", document.location).pathname;
+			staticRoot = new URL("__STATIC_ROOT__", document.location).href;
 			function scriptError(err) {
 				console.error(err);
 				let bootstrapMsg = document.getElementById("bootstrap-msg");

packages/lib/src/config/definitions.ts

@@ -18,6 +18,7 @@ export interface ControllerConfigFields {
 	"controller.bind_address": string | null;
 	"controller.trusted_proxies": string | null;
 	"controller.public_url": string | null;
+	"controller.static_url": string | null;
 	"controller.grafana_url": string | null;
 	"controller.tls_certificate": string | null;
 	"controller.tls_private_key": string | null;
@@ -138,6 +139,21 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
 			optional: true,
 			// TODO extendedValidation, valid url including the protocol
 		},
+		"controller.static_url": {
+			title: "Static URL",
+			description: "Override the URL to where static assets are hosted (must have a path consisting of /static/)",
+			type: "string",
+			optional: true,
+			validator: (value) => {
+				if (value) {
+					// This is unfortunately a limitiation of how the webpack build is set up right now.
+					const pathname = new URL(value, "http://localhost/").pathname;
+					if (pathname !== "/static/") {
+						throw new Error(`Path component of static_url must be /static/ not ${pathname}.`);
+					}
+				}
+			},
+		},
 		"controller.grafana_url": {
 			title: "Grafana URL",
 			description: "URL of a Grafana dashboard to link from the overview page, including the protocol.",

packages/lib/src/plugin.ts

@@ -41,6 +41,11 @@ export type PluginDeclaration = {
 }
 
 export type PluginNodeEnvInfo = PluginDeclaration & {
+	/**
+	 * Path to the folder with the static files that should be hosted on the web
+	 * server in order for the web interface to be able to load the plugin.
+	 */
+	webStaticPath: string;
 	requirePath: string;
 	version: string;
 	manifest: any;

packages/lib/src/plugin_loader.ts

@@ -69,6 +69,12 @@ export async function loadPluginInfos(pluginList: Map<string, string>) {
 			continue;
 		}
 
+		pluginInfo.webStaticPath = path.join(
+			path.dirname(
+				require.resolve(path.posix.join(pluginPath, "package.json"))
+			),
+			"dist", "web", "static",
+		);
 		pluginInfo.requirePath = pluginPath;
 		pluginInfo.version = pluginPackage.version;
 		pluginInfo.npmPackage = !pluginPackage.private && pluginPath === pluginPackage.name ? pluginPath : undefined;

test/integration/clusterio.js

@@ -140,6 +140,25 @@ describe("Integration of Clusterio", function() {
 			});
 		});
 
+		describe("copy-static", function() {
+			it("should copy the files from dist/web/static", async function() {
+				const dir = "temp/test/copy-static";
+				await fs.rm(dir, { force: true, recursive: true, maxRetries: 10 });
+				await execController("copy-static copy-static");
+				const controllerFiles = await fs.readdir("packages/controller/dist/web/static");
+				const globalChatFiles = await fs.readdir("plugins/global_chat/dist/web/static");
+				const targetFiles = await fs.readdir(dir);
+				assert(
+					controllerFiles.every(name => targetFiles.includes(name)),
+					"Missing files from controller static folder",
+				);
+				assert(
+					globalChatFiles.every(name => targetFiles.includes(name)),
+					"Missing files from global_chat's static folder",
+				);
+			});
+		});
+
 		describe("config", function() {
 			it("can read the config file", async function() {
 				await execController("config list");

test/lib/config/validators.test.js

@@ -184,7 +184,35 @@ describe("lib/config/validators", function() {
 
 describe("lib/config/definitions/validators", function() {
 	describe("Controller Config", function() {
+		it("should validate controller.static_url", function() {
+			const config = new ControllerConfig("controller");
 
+			// valid cases
+			config.set("controller.static_url", "");
+			config.set("controller.static_url", null);
+			config.set("controller.static_url", "static/");
+			config.set("controller.static_url", "./static/");
+			config.set("controller.static_url", "/static/");
+			config.set("controller.static_url", "http://example/static/");
+
+			// invalid cases
+			assert.throws(
+				() => config.set("controller.static_url", "/foo/"),
+				"Path component of static_url must be \/static\/ not \/foo\/."
+			);
+			assert.throws(
+				() => config.set("controller.static_url", "http://example/foo/static/"),
+				"Path component of static_url must be \/static\/ not \/foo\/static\/."
+			);
+			assert.throws(
+				() => config.set("controller.static_url", "http://example/static"),
+				"Path component of static_url must be \/static\/ not \/static."
+			);
+			assert.throws(
+				() => config.set("controller.static_url", "http://:80"),
+				/Invalid URL/,
+			);
+		});
 	});
 
 	describe("Host Config", function() {

test/lib/plugin_loader.js

@@ -43,7 +43,13 @@ describe("lib/plugin_loader", function() {
 		it("should load test plugin", async function() {
 			assert.deepEqual(
 				await lib.loadPluginInfos(new Map([["test", path.resolve(testPlugin)]])),
-				[{ name: "test", version: "0.0.1", npmPackage: undefined, requirePath: path.resolve(testPlugin) }]
+				[{
+					name: "test",
+					version: "0.0.1",
+					npmPackage: undefined,
+					requirePath: path.resolve(testPlugin),
+					webStaticPath: path.resolve(path.join(testPlugin, "dist", "web", "static")),
+				}]
 			);
 		});
 		it("should reject on broken plugin", async function() {