Fix running Clusterio from published packages (#1070)

* Fix running Clusterio from published packages

Since the move to ES modules and pnpm 12, packages installed from npm
fail to start:

- The package imports of @clusterio/lib point Node at TypeScript
  sources, which Node refuses to strip types from inside node_modules.
- The controller, host and ctl import their package.json from next to
  the compiled entrypoint, a copy pnpm leaves out of the package.
- ctl, host and web_ui have no .npmignore, so pnpm packs them using the
  repository .gitignore and leaves out nearly all of dist/.

Resolve the package imports to the compiled files at runtime while
keeping the sources for type checking, import package.json by package
name so it resolves to the real file, and add .npmignore files so pnpm
packs the same files as npm.

* Load plugins installed from npm by package name

Plugins installed from npm are listed by package name, but the plugin
loader joined that name onto the main and entrypoint paths and imported
the result as a file, which resolves relative to the working directory.
Every npm installed plugin then failed to load. Resolve these paths from
the directory of the plugin's package.json instead.

packages/controller/controller.ts

@@ -32,7 +32,7 @@ import { ConsoleTransport, levels, logger } from "@clusterio/lib";
 import Controller from "./src/Controller.js";
 import UserManager from "./src/UserManager.js";
 import UserRecord from "./src/UserRecord.js";
-import packageConfig from "./package.json" with { type: "json" };
+import packageConfig from "@clusterio/controller/package.json" with { type: "json" };
 
 // globals
 let controller: Controller;

packages/controller/src/WsServer.ts

@@ -12,7 +12,7 @@ import ControlConnection from "./ControlConnection.js";
 import HostConnection from "./HostConnection.js";
 import WsServerConnector from "./WsServerConnector.js";
 
-import packageConfig from "../package.json" with { type: "json" };
+import packageConfig from "@clusterio/controller/package.json" with { type: "json" };
 
 const wsMessageCounter = new lib.Counter(
 	"clusterio_controller_websocket_message_total",

packages/ctl/.npmignore

@@ -0,0 +1 @@
+# Without an .npmignore pnpm packs using the repository .gitignore, which leaves out dist/

packages/ctl/ctl.ts

@@ -7,7 +7,7 @@
 import fs from "node:fs/promises";
 import yargs, { type Argv } from "yargs";
 import path from "path";
-import packageConfig from "./package.json" with { type: "json" };
+import packageConfig from "@clusterio/ctl/package.json" with { type: "json" };
 import { strict as assert } from "assert";
 
 // Reduce startup time by lazy compiling schemas.

packages/host/.npmignore

@@ -0,0 +1 @@
+# Without an .npmignore pnpm packs using the repository .gitignore, which leaves out dist/

packages/host/host.ts

@@ -15,7 +15,7 @@
 import fs from "node:fs/promises";
 import path from "path";
 import yargs from "yargs";
-import packageConfig from "./package.json" with { type: "json" };
+import packageConfig from "@clusterio/host/package.json" with { type: "json" };
 import winston from "winston";
 import "winston-daily-rotate-file";
 

packages/lib/package.json

@@ -25,11 +25,17 @@
 	"type": "module",
 	"imports": {
 		"#WebSocket": {
-			"node": "./src/link/WebSocket.node.ts",
+			"node": {
+				"types": "./src/link/WebSocket.node.ts",
+				"default": "./dist/node/src/link/WebSocket.node.js"
+			},
 			"import": "./src/link/WebSocket.browser.ts"
 		},
 		"#is_deep_strict_equal": {
-			"node": "./src/is_deep_strict_equal.node.ts",
+			"node": {
+				"types": "./src/is_deep_strict_equal.node.ts",
+				"default": "./dist/node/src/is_deep_strict_equal.node.js"
+			},
 			"import": "./src/is_deep_strict_equal.browser.ts"
 		}
 	},

packages/lib/src/plugin_loader.ts

@@ -51,8 +51,9 @@ export async function loadPluginInfos(pluginList: Map<string, string>) {
 
 		try {
 			pluginPackage = (await import(pathToFileURL(packagePath).href, { with: { type: "json" }})).default;
+			// pluginPath is a package name for plugins installed from npm, so resolve from the package itself
 			pluginInfo = (await import(
-				pathToFileURL(path.posix.join(pluginPath, pluginPackage.main ?? "index.js")).href
+				pathToFileURL(path.join(path.dirname(packagePath), pluginPackage.main ?? "index.js")).href
 			)).plugin;
 		} catch (err: any) {
 			if (err.code === "InstallationError") {
@@ -106,7 +107,7 @@ export async function loadPlugin<
 		return;
 	}
 
-	const module = await import(pathToFileURL(path.posix.join(pluginInfo.requirePath, requirePath)).href);
+	const module = await import(pathToFileURL(path.join(path.dirname(pluginInfo.packagePath), requirePath)).href);
 	const pluginContext: libPlugin.PluginLoadContext<Context> = {
 		...context,
 		plugin: pluginInfo,

packages/web_ui/.npmignore

@@ -0,0 +1 @@
+# Without an .npmignore pnpm packs using the repository .gitignore, which leaves out dist/

test/lib/plugin_loader.js

@@ -66,6 +66,43 @@ describe("lib/plugin_loader", function() {
 				{ message: `PluginError: ${brokenMessage}` }
 			);
 		});
+		describe("plugin installed from npm", function() {
+			// Plugins from npm are listed by package name, so they must be resolved like a dependency
+			const packageDir = path.resolve("node_modules", "@clusterio-test", "npm_plugin");
+			before(async function() {
+				await fs.mkdir(path.join(packageDir, "dist"), { recursive: true });
+				await fs.writeFile(path.join(packageDir, "package.json"), JSON.stringify({
+					name: "@clusterio-test/npm_plugin", version: "0.0.2", main: "dist/index.js", type: "module",
+				}));
+				await fs.writeFile(
+					path.join(packageDir, "dist", "index.js"),
+					'export const plugin = { name: "npm_plugin", controllerEntrypoint: "dist/controller.js" };',
+				);
+				await fs.writeFile(
+					path.join(packageDir, "dist", "controller.js"),
+					"export default async function(context) { context.controller.loadedNpmPlugin = true; }",
+				);
+			});
+			after(async function() {
+				await fs.rm(path.dirname(packageDir), { recursive: true, force: true });
+			});
+
+			it("should load the plugin and its entrypoints", async function() {
+				const [pluginInfo] = await lib.loadPluginInfos(
+					new Map([["npm_plugin", "@clusterio-test/npm_plugin"]])
+				);
+				assert.equal(pluginInfo.name, "npm_plugin");
+				assert.equal(pluginInfo.npmPackage, "@clusterio-test/npm_plugin");
+				assert.equal(pluginInfo.packagePath, path.join(packageDir, "package.json"));
+
+				const controller = new mock.MockController();
+				await lib.loadPlugin(
+					pluginInfo, "controller", { controller, metrics: {}, logger: new mock.MockLogger() },
+					"ControllerPlugin", BaseControllerPlugin,
+				);
+				assert.equal(controller.loadedNpmPlugin, true);
+			});
+		});
 		it("should reject on invalid plugin", async function() {
 			await assert.rejects(
 				lib.loadPluginInfos(new Map([["invalid", path.resolve(invalidPlugin)]])),
@@ -122,7 +159,12 @@ describe("lib/plugin_loader", function() {
 		});
 
 		function info(requirePath, entrypoint = "controller.js") {
-			return { name: "test", requirePath, controllerEntrypoint: entrypoint };
+			return {
+				name: "test",
+				requirePath,
+				packagePath: path.join(requirePath, "package.json"),
+				controllerEntrypoint: entrypoint,
+			};
 		}
 
 		it("should do nothing when the entrypoint is not set", async function() {