Merge pull request #692 from clusterio/plugin-management

Add plugins automatically

packages/controller/controller.ts

@@ -317,14 +317,7 @@ async function initialize(): Promise<InitializeParameters> {
 	}
 
 	logger.info(`Loading available plugins from ${args.pluginList}`);
-	let pluginList = new Map();
-	try {
-		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList, { encoding: "utf8" })));
-	} catch (err: any) {
-		if (err.code !== "ENOENT") {
-			throw err;
-		}
-	}
+	let pluginList = await lib.loadPluginList(args.pluginList);
 
 	// If the command is plugin management we don't try to load plugins
 	if (command === "plugin") {

packages/controller/package.json

@@ -13,6 +13,7 @@
 		"clusteriocontroller": "./dist/node/controller.js"
 	},
 	"keywords": [
+		"clusterio",
 		"factorio"
 	],
 	"engines": {

packages/create/package.json

@@ -9,8 +9,8 @@
 		"clusteriocreate": "./create.js"
 	},
 	"keywords": [
-		"factorio",
-		"clusterio"
+		"clusterio",
+		"factorio"
 	],
 	"engines": {
 		"node": ">=18"

packages/create/templates/common/package.json

@@ -48,6 +48,7 @@
 	},
 	"keywords": [
 		"clusterio",
+		"clusterio-plugin",
 		"factorio"
 	]
 }

packages/ctl/ctl.ts

@@ -188,14 +188,7 @@ async function startControl() {
 	lib.handleUnhandledErrors();
 
 	logger.verbose(`Loading available plugins from ${args.pluginList}`);
-	let pluginList = new Map();
-	try {
-		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList, "utf8")));
-	} catch (err: any) {
-		if (err.code !== "ENOENT") {
-			throw err;
-		}
-	}
+	let pluginList = await lib.loadPluginList(args.pluginList);
 
 	// If the command is plugin management we don't try to load plugins
 	if (args._[0] === "plugin") {

packages/ctl/package.json

@@ -13,6 +13,7 @@
 		"clusterioctl": "./dist/node/ctl.js"
 	},
 	"keywords": [
+		"clusterio",
 		"factorio"
 	],
 	"engines": {

packages/host/host.ts

@@ -147,14 +147,7 @@ async function startHost() {
 	}
 
 	logger.info(`Loading available plugins from ${args.pluginList}`);
-	let pluginList = new Map();
-	try {
-		pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList, "utf8")));
-	} catch (err: any) {
-		if (err.code !== "ENOENT") {
-			throw err;
-		}
-	}
+	let pluginList = await lib.loadPluginList(args.pluginList);
 
 	// If the command is plugin management we don't try to load plugins
 	if (command === "plugin") {

packages/host/package.json

@@ -13,6 +13,7 @@
 		"clusteriohost": "./dist/node/host.js"
 	},
 	"keywords": [
+		"clusterio",
 		"factorio"
 	],
 	"engines": {

packages/lib/index.ts

@@ -29,6 +29,7 @@ export * from "./src/system_collectors";
 export * from "./src/zip_ops";
 export * from "./src/subscriptions";
 export * from "./src/datastore";
+export * from "./src/load_plugin_list";
 
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as ModStore } from "./src/ModStore";

packages/lib/package.json

@@ -11,6 +11,7 @@
 	"main": "dist/node/index.js",
 	"browser": "browser.ts",
 	"keywords": [
+		"clusterio",
 		"factorio"
 	],
 	"engines": {

packages/lib/src/load_plugin_list.ts

@@ -0,0 +1,186 @@
+import fs from "fs-extra";
+import path from "path";
+import { logger } from "./logging";
+import * as libFileOps from "./file_ops";
+import vm from "vm";
+
+/**
+ * Searches for local plugins in the plugins/ and external_plugins/ directories
+ * and adds them to the plugin list if they contain a package.json file.
+ *
+ * @param pluginList - Map of plugin names to their file paths
+ * @returns Promise that resolves to true if any plugins were added to the list, false otherwise
+ */
+async function findLocalPlugins(pluginList: Map<string, string>): Promise<boolean> {
+	// Check folders for plugins
+	const pluginFolders = [
+		"plugins",
+		"external_plugins",
+	];
+
+	let hasChanged = 0;
+	const ignoredFolders = new Set(["node_modules", ".git", "dist", "build", "disabled"]);
+	const maxDepth = 3;
+
+	for (const pluginFolder of pluginFolders) {
+		let pluginNames;
+		try {
+			pluginNames = await fs.readdir(pluginFolder);
+		} catch (err: any) {
+			if (err.code === "ENOENT") {
+				// Skip if folder doesn't exist
+				continue;
+			}
+			throw err;
+		}
+		for (const pluginName of pluginNames) {
+			const pluginPath = path.join(pluginFolder, pluginName);
+			if (pluginList.has(pluginName)) {
+				// If the one in the list is not the same as this one, warn
+				if (pluginList.get(pluginName) !== path.resolve(pluginPath)) {
+					logger.warn(
+						`${pluginName} is already in the plugin list, but with a different path - ` +
+						`using ${pluginList.get(pluginName)}`
+					);
+				}
+				continue;
+			}
+			const stats = await fs.stat(pluginPath);
+			if (stats.isDirectory()) {
+				// Check for package.json and plugin keyword
+				if (await checkPackageJson(pluginPath)) {
+					pluginList.set(pluginName, path.resolve(pluginPath));
+					logger.info(`Added ${pluginName} from ${pluginFolder}`);
+					hasChanged += 1;
+					continue;
+				}
+
+				if (pluginPath.split(path.sep).length < maxDepth && !ignoredFolders.has(pluginName)) {
+					logger.warn(`Possible monorepo detected, scanning ${pluginPath} for clusterio-plugins`);
+					pluginFolders.push(pluginPath);
+				}
+			}
+		}
+	}
+
+	return hasChanged > 0;
+}
+
+function getPluginName(requireSpec: string) {
+	const context = vm.createContext({ require: require, pluginInfo: null });
+	const code = `pluginInfo = require(${JSON.stringify(requireSpec)}).plugin;`;
+	vm.runInContext(code, context);
+	return context.pluginInfo.name;
+}
+
+/**
+ * Find NPM packages in the root package.json that satisfies the requirements for plugins.
+ * Adds discovered plugins to the plugin list.
+ *
+ * @param pluginList - Map of plugin names to their file paths
+ * @returns Promise that resolves to true if any plugins were added to the list, false otherwise
+ */
+async function findNpmPlugins(pluginList: Map<string, string>): Promise<boolean> {
+	let dependencies;
+	try {
+		const rootPackageJson = JSON.parse(await fs.readFile(path.resolve("package.json"), { encoding: "utf8" }));
+		dependencies = rootPackageJson.dependencies;
+	} catch (err: any) {
+		if (err.code === "ENOENT") {
+			// Skip if package.json doesn't exist
+			return false;
+		}
+		throw err;
+	}
+
+	if (!dependencies) {
+		return false;
+	}
+
+	let hasChanged = 0;
+	for (const [packageName, packageVersion] of Object.entries(dependencies)) {
+		if ([...pluginList.values()].includes(packageName)) {
+			continue; // This npm module is already in the plugin list
+		}
+		// Find the package in node_modules and read the package.json to check if it's a clusterio-plugin
+		if (await checkPackageJson(path.resolve("node_modules", packageName))) {
+			const pluginName = getPluginName(path.resolve("node_modules", packageName));
+			if (pluginList.has(pluginName)) {
+				logger.warn(
+					`${pluginName} provided by ${packageName}@${packageVersion} is already in the plugin list, ` +
+					`but with a different path - using ${pluginList.get(pluginName)}`
+				);
+			} else {
+				pluginList.set(pluginName, packageName);
+				logger.info(`Added ${pluginName} from NPM`);
+				hasChanged += 1;
+			}
+		} else if (![
+			"@clusterio/controller",
+			"@clusterio/ctl",
+			"@clusterio/host",
+			"@clusterio/lib",
+			"@clusterio/web_ui",
+		].includes(packageName)) {
+			logger.warn(`${packageName}@${packageVersion} is not a clusterio-plugin`);
+		}
+	}
+	return hasChanged > 0;
+}
+
+/**
+ * Loads the plugin list from the specified file and optionally discovers local plugins
+ *
+ * @param pluginListPath - Path to the plugin list JSON file
+ * @param findLocal - Whether to look for local plugins in plugins/ and external_plugins/ directories
+ * @returns Map of plugin name to plugin path
+ */
+export async function loadPluginList(pluginListPath: string, findLocal = true): Promise<Map<string, string>> {
+	let pluginList = new Map<string, string>();
+
+	// Try to load existing plugin list
+	try {
+		pluginList = new Map(JSON.parse(await fs.readFile(pluginListPath, { encoding: "utf8" })));
+		logger.info(`Loaded ${pluginList.size} plugins from ${pluginListPath}`);
+	} catch (err: any) {
+		if (err.code !== "ENOENT") {
+			throw err;
+		}
+		logger.info(`No existing plugin list found at ${pluginListPath}`);
+	}
+
+	// Track if any changes were made
+	let hasChanges = false;
+
+	// Optionally discover local plugins
+	if (findLocal) {
+		hasChanges = await findLocalPlugins(pluginList) || hasChanges;
+	}
+
+	// Discover NPM plugins
+	hasChanges = await findNpmPlugins(pluginList) || hasChanges;
+
+	if (hasChanges) {
+		await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, "\t"));
+	}
+
+	return pluginList;
+}
+
+/**
+ * Helper function to check if a package.json file exists and contains the clusterio-plugin keyword
+ * @param pluginPath - Path to the plugin directory
+ * @returns Promise that resolves to true if package.json exists and has clusterio-plugin keyword
+ */
+async function checkPackageJson(pluginPath: string): Promise<boolean> {
+	const packageJsonPath = path.join(pluginPath, "package.json");
+	if (!await fs.exists(packageJsonPath)) {
+		return false;
+	}
+
+	// Read and parse package.json
+	const packageJson = JSON.parse(await fs.readFile(packageJsonPath, { encoding: "utf8" }));
+
+	// Check if package has the clusterio-plugin keyword
+	return packageJson.keywords?.includes("clusterio-plugin");
+}

packages/lib/src/plugin_loader.ts

@@ -26,6 +26,15 @@ export async function loadPluginInfos(pluginList: Map<string, string>) {
 		let pluginInfo: libPlugin.PluginNodeEnvInfo;
 		let pluginPackage: { name?: string, version: string, main?: string, private?: boolean };
 
+		// Check if plugin path exists, otherwise remove it
+		try {
+			require.resolve(pluginPath);
+		} catch (err) {
+			logger.error(`Plugin path ${pluginPath} does not exist, not loading ${pluginName}`);
+			pluginList.delete(pluginName);
+			continue;
+		}
+
 		try {
 			pluginInfo = require(pluginPath).plugin;
 			pluginPackage = require(path.posix.join(pluginPath, "package.json"));

packages/web_ui/package.json

@@ -10,7 +10,8 @@
 		"test": "echo \"Error: no test specified\" && exit 1"
 	},
 	"keywords": [
-		"clusterio"
+		"clusterio",
+		"factorio"
 	],
 	"engines": {
 		"node": ">=12"

plugins/global_chat/package.json

@@ -7,6 +7,11 @@
 	"license": "MIT",
 	"repository": "clusterio/clusterio",
 	"main": "dist/node/index.js",
+	"keywords": [
+		"clusterio",
+		"clusterio-plugin",
+		"factorio"
+	],
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --env production",
 		"test": "echo \"Error: run tests from root\" && exit 1"

plugins/inventory_sync/package.json

@@ -7,6 +7,11 @@
 	"license": "MIT",
 	"repository": "clusterio/factorioClusterio",
 	"main": "dist/node/index.js",
+	"keywords": [
+		"clusterio",
+		"clusterio-plugin",
+		"factorio"
+	],
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --env production",
 		"test": "echo \"Error: run tests from root\" && exit 1"

plugins/player_auth/package.json

@@ -7,6 +7,11 @@
 	"license": "MIT",
 	"repository": "clusterio/clusterio",
 	"main": "dist/node/index.js",
+	"keywords": [
+		"clusterio",
+		"clusterio-plugin",
+		"factorio"
+	],
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --env production",
 		"test": "echo \"Error: run tests from root\" && exit 1"

plugins/research_sync/package.json

@@ -7,6 +7,11 @@
 	"license": "MIT",
 	"repository": "clusterio/clusterio",
 	"main": "dist/node/index.js",
+	"keywords": [
+		"clusterio",
+		"clusterio-plugin",
+		"factorio"
+	],
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --env production",
 		"test": "echo \"Error: run tests from root\" && exit 1"

plugins/statistics_exporter/package.json

@@ -7,6 +7,11 @@
 	"license": "MIT",
 	"repository": "clusterio/clusterio",
 	"main": "dist/node/index.js",
+	"keywords": [
+		"clusterio",
+		"clusterio-plugin",
+		"factorio"
+	],
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --env production",
 		"test": "echo \"Error: run tests from root\" && exit 1"

plugins/subspace_storage/package.json

@@ -7,6 +7,11 @@
 	"license": "MIT",
 	"repository": "clusterio/clusterio",
 	"main": "dist/node/index.js",
+	"keywords": [
+		"clusterio",
+		"clusterio-plugin",
+		"factorio"
+	],
 	"scripts": {
 		"prepare": "tsc --build && webpack-cli --env production",
 		"test": "echo \"Error: run tests from root\" && exit 1"

test/lib/plugin_loader.js

@@ -33,11 +33,9 @@ describe("lib/plugin_loader", function() {
 			await writePlugin(invalidPlugin, "wrong");
 		});
 
-		it("should throw on missing plugin", async function() {
-			await assert.rejects(
-				lib.loadPluginInfos(new Map([["missing", missingPlugin]]), []),
-				new RegExp(`^Error: PluginError: Cannot find module '${escapeRegExp(missingPlugin)}'`)
-			);
+		it("should ignore missing plugins", async function() {
+			const result = await lib.loadPluginInfos(new Map([["missing", missingPlugin]]), []);
+			assert.deepEqual(result, []);
 		});
 		it("should load test plugin", async function() {
 			assert.deepEqual(
@@ -112,4 +110,113 @@ describe("lib/plugin_loader", function() {
 			);
 		});
 	});
+	describe("loadPluginList()", async function() {
+		const old_cwd = process.cwd();
+		const baseDir = path.join("temp", "test", "plugin_list");
+		const pluginListPath = path.join(baseDir, "plugin_list.json");
+		const localPluginPath = path.join(baseDir, "plugins", "local_plugin");
+		const localPluginPathAbs = path.resolve(localPluginPath);
+		const externalPluginPath = path.join(baseDir, "external_plugins", "external_plugin");
+		const externalPluginPathAbs = path.resolve(externalPluginPath);
+		const npmPluginPath = path.join(baseDir, "node_modules", "npm-plugin");
+		const monorepoPluginPath = path.join(baseDir, "external_plugins", "monorepo", "monorepo-plugin");
+		const monorepoPluginPathAbs = path.resolve(monorepoPluginPath);
+		let pluginList;
+
+		before(async function() {
+			// Setup test plugins
+			async function writePlugin(pluginPath, name) {
+				await fs.outputFile(
+					path.join(pluginPath, "index.js"),
+					`module.exports.plugin = { name: "${name}" };`
+				);
+				await fs.outputFile(
+					path.join(pluginPath, "package.json"),
+					JSON.stringify({
+						name: path.basename(pluginPath),
+						version: "0.0.1",
+						keywords: ["clusterio-plugin"],
+					})
+				);
+			}
+
+			// Create local plugin
+			await writePlugin(localPluginPath, "local_plugin");
+			// Create external plugin
+			await writePlugin(externalPluginPath, "external_plugin");
+			// Create npm plugin
+			await writePlugin(npmPluginPath, "npm");
+			// Write a monorepo plugin
+			await writePlugin(monorepoPluginPath, "monorepo-plugin");
+			// Create root package.json
+			await fs.outputFile(
+				path.join(baseDir, "package.json"),
+				JSON.stringify({
+					dependencies: {
+						"npm-plugin": "^1.0.0",
+						"not-a-plugin": "^1.0.0",
+					},
+				})
+			);
+			// Create an npm module that is not a plugin
+			await fs.outputFile(
+				path.join(baseDir, "node_modules", "not-a-plugin", "package.json"),
+				JSON.stringify({ name: "not-a-plugin", version: "1.0.0" })
+			);
+
+			process.chdir(baseDir);
+			pluginList = await lib.loadPluginList(pluginListPath, true);
+		});
+		beforeEach(async function() {
+			await fs.remove(pluginListPath);
+		});
+
+		it("should discover local plugins", async function() {
+			assert.ok(pluginList.has("local_plugin"));
+			assert.ok(pluginList.has("external_plugin"));
+			assert.strictEqual(pluginList.get("local_plugin"), localPluginPathAbs);
+			assert.strictEqual(pluginList.get("external_plugin"), externalPluginPathAbs);
+		});
+
+		it("should discover npm plugins", async function() {
+			assert.ok(pluginList.has("npm"));
+			assert.strictEqual(pluginList.get("npm"), "npm-plugin");
+			// Check that it does not contain not-a-plugin
+			assert.strictEqual(pluginList.get("not-a-plugin"), undefined);
+		});
+
+		it("should load existing plugin list", async function() {
+			const existingList = new Map([["test", "/test/path"]]);
+			await fs.outputFile(pluginListPath, JSON.stringify([...existingList]));
+			const loadedPlugins = await lib.loadPluginList(pluginListPath, false);
+			assert.ok(loadedPlugins.has("test"));
+			assert.strictEqual(loadedPlugins.get("test"), "/test/path");
+		});
+
+		it("should support monorepo plugins", async function() {
+			assert.ok(pluginList.has("monorepo-plugin"));
+			assert.strictEqual(pluginList.get("monorepo-plugin"), monorepoPluginPathAbs);
+		});
+
+		it("should not throw when package.json has no dependencies field", async function() {
+			// Create a package.json without dependencies
+			await fs.outputFile(
+				path.join("package.json"),
+				JSON.stringify({
+					name: "test-package",
+					version: "1.0.0",
+				})
+			);
+
+			// Should not throw
+			await assert.doesNotReject(async () => {
+				await lib.loadPluginList(pluginListPath, true);
+			});
+		});
+
+		after(async function() {
+			process.chdir(old_cwd);
+			await fs.remove(baseDir);
+		});
+	});
 });