diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index d9bdaf9c..6587e01d 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -317,14 +317,7 @@ async function initialize(): Promise { } logger.info(`Loading available plugins from ${args.pluginList}`); - let pluginList = new Map(); - try { - pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList, { encoding: "utf8" }))); - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - } + let pluginList = await lib.loadPluginList(args.pluginList); // If the command is plugin management we don't try to load plugins if (command === "plugin") { diff --git a/packages/controller/package.json b/packages/controller/package.json index ccda8907..9aeb29bf 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -13,6 +13,7 @@ "clusteriocontroller": "./dist/node/controller.js" }, "keywords": [ + "clusterio", "factorio" ], "engines": { diff --git a/packages/create/package.json b/packages/create/package.json index 814e26f0..8fc99cb5 100644 --- a/packages/create/package.json +++ b/packages/create/package.json @@ -9,8 +9,8 @@ "clusteriocreate": "./create.js" }, "keywords": [ - "factorio", - "clusterio" + "clusterio", + "factorio" ], "engines": { "node": ">=18" diff --git a/packages/create/templates/common/package.json b/packages/create/templates/common/package.json index b05122b3..ada0a72b 100644 --- a/packages/create/templates/common/package.json +++ b/packages/create/templates/common/package.json @@ -48,6 +48,7 @@ }, "keywords": [ "clusterio", + "clusterio-plugin", "factorio" ] } diff --git a/packages/ctl/ctl.ts b/packages/ctl/ctl.ts index b5edb31c..70d49964 100755 --- a/packages/ctl/ctl.ts +++ b/packages/ctl/ctl.ts @@ -188,14 +188,7 @@ async function startControl() { lib.handleUnhandledErrors(); logger.verbose(`Loading available plugins from ${args.pluginList}`); - let pluginList = new Map(); - try { - pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList, "utf8"))); - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - } + let pluginList = await lib.loadPluginList(args.pluginList); // If the command is plugin management we don't try to load plugins if (args._[0] === "plugin") { diff --git a/packages/ctl/package.json b/packages/ctl/package.json index 28c79023..e26219d3 100644 --- a/packages/ctl/package.json +++ b/packages/ctl/package.json @@ -13,6 +13,7 @@ "clusterioctl": "./dist/node/ctl.js" }, "keywords": [ + "clusterio", "factorio" ], "engines": { diff --git a/packages/host/host.ts b/packages/host/host.ts index b0123964..0247e9aa 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -147,14 +147,7 @@ async function startHost() { } logger.info(`Loading available plugins from ${args.pluginList}`); - let pluginList = new Map(); - try { - pluginList = new Map(JSON.parse(await fs.readFile(args.pluginList, "utf8"))); - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - } + let pluginList = await lib.loadPluginList(args.pluginList); // If the command is plugin management we don't try to load plugins if (command === "plugin") { diff --git a/packages/host/package.json b/packages/host/package.json index ea44fdc9..f5bc92d8 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -13,6 +13,7 @@ "clusteriohost": "./dist/node/host.js" }, "keywords": [ + "clusterio", "factorio" ], "engines": { diff --git a/packages/lib/index.ts b/packages/lib/index.ts index bec088ad..9cd87213 100644 --- a/packages/lib/index.ts +++ b/packages/lib/index.ts @@ -29,6 +29,7 @@ export * from "./src/system_collectors"; export * from "./src/zip_ops"; export * from "./src/subscriptions"; export * from "./src/datastore"; +export * from "./src/load_plugin_list"; export { default as ExponentialBackoff } from "./src/ExponentialBackoff"; export { default as ModStore } from "./src/ModStore"; diff --git a/packages/lib/package.json b/packages/lib/package.json index d5c58364..0f0ed1b5 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -11,6 +11,7 @@ "main": "dist/node/index.js", "browser": "browser.ts", "keywords": [ + "clusterio", "factorio" ], "engines": { diff --git a/packages/lib/src/load_plugin_list.ts b/packages/lib/src/load_plugin_list.ts new file mode 100644 index 00000000..22d3fbaa --- /dev/null +++ b/packages/lib/src/load_plugin_list.ts @@ -0,0 +1,186 @@ +import fs from "fs-extra"; +import path from "path"; +import { logger } from "./logging"; +import * as libFileOps from "./file_ops"; +import vm from "vm"; + +/** + * Searches for local plugins in the plugins/ and external_plugins/ directories + * and adds them to the plugin list if they contain a package.json file. + * + * @param pluginList - Map of plugin names to their file paths + * @returns Promise that resolves to true if any plugins were added to the list, false otherwise + */ +async function findLocalPlugins(pluginList: Map): Promise { + // Check folders for plugins + const pluginFolders = [ + "plugins", + "external_plugins", + ]; + + let hasChanged = 0; + const ignoredFolders = new Set(["node_modules", ".git", "dist", "build", "disabled"]); + const maxDepth = 3; + + for (const pluginFolder of pluginFolders) { + let pluginNames; + try { + pluginNames = await fs.readdir(pluginFolder); + } catch (err: any) { + if (err.code === "ENOENT") { + // Skip if folder doesn't exist + continue; + } + throw err; + } + for (const pluginName of pluginNames) { + const pluginPath = path.join(pluginFolder, pluginName); + if (pluginList.has(pluginName)) { + // If the one in the list is not the same as this one, warn + if (pluginList.get(pluginName) !== path.resolve(pluginPath)) { + logger.warn( + `${pluginName} is already in the plugin list, but with a different path - ` + + `using ${pluginList.get(pluginName)}` + ); + } + continue; + } + const stats = await fs.stat(pluginPath); + if (stats.isDirectory()) { + // Check for package.json and plugin keyword + if (await checkPackageJson(pluginPath)) { + pluginList.set(pluginName, path.resolve(pluginPath)); + logger.info(`Added ${pluginName} from ${pluginFolder}`); + hasChanged += 1; + continue; + } + + if (pluginPath.split(path.sep).length < maxDepth && !ignoredFolders.has(pluginName)) { + logger.warn(`Possible monorepo detected, scanning ${pluginPath} for clusterio-plugins`); + pluginFolders.push(pluginPath); + } + } + } + } + + return hasChanged > 0; +} + +function getPluginName(requireSpec: string) { + const context = vm.createContext({ require: require, pluginInfo: null }); + const code = `pluginInfo = require(${JSON.stringify(requireSpec)}).plugin;`; + vm.runInContext(code, context); + return context.pluginInfo.name; +} + +/** + * Find NPM packages in the root package.json that satisfies the requirements for plugins. + * Adds discovered plugins to the plugin list. + * + * @param pluginList - Map of plugin names to their file paths + * @returns Promise that resolves to true if any plugins were added to the list, false otherwise + */ +async function findNpmPlugins(pluginList: Map): Promise { + let dependencies; + try { + const rootPackageJson = JSON.parse(await fs.readFile(path.resolve("package.json"), { encoding: "utf8" })); + dependencies = rootPackageJson.dependencies; + } catch (err: any) { + if (err.code === "ENOENT") { + // Skip if package.json doesn't exist + return false; + } + throw err; + } + + if (!dependencies) { + return false; + } + + let hasChanged = 0; + for (const [packageName, packageVersion] of Object.entries(dependencies)) { + if ([...pluginList.values()].includes(packageName)) { + continue; // This npm module is already in the plugin list + } + // Find the package in node_modules and read the package.json to check if it's a clusterio-plugin + if (await checkPackageJson(path.resolve("node_modules", packageName))) { + const pluginName = getPluginName(path.resolve("node_modules", packageName)); + if (pluginList.has(pluginName)) { + logger.warn( + `${pluginName} provided by ${packageName}@${packageVersion} is already in the plugin list, ` + + `but with a different path - using ${pluginList.get(pluginName)}` + ); + } else { + pluginList.set(pluginName, packageName); + logger.info(`Added ${pluginName} from NPM`); + hasChanged += 1; + } + } else if (![ + "@clusterio/controller", + "@clusterio/ctl", + "@clusterio/host", + "@clusterio/lib", + "@clusterio/web_ui", + ].includes(packageName)) { + logger.warn(`${packageName}@${packageVersion} is not a clusterio-plugin`); + } + } + return hasChanged > 0; +} + +/** + * Loads the plugin list from the specified file and optionally discovers local plugins + * + * @param pluginListPath - Path to the plugin list JSON file + * @param findLocal - Whether to look for local plugins in plugins/ and external_plugins/ directories + * @returns Map of plugin name to plugin path + */ +export async function loadPluginList(pluginListPath: string, findLocal = true): Promise> { + let pluginList = new Map(); + + // Try to load existing plugin list + try { + pluginList = new Map(JSON.parse(await fs.readFile(pluginListPath, { encoding: "utf8" }))); + logger.info(`Loaded ${pluginList.size} plugins from ${pluginListPath}`); + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + logger.info(`No existing plugin list found at ${pluginListPath}`); + } + + // Track if any changes were made + let hasChanges = false; + + // Optionally discover local plugins + if (findLocal) { + hasChanges = await findLocalPlugins(pluginList) || hasChanges; + } + + // Discover NPM plugins + hasChanges = await findNpmPlugins(pluginList) || hasChanges; + + if (hasChanges) { + await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, "\t")); + } + + return pluginList; +} + +/** + * Helper function to check if a package.json file exists and contains the clusterio-plugin keyword + * @param pluginPath - Path to the plugin directory + * @returns Promise that resolves to true if package.json exists and has clusterio-plugin keyword + */ +async function checkPackageJson(pluginPath: string): Promise { + const packageJsonPath = path.join(pluginPath, "package.json"); + if (!await fs.exists(packageJsonPath)) { + return false; + } + + // Read and parse package.json + const packageJson = JSON.parse(await fs.readFile(packageJsonPath, { encoding: "utf8" })); + + // Check if package has the clusterio-plugin keyword + return packageJson.keywords?.includes("clusterio-plugin"); +} diff --git a/packages/lib/src/plugin_loader.ts b/packages/lib/src/plugin_loader.ts index 3c1af67f..fa5307cc 100644 --- a/packages/lib/src/plugin_loader.ts +++ b/packages/lib/src/plugin_loader.ts @@ -26,6 +26,15 @@ export async function loadPluginInfos(pluginList: Map) { let pluginInfo: libPlugin.PluginNodeEnvInfo; let pluginPackage: { name?: string, version: string, main?: string, private?: boolean }; + // Check if plugin path exists, otherwise remove it + try { + require.resolve(pluginPath); + } catch (err) { + logger.error(`Plugin path ${pluginPath} does not exist, not loading ${pluginName}`); + pluginList.delete(pluginName); + continue; + } + try { pluginInfo = require(pluginPath).plugin; pluginPackage = require(path.posix.join(pluginPath, "package.json")); diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index f61a632f..1921c733 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -10,7 +10,8 @@ "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [ - "clusterio" + "clusterio", + "factorio" ], "engines": { "node": ">=12" diff --git a/plugins/global_chat/package.json b/plugins/global_chat/package.json index b8d97aa2..e4620e37 100644 --- a/plugins/global_chat/package.json +++ b/plugins/global_chat/package.json @@ -7,6 +7,11 @@ "license": "MIT", "repository": "clusterio/clusterio", "main": "dist/node/index.js", + "keywords": [ + "clusterio", + "clusterio-plugin", + "factorio" + ], "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/inventory_sync/package.json b/plugins/inventory_sync/package.json index e184e734..013b6c6e 100644 --- a/plugins/inventory_sync/package.json +++ b/plugins/inventory_sync/package.json @@ -7,6 +7,11 @@ "license": "MIT", "repository": "clusterio/factorioClusterio", "main": "dist/node/index.js", + "keywords": [ + "clusterio", + "clusterio-plugin", + "factorio" + ], "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/player_auth/package.json b/plugins/player_auth/package.json index a7c40a5b..8b40b929 100644 --- a/plugins/player_auth/package.json +++ b/plugins/player_auth/package.json @@ -7,6 +7,11 @@ "license": "MIT", "repository": "clusterio/clusterio", "main": "dist/node/index.js", + "keywords": [ + "clusterio", + "clusterio-plugin", + "factorio" + ], "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/research_sync/package.json b/plugins/research_sync/package.json index 955874ae..5cf8a269 100644 --- a/plugins/research_sync/package.json +++ b/plugins/research_sync/package.json @@ -7,6 +7,11 @@ "license": "MIT", "repository": "clusterio/clusterio", "main": "dist/node/index.js", + "keywords": [ + "clusterio", + "clusterio-plugin", + "factorio" + ], "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/statistics_exporter/package.json b/plugins/statistics_exporter/package.json index cb4e9d0c..691c8c38 100644 --- a/plugins/statistics_exporter/package.json +++ b/plugins/statistics_exporter/package.json @@ -7,6 +7,11 @@ "license": "MIT", "repository": "clusterio/clusterio", "main": "dist/node/index.js", + "keywords": [ + "clusterio", + "clusterio-plugin", + "factorio" + ], "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/plugins/subspace_storage/package.json b/plugins/subspace_storage/package.json index e6de5b57..ee517d3d 100644 --- a/plugins/subspace_storage/package.json +++ b/plugins/subspace_storage/package.json @@ -7,6 +7,11 @@ "license": "MIT", "repository": "clusterio/clusterio", "main": "dist/node/index.js", + "keywords": [ + "clusterio", + "clusterio-plugin", + "factorio" + ], "scripts": { "prepare": "tsc --build && webpack-cli --env production", "test": "echo \"Error: run tests from root\" && exit 1" diff --git a/test/lib/plugin_loader.js b/test/lib/plugin_loader.js index b8f86ea4..fd5126bb 100644 --- a/test/lib/plugin_loader.js +++ b/test/lib/plugin_loader.js @@ -33,11 +33,9 @@ describe("lib/plugin_loader", function() { await writePlugin(invalidPlugin, "wrong"); }); - it("should throw on missing plugin", async function() { - await assert.rejects( - lib.loadPluginInfos(new Map([["missing", missingPlugin]]), []), - new RegExp(`^Error: PluginError: Cannot find module '${escapeRegExp(missingPlugin)}'`) - ); + it("should ignore missing plugins", async function() { + const result = await lib.loadPluginInfos(new Map([["missing", missingPlugin]]), []); + assert.deepEqual(result, []); }); it("should load test plugin", async function() { assert.deepEqual( @@ -112,4 +110,113 @@ describe("lib/plugin_loader", function() { ); }); }); + describe("loadPluginList()", async function() { + const old_cwd = process.cwd(); + const baseDir = path.join("temp", "test", "plugin_list"); + const pluginListPath = path.join(baseDir, "plugin_list.json"); + const localPluginPath = path.join(baseDir, "plugins", "local_plugin"); + const localPluginPathAbs = path.resolve(localPluginPath); + const externalPluginPath = path.join(baseDir, "external_plugins", "external_plugin"); + const externalPluginPathAbs = path.resolve(externalPluginPath); + const npmPluginPath = path.join(baseDir, "node_modules", "npm-plugin"); + const monorepoPluginPath = path.join(baseDir, "external_plugins", "monorepo", "monorepo-plugin"); + const monorepoPluginPathAbs = path.resolve(monorepoPluginPath); + let pluginList; + + before(async function() { + // Setup test plugins + async function writePlugin(pluginPath, name) { + await fs.outputFile( + path.join(pluginPath, "index.js"), + `module.exports.plugin = { name: "${name}" };` + ); + await fs.outputFile( + path.join(pluginPath, "package.json"), + JSON.stringify({ + name: path.basename(pluginPath), + version: "0.0.1", + keywords: ["clusterio-plugin"], + }) + ); + } + + // Create local plugin + await writePlugin(localPluginPath, "local_plugin"); + // Create external plugin + await writePlugin(externalPluginPath, "external_plugin"); + // Create npm plugin + await writePlugin(npmPluginPath, "npm"); + // Write a monorepo plugin + await writePlugin(monorepoPluginPath, "monorepo-plugin"); + // Create root package.json + await fs.outputFile( + path.join(baseDir, "package.json"), + JSON.stringify({ + dependencies: { + "npm-plugin": "^1.0.0", + "not-a-plugin": "^1.0.0", + }, + }) + ); + // Create an npm module that is not a plugin + await fs.outputFile( + path.join(baseDir, "node_modules", "not-a-plugin", "package.json"), + JSON.stringify({ name: "not-a-plugin", version: "1.0.0" }) + ); + + process.chdir(baseDir); + pluginList = await lib.loadPluginList(pluginListPath, true); + }); + beforeEach(async function() { + await fs.remove(pluginListPath); + }); + + it("should discover local plugins", async function() { + assert.ok(pluginList.has("local_plugin")); + assert.ok(pluginList.has("external_plugin")); + assert.strictEqual(pluginList.get("local_plugin"), localPluginPathAbs); + assert.strictEqual(pluginList.get("external_plugin"), externalPluginPathAbs); + }); + + it("should discover npm plugins", async function() { + assert.ok(pluginList.has("npm")); + assert.strictEqual(pluginList.get("npm"), "npm-plugin"); + // Check that it does not contain not-a-plugin + assert.strictEqual(pluginList.get("not-a-plugin"), undefined); + }); + + it("should load existing plugin list", async function() { + const existingList = new Map([["test", "/test/path"]]); + await fs.outputFile(pluginListPath, JSON.stringify([...existingList])); + const loadedPlugins = await lib.loadPluginList(pluginListPath, false); + assert.ok(loadedPlugins.has("test")); + assert.strictEqual(loadedPlugins.get("test"), "/test/path"); + }); + + it("should support monorepo plugins", async function() { + assert.ok(pluginList.has("monorepo-plugin")); + assert.strictEqual(pluginList.get("monorepo-plugin"), monorepoPluginPathAbs); + }); + + it("should not throw when package.json has no dependencies field", async function() { + // Create a package.json without dependencies + await fs.outputFile( + path.join("package.json"), + JSON.stringify({ + name: "test-package", + version: "1.0.0", + }) + ); + + // Should not throw + await assert.doesNotReject(async () => { + await lib.loadPluginList(pluginListPath, true); + }); + }); + + after(async function() { + process.chdir(old_cwd); + await fs.remove(baseDir); + }); + }); });