diff --git a/docs/writing-plugins.md b/docs/writing-plugins.md index 18d32b64..690a247a 100644 --- a/docs/writing-plugins.md +++ b/docs/writing-plugins.md @@ -212,6 +212,32 @@ An error thrown by a handler is logged and does not affect the other handlers, t For TypeScript the context types are exported as `ControllerPluginContext`, `HostPluginContext`, `InstancePluginContext`, `CtlPluginContext` and `WebPluginContext` from their respective packages. +### Depending on other plugins + +Plugins are loaded at the same time, so there is no order between them. +A plugin that uses another plugin can wait for it with `waitForPlugin(waiter, name, timeoutMs)` on the controller, host or instance it is loaded for. +`waiter` is the name of the plugin waiting, `name` is the plugin to wait for, and `timeoutMs` defaults to 30 seconds. +It resolves to the value returned by the other plugin's entrypoint for the same controller, host or instance, so a plugin that offers an API to other plugins should return it from its entrypoint. +For deprecated class based plugins it resolves to the plugin instance. + +```js +export default async function({ controller, plugin }) { + const other = await controller.waitForPlugin(plugin.name, "other_plugin"); + if (other) { + other.doSomething(); + } +} +``` + +If the plugin is already loaded it resolves right away. +If the plugin is not installed, has load_plugin set to false, or plugins are disabled by recovery mode, it resolves to undefined once all plugins have loaded. +If it times out it resolves to undefined. +In both cases a warning naming both plugins is logged. + +Startup does not continue until every entrypoint has returned. +Two plugins awaiting each other in their entrypoints hold up startup until one of them times out. +Use `.then` or wait from a hook instead if the other plugin may also wait on yours. + ### Class based plugins Plugins written as classes deriving from `BaseControllerPlugin`, `BaseHostPlugin`, `BaseInstancePlugin`, `BaseCtlPlugin` or `BaseWebPlugin` still load, but this way of defining plugins is deprecated and a warning is logged when such a plugin is loaded. diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index da5f2369..28bb6128 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -77,6 +77,7 @@ export default class Controller { hooks = new ControllerHooks(logger); /** Plugins which are currently loaded */ loadedPlugins: Set = new Set(); + private _pluginLoader = new lib.PluginLoader(logger); /** Latest versions of plugins on npm, as shown on the plugins page */ pluginLatestVersionCache: lib.LatestVersionCache = new Map(); @@ -1317,8 +1318,22 @@ export default class Controller { } } + /** + * Wait for another plugin to load + * + * @param waiter - Name of the plugin waiting, used in warnings. + * @param name - Name of the plugin to wait for. + * @param timeoutMs - Time to wait before giving up. + * @returns + * What the plugin's entrypoint returned, or undefined with a + * warning logged if it did not load. + */ + waitForPlugin(waiter: string, name: string, timeoutMs?: number) { + return this._pluginLoader.waitFor(waiter, name, timeoutMs); + } + async loadPlugins() { - const context = { metrics, logger, controller: this }; + const toLoad: lib.PluginNodeEnvInfo[] = []; for (let pluginInfo of this.pluginInfos) { try { const manifestPath = path.posix.join(pluginInfo.requirePath, "dist", "web", "manifest.json"); @@ -1341,17 +1356,20 @@ export default class Controller { continue; } + toLoad.push(pluginInfo); + } + + const context = { metrics, logger, controller: this }; + await this._pluginLoader.load(toLoad, async pluginInfo => { + let api; try { - await lib.loadPlugin( + api = await lib.loadPlugin( pluginInfo, "controller", context, "ControllerPlugin", BaseControllerPlugin, ); - - this.loadedPlugins.add(pluginInfo); - } catch (err: any) { if (err.code === "InstallationError") { throw err; @@ -1360,6 +1378,11 @@ export default class Controller { } logger.info(`Loaded plugin ${pluginInfo.name}`); + return api; + }); + + for (const pluginInfo of toLoad) { + this.loadedPlugins.add(pluginInfo); } } diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index aca92062..80d66a60 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -331,6 +331,7 @@ export default class Host extends lib.Link { hooks = new HostHooks(logger); /** Plugins which are currently loaded */ loadedPlugins: Set = new Set(); + private _pluginLoader = new lib.PluginLoader(logger); /** A map from instance id to instance connection. Only present when instance is running. */ instanceConnections = new Map(); @@ -487,8 +488,22 @@ export default class Host extends lib.Link { this.handle(lib.InstanceDeleteInternalRequest, this.handleInstanceDeleteInternalRequest.bind(this)); } + /** + * Wait for another host plugin to load + * + * @param waiter - Name of the plugin waiting, used in warnings. + * @param name - Name of the plugin to wait for. + * @param timeoutMs - Time to wait before giving up. + * @returns + * What the plugin's host entrypoint returned, or undefined with + * a warning logged if it did not load. + */ + waitForPlugin(waiter: string, name: string, timeoutMs?: number) { + return this._pluginLoader.waitFor(waiter, name, timeoutMs); + } + async loadPlugins() { - const context = { logger, host: this }; + const toLoad: lib.PluginNodeEnvInfo[] = []; for (let pluginInfo of this.pluginInfos) { if ( !pluginInfo.hostEntrypoint && !pluginInfo.instanceEntrypoint @@ -502,17 +517,20 @@ export default class Host extends lib.Link { continue; } + toLoad.push(pluginInfo); + } + + const context = { logger, host: this }; + await this._pluginLoader.load(toLoad, async pluginInfo => { + let api; try { - await lib.loadPlugin( + api = await lib.loadPlugin( pluginInfo, "host", context, "HostPlugin", BaseHostPlugin, ); - - this.loadedPlugins.add(pluginInfo); - } catch (err: any) { if (err.code === "InstallationError") { throw err; @@ -521,6 +539,11 @@ export default class Host extends lib.Link { } logger.info(`Loaded plugin ${pluginInfo.name}`); + return api; + }); + + for (const pluginInfo of toLoad) { + this.loadedPlugins.add(pluginInfo); } } diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index d6cfe046..89d5a21b 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -181,6 +181,7 @@ export default class Instance extends lib.Link { hooks: InstanceHooks; /** Plugins which are currently loaded */ loadedPlugins: Set = new Set(); + private _pluginLoader: lib.PluginLoader; _host: Host; @@ -211,6 +212,7 @@ export default class Instance extends lib.Link { this.id = this.config.get("instance.id"); this.logger = instanceLogger(instanceConfig); + this._pluginLoader = new lib.PluginLoader(this.logger); this.hooks = new InstanceHooks(this.logger); @@ -766,14 +768,27 @@ end`.replace(/\r?\n/g, " "); return instance; } + /** + * Wait for another instance plugin to load + * + * @param waiter - Name of the plugin waiting, used in warnings. + * @param name - Name of the plugin to wait for. + * @param timeoutMs - Time to wait before giving up. + * @returns + * What the plugin's instance entrypoint returned, or undefined with + * a warning logged if it did not load. + */ + waitForPlugin(waiter: string, name: string, timeoutMs?: number) { + return this._pluginLoader.waitFor(waiter, name, timeoutMs); + } + /** * Load the instance plugins and report the instance as initialised * * Must be called once the instance is connected to the host. */ async loadPlugins(pluginInfos: lib.PluginNodeEnvInfo[]) { - // load plugins - const plugins: Record = {}; + const toLoad: lib.PluginNodeEnvInfo[] = []; for (let pluginInfo of pluginInfos) { if ( !pluginInfo.instanceEntrypoint @@ -788,26 +803,33 @@ end`.replace(/\r?\n/g, " "); continue; } - try { + toLoad.push(pluginInfo); + } + + try { + await this._pluginLoader.load(toLoad, async pluginInfo => { let pluginLoadStartedMs = Date.now(); - await lib.loadPlugin( + const api = await lib.loadPlugin( pluginInfo, "instance", { logger: this.logger, host: this._host, instance: this }, "InstancePlugin", BaseInstancePlugin, ); - - this.loadedPlugins.add(pluginInfo); - plugins[pluginInfo.name] = pluginInfo.version; this.logger.info(`Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStartedMs}ms`); - } catch (err) { - this.notifyExit(); - await this.sendSaveListUpdate(); - throw err; - } + return api; + }); + } catch (err) { + this.notifyExit(); + await this.sendSaveListUpdate(); + throw err; } + const plugins: Record = {}; + for (const pluginInfo of toLoad) { + this.loadedPlugins.add(pluginInfo); + plugins[pluginInfo.name] = pluginInfo.version; + } this.send(new lib.InstanceInitialisedEvent(plugins)); } diff --git a/packages/lib/index.ts b/packages/lib/index.ts index ca6f0540..081c9e29 100644 --- a/packages/lib/index.ts +++ b/packages/lib/index.ts @@ -24,6 +24,7 @@ export * from "./src/lua_tools.js"; export * from "./src/permissions.js"; export * from "./src/plugin.js"; export * from "./src/plugin_loader.js"; +export * from "./src/PluginLoader.js"; export * from "./src/loadPlugin.js"; export * from "./src/prometheus.js"; export * from "./src/schema.js"; diff --git a/packages/lib/src/PluginLoader.ts b/packages/lib/src/PluginLoader.ts new file mode 100644 index 00000000..836d2731 --- /dev/null +++ b/packages/lib/src/PluginLoader.ts @@ -0,0 +1,111 @@ +import { logger as defaultLogger, type Logger } from "./logging.js"; +import type { PluginNodeEnvInfo } from "./plugin.js"; + +type Waiter = { + resolve: (api: unknown) => void, + timeout: ReturnType, + waiter: string, +}; + +/** + * Loads plugins concurrently and lets them wait for each other + * + * Owned by the object plugins are loaded into (controller, host or + * instance). + */ +export class PluginLoader { + private _apis = new Map(); + private _waiters = new Map(); + private _finished = false; + + constructor( + private _logger: Logger = defaultLogger, + ) {} + + /** + * Load plugins concurrently + * + * Waits for all loads to settle, then throws the error of the first + * plugin in the list that failed. + * + * @param pluginInfos - Plugins to load. + * @param loadPlugin - Loads one plugin and returns what its entrypoint returned. + */ + async load(pluginInfos: PluginNodeEnvInfo[], loadPlugin: (pluginInfo: PluginNodeEnvInfo) => Promise) { + try { + const results = await Promise.allSettled(pluginInfos.map(async pluginInfo => { + this._loaded(pluginInfo.name, await loadPlugin(pluginInfo)); + })); + const failed = results.find(result => result.status === "rejected"); + if (failed) { + throw failed.reason; + } + } finally { + this._finish(); + } + } + + private _loaded(name: string, api: unknown) { + this._apis.set(name, api); + for (const entry of this._waiters.get(name) ?? []) { + clearTimeout(entry.timeout); + entry.resolve(api); + } + this._waiters.delete(name); + } + + private _finish() { + this._finished = true; + for (const [name, entries] of this._waiters) { + for (const entry of entries) { + clearTimeout(entry.timeout); + this._warnNotLoaded(entry.waiter, name); + entry.resolve(undefined); + } + } + this._waiters.clear(); + } + + /** + * Wait for a plugin to load + * + * @param waiter - Name of the plugin waiting, used in warnings. + * @param name - Name of the plugin to wait for. + * @param timeoutMs - Time to wait before giving up. + * @returns + * What the plugin's entrypoint returned, or undefined with a + * warning logged if it did not load. + */ + waitFor(waiter: string, name: string, timeoutMs = 30_000): Promise { + if (this._apis.has(name)) { + return Promise.resolve(this._apis.get(name) as T); + } + if (this._finished) { + this._warnNotLoaded(waiter, name); + return Promise.resolve(undefined); + } + + return new Promise(resolve => { + const entry: Waiter = { + resolve: resolve as (api: unknown) => void, + waiter, + timeout: setTimeout(() => { + const entries = this._waiters.get(name)!; + entries.splice(entries.indexOf(entry), 1); + this._logger.warn(`Plugin ${waiter} timed out after ${timeoutMs}ms waiting for plugin ${name}`); + resolve(undefined); + }, timeoutMs), + }; + let entries = this._waiters.get(name); + if (!entries) { + entries = []; + this._waiters.set(name, entries); + } + entries.push(entry); + }); + } + + private _warnNotLoaded(waiter: string, name: string) { + this._logger.warn(`Plugin ${waiter} waited for plugin ${name} which is not loaded`); + } +} diff --git a/packages/lib/src/loadPlugin.ts b/packages/lib/src/loadPlugin.ts index 7a6f0a87..1b450908 100644 --- a/packages/lib/src/loadPlugin.ts +++ b/packages/lib/src/loadPlugin.ts @@ -17,7 +17,7 @@ export async function loadPluginEntrypoint< throw new Error(`Expected ${pluginType} plugin ${pluginInfo.name} to export a default function`); } - await init(context); + return await init(context) as unknown; } export async function loadPluginClass< diff --git a/packages/lib/src/plugin_loader.ts b/packages/lib/src/plugin_loader.ts index 1aec0eee..7ea6747a 100644 --- a/packages/lib/src/plugin_loader.ts +++ b/packages/lib/src/plugin_loader.ts @@ -256,15 +256,13 @@ export async function loadPlugin< }; if (typeof module.default === "function") { - await loadPluginEntrypoint(pluginInfo, pluginType, pluginContext, module); - return; + return await loadPluginEntrypoint(pluginInfo, pluginType, pluginContext, module); } // migrate: accept plugins which export classes if (module[exportName]) { pluginContext.logger.warn(`Plugin ${pluginInfo.name} is using deprecated class export`); - await loadPluginClass(pluginInfo, pluginType, pluginContext, module, exportName, baseClass); - return; + return await loadPluginClass(pluginInfo, pluginType, pluginContext, module, exportName, baseClass); } throw new Error(`Plugin ${pluginInfo.name} must export either a default function or ${exportName} class`); diff --git a/test/controller/Controller.js b/test/controller/Controller.js index 1eaa26d9..a3528a06 100644 --- a/test/controller/Controller.js +++ b/test/controller/Controller.js @@ -239,6 +239,66 @@ describe("controller/src/Controller", function() { }); }); }); + describe(".waitForPlugin()", function() { + const names = ["wait_early", "wait_late", "wait_off"]; + let dir; + before(async function() { + addPluginConfigFields(names.map(name => ({ name, version: "1.0.0", controllerEntrypoint: "x" }))); + dir = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-wait-")); + const entrypoints = { + wait_early: `export default async function({ controller, plugin }) { + controller.lateApi = await controller.waitForPlugin(plugin.name, "wait_late"); + }`, + wait_late: "export default function() { return { late: true }; }", + wait_off: "export default function() { return { off: true }; }", + }; + for (const [name, content] of Object.entries(entrypoints)) { + await fs.mkdir(path.join(dir, name, "dist", "web"), { recursive: true }); + await fs.writeFile(path.join(dir, name, "dist", "web", "manifest.json"), "{}"); + await fs.writeFile(path.join(dir, name, "controller.js"), content); + } + }); + after(async function() { + for (const name of Object.keys(lib.ControllerConfig.fieldDefinitions)) { + if (names.some(plugin => name.startsWith(`${plugin}.`))) { + delete lib.ControllerConfig.fieldDefinitions[name]; + } + } + await fs.rm(dir, { recursive: true, force: true }); + }); + function pluginController(recoveryMode = false) { + const config = new ControllerConfig("controller", { + "controller.version": controllerVersion, "wait_off.load_plugin": false, + }); + const infos = names.map(name => ({ + name, + requirePath: path.join(dir, name), + packagePath: path.join(dir, name, "package.json"), + controllerEntrypoint: "controller.js", + })); + return new Controller({}, infos, config, false, recoveryMode); + } + it("should resolve to what a plugin later in the list returned", async function() { + const pluginsController = pluginController(); + await pluginsController.loadPlugins(); + assert.deepEqual(pluginsController.lateApi, { late: true }); + assert.deepEqual(await pluginsController.waitForPlugin("test", "wait_late"), { late: true }); + }); + it("should resolve undefined once loading finishes for plugins not loaded", async function() { + const pluginsController = pluginController(); + const off = pluginsController.waitForPlugin("test", "wait_off", 60_000); + const missing = pluginsController.waitForPlugin("test", "missing", 60_000); + await pluginsController.loadPlugins(); + assert.equal(await off, undefined); + assert.equal(await missing, undefined); + }); + it("should resolve undefined once loading finishes in recovery mode", async function() { + const pluginsController = pluginController(true); + const late = pluginsController.waitForPlugin("test", "wait_late", 60_000); + await pluginsController.loadPlugins(); + assert.equal(await late, undefined); + }); + }); describe(".validateExports()", function() { let dir; before(async function() { diff --git a/test/host/Host.test.js b/test/host/Host.test.js index 0dcdc1f9..d84f19e0 100644 --- a/test/host/Host.test.js +++ b/test/host/Host.test.js @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import dgram from "node:dgram"; import events from "node:events"; import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -250,6 +251,41 @@ describe("host/src/Host", function() { assert.equal(host.config.restartRequired, true); }); }); + describe(".waitForPlugin()", function() { + let dir; + before(async function() { + lib.addPluginConfigFields([ + { name: "host_wait_early", version: "1.0.0", hostEntrypoint: "x" }, + { name: "host_wait_late", version: "1.0.0", hostEntrypoint: "x" }, + ]); + dir = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-host-wait-")); + await fs.writeFile(path.join(dir, "early.js"), `export default async function({ host, plugin }) { + host.lateApi = await host.waitForPlugin(plugin.name, "host_wait_late"); + }`); + await fs.writeFile(path.join(dir, "late.js"), "export default function() { return { late: true }; }"); + }); + after(async function() { + for (const name of Object.keys(lib.HostConfig.fieldDefinitions)) { + if (name.startsWith("host_wait_")) { + delete lib.HostConfig.fieldDefinitions[name]; + } + } + await fs.rm(dir, { recursive: true, force: true }); + }); + it("should resolve to what a host plugin later in the list returned", async function() { + const pluginInfos = [["host_wait_early", "early.js"], ["host_wait_late", "late.js"]].map( + ([name, hostEntrypoint]) => ({ + name, hostEntrypoint, packagePath: path.resolve(dir, "package.json"), + }) + ); + const hostConfig = new lib.HostConfig("host"); + const host = new Host(new HostConnector(hostConfig, pluginInfos), hostConfig, pluginInfos); + const missing = host.waitForPlugin("test", "missing", 60_000); + await host.loadPlugins(); + assert.deepEqual(host.lateApi, { late: true }); + assert.equal(await missing, undefined); + }); + }); describe(".assignFreeGamePort()", function() { let host; let held; diff --git a/test/lib/PluginLoader.test.js b/test/lib/PluginLoader.test.js new file mode 100644 index 00000000..9469df1b --- /dev/null +++ b/test/lib/PluginLoader.test.js @@ -0,0 +1,72 @@ +import assert from "node:assert/strict"; +import * as lib from "@clusterio/lib"; + +describe("lib/PluginLoader", function() { + let logger; + let loader; + beforeEach(function() { + logger = { warnings: [], warn(msg) { this.warnings.push(msg); } }; + loader = new lib.PluginLoader(logger); + }); + const infos = (...names) => names.map(name => ({ name })); + + it("should let a plugin await a plugin later in the list while loading", async function() { + const apis = { b: { b: true } }; + await loader.load(infos("a", "b"), async ({ name }) => { + if (name === "a") { + apis.a = { gotB: await loader.waitFor("a", "b") }; + } + return apis[name]; + }); + assert.deepEqual(apis.a, { gotB: { b: true } }); + assert.equal(await loader.waitFor("c", "a"), apis.a); + assert.deepEqual(logger.warnings, []); + }); + it("should resolve undefined with a warning once loading finishes without the plugin", async function() { + const result = loader.waitFor("a", "b"); + await loader.load(infos("c"), async () => ({})); + assert.equal(await result, undefined); + assert.equal(await loader.waitFor("a", "d"), undefined); + assert.deepEqual(logger.warnings, [ + "Plugin a waited for plugin b which is not loaded", + "Plugin a waited for plugin d which is not loaded", + ]); + }); + it("should resolve undefined with a warning on timeout", async function() { + const result = loader.waitFor("a", "b", 10); + const other = loader.waitFor("c", "b"); + assert.equal(await result, undefined); + assert.deepEqual(logger.warnings, ["Plugin a timed out after 10ms waiting for plugin b"]); + await loader.load(infos("b"), async () => "api"); + assert.equal(await other, "api"); + }); + it("should break a wait cycle with the timeout", async function() { + const got = {}; + await loader.load(infos("a", "b"), async ({ name }) => { + got[name] = await loader.waitFor(name, name === "a" ? "b" : "a", name === "a" ? 10 : 1000); + return name; + }); + assert.deepEqual(got, { a: undefined, b: "a" }); + assert.deepEqual(logger.warnings, ["Plugin a timed out after 10ms waiting for plugin b"]); + }); + it("should finish loading the rest before throwing the first error in list order", async function() { + const loaded = []; + await assert.rejects( + loader.load(infos("a", "b", "c"), async ({ name }) => { + if (name === "a") { + await new Promise(resolve => setTimeout(resolve, 10)); + throw new Error("a failed"); + } + if (name === "b") { + throw new Error("b failed"); + } + loaded.push(name); + return name; + }), + { message: "a failed" }, + ); + assert.deepEqual(loaded, ["c"]); + assert.equal(await loader.waitFor("x", "a"), undefined); + assert.deepEqual(logger.warnings, ["Plugin x waited for plugin a which is not loaded"]); + }); +}); diff --git a/test/lib/plugin_loader.js b/test/lib/plugin_loader.js index ba999482..87156bc5 100644 --- a/test/lib/plugin_loader.js +++ b/test/lib/plugin_loader.js @@ -185,6 +185,7 @@ describe("lib/plugin_loader", function() { await writeEntrypoint(functionPlugin, ` export default async function(context) { context.controller.loadedWith = context; + return { api: true }; } `); await writeEntrypoint(classPlugin, ` @@ -231,7 +232,10 @@ describe("lib/plugin_loader", function() { }); it("should call the default export with the load context", async function() { const pluginInfo = info(functionPlugin); - await lib.loadPlugin(pluginInfo, "controller", context, "ControllerPlugin", BaseControllerPlugin); + const result = await lib.loadPlugin( + pluginInfo, "controller", context, "ControllerPlugin", BaseControllerPlugin, + ); + assert.deepEqual(result, { api: true }); assert.equal(controller.loadedWith.controller, controller); assert.equal(controller.loadedWith.plugin, pluginInfo); assert.equal(controller.loadedWith.logger, logger); @@ -239,7 +243,10 @@ describe("lib/plugin_loader", function() { }); it("should load a deprecated class export", async function() { const pluginInfo = info(classPlugin); - await lib.loadPlugin(pluginInfo, "controller", context, "ControllerPlugin", BaseControllerPlugin); + const result = await lib.loadPlugin( + pluginInfo, "controller", context, "ControllerPlugin", BaseControllerPlugin, + ); + assert.equal(result, controller.loadedWith); assert(controller.loadedWith instanceof BaseControllerPlugin); assert.equal(controller.loadedWith.info, pluginInfo); assert.deepEqual([...controller.hooks.save.attached], ["test"]);