Merge pull request #1232 from bbassie/feature/wait-for-plugin

Add waitForPlugin and load plugins concurrently
This commit is contained in:
Cooldude2606 authored and GitHub committed 2026-11-22 13:14:30 +00:00
commit 1a016372c1
20 files changed
+800 -411

No files matched your search

+14 -19
View File
@@ -1585,7 +1585,7 @@ export default class Controller {
}
async loadPlugins() {
const context = { metrics, logger, controller: this };
const toLoad: lib.PluginNodeEnvInfo[] = [];
for (let pluginInfo of this.pluginInfos) {
try {
const manifestPath = path.posix.join(pluginInfo.requirePath, "dist", "web", "manifest.json");
@@ -1608,25 +1608,20 @@ export default class Controller {
continue;
}
try {
await lib.loadPlugin(
pluginInfo,
"controller",
context,
"ControllerPlugin",
BaseControllerPlugin,
);
toLoad.push(pluginInfo);
}
this.loadedPlugins.add(pluginInfo);
} catch (err: any) {
if (err.code === "InstallationError") {
throw err;
}
throw new lib.PluginError(pluginInfo.name, err);
}
logger.info(`Loaded plugin ${pluginInfo.name}`);
const { loaded } = await lib.PluginLoader.load({
type: "controller",
context: { metrics, controller: this },
logger,
installed: this.pluginInfos,
load: toLoad,
exportName: "ControllerPlugin",
baseClass: BaseControllerPlugin,
});
for (const pluginInfo of loaded.keys()) {
this.loadedPlugins.add(pluginInfo);
}
}
+10 -12
View File
@@ -118,18 +118,16 @@ async function loadPlugins(pluginList: Map<string, string>, hooks: CtlHooks) {
lib.registerPluginPermissions(pluginInfos);
lib.addPluginConfigFields(pluginInfos);
const ctlPlugins = new Set<lib.PluginNodeEnvInfo>();
for (let pluginInfo of pluginInfos) {
if (!pluginInfo.ctlEntrypoint) {
continue;
}
const context = { hooks, logger, plugin: pluginInfo };
await lib.loadPlugin(pluginInfo, "ctl", context, "CtlPlugin", BaseCtlPlugin);
ctlPlugins.add(pluginInfo);
}
return ctlPlugins;
const { loaded } = await lib.PluginLoader.load({
type: "ctl",
context: { hooks },
logger,
installed: pluginInfos,
load: pluginInfos.filter(pluginInfo => pluginInfo.ctlEntrypoint),
exportName: "CtlPlugin",
baseClass: BaseCtlPlugin,
});
return new Set(loaded.keys());
}
interface CtlArguments {
+14 -19
View File
@@ -490,7 +490,7 @@ export default class Host extends lib.Link {
}
async loadPlugins() {
const context = { logger, host: this };
const toLoad: lib.PluginNodeEnvInfo[] = [];
for (let pluginInfo of this.pluginInfos) {
if (
!pluginInfo.hostEntrypoint && !pluginInfo.instanceEntrypoint
@@ -504,25 +504,20 @@ export default class Host extends lib.Link {
continue;
}
try {
await lib.loadPlugin(
pluginInfo,
"host",
context,
"HostPlugin",
BaseHostPlugin,
);
toLoad.push(pluginInfo);
}
this.loadedPlugins.add(pluginInfo);
} catch (err: any) {
if (err.code === "InstallationError") {
throw err;
}
throw new lib.PluginError(pluginInfo.name, err);
}
logger.info(`Loaded plugin ${pluginInfo.name}`);
const { loaded } = await lib.PluginLoader.load({
type: "host",
context: { host: this },
logger,
installed: this.pluginInfos,
load: toLoad,
exportName: "HostPlugin",
baseClass: BaseHostPlugin,
});
for (const pluginInfo of loaded.keys()) {
this.loadedPlugins.add(pluginInfo);
}
}
+24 -20
View File
@@ -804,8 +804,7 @@ end`.replace(/\r?\n/g, " ");
* Must be called once the instance is connected to the host.
*/
async loadPlugins(pluginInfos: lib.PluginNodeEnvInfo[]) {
// load plugins
const plugins: Record<string, string> = {};
const toLoad: lib.PluginNodeEnvInfo[] = [];
for (let pluginInfo of pluginInfos) {
if (
!pluginInfo.instanceEntrypoint
@@ -820,26 +819,31 @@ end`.replace(/\r?\n/g, " ");
continue;
}
try {
let pluginLoadStartedMs = Date.now();
await lib.loadPlugin(
pluginInfo,
"instance",
{ logger: this.logger, host: this._host, instance: this },
"InstancePlugin",
BaseInstancePlugin,
);
this.loadedPlugins.add(pluginInfo);
plugins[pluginInfo.name] = pluginInfo.version;
this.logger.info(`Loaded plugin ${pluginInfo.name} in ${Date.now() - pluginLoadStartedMs}ms`);
} catch (err) {
this.notifyExit();
await this.sendSaveListUpdate();
throw err;
}
toLoad.push(pluginInfo);
}
let loaded;
try {
({ loaded } = await lib.PluginLoader.load({
type: "instance",
context: { host: this._host, instance: this },
logger: this.logger,
installed: pluginInfos,
load: toLoad,
exportName: "InstancePlugin",
baseClass: BaseInstancePlugin,
}));
} catch (err) {
this.notifyExit();
await this.sendSaveListUpdate();
throw err;
}
const plugins: Record<string, string> = {};
for (const pluginInfo of loaded.keys()) {
this.loadedPlugins.add(pluginInfo);
plugins[pluginInfo.name] = pluginInfo.version;
}
this.send(new lib.InstanceInitialisedEvent(plugins));
}
+1 -1
View File
@@ -13,7 +13,7 @@ export * from "./src/link";
export * from "./src/lua_tools";
export * from "./src/logging";
export * from "./src/permissions";
export * from "./src/loadPlugin";
export * from "./src/PluginLoader";
export * from "./src/schema";
export * from "./src/subscriptions";
+2 -2
View File
@@ -23,8 +23,8 @@ export * from "./src/logging_utils.js";
export * from "./src/lua_tools.js";
export * from "./src/permissions.js";
export * from "./src/plugin.js";
export * from "./src/plugin_loader.js";
export * from "./src/loadPlugin.js";
export * from "./src/plugin_info.js";
export * from "./src/PluginLoader.js";
export * from "./src/prometheus.js";
export * from "./src/schema.js";
export * from "./src/shared_commands.js";
+7
View File
@@ -38,6 +38,13 @@
},
"import": "./src/zlib_sync.browser.ts"
},
"#import_plugin": {
"node": {
"types": "./src/import_plugin.node.ts",
"default": "./dist/node/src/import_plugin.node.js"
},
"import": "./src/import_plugin.browser.ts"
},
"#is_deep_strict_equal": {
"node": {
"types": "./src/is_deep_strict_equal.node.ts",
+264
View File
@@ -0,0 +1,264 @@
// Shared by node and the web interface, node specifics go in #import_plugin
import importPlugin from "#import_plugin";
import { Config } from "./config/classes.js";
import { ControllerConfig, HostConfig, InstanceConfig } from "./config/definitions.js";
import Role from "./data/Role.js";
import { PluginError } from "./errors.js";
import { logger, type Logger } from "./logging.js";
import type {
MigrationHooks, PluginClass, PluginDeclaration, PluginLoadContext, PluginNodeEnvInfo, PluginRegistry, PluginType,
} from "./plugin.js";
type Waiter = {
resolve: (api: unknown) => void,
timeout: ReturnType<typeof setTimeout>,
waiter: string,
};
export type PluginLoadOptions<Context extends object, Info extends PluginDeclaration> = {
/** Entrypoint to load, e.g. controller loads controllerEntrypoint */
type: PluginType,
/** Context passed to each plugin, plugin, logger and plugins are added per plugin */
context: Context | ((pluginInfo: Info) => Context),
/** Logger for warnings, plugins get a child of it */
logger: Logger,
/** All installed plugins, used by checkInstalled */
installed: Info[],
/** Plugins to load, used by checkEnabled */
load: Info[],
/** Name of the deprecated class export, class exports are not accepted if unset */
exportName?: string,
baseClass?: PluginClass<any, any>,
/** Return failed plugins instead of throwing the first error */
continueOnError?: boolean,
};
export type PluginLoadResult<Info> = {
/** What each loaded plugin's entrypoint returned, in load order */
loaded: Map<Info, unknown>,
failed: Map<Info, Error>,
};
/**
* Loads plugins concurrently and lets them wait for each other
*
* Created for one load of plugins with {@link PluginLoader.load}.
*/
export class PluginLoader<Context extends object, Info extends PluginDeclaration> {
private _apis = new Map<string, unknown>();
private _waiters = new Map<string, Waiter[]>();
private _installed: Set<string>;
private _enabled: Set<string>;
/** Plugins still loading */
private _pending: Set<string>;
private constructor(
private _options: PluginLoadOptions<Context, Info>,
) {
this._installed = new Set(_options.installed.map(pluginInfo => pluginInfo.name));
this._enabled = new Set(_options.load.map(pluginInfo => pluginInfo.name));
this._pending = new Set(this._enabled);
}
/**
* Load plugins concurrently
*
* Waits for all plugins to settle, then throws the error of the first
* plugin in the list that failed unless continueOnError is set.
* Errors are wrapped in PluginError.
*
* @param options - What to load and how.
* @returns Loaded and failed plugins.
*/
static async load<Context extends object, Info extends PluginDeclaration>(
options: PluginLoadOptions<Context, Info>
): Promise<PluginLoadResult<Info>> {
return await new PluginLoader(options)._load();
}
private async _load() {
const results = await Promise.allSettled(this._options.load.map(async pluginInfo => {
const startedMs = Date.now();
try {
const api = await this._loadPlugin(pluginInfo);
if (this._options.type !== "migration" && pluginInfo[`${this._options.type}Entrypoint`]) {
this._options.logger.info(`Loaded plugin ${pluginInfo.name} in ${Date.now() - startedMs}ms`);
}
this._settle(pluginInfo.name, true, api);
return api;
} catch (err) {
this._settle(pluginInfo.name, false);
throw err;
}
}));
const loaded = new Map<Info, unknown>();
const failed = new Map<Info, Error>();
for (const [index, result] of results.entries()) {
const pluginInfo = this._options.load[index];
if (result.status === "fulfilled") {
loaded.set(pluginInfo, result.value);
} else {
failed.set(pluginInfo, result.reason);
}
}
if (failed.size && !this._options.continueOnError) {
throw failed.values().next().value;
}
return { loaded, failed };
}
private async _loadPlugin(pluginInfo: Info) {
const { type, exportName, baseClass } = this._options;
const entrypoint = pluginInfo[`${type}Entrypoint`];
if (!entrypoint) {
return undefined;
}
const options = this._options;
const context = typeof options.context === "function" ? options.context(pluginInfo) : options.context;
const pluginContext: PluginLoadContext<Context, Info> = {
...context,
plugin: pluginInfo,
logger: options.logger.child({ plugin: pluginInfo.name }),
plugins: this._registry(pluginInfo.name),
};
try {
const module = await importPlugin(pluginInfo, entrypoint);
if (typeof module.default === "function") {
return await module.default(pluginContext) as unknown;
}
// migrate: accept plugins which export classes
if (exportName && baseClass && module[exportName]) {
pluginContext.logger.warn(`Plugin ${pluginInfo.name} is using deprecated class export`);
return await loadPluginClass(pluginInfo, pluginContext, module[exportName], exportName, baseClass);
}
throw new Error(exportName
? `Plugin ${pluginInfo.name} must export either a default function or ${exportName} class`
: `Expected ${type} plugin ${pluginInfo.name} to export a default function`
);
} catch (err: any) {
if (err.code === "InstallationError") {
throw err;
}
throw new PluginError(pluginInfo.name, err);
}
}
private _registry(waiter: string): PluginRegistry {
return {
waitFor: (name, timeoutMs) => this._waitFor(waiter, name, timeoutMs),
checkInstalled: name => this._installed.has(name),
checkEnabled: name => this._enabled.has(name),
};
}
private _settle(name: string, loaded: boolean, api?: unknown) {
this._pending.delete(name);
if (loaded) {
this._apis.set(name, api);
}
for (const entry of this._waiters.get(name) ?? []) {
clearTimeout(entry.timeout);
if (!loaded) {
this._warnNotLoaded(entry.waiter, name);
}
entry.resolve(api);
}
this._waiters.delete(name);
}
private _waitFor<T>(waiter: string, name: string, timeoutMs = 30_000): Promise<T | undefined> {
if (this._apis.has(name)) {
return Promise.resolve(this._apis.get(name) as T);
}
if (!this._pending.has(name)) {
this._warnNotLoaded(waiter, name);
return Promise.resolve(undefined);
}
return new Promise(resolve => {
const entry: Waiter = {
resolve: resolve as (api: unknown) => void,
waiter,
timeout: setTimeout(() => {
const entries = this._waiters.get(name)!;
entries.splice(entries.indexOf(entry), 1);
this._options.logger.warn(
`Plugin ${waiter} timed out after ${timeoutMs}ms waiting for plugin ${name}`
);
resolve(undefined);
}, timeoutMs),
};
let entries = this._waiters.get(name);
if (!entries) {
entries = [];
this._waiters.set(name, entries);
}
entries.push(entry);
});
}
private _warnNotLoaded(waiter: string, name: string) {
this._options.logger.warn(`Plugin ${waiter} waited for plugin ${name} which is not loaded`);
}
}
async function loadPluginClass(
pluginInfo: PluginDeclaration,
context: PluginLoadContext<object, any>,
pluginClass: unknown,
exportName: string,
baseClass: PluginClass<any, any>,
) {
if (typeof pluginClass !== "function") {
throw new Error(`Expected plugin ${pluginInfo.name} to export a class named ${exportName}`);
}
if (!(pluginClass.prototype instanceof baseClass)) {
throw new Error(`Expected ${exportName} exported from ${pluginInfo.name} to extend ${baseClass.name}`);
}
const plugin = (pluginClass as PluginClass<any, any>).fromContext(context);
try {
await plugin.init();
} catch (err) {
plugin.detachHooks();
throw err;
}
return plugin;
}
/**
* Load the migration entrypoints of plugins
*
* Must be called before configs and roles are loaded for the migrations
* to run on them. Loaded for every plugin, including those disabled with
* load_plugin. Also registers the plugin versions written to each config
* after the migrations run.
*
* @param pluginInfos - Plugins to load migrations for.
*/
export async function loadPluginMigrations(pluginInfos: PluginNodeEnvInfo[]) {
const hooks: MigrationHooks = {
controllerConfig: ControllerConfig.pluginMigrations,
hostConfig: HostConfig.pluginMigrations,
instanceConfig: InstanceConfig.pluginMigrations,
role: Role.pluginMigrations,
};
for (const pluginInfo of pluginInfos) {
Config.pluginVersions.set(pluginInfo.name, pluginInfo.version);
}
await PluginLoader.load({
type: "migration",
context: { hooks },
logger,
installed: pluginInfos,
load: pluginInfos,
});
}
@@ -0,0 +1,7 @@
import type { PluginDeclaration, PluginWebEnvInfo } from "./plugin";
export default async function importPlugin(
pluginInfo: PluginDeclaration, entrypoint: string
): Promise<Record<string, any>> {
return await (pluginInfo as PluginWebEnvInfo).container.get(entrypoint);
}
+10
View File
@@ -0,0 +1,10 @@
import path from "node:path";
import { pathToFileURL } from "node:url";
import type { PluginDeclaration, PluginNodeEnvInfo } from "./plugin.js";
export default async function importPlugin(
pluginInfo: PluginDeclaration, entrypoint: string
): Promise<Record<string, any>> {
const packagePath = (pluginInfo as PluginNodeEnvInfo).packagePath;
return await import(pathToFileURL(path.join(path.dirname(packagePath), entrypoint)).href);
}
-54
View File
@@ -1,54 +0,0 @@
// Contained in its own file to allow use within web build
import type { PluginDeclaration, PluginType, PluginLoadContext, PluginClass } from "./plugin.js";
export async function loadPluginEntrypoint<
Context extends object,
Info extends PluginDeclaration
>(
pluginInfo: Info,
pluginType: PluginType,
context: PluginLoadContext<Context, Info>,
module: Record<string, unknown>
) {
const init = module.default;
if (typeof init !== "function") {
throw new Error(`Expected ${pluginType} plugin ${pluginInfo.name} to export a default function`);
}
await init(context);
}
export async function loadPluginClass<
Context extends object,
Class extends PluginClass<Context, Info>,
Info extends PluginDeclaration
>(
pluginInfo: Info,
pluginType: PluginType,
context: PluginLoadContext<Context, Info>,
module: Record<string, unknown>,
exportName: string,
baseClass: Class
) {
const pluginClass = module[exportName];
if (typeof pluginClass !== "function") {
throw new Error(`Expected ${pluginType} plugin ${pluginInfo.name} to export a class named ${exportName}`);
}
if (!(pluginClass.prototype instanceof baseClass)) {
throw new Error(`Expected ${exportName} exported from ${pluginInfo.name} to extend ${baseClass.name}`);
}
const plugin = (pluginClass as Class).fromContext(context);
try {
await plugin.init();
} catch (err) {
plugin.detachHooks();
throw err;
}
return plugin as InstanceType<Class>;
}
+19
View File
@@ -142,12 +142,31 @@ export interface PlayerEvent {
stats: PlayerStats,
}
/** Access to the other plugins loaded alongside a plugin */
export interface PluginRegistry {
/**
* Wait for another plugin to load
*
* @param name - Name of the plugin to wait for.
* @param timeoutMs - Time to wait before giving up, defaults to 30 seconds.
* @returns
* What the plugin's entrypoint returned, or undefined with a
* warning logged if it did not load.
*/
waitFor<T = unknown>(name: string, timeoutMs?: number): Promise<T | undefined>;
/** Check if a plugin is installed, it may still be disabled */
checkInstalled(name: string): boolean;
/** Check if a plugin is being loaded alongside this one */
checkEnabled(name: string): boolean;
}
export type PluginLoadContext<
Context extends object,
Info extends PluginDeclaration = PluginNodeEnvInfo
> = Context & {
logger: Logger;
plugin: Info;
plugins: PluginRegistry;
};
/** Hooks for migrating data that Clusterio stores on behalf of plugins */
@@ -1,18 +1,14 @@
/**
* Helpers for loading plugins in Node.js
* @module lib/plugin_loader
* Helpers for finding installed plugins and reading their info in Node.js
* @module lib/plugin_info
*/
import path from "path";
import fs from "node:fs/promises";
import { fileURLToPath, pathToFileURL } from "node:url";
import * as libErrors from "./errors.js";
import * as libPlugin from "./plugin.js";
import { type Logger, logger } from "./logging.js";
import { loadPluginEntrypoint, loadPluginClass } from "./loadPlugin.js";
import { Config } from "./config/classes.js";
import { ControllerConfig, HostConfig, InstanceConfig } from "./config/definitions.js";
import Role from "./data/Role.js";
import { logger } from "./logging.js";
import { PluginDetails } from "./data/messages_plugin.js";
import {
discoverNpmPluginPackages, findNonRegistryDirs, installedDependencies, readPackageJson, realDir, resolvePackagePath,
@@ -196,82 +192,3 @@ export async function listPluginDetails(
return plugins;
}
/**
* Load the migration entrypoints of plugins
*
* Must be called before configs and roles are loaded for the migrations
* to run on them. Loaded for every plugin, including those disabled with
* load_plugin. Also registers the plugin versions written to each config
* after the migrations run.
*
* @param pluginInfos - Plugins to load migrations for.
*/
export async function loadPluginMigrations(pluginInfos: libPlugin.PluginNodeEnvInfo[]) {
const hooks: libPlugin.MigrationHooks = {
controllerConfig: ControllerConfig.pluginMigrations,
hostConfig: HostConfig.pluginMigrations,
instanceConfig: InstanceConfig.pluginMigrations,
role: Role.pluginMigrations,
};
for (const pluginInfo of pluginInfos) {
Config.pluginVersions.set(pluginInfo.name, pluginInfo.version);
if (!pluginInfo.migrationEntrypoint) {
continue;
}
try {
const module = await import(pathToFileURL(
path.join(path.dirname(pluginInfo.packagePath), pluginInfo.migrationEntrypoint)
).href);
const context: libPlugin.MigrationContext = {
hooks,
plugin: pluginInfo,
logger: logger.child({ plugin: pluginInfo.name }),
};
await loadPluginEntrypoint(pluginInfo, "migration", context, module);
} catch (err: any) {
throw new libErrors.PluginError(pluginInfo.name, err);
}
}
}
export async function loadPlugin<
Context extends { logger: Logger },
Class extends libPlugin.PluginClass<Context, libPlugin.PluginNodeEnvInfo>,
> (
pluginInfo: libPlugin.PluginNodeEnvInfo,
pluginType: libPlugin.PluginType,
context: Context,
exportName: `${string}Plugin`,
baseClass: Class,
) {
const entrypoint = `${pluginType}Entrypoint` as const;
const requirePath = pluginInfo[entrypoint];
if (!requirePath) {
return;
}
const module = await import(pathToFileURL(path.join(path.dirname(pluginInfo.packagePath), requirePath)).href);
const pluginContext: libPlugin.PluginLoadContext<Context> = {
...context,
plugin: pluginInfo,
logger: context.logger.child({ plugin: pluginInfo.name }),
};
if (typeof module.default === "function") {
await loadPluginEntrypoint(pluginInfo, pluginType, pluginContext, module);
return;
}
// migrate: accept plugins which export classes
if (module[exportName]) {
pluginContext.logger.warn(`Plugin ${pluginInfo.name} is using deprecated class export`);
await loadPluginClass(pluginInfo, pluginType, pluginContext, module, exportName, baseClass);
return;
}
throw new Error(`Plugin ${pluginInfo.name} must export either a default function or ${exportName} class`);
}
+2 -1
View File
@@ -25,7 +25,8 @@
"./src/logging.ts",
"./src/permissions.ts",
"./src/prometheus.ts",
"./src/loadPlugin.ts",
"./src/PluginLoader.ts",
"./src/import_plugin.browser.ts",
"./src/plugin.ts",
"./src/schema.ts",
"./src/subscriptions.ts",
+26 -39
View File
@@ -71,49 +71,36 @@ async function loadPlugins(
pluginInfos: lib.PluginWebEnvInfo[],
control: Control
) {
const plugins = new Map<string, WebPlugin.BaseWebPlugin>();
const { loaded, failed } = await lib.PluginLoader.load({
type: "web",
context: (pluginInfo: lib.PluginWebEnvInfo) => ({
control,
container: pluginInfo.container,
package: pluginInfo.package,
}),
logger,
installed: pluginInfos,
load: pluginInfos.filter(pluginInfo => pluginInfo.enabled && pluginInfo.webEntrypoint),
exportName: "WebPlugin",
baseClass: WebPlugin.BaseWebPlugin,
continueOnError: true,
});
for (const pluginInfo of pluginInfos) {
if (!pluginInfo.enabled || !pluginInfo.webEntrypoint) {
continue;
}
try {
const webModule = await pluginInfo.container.get(pluginInfo.webEntrypoint);
const pluginContext: WebPlugin.WebPluginContext = {
control,
plugin: pluginInfo,
container: pluginInfo.container,
package: pluginInfo.package,
logger: logger.child({ plugin: pluginInfo.name }),
};
if (typeof webModule.default === "function") {
await lib.loadPluginEntrypoint(pluginInfo, "web", pluginContext, webModule);
continue;
}
// migrate: accept plugins which export classes
if (webModule.WebPlugin) {
logger.warn(`Plugin ${pluginInfo.name} is using deprecated class hooks`);
plugins.set(pluginInfo.name, await lib.loadPluginClass(
pluginInfo, "web", pluginContext, webModule, "WebPlugin", WebPlugin.BaseWebPlugin
));
continue;
}
throw new Error(`Plugin ${pluginInfo.name} must export either a default function or WebPlugin`);
} catch (err: any) {
pluginInfo.error = `Error loading plugin: ${err.message}`;
logger.error(`Failed to load plugin ${pluginInfo.name}`);
if (err.stack) {
logger.error(err.stack);
}
for (const [pluginInfo, err] of failed) {
const cause = err instanceof lib.PluginError ? err.original : err;
pluginInfo.error = `Error loading plugin: ${cause.message}`;
logger.error(`Failed to load plugin ${pluginInfo.name}`);
if (cause.stack) {
logger.error(cause.stack);
}
}
const plugins = new Map<string, WebPlugin.BaseWebPlugin>();
for (const [pluginInfo, api] of loaded) {
if (api instanceof WebPlugin.BaseWebPlugin) {
plugins.set(pluginInfo.name, api);
}
}
return plugins;
}