Implement host plugins

Add ability to define plugin code that runs in the context of a host,
starts up with the host does, and stops when the host shuts down.

docs/writing-plugins.md

@@ -136,8 +136,8 @@ module.exports = {
 }
 ```
 
-For the instance plugin it's exactly the same except "Controller" is replaced with "Instance", and for the clusterioctl plugin "Ctl" is used.
-The available hooks that you can override are documented in the base class [in lib/plugin.js](/packages/lib/plugin.js).
+For the instance plugin it's exactly the same except "Controller" is replaced with "Instance", for host plugins "Host" is used and for the clusterioctl plugin "Ctl" is used.
+The available hooks that you can override are documented in the base classes.
 
 It's best to avoid defining a constructor, but if you insist on defining one, forward all arguments to the base class.
 E.g.:

packages/host/host.ts

@@ -203,7 +203,7 @@ async function startHost() {
 	}
 
 	let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos);
-	let host = new Host(hostConnector, hostConfig, tlsCa, pluginInfos);
+	let host = new Host(hostConnector, args.config, hostConfig, tlsCa, pluginInfos);
 
 	// Handle interrupts
 	let secondSigint = false;
@@ -237,6 +237,13 @@ async function startHost() {
 		host.shutdown();
 	});
 
+	try {
+		await host.loadPlugins();
+	} catch (err) {
+		await host.shutdown();
+		throw err;
+	}
+
 	hostConnector.once("connect", () => {
 		logger.add(new lib.LinkTransport({ link: host }));
 	});

packages/host/index.ts

@@ -1,4 +1,5 @@
 import { bootstrap } from "./host";
+export { default as BaseHostPlugin } from "./src/BaseHostPlugin";
 export { default as BaseInstancePlugin } from "./src/BaseInstancePlugin";
 
 if (module === require.main) {

packages/host/src/BaseHostPlugin.ts

@@ -0,0 +1,128 @@
+import type {
+	ConfigGroup, CollectorResult, Logger, ParsedFactorioOutput, PlayerEvent, PluginNodeEnvInfo,
+} from "@clusterio/lib";
+import type Host from "./Host";
+
+/**
+ * Base class for host plugins
+ *
+ * Host plugins are subclasses of this class which get instantiated by
+ * the host when it starts up with the plugin enabled in the config.  To be
+ * discovered the class must be exported under the name `HostPlugin` in the
+ * module specified by the `hostEntrypoint` in the plugin's info.js file.
+ */
+export default class BaseHostPlugin {
+	/**
+	 * Logger for this plugin
+	 *
+	 * Instance of winston Logger for sending log messages from this
+	 * plugin.  Supported methods and their corresponding log levels are
+	 * `error`, `warn`, `audit`, `info` and `verbose`.
+	 */
+	logger: Logger;
+
+	constructor(
+		/**
+		 * The plugin's own info module
+		 */
+		public info: PluginNodeEnvInfo,
+		/**
+		 * Host the plugin started for
+		 */
+		public host: Host,
+		logger: Logger,
+	) {
+		this.logger = logger.child({ plugin: this.info.name }) as unknown as Logger;
+	}
+
+	/**
+	 * Called immediately after the class is instantiated
+	 */
+	async init() { }
+
+	/**
+	 * Called when the value of a host config field changed.
+	 *
+	 * Invoked after the value of the config field given by `field` has
+	 * changed on this host.
+	 *
+	 * @param group -
+	 *     The group who's field got changed.
+	 * @param field - Name of the field that changed.
+	 * @param prev - The previous value of the field.
+	 */
+	async onHostConfigFieldChanged(group: ConfigGroup, field: string, prev: unknown) { }
+
+	/**
+	 * Called before collecting Prometheus metrics
+	 *
+	 * Invoked before the default metrics of prometheus is collected.  Note
+	 * that since this is done while replying to /metrics this should not
+	 * take a long time, otherwise the requst will time out.
+	 *
+	 * Although you can return the results of collecting your prometheus
+	 * collectors directly, all gauges in the default registry is already
+	 * automatically collected.  It's therefore recommended to only update
+	 * the Gauges/Counters/etc in onMetric and let Clusterio deal will
+	 * collecting the values.
+	 *
+	 * @returns an async iterator of prometheus metric results or undefined.
+	 */
+	async onMetrics(): Promise<void | AsyncIterator<CollectorResult>> { }
+
+	/**
+	 * Called when the host is shutting down
+	 */
+	async onShutdown() { }
+
+	/**
+	 * Called when an event on the controller connection happens
+	 *
+	 * The event param may be one of connect, drop, resume and close and has
+	 * the following meaning:
+	 *
+	 * ##### connect
+	 *
+	 * Invoked when a new connection to the controller has been established.
+	 *
+	 * ##### drop
+	 *
+	 * Invoked when a connection loss is detected between the host and the
+	 * controller.  Plugins should respond to this event by throtteling messages
+	 * it is sending to the controller to an absolute minimum.
+	 *
+	 * Messages sent over a dropped controller connection will get queued up in
+	 * memory on the host and sent all in one go when the connection is
+	 * re-established again.
+	 *
+	 * ##### resume
+	 *
+	 * Invoked when the connection that had previously dropped is
+	 * re-established.
+	 *
+	 * ##### close
+	 *
+	 * Invoked when the connection to the controller has been closed.  This
+	 * typically means the controller has shut down.  Plugins should not
+	 * send any messages that goes to or via the controller after the
+	 * connection has been closed and before a new one is established.
+	 *
+	 * @param event - one of connect, drop, resume and close
+	 */
+	onControllerConnectionEvent(event: "connect" | "drop" | "resume" | "close") { }
+
+	/**
+	 * Called when the controller is preparing to disconnect from the host
+	 *
+	 * Invoked when the controller has requested the disconnection from
+	 * the host.  This typically happens when it is in the process of
+	 * shutting down.
+	 *
+	 * Plugins must stop sending messages to the controller, or are forwarded
+	 * via the controller after the prepare disconnect has been handled.
+	 *
+	 * @param connection -
+	 *     The connection to the host preparing to disconnect.
+	 */
+	async onPrepareControllerDisconnect(connection: Host) { }
+}

packages/host/src/Host.ts

@@ -14,6 +14,7 @@ import { logger } from "@clusterio/lib";
 import type { HostConnector } from "../host";
 import Instance from "./Instance";
 import InstanceConnection from "./InstanceConnection";
+import BaseHostPlugin from "./BaseHostPlugin";
 
 const finished = util.promisify(stream.finished);
 
@@ -241,8 +242,12 @@ export default class Host extends lib.Link {
 	 */
 	tlsCa?: string;
 	pluginInfos: lib.PluginNodeEnvInfo[];
+	configPath: string;
 	config: lib.HostConfig;
 
+	/** Mapping of plugin name to loaded plugin */
+	plugins: Map<string, BaseHostPlugin> = new Map();
+
 	instanceConnections = new Map<number, InstanceConnection>();
 	discoveredInstanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
 	instanceInfos = new Map<number, { path: string, config: lib.InstanceConfig }>();
@@ -260,6 +265,7 @@ export default class Host extends lib.Link {
 
 	constructor(
 		connector: HostConnector,
+		hostConfigPath: string,
 		hostConfig: lib.HostConfig,
 		tlsCa: string | undefined,
 		pluginInfos: lib.PluginNodeEnvInfo[]
@@ -268,8 +274,13 @@ export default class Host extends lib.Link {
 		this.tlsCa = tlsCa;
 
 		this.pluginInfos = pluginInfos;
+		this.configPath = hostConfigPath;
 		this.config = hostConfig;
 
+		this.config.on("fieldChanged", (group, field, prev) => {
+			lib.invokeHook(this.plugins, "onHostConfigFieldChanged", group, field, prev);
+		});
+
 		this.connector.on("hello", data => {
 			this.serverVersion = data.version;
 			this.serverPlugins = new Map(Object.entries(data.plugins));
@@ -308,12 +319,15 @@ export default class Host extends lib.Link {
 			}
 		});
 
-		for (let event of ["connect", "drop", "resume", "close"]) {
+		for (let event of ["connect", "drop", "resume", "close"] as const) {
 			this.connector.on(event, () => {
 				let message = new lib.ControllerConnectionEvent(event);
 				for (let instanceConnection of this.instanceConnections.values()) {
 					instanceConnection.send(message);
 				}
+				for (let plugin of this.plugins.values()) {
+					plugin.onControllerConnectionEvent(event);
+				}
 			});
 		}
 
@@ -334,6 +348,35 @@ export default class Host extends lib.Link {
 		this.handle(lib.InstanceDeleteInternalRequest, this.handleInstanceDeleteInternalRequest.bind(this));
 	}
 
+	async loadPlugins() {
+		for (let pluginInfo of this.pluginInfos) {
+			if (!this.config.group(pluginInfo.name).get("load_plugin")) {
+				continue;
+			}
+
+			let HostPluginClass = BaseHostPlugin;
+			try {
+				if (pluginInfo.hostEntrypoint) {
+					HostPluginClass = await lib.loadPluginClass(
+						pluginInfo.name,
+						path.posix.join(pluginInfo.requirePath, pluginInfo.hostEntrypoint),
+						"HostPlugin",
+						BaseHostPlugin,
+					);
+				}
+
+				let hostPlugin = new HostPluginClass(pluginInfo, this, logger);
+				await hostPlugin.init();
+				this.plugins.set(pluginInfo.name, hostPlugin);
+
+			} catch (err: any) {
+				throw new lib.PluginError(pluginInfo.name, err);
+			}
+
+			logger.info(`Loaded plugin ${pluginInfo.name}`);
+		}
+	}
+
 	async _createNewInstanceDir(name: string) {
 		name = lib.cleanFilename(name);
 		try {
@@ -551,6 +594,14 @@ export default class Host extends lib.Link {
 		}
 
 		let results = [];
+		type ResultIterator = AsyncIterable<lib.CollectorResult> | Iterable<lib.CollectorResult>
+		let pluginResults = await lib.invokeHook(this.plugins, "onMetrics") as ResultIterator[];
+		for (let metricIterator of pluginResults) {
+			for await (let metric of metricIterator) {
+				results.push(metric);
+			}
+		}
+
 		for (let response of await Promise.all(requests)) {
 			results.push(...response.results);
 		}
@@ -808,6 +859,7 @@ export default class Host extends lib.Link {
 	}
 
 	async prepareDisconnect() {
+		await lib.invokeHook(this.plugins, "onPrepareControllerDisconnect");
 		for (let instanceConnection of this.instanceConnections.values()) {
 			await instanceConnection.send(new lib.PrepareControllerDisconnectRequest());
 		}
@@ -824,6 +876,8 @@ export default class Host extends lib.Link {
 		}
 		this._shuttingDown = true;
 
+		await lib.invokeHook(this.plugins, "onShutdown");
+
 		for (let instanceConnection of this.instanceConnections.values()) {
 			try {
 				await instanceConnection.instance.stop();
@@ -840,6 +894,9 @@ export default class Host extends lib.Link {
 			}
 		}
 
+		logger.info("Saving config");
+		await lib.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, "\t"));
+
 		try {
 			// Clear silly interval in pidfile library.
 			pidusage.clear();

packages/lib/src/config/definitions.ts

@@ -505,7 +505,7 @@ ControlConfig.registerGroup(ControlGroup);
 
 function validateGroup(
 	pluginInfo: PluginNodeEnvInfo | PluginWebpackEnvInfo,
-	groupName: "ControllerConfigGroup"|"InstanceConfigGroup",
+	groupName: "ControllerConfigGroup" | "HostConfigGroup" | "InstanceConfigGroup",
 ) {
 	if (!(pluginInfo[groupName]?.prototype instanceof classes.PluginConfigGroup)) {
 		throw new Error(
@@ -525,32 +525,31 @@ function validateGroup(
  *
  * @param {Array<Object>} pluginInfos - Array of plugin info objects.
  */
-export function registerPluginConfigGroups(pluginInfos: PluginNodeEnvInfo[]|PluginWebpackEnvInfo[]) {
-	for (let pluginInfo of pluginInfos) {
-		if (pluginInfo.ControllerConfigGroup) {
-			validateGroup(pluginInfo, "ControllerConfigGroup");
-			ControllerConfig.registerGroup(pluginInfo.ControllerConfigGroup);
+export function registerPluginConfigGroups(pluginInfos: PluginNodeEnvInfo[] | PluginWebpackEnvInfo[]) {
+	function pluginConfig(
+		pluginInfo: PluginNodeEnvInfo | PluginWebpackEnvInfo,
+		group: "ControllerConfigGroup" | "HostConfigGroup" | "InstanceConfigGroup",
+		Config: typeof classes.Config,
+	) {
+		if (pluginInfo[group]) {
+			validateGroup(pluginInfo, group);
+			Config.registerGroup(pluginInfo[group]!);
 
 		} else {
-			class ControllerConfigGroup extends classes.PluginConfigGroup { }
-			ControllerConfigGroup.defaultAccess = ["controller", "host", "control"];
-			ControllerConfigGroup.groupName = pluginInfo.name;
-			ControllerConfigGroup.finalize();
-			ControllerConfig.registerGroup(ControllerConfigGroup);
+			class ConfigGroup extends classes.PluginConfigGroup { }
+			ConfigGroup.defaultAccess = ["controller", "host", "control"];
+			ConfigGroup.groupName = pluginInfo.name;
+			ConfigGroup.finalize();
+			Config.registerGroup(ConfigGroup);
+		}
+	}
+	for (let pluginInfo of pluginInfos) {
+		pluginConfig(pluginInfo, "ControllerConfigGroup", ControllerConfig);
+		if (pluginInfo.hostEntrypoint || pluginInfo.instanceEntrypoint) {
+			pluginConfig(pluginInfo, "HostConfigGroup", HostConfig);
 		}
-
 		if (pluginInfo.instanceEntrypoint) {
-			if (pluginInfo.InstanceConfigGroup) {
-				validateGroup(pluginInfo, "InstanceConfigGroup");
-				InstanceConfig.registerGroup(pluginInfo.InstanceConfigGroup);
-
-			} else {
-				class InstanceConfigGroup extends classes.PluginConfigGroup { }
-				InstanceConfigGroup.defaultAccess = ["controller", "host", "control"];
-				InstanceConfigGroup.groupName = pluginInfo.name;
-				InstanceConfigGroup.finalize();
-				InstanceConfig.registerGroup(InstanceConfigGroup);
-			}
+			pluginConfig(pluginInfo, "InstanceConfigGroup", InstanceConfig);
 		}
 	}
 }

packages/lib/src/plugin.ts

@@ -12,12 +12,15 @@ export type PluginDeclaration = {
 	title: string;
 	description?: string;
 
-	instanceEntrypoint?: string;
-	InstanceConfigGroup?: typeof PluginConfigGroup;
-
 	controllerEntrypoint?: string;
 	ControllerConfigGroup?: typeof PluginConfigGroup;
 
+	hostEntrypoint?: string;
+	HostConfigGroup?: typeof PluginConfigGroup;
+
+	instanceEntrypoint?: string;
+	InstanceConfigGroup?: typeof PluginConfigGroup;
+
 	ctlEntrypoint?: string;
 	webEntrypoint?: string;
 	ControlConfigGroup?: typeof PluginConfigGroup;

test/host/BaseHostPlugin.js

@@ -0,0 +1,22 @@
+"use strict";
+const assert = require("assert").strict;
+const mock = require("../mock");
+
+const host = require("@clusterio/host");
+
+describe("host/src/BaseHostPlugin", function() {
+	describe("class BaseHostPlugin", function() {
+		let hostPlugin;
+		it("should be constructible", async function() {
+			hostPlugin = new host.BaseHostPlugin({}, new mock.MockHost(), new mock.MockLogger());
+			await hostPlugin.init();
+		});
+		it("should define defaults for hooks", async function() {
+			await hostPlugin.onHostConfigFieldChanged();
+			await hostPlugin.onMetrics();
+			await hostPlugin.onShutdown();
+			hostPlugin.onControllerConnectionEvent("connect");
+			await hostPlugin.onPrepareControllerDisconnect();
+		});
+	});
+});