mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-18 15:33:12 +09:00
Let plugins migrate the data core stores for them
Plugins had no way to update the config fields and permissions core stores for them, so renaming inventory_sync needed migrations in core packages. A plugin can now declare a migrationEntrypoint. It's loaded on the controller and hosts before configs and roles are read, for every plugin in the plugin list, and attaches handlers to sync hooks for the controller, host and instance configs and roles. lib gets helpers to rename config fields and permissions, and each config gets a hidden <plugin>.version field set after load so migrations can tell which version saved the data. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
24586a4b7c
commit
aec0fc7a1e
19 files changed
+562
-7
No files matched your search
@@ -419,6 +419,9 @@ async function initialize(): Promise<InitializeParameters> {
|
||||
lib.registerPluginMessages(pluginInfos);
|
||||
lib.registerPluginPermissions(pluginInfos);
|
||||
lib.addPluginConfigFields(pluginInfos);
|
||||
if (!args.recovery) {
|
||||
await lib.loadPluginMigrations(pluginInfos);
|
||||
}
|
||||
|
||||
const [controllerConfig, controllerConfigLock] = await lib.loadConfigFromArgs(
|
||||
args,
|
||||
@@ -427,6 +430,9 @@ async function initialize(): Promise<InitializeParameters> {
|
||||
);
|
||||
|
||||
controllerConfig.set("controller.version", packageConfig.version); // Allows tracking last loaded version
|
||||
if (!args.recovery) {
|
||||
lib.setPluginVersions(controllerConfig, pluginInfos);
|
||||
}
|
||||
|
||||
if (!controllerConfig.get("controller.auth_secret")) {
|
||||
logger.info("Generating new controller authentication secret");
|
||||
@@ -477,7 +483,7 @@ async function startup() {
|
||||
controllerConfig,
|
||||
Boolean(args.canRestart),
|
||||
Boolean(args.recovery),
|
||||
...await Controller.bootstrap(controllerConfig)
|
||||
...await Controller.bootstrap(controllerConfig, args.recovery ? [] : pluginInfos)
|
||||
);
|
||||
|
||||
// Refuse to start if there are no users loaded
|
||||
|
||||
@@ -148,7 +148,15 @@ export default class Controller {
|
||||
}
|
||||
});
|
||||
|
||||
static async bootstrap(config: lib.ControllerConfig) {
|
||||
/**
|
||||
* Load the controller datastores
|
||||
*
|
||||
* @param config - Controller config.
|
||||
* @param pluginInfos -
|
||||
* Plugins whose migrations ran, their versions are saved to the
|
||||
* instance configs.
|
||||
*/
|
||||
static async bootstrap(config: lib.ControllerConfig, pluginInfos: lib.PluginNodeEnvInfo[] = []) {
|
||||
let databaseDirectory = config.get("controller.database_directory");
|
||||
await fs.mkdir(databaseDirectory, { recursive: true });
|
||||
|
||||
@@ -175,6 +183,11 @@ export default class Controller {
|
||||
InstanceRecord.fromJSON.bind(InstanceRecord),
|
||||
this.migrateInstances, this.finaliseInstances,
|
||||
).bootstrap());
|
||||
for (const instance of instances.values()) {
|
||||
if (lib.setPluginVersions(instance.config, pluginInfos)) {
|
||||
instances.set(instance);
|
||||
}
|
||||
}
|
||||
|
||||
const saves = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
|
||||
path.join(databaseDirectory, "saves.json"),
|
||||
@@ -822,7 +835,7 @@ export default class Controller {
|
||||
}
|
||||
return permission;
|
||||
});
|
||||
return json;
|
||||
return lib.Role.pluginMigrations.reduce(json);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -177,6 +177,9 @@ async function startHost() {
|
||||
lib.registerPluginMessages(pluginInfos);
|
||||
lib.registerPluginPermissions(pluginInfos);
|
||||
lib.addPluginConfigFields(pluginInfos);
|
||||
if (!args.recovery) {
|
||||
await lib.loadPluginMigrations(pluginInfos);
|
||||
}
|
||||
|
||||
const [hostConfig, hostConfigLock] = await lib.loadConfigFromArgs(
|
||||
args,
|
||||
@@ -185,6 +188,9 @@ async function startHost() {
|
||||
);
|
||||
|
||||
hostConfig.set("host.version", packageConfig.version); // Allows tracking last loaded version
|
||||
if (!args.recovery) {
|
||||
lib.setPluginVersions(hostConfig, pluginInfos);
|
||||
}
|
||||
|
||||
if (command === "config") {
|
||||
await lib.handleConfigCommand(args, hostConfig, hostConfigLock);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
export * from "./src/api";
|
||||
export * from "./src/AsyncHook";
|
||||
export * from "./src/SyncHook";
|
||||
export * from "./src/config";
|
||||
export * from "./src/data";
|
||||
export * from "./src/errors";
|
||||
|
||||
@@ -37,6 +37,8 @@ export * from "./src/load_plugin_list.js";
|
||||
export * from "./src/rce_ops.js";
|
||||
export * from "./src/LockFile.js";
|
||||
export * from "./src/AsyncHook.js";
|
||||
export * from "./src/SyncHook.js";
|
||||
export * from "./src/migrations.js";
|
||||
|
||||
export { default as ExponentialBackoff } from "./src/ExponentialBackoff.js";
|
||||
export { default as ModStore } from "./src/ModStore.js";
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { PluginError } from "./errors.js";
|
||||
|
||||
export type SyncHookHandler<Value> = (value: Value) => Value;
|
||||
|
||||
/**
|
||||
* Hook that passes a value through each attached handler in turn.
|
||||
*
|
||||
* Handlers run synchronously in the order they were attached, for places
|
||||
* that can't wait on a promise such as the Config constructor.
|
||||
*/
|
||||
export class SyncHook<Value> {
|
||||
private _handlers = new Map<string, SyncHookHandler<Value>>();
|
||||
|
||||
get size() {
|
||||
return this._handlers.size;
|
||||
}
|
||||
|
||||
get attached() {
|
||||
return this._handlers.keys();
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach a handler to this hook.
|
||||
*
|
||||
* An error is thrown if another handler with the same name is already attached.
|
||||
*/
|
||||
attach(name: string, handler: SyncHookHandler<Value>) {
|
||||
if (this._handlers.has(name)) {
|
||||
throw new Error(`Handler with name ${name} is already attached`);
|
||||
}
|
||||
this._handlers.set(name, handler);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a previously attached handler.
|
||||
*
|
||||
* If no handler exists, then no action is performed.
|
||||
*/
|
||||
detach(name: string) {
|
||||
this._handlers.delete(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove all attached handlers.
|
||||
*/
|
||||
clear() {
|
||||
this._handlers.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass a value through all attached handlers.
|
||||
*
|
||||
* Each handler gets the value returned by the one before it. Errors are
|
||||
* rethrown as a PluginError named after the handler that threw.
|
||||
*
|
||||
* @param value - Value given to the first handler.
|
||||
* @returns Value returned by the last handler.
|
||||
*/
|
||||
reduce(value: Value) {
|
||||
for (const [name, handler] of this._handlers) {
|
||||
try {
|
||||
value = handler(value);
|
||||
} catch (err: any) {
|
||||
throw new PluginError(name, err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ import { StringEnum } from "../data/composites.js";
|
||||
import { safeOutputFile } from "../file_ops.js";
|
||||
import * as validators from "./validators.js";
|
||||
import { RequestError } from "../errors.js";
|
||||
import type { SyncHook } from "../SyncHook.js";
|
||||
|
||||
const ConfigLocation = StringEnum(["controller", "host", "control"]);
|
||||
export type ConfigLocation = Static<typeof ConfigLocation>;
|
||||
@@ -224,6 +225,12 @@ export class Config<
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Migrations attached by plugins, run after {@link Config.migrations}.
|
||||
* Declared by each subclass that plugins can add fields to.
|
||||
*/
|
||||
declare static pluginMigrations?: SyncHook<ConfigSchema>;
|
||||
|
||||
/** Defines the default access locations for the config class */
|
||||
static defaultAccess: ConfigLocation[] = ["controller", "host", "control"];
|
||||
|
||||
@@ -269,7 +276,11 @@ export class Config<
|
||||
) as Fields;
|
||||
|
||||
if (fields) {
|
||||
this.update(this.constructor.migrations(fields), false, location);
|
||||
let migrated = this.constructor.migrations(fields);
|
||||
if (this.constructor.pluginMigrations) {
|
||||
migrated = this.constructor.pluginMigrations.reduce(migrated);
|
||||
}
|
||||
this.update(migrated, false, location);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import * as validators from "./validators.js";
|
||||
import type { PluginNodeEnvInfo, PluginWebpackEnvInfo } from "../plugin.js";
|
||||
import { Static } from "@sinclair/typebox";
|
||||
import { isTargetVersion } from "../data/version.js";
|
||||
import { SyncHook } from "../SyncHook.js";
|
||||
|
||||
type configFromJSON<T> = (...args: Parameters<typeof classes.Config.fromJSON>) => T;
|
||||
type configFromFile<T> = (...args: Parameters<typeof classes.Config.fromFile>) => Promise<T>;
|
||||
@@ -42,6 +43,7 @@ export interface ControllerConfigFields {
|
||||
"controller.allow_plugin_updates": boolean;
|
||||
"controller.allow_plugin_install": boolean;
|
||||
[key: `${string}.load_plugin`]: boolean;
|
||||
[key: `${string}.version`]: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -51,6 +53,7 @@ export interface ControllerConfigFields {
|
||||
export class ControllerConfig extends classes.Config<ControllerConfigFields> {
|
||||
declare static fromJSON: configFromJSON<ControllerConfig>;
|
||||
declare static fromFile: configFromFile<ControllerConfig>;
|
||||
static pluginMigrations = new SyncHook<classes.ConfigSchema>();
|
||||
static migrations(config: Static<typeof this.jsonSchema>) {
|
||||
if (config.hasOwnProperty("controller.external_address")) {
|
||||
config["controller.public_url"] = config["controller.external_address"];
|
||||
@@ -334,6 +337,7 @@ export interface HostConfigFields {
|
||||
"host.allow_plugin_updates": boolean;
|
||||
"host.allow_plugin_install": boolean;
|
||||
[key: `${string}.load_plugin`]: boolean;
|
||||
[key: `${string}.version`]: string;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -343,6 +347,7 @@ export interface HostConfigFields {
|
||||
export class HostConfig extends classes.Config<HostConfigFields> {
|
||||
declare static fromJSON: configFromJSON<HostConfig>;
|
||||
declare static fromFile: configFromFile<HostConfig>;
|
||||
static pluginMigrations = new SyncHook<classes.ConfigSchema>();
|
||||
static fieldDefinitions: classes.ConfigDefs<HostConfigFields> = {
|
||||
"host.version": {
|
||||
type: "string",
|
||||
@@ -468,6 +473,7 @@ export interface InstanceConfigFields {
|
||||
"instance.auto_start": boolean;
|
||||
"instance.exclude_from_start_all": boolean;
|
||||
[key: `${string}.load_plugin`]: boolean;
|
||||
[key: `${string}.version`]: string;
|
||||
|
||||
"factorio.version": string;
|
||||
"factorio.executable_path": string | null;
|
||||
@@ -500,6 +506,7 @@ export interface InstanceConfigFields {
|
||||
export class InstanceConfig extends classes.Config<InstanceConfigFields> {
|
||||
declare static fromJSON: configFromJSON<InstanceConfig>;
|
||||
declare static fromFile: configFromFile<InstanceConfig>;
|
||||
static pluginMigrations = new SyncHook<classes.ConfigSchema>();
|
||||
static migrations(config: Static<typeof this.jsonSchema>) {
|
||||
function boolToEnableDisable(name: string) {
|
||||
if (config.hasOwnProperty(name) && typeof config[name] === "boolean") {
|
||||
@@ -805,6 +812,14 @@ export function addPluginFieldDefinitions(
|
||||
|
||||
Config.fieldDefinitions[`${pluginInfo.name}.load_plugin`] = fieldDef;
|
||||
|
||||
// Version of the plugin the config was last loaded with, see setPluginVersions
|
||||
Config.fieldDefinitions[`${pluginInfo.name}.version`] = {
|
||||
type: "string",
|
||||
initialValue: "version" in pluginInfo ? pluginInfo.version : "",
|
||||
readonly: ["controller", "host"],
|
||||
hidden: true,
|
||||
};
|
||||
|
||||
// Merge in other fields from the plugin
|
||||
const fields = pluginInfo[kind];
|
||||
if (fields) {
|
||||
@@ -829,3 +844,25 @@ export function addPluginConfigFields(pluginInfos: PluginNodeEnvInfo[] | PluginW
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the version field of each plugin to the version that was loaded
|
||||
*
|
||||
* Done after loading a config so the plugin migrations can compare the
|
||||
* stored version against the one now running.
|
||||
*
|
||||
* @param config - Config to update.
|
||||
* @param pluginInfos - Plugins loaded.
|
||||
* @returns true if any version changed.
|
||||
*/
|
||||
export function setPluginVersions(config: classes.Config<any>, pluginInfos: PluginNodeEnvInfo[]) {
|
||||
let changed = false;
|
||||
for (const pluginInfo of pluginInfos) {
|
||||
const name = `${pluginInfo.name}.version`;
|
||||
if (config.constructor.fieldDefinitions[name] && config.get(name) !== pluginInfo.version) {
|
||||
config.set(name, pluginInfo.version);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
return changed;
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Static, Type } from "@sinclair/typebox";
|
||||
import { permissions as globalPermissions } from "../permissions.js";
|
||||
import { SyncHook } from "../SyncHook.js";
|
||||
|
||||
interface RoleCollection {
|
||||
get(id: number): Role;
|
||||
@@ -31,6 +32,9 @@ export default class Role {
|
||||
is_deleted: Type.Optional(Type.Boolean()),
|
||||
});
|
||||
|
||||
/** Migrations attached by plugins, run on roles loaded by the controller */
|
||||
static pluginMigrations = new SyncHook<Static<typeof Role.jsonSchema>>();
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
return new this(
|
||||
json.id,
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
/**
|
||||
* Helpers for plugin migrations
|
||||
* @module lib/migrations
|
||||
*/
|
||||
import type { Static } from "@sinclair/typebox";
|
||||
import type { ConfigSchema } from "./config/classes.js";
|
||||
import type Role from "./data/Role.js";
|
||||
|
||||
type SerializedRole = Static<typeof Role.jsonSchema>;
|
||||
|
||||
/**
|
||||
* Rename a field in a serialized config
|
||||
*
|
||||
* The value is moved if the old field exists and the new one doesn't. The
|
||||
* old field is removed either way.
|
||||
*
|
||||
* @param config - Serialized config to update.
|
||||
* @param from - Current name of the field.
|
||||
* @param to - New name of the field.
|
||||
* @returns The config passed in.
|
||||
*/
|
||||
export function renameConfigField(config: ConfigSchema, from: string, to: string) {
|
||||
if (Object.hasOwn(config, from)) {
|
||||
if (!Object.hasOwn(config, to)) {
|
||||
config[to] = config[from];
|
||||
}
|
||||
delete config[from];
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rename all fields starting with a prefix in a serialized config
|
||||
*
|
||||
* @example
|
||||
* lib.renameConfigPrefix(config, "inventory_sync.", "player_sync.");
|
||||
*
|
||||
* @param config - Serialized config to update.
|
||||
* @param from - Current prefix of the fields.
|
||||
* @param to - New prefix of the fields.
|
||||
* @returns The config passed in.
|
||||
*/
|
||||
export function renameConfigPrefix(config: ConfigSchema, from: string, to: string) {
|
||||
for (const name of Object.keys(config)) {
|
||||
if (name.startsWith(from)) {
|
||||
renameConfigField(config, name, to + name.slice(from.length));
|
||||
}
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rename a permission in a serialized role
|
||||
*
|
||||
* @param role - Serialized role to update.
|
||||
* @param from - Current name of the permission.
|
||||
* @param to - New name of the permission.
|
||||
* @returns The role passed in.
|
||||
*/
|
||||
export function renamePermission(role: SerializedRole, from: string, to: string) {
|
||||
return renamePermissionsWith(role, name => (name === from ? to : name));
|
||||
}
|
||||
|
||||
/**
|
||||
* Rename all permissions starting with a prefix in a serialized role
|
||||
*
|
||||
* @param role - Serialized role to update.
|
||||
* @param from - Current prefix of the permissions.
|
||||
* @param to - New prefix of the permissions.
|
||||
* @returns The role passed in.
|
||||
*/
|
||||
export function renamePermissionPrefix(role: SerializedRole, from: string, to: string) {
|
||||
return renamePermissionsWith(role, name => (name.startsWith(from) ? to + name.slice(from.length) : name));
|
||||
}
|
||||
|
||||
function renamePermissionsWith(role: SerializedRole, rename: (name: string) => string) {
|
||||
role.permissions = [...new Set(role.permissions.map(rename))];
|
||||
return role;
|
||||
}
|
||||
@@ -2,10 +2,12 @@
|
||||
* Plugin interfaces and utilities.
|
||||
* @module lib/plugin
|
||||
*/
|
||||
import type { Static } from "@sinclair/typebox";
|
||||
import type { Logger } from "./logging.js";
|
||||
import type { FieldDefinition } from "./config/index.js";
|
||||
import type { ConfigSchema, FieldDefinition } from "./config/index.js";
|
||||
import type { PermissionDefinition } from "./permissions.js";
|
||||
import type { PlayerStats } from "./data/index.js";
|
||||
import type { PlayerStats, Role } from "./data/index.js";
|
||||
import type { SyncHook } from "./SyncHook.js";
|
||||
|
||||
export const PluginFeatureFlags = [
|
||||
/** The plugin requires module code to be patched into the save */
|
||||
@@ -33,6 +35,8 @@ export type PluginDeclaration = {
|
||||
webEntrypoint?: string;
|
||||
ctlConfigFields?: Record<string, FieldDefinition>;
|
||||
|
||||
migrationEntrypoint?: string;
|
||||
|
||||
features?: (typeof PluginFeatureFlags)[number][];
|
||||
|
||||
messages?: any[];
|
||||
@@ -114,6 +118,16 @@ export type PluginLoadContext<
|
||||
plugin: Info;
|
||||
};
|
||||
|
||||
/** Hooks for migrating data that Clusterio stores on behalf of plugins */
|
||||
export type MigrationHooks = {
|
||||
controllerConfig: SyncHook<ConfigSchema>;
|
||||
hostConfig: SyncHook<ConfigSchema>;
|
||||
instanceConfig: SyncHook<ConfigSchema>;
|
||||
role: SyncHook<Static<typeof Role.jsonSchema>>;
|
||||
};
|
||||
|
||||
export type MigrationContext = PluginLoadContext<{ hooks: MigrationHooks }>;
|
||||
|
||||
export type PluginClass<
|
||||
Context extends object,
|
||||
Info extends PluginDeclaration,
|
||||
|
||||
@@ -10,6 +10,8 @@ import * as libErrors from "./errors.js";
|
||||
import * as libPlugin from "./plugin.js";
|
||||
import { type Logger, logger } from "./logging.js";
|
||||
import { loadPluginEntrypoint, loadPluginClass } from "./loadPlugin.js";
|
||||
import { ControllerConfig, HostConfig, InstanceConfig } from "./config/definitions.js";
|
||||
import Role from "./data/Role.js";
|
||||
|
||||
|
||||
/**
|
||||
@@ -90,6 +92,44 @@ export async function loadPluginInfos(pluginList: Map<string, string>) {
|
||||
return plugins;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the migration entrypoints of plugins
|
||||
*
|
||||
* Must be called before configs and roles are loaded for the migrations
|
||||
* to run on them. Loaded for every plugin, including those disabled with
|
||||
* load_plugin.
|
||||
*
|
||||
* @param pluginInfos - Plugins to load migrations for.
|
||||
*/
|
||||
export async function loadPluginMigrations(pluginInfos: libPlugin.PluginNodeEnvInfo[]) {
|
||||
const hooks: libPlugin.MigrationHooks = {
|
||||
controllerConfig: ControllerConfig.pluginMigrations,
|
||||
hostConfig: HostConfig.pluginMigrations,
|
||||
instanceConfig: InstanceConfig.pluginMigrations,
|
||||
role: Role.pluginMigrations,
|
||||
};
|
||||
|
||||
for (const pluginInfo of pluginInfos) {
|
||||
if (!pluginInfo.migrationEntrypoint) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const module = await import(pathToFileURL(
|
||||
path.join(path.dirname(pluginInfo.packagePath), pluginInfo.migrationEntrypoint)
|
||||
).href);
|
||||
const context: libPlugin.MigrationContext = {
|
||||
hooks,
|
||||
plugin: pluginInfo,
|
||||
logger: logger.child({ plugin: pluginInfo.name }),
|
||||
};
|
||||
await loadPluginEntrypoint(pluginInfo, "migration", context, module);
|
||||
} catch (err: any) {
|
||||
throw new libErrors.PluginError(pluginInfo.name, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function loadPlugin<
|
||||
Context extends { logger: Logger },
|
||||
Class extends libPlugin.PluginClass<Context, libPlugin.PluginNodeEnvInfo>,
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
"package.json",
|
||||
"./src/api.ts",
|
||||
"./src/AsyncHook.ts",
|
||||
"./src/SyncHook.ts",
|
||||
"./src/config/*.ts",
|
||||
"./src/data/*.ts",
|
||||
"./src/external/*.ts",
|
||||
|
||||
Reference in new issue
Block a user