Merge pull request #1010 from bbassie/feature/plugin-browser

Plugin management through web interface
This commit is contained in:
Bastiaan authored and GitHub committed 2026-10-27 21:32:51 +00:00
1 parent e36a76acc5
commit ffe82f81d8
48 files changed
+4275 -536

No files matched your search

+142 -31
View File
@@ -1,5 +1,5 @@
import { FileHandle, open, unlink, readFile } from "node:fs/promises";
import { unlinkSync } from "node:fs";
import fs, { type FileHandle } from "node:fs/promises";
import { readFileSync, unlinkSync } from "node:fs";
import assert from "assert";
/**
@@ -24,23 +24,99 @@ export class LockFileBusyError extends Error {
}
}
/**
* Error thrown when a process exited while clearing a stale lock, which needs the recovery file removed by hand.
*/
export class LockFileRecoveryError extends Error {
constructor(
readonly recoveryPath: string
) {
super(`Lock file recovery was interrupted, remove ${recoveryPath} to allow the lock to be acquired`);
this.name = "LockFileRecoveryError";
}
}
type LockState = "idle" | "acquiring" | "acquired" | "releasing";
type LockSnapshot = { content: string, ino: bigint, mtimeNs: bigint };
/**
* Cross-platform lock file utility for Node.js.
*/
export class LockFile {
/**
* Content written to the lock file. The start time tells this process apart from an earlier one with
* the same pid, and is the same for every copy of lib loaded in this process.
*/
private static readonly content = `${process.pid} ${performance.timeOrigin}\n`;
// A lock file without a valid pid is left alone for this long, as it may still be being written
private static readonly writeGraceMs = 10e3;
private handle: FileHandle | null = null;
private state: LockState = "idle";
/**
* @param filePath - Path of the lock file
* @param handleSignals -
* Release the lock on SIGINT, SIGTERM, SIGHUP and uncaught exceptions. Listening to these
* changes how the process exits, so without it the lock is only removed on exit.
*/
constructor(
private readonly filePath: string
private readonly filePath: string,
private readonly handleSignals = true,
) {}
/** Content and identity of a lock file, read through one handle so both belong to the same file */
private static async readLock(filePath: string): Promise<LockSnapshot | undefined> {
let handle;
try {
handle = await fs.open(filePath, "r");
} catch (err: any) {
if (err.code === "ENOENT") {
return undefined;
}
throw err;
}
try {
const stat = await handle.stat({ bigint: true });
return { content: await handle.readFile("utf8"), ino: stat.ino, mtimeNs: stat.mtimeNs };
} finally {
await handle.close();
}
}
private static isSameLock(a: LockSnapshot, b: LockSnapshot) {
return a.content === b.content && a.ino === b.ino && a.mtimeNs === b.mtimeNs;
}
private static lockPid(lock: LockSnapshot) {
const pid = Number(lock.content.split(" ")[0]);
return Number.isInteger(pid) && pid > 0 ? pid : undefined;
}
/** True if the lock names a process that is still running, or may still be written to */
private static lockIsLive(lock: LockSnapshot) {
const pid = LockFile.lockPid(lock);
if (pid === undefined) {
return Date.now() - Number(lock.mtimeNs / 1000000n) < LockFile.writeGraceMs;
}
if (pid === process.pid) {
// Held by this process, or left by an earlier process with the same pid
return lock.content === LockFile.content;
}
try {
// Signal 0 only checks that the process exists
process.kill(pid, 0);
return true;
} catch (err: any) {
return err.code !== "ESRCH";
}
}
/**
* Acquire the lock file.
* Throws LockFileExistsError if already locked.
* Throws LockFileBusyError if not in idle state.
* Throws LockFileRecoveryError if clearing a stale lock was interrupted.
*/
async acquire(): Promise<void> {
if (this.state !== "idle") {
@@ -49,13 +125,10 @@ export class LockFile {
this.state = "acquiring";
if (await this.isStale()) {
await unlink(this.filePath).catch(() => {}); // ignore errors, may already be removed
}
try {
this.handle = await open(this.filePath, "wx");
await this.handle.write(`${process.pid}\n`);
await this.removeStale();
this.handle = await fs.open(this.filePath, "wx");
await this.handle.write(LockFile.content);
this.attachExitHandlers();
this.state = "acquired";
} catch (err: any) {
@@ -81,7 +154,9 @@ export class LockFile {
// Ignored: file may have already been removed by another process
try { await this.handle?.close(); } catch {}
try { await unlink(this.filePath); } catch {}
if (await this.isHeldFile()) {
await fs.unlink(this.filePath).catch(() => {});
}
this.handle = null;
this.state = "idle";
@@ -89,19 +164,20 @@ export class LockFile {
/**
* Check whether the lock is currently held by this process.
* True only if the lock is acquired and the PID in the file
* matches the current process.
* True only if the lock is acquired and the lock file was written by this process.
*/
async isHeld(): Promise<boolean> {
if (this.state !== "acquired") {
return false;
}
return await this.isHeldFile();
}
private async isHeldFile() {
try {
const pidInLock = Number(await readFile(this.filePath, "utf8"));
return pidInLock === process.pid;
return (await LockFile.readLock(this.filePath))?.content === LockFile.content;
} catch {
// Ignored: file may not exist or be unreadable, so lock is not held
// Ignored: file may be unreadable, so lock is not held
return false;
}
}
@@ -110,14 +186,45 @@ export class LockFile {
* Determine whether an existing lock is stale.
*/
async isStale(): Promise<boolean> {
const lock = await LockFile.readLock(this.filePath);
return lock === undefined || !LockFile.lockIsLive(lock);
}
/** Pid of the process holding the lock, if it has written it */
async holderPid(): Promise<number | undefined> {
const lock = await LockFile.readLock(this.filePath);
return lock && LockFile.lockPid(lock);
}
/**
* Remove a stale lock file if it is still the same file. Clearing is done while holding a separate
* recovery lock, so a lock another process took after the stale check is never removed.
*/
private async removeStale(): Promise<void> {
const stale = await LockFile.readLock(this.filePath);
if (!stale || LockFile.lockIsLive(stale)) {
return;
}
const recoveryPath = `${this.filePath}.recovery`;
try {
const pidInLock = Number(await readFile(this.filePath, "utf8"));
if (!Number.isFinite(pidInLock)) { return true; }
// signal 0 is a special case to test if a process exists
process.kill(pidInLock, 0);
return false;
} catch {
return true;
await fs.writeFile(recoveryPath, LockFile.content, { flag: "wx" });
} catch (err: any) {
if (err.code !== "EEXIST") {
throw err;
}
const holder = await LockFile.readLock(recoveryPath);
if (holder && !LockFile.lockIsLive(holder)) {
throw new LockFileRecoveryError(recoveryPath);
}
return; // Another process is clearing it
}
try {
const current = await LockFile.readLock(this.filePath);
if (current && LockFile.isSameLock(current, stale)) {
await fs.unlink(this.filePath);
}
} finally {
await fs.unlink(recoveryPath).catch(() => {});
}
}
@@ -126,10 +233,12 @@ export class LockFile {
*/
private attachExitHandlers(): void {
assert(this.state === "acquiring");
process.on("SIGINT", this.handleExit);
process.on("SIGTERM", this.handleExit);
process.on("SIGHUP", this.handleExit);
process.on("uncaughtException", this.handleExit);
if (this.handleSignals) {
process.on("SIGINT", this.handleExit);
process.on("SIGTERM", this.handleExit);
process.on("SIGHUP", this.handleExit);
process.on("uncaughtException", this.handleExit);
}
// eslint-disable-next-line node/no-sync
process.on("exit", this.handleExitSync);
}
@@ -137,7 +246,7 @@ export class LockFile {
/**
* Detach process handlers, called when transitioning to "idle"
*/
private async detachExitHandlers(): Promise<void> {
private detachExitHandlers(): void {
assert(this.state === "releasing");
process.off("SIGINT", this.handleExit);
process.off("SIGTERM", this.handleExit);
@@ -147,17 +256,19 @@ export class LockFile {
process.off("exit", this.handleExitSync);
}
// Event handler for async signals / exceptions
// Event handler for async signals / exceptions
private handleExit = async (): Promise<void> => {
await this.release();
};
// Event handler for synchronous process exit
// Event handler for synchronous process exit, readLock can't be used as it is async
private handleExitSync = (): void => {
try {
unlinkSync(this.filePath);
if (readFileSync(this.filePath, "utf8") === LockFile.content) {
unlinkSync(this.filePath);
}
} catch (err) {
// Ignored: file may have already been removed by another process
// Ignored: file may have already been removed by another process
}
};
}
@@ -3,6 +3,7 @@ import { JsonString, StringEnum, StringKey, plainJson } from "./composites.js";
import { type LogEntry, levels } from "../logging.js";
import { ControllerConfig, HostConfig } from "../config/index.js";
import { ExternalFactorioVersionSchema, LatestReleasesSchema } from "../external/index.js";
import { PluginDetails } from "./messages_plugin.js";
const LogEntrySchema = Type.Object({
"level": StringKey(levels),
@@ -298,6 +299,8 @@ export class SystemInfo {
/** Millisecond Unix timestamp this entry was last updated at */
public updatedAtMs: number,
public isDeleted: boolean,
/** Plugins on the machine when last collected, kept while it is offline */
public plugins?: PluginDetails[],
) { }
static jsonSchema = Type.Object({
@@ -318,6 +321,7 @@ export class SystemInfo {
"processStartedAtMs": Type.Number(),
"updatedAtMs": Type.Number(),
"isDeleted": Type.Boolean(),
"plugins": Type.Optional(Type.Array(PluginDetails.jsonSchema)),
});
static fromJSON(json: Static<typeof this.jsonSchema>) {
@@ -339,6 +343,7 @@ export class SystemInfo {
json.processStartedAtMs,
json.updatedAtMs,
json.isDeleted,
json.plugins?.map(plugin => PluginDetails.fromJSON(plugin)),
);
}
+137 -1
View File
@@ -14,6 +14,18 @@ export class PluginDetails {
public enabled: boolean,
public description?: string,
public npmPackage?: string,
/** Version installed on disk, differs from version when an update is waiting for a restart */
public installedVersion?: string,
/** Installed after the machine started, will be loaded on the next restart */
public pendingInstall?: boolean,
/** Uninstalled after the machine started, will not be loaded on the next restart */
public removed?: boolean,
/** Name in the root package.json dependencies, differs from npmPackage for aliases */
public dependencyName?: string,
/** Has no entrypoint for this machine, so it is never loaded here */
public noEntrypoint?: boolean,
/** Not loaded as another package provides a plugin with the same name */
public nameTaken?: boolean,
) {}
static jsonSchema = Type.Object({
@@ -24,12 +36,19 @@ export class PluginDetails {
enabled: Type.Boolean(),
description: Type.Optional(Type.String()),
npmPackage: Type.Optional(Type.String()),
installedVersion: Type.Optional(Type.String()),
pendingInstall: Type.Optional(Type.Boolean()),
removed: Type.Optional(Type.Boolean()),
dependencyName: Type.Optional(Type.String()),
noEntrypoint: Type.Optional(Type.Boolean()),
nameTaken: Type.Optional(Type.Boolean()),
});
static fromJSON(json: Static<typeof this.jsonSchema>) {
return new this(
json.name, json.title, json.version, json.loaded, json.enabled,
json.description, json.npmPackage
json.description, json.npmPackage, json.installedVersion, json.pendingInstall, json.removed,
json.dependencyName, json.noEntrypoint, json.nameTaken,
);
}
@@ -39,6 +58,15 @@ export class PluginDetails {
pluginInfo.description, pluginInfo.npmPackage,
);
}
/** Plugin package installed since startup, it is loaded on the next restart */
static fromPackageJson(packageName: string, packageJson: any, npmPackage?: string, dependencyName?: string) {
const description = typeof packageJson.description === "string" ? packageJson.description : undefined;
return new this(
packageName, packageName, packageJson.version, false, false,
description, npmPackage, packageJson.version, true, undefined, dependencyName,
);
}
}
export class PluginListRequest {
@@ -94,6 +122,114 @@ export class PluginInstallRequest {
}
}
/* Plugin package listed on the npm registry */
export class PluginSearchResult {
constructor(
public name: string,
public version: string,
public description?: string,
public publisher?: string,
public date?: string,
public homepage?: string,
public repository?: string,
public npm?: string,
) {}
static jsonSchema = Type.Object({
name: Type.String(),
version: Type.String(),
description: Type.Optional(Type.String()),
publisher: Type.Optional(Type.String()),
date: Type.Optional(Type.String()),
homepage: Type.Optional(Type.String()),
repository: Type.Optional(Type.String()),
npm: Type.Optional(Type.String()),
});
static fromJSON(json: Static<typeof this.jsonSchema>) {
return new this(
json.name, json.version, json.description, json.publisher,
json.date, json.homepage, json.repository, json.npm,
);
}
}
export class PluginSearchRequest {
declare ["constructor"]: typeof PluginSearchRequest;
static type = "request" as const;
static src = "control" as const;
static dst = "controller" as const;
static permission = "core.plugin.search";
constructor(
public query: string = "",
public page: number = 1,
public pageSize: number = 20,
) {}
static jsonSchema = Type.Object({
query: Type.String(),
page: Type.Integer(),
pageSize: Type.Integer(),
});
static fromJSON(json: Static<typeof this.jsonSchema>) {
return new this(json.query, json.page, json.pageSize);
}
static Response = class Response {
constructor(
public total: number,
public results: PluginSearchResult[],
) {}
static jsonSchema = Type.Object({
total: Type.Integer(),
results: Type.Array(PluginSearchResult.jsonSchema),
});
static fromJSON(json: Static<typeof this.jsonSchema>) {
return new this(json.total, json.results.map(r => PluginSearchResult.fromJSON(r)));
}
};
}
/* Latest version of each package published on npm, used to offer plugin updates */
export class PluginLatestVersionsRequest {
declare ["constructor"]: typeof PluginLatestVersionsRequest;
static type = "request" as const;
static src = "control" as const;
static dst = "controller" as const;
static permission = "core.plugin.list";
constructor(
public packages: string[],
) {}
static jsonSchema = Type.Object({
packages: Type.Array(Type.String()),
});
static fromJSON(json: Static<typeof this.jsonSchema>) {
return new this(json.packages);
}
static Response = class Response {
constructor(
/** Package name to its latest version, packages not found on npm are left out */
public versions: Record<string, string>,
) {}
static jsonSchema = Type.Object({
versions: Type.Record(Type.String(), Type.String()),
});
static fromJSON(json: Static<typeof this.jsonSchema>) {
return new this(json.versions);
}
};
}
export class UpdateAllRequest {
declare ["constructor"]: typeof UpdateAllRequest;
static type = "request" as const;
+51
View File
@@ -321,6 +321,57 @@ export function redactUrl(url: URL | string) {
return isPath ? parsed.pathname + parsed.search + parsed.hash : parsed.toString();
}
const gitProtocols = new Set(["git:", "git+http:", "git+https:", "git+ssh:", "ssh:"]);
const repositoryShorthands = new Map([
["github:", "https://github.com/"],
["gitlab:", "https://gitlab.com/"],
["bitbucket:", "https://bitbucket.org/"],
]);
/**
* Convert a link from untrusted metadata, like a package's homepage or repository, into a URL
* that is safe to use as the href of a link
*
* Only http and https URLs are returned, so links like `javascript:` can't run code when
* clicked. Accepts the formats npm uses for repositories: a string or an object with a `url`
* property, `git+https://`, `git://` and `git+ssh://` URLs, `git@host:path` and shorthands
* like `github:owner/repo`, which are converted to https URLs. Credentials are removed.
*
* @param link - URL string or object with a `url` property.
* @returns http(s) URL, or undefined if the link can't be converted to one.
*/
export function safeExternalUrl(link: unknown): string | undefined {
const value = typeof link === "object" && link !== null ? (link as { url?: unknown }).url : link;
if (typeof value !== "string") {
return undefined;
}
let url = value.trim();
const scpLike = /^[\w.-]+@([\w.-]+):(?!\/)(.+)$/.exec(url);
if (scpLike) {
url = `ssh://${scpLike[1]}/${scpLike[2]}`;
}
let parsed: URL;
try {
parsed = new URL(url);
const shorthand = repositoryShorthands.get(parsed.protocol);
if (shorthand) {
parsed = new URL(url.slice(parsed.protocol.length).replace(/^\/+/, ""), shorthand);
} else if (gitProtocols.has(parsed.protocol)) {
// The protocol setter can't change a URL into an https URL, so parse it again
parsed = new URL(`https://${parsed.host}${parsed.pathname.replace(/\.git$/, "")}`);
}
} catch {
return undefined;
}
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
return undefined;
}
parsed.username = "";
parsed.password = "";
return parsed.href;
}
/**
* Copy of parsed query parameters with credentials replaced by `REDACTED`
*
+2
View File
@@ -147,5 +147,7 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
plugin.PluginListRequest,
plugin.PluginUpdateRequest,
plugin.PluginInstallRequest,
plugin.PluginSearchRequest,
plugin.PluginLatestVersionsRequest,
plugin.UpdateAllRequest,
];
+206 -103
View File
@@ -5,6 +5,96 @@ import { logger } from "./logging.js";
import * as libFileOps from "./file_ops.js";
import vm from "vm";
const corePackages = new Set([
"@clusterio/controller",
"@clusterio/ctl",
"@clusterio/host",
"@clusterio/lib",
"@clusterio/web_ui",
]);
function isPluginPackageJson(packageJson: any): boolean {
return Boolean(packageJson?.keywords?.includes?.("clusterio-plugin"));
}
/**
* Read the package.json in a package directory.
* @param packagePath - Directory of the package
* @param tolerant - Return undefined instead of throwing if the file can't be parsed, as npm may be writing it
* @returns The parsed package.json, or undefined if it does not exist
*/
export async function readPackageJson(packagePath: string, tolerant: boolean): Promise<any> {
let content;
try {
content = await fs.readFile(path.join(packagePath, "package.json"), "utf8");
} catch (err: any) {
return undefined;
}
try {
return JSON.parse(content);
} catch (err: any) {
if (tolerant) {
return undefined;
}
throw err;
}
}
/** Dependencies npm installs, an optionalDependencies entry wins over a dependencies entry of the same name */
export function installedDependencies(packageJson: any): Record<string, unknown> {
return { ...packageJson?.dependencies, ...packageJson?.optionalDependencies };
}
/** Whether a dependency spec installs from the registry, as opposed to a path, git, url, tarball or alias */
export function isRegistrySpec(spec: unknown) {
return typeof spec === "string" && !/[:/\\]/.test(spec) && !spec.startsWith(".")
&& !/\.(?:tgz|tar\.gz|tar)$/i.test(spec);
}
/** Real path of a directory, used to compare packages as npm and pnpm install them behind symlinks */
export async function realDir(dir: string) {
try {
return await fs.realpath(dir);
} catch {
// Removed since, nothing else can be at this path
return path.resolve(dir);
}
}
/** Directory of a package as require resolves it from fromPath, or undefined if not found */
export function resolvePackagePath(packageName: string, fromPath: string, tolerant: boolean): string | undefined {
const packageRequire = createRequire(path.join(fromPath, "package.json"));
try {
return path.dirname(packageRequire.resolve(`${packageName}/package.json`));
} catch (err: any) {
// A package.json npm is still writing fails with ERR_INVALID_PACKAGE_CONFIG
if (tolerant || err.code === "MODULE_NOT_FOUND" || err.code === "ERR_PACKAGE_PATH_NOT_EXPORTED") {
return undefined;
}
throw err;
}
}
async function pluginPathExists(pluginPath: string) {
if (!path.isAbsolute(pluginPath)) {
return resolvePackagePath(pluginPath, process.cwd(), true) !== undefined;
}
try {
await fs.access(path.join(pluginPath, "package.json"));
return true;
} catch {
return false;
}
}
const require = createRequire(import.meta.url);
function getPluginName(requireSpec: string) {
const context = vm.createContext({ require: require, pluginInfo: null });
const code = `pluginInfo = require(${JSON.stringify(requireSpec)}).plugin;`;
vm.runInContext(code, context);
return context.pluginInfo.name;
}
/**
* Searches for local plugins in the plugins/ and external_plugins/ directories
* and adds them to the plugin list if they contain a package.json file.
@@ -49,7 +139,7 @@ async function findLocalPlugins(pluginList: Map<string, string>): Promise<boolea
const stats = await fs.stat(pluginPath);
if (stats.isDirectory()) {
// Check for package.json and plugin keyword
if (await checkPackageJson(pluginPath)) {
if (isPluginPackageJson(await readPackageJson(pluginPath, false))) {
pluginList.set(pluginName, path.resolve(pluginPath));
logger.info(`Added ${pluginName} from ${pluginFolder}`);
hasChanged += 1;
@@ -67,24 +157,85 @@ async function findLocalPlugins(pluginList: Map<string, string>): Promise<boolea
return hasChanged > 0;
}
const require = createRequire(import.meta.url);
function getPluginName(requireSpec: string) {
const context = vm.createContext({ require: require, pluginInfo: null });
const code = `pluginInfo = require(${JSON.stringify(requireSpec)}).plugin;`;
vm.runInContext(code, context);
return context.pluginInfo.name;
type NpmPluginPackage = {
packageName: string,
packageVersion: unknown,
packagePath: string,
packageJson: any,
};
/**
* Walk the dependencies of the root package.json and of every plugin package found, without
* traversing packages that aren't plugins. Does not execute any plugin code.
*
* @param dependencies - Dependencies of the root package.json
* @param tolerant - Skip packages with a package.json that can't be parsed instead of throwing,
* for use while npm may be rewriting them
* @returns Plugin packages found, and root dependencies that are not plugins
*/
export async function discoverNpmPluginPackages(dependencies: Record<string, unknown>, tolerant = false) {
const plugins: NpmPluginPackage[] = [];
const nonPluginRoots: { packageName: string, packageVersion: unknown }[] = [];
const pendingPackages = Object.entries(dependencies).map(([packageName, packageVersion]) => ({
packageName,
packageVersion,
parentPath: process.cwd(),
isRootDependency: true,
}));
const visitedPackages = new Set<string>();
// Directories some dependent installs from a path, git, url, tarball or alias
const nonRegistryDirs = new Set<string>();
while (pendingPackages.length) {
const { packageName, packageVersion, parentPath, isRootDependency } = pendingPackages.shift()!;
const packagePath = resolvePackagePath(packageName, parentPath, tolerant);
if (!packagePath) {
continue;
}
if (!isRegistrySpec(packageVersion)) {
nonRegistryDirs.add(await realDir(packagePath));
}
if (visitedPackages.has(packageName)) {
continue;
}
visitedPackages.add(packageName);
const packageJson = await readPackageJson(packagePath, tolerant);
if (packageJson && isPluginPackageJson(packageJson)) {
plugins.push({ packageName, packageVersion, packagePath, packageJson });
for (const [dependencyName, dependencyVersion] of Object.entries(installedDependencies(packageJson))) {
pendingPackages.push({
packageName: dependencyName,
packageVersion: dependencyVersion,
parentPath: packagePath,
isRootDependency: false,
});
}
} else if (isRootDependency) {
nonPluginRoots.push({ packageName, packageVersion });
}
}
return { plugins, nonPluginRoots, nonRegistryDirs };
}
function resolvePackagePath(packageName: string, fromPath: string): string | undefined {
const packageRequire = createRequire(path.join(fromPath, "package.json"));
try {
return path.dirname(packageRequire.resolve(`${packageName}/package.json`));
} catch (err: any) {
if (err.code === "MODULE_NOT_FOUND" || err.code === "ERR_PACKAGE_PATH_NOT_EXPORTED") {
return undefined;
}
throw err;
/** Real paths of plugin packages some dependent installs from a path, git, url, tarball or alias */
export async function findNonRegistryDirs() {
const dependencies = installedDependencies(await readPackageJson(process.cwd(), true));
return (await discoverNpmPluginPackages(dependencies, true)).nonRegistryDirs;
}
/** Whether npm install name would replace a plugin package not installed from the registry */
export async function isNonRegistryPackage(name: string) {
const dependencies = installedDependencies(await readPackageJson(process.cwd(), true));
if (Object.hasOwn(dependencies, name) && !isRegistrySpec(dependencies[name])) {
return true;
}
const packagePath = resolvePackagePath(name, process.cwd(), true);
if (!packagePath) {
return false;
}
return (await findNonRegistryDirs()).has(await realDir(packagePath));
}
/**
@@ -92,79 +243,46 @@ function resolvePackagePath(packageName: string, fromPath: string): string | und
* Adds discovered plugins to the plugin list.
*
* @param pluginList - Map of plugin names to their file paths
* @param skippedNpmPlugins - Filled with the package directory by plugin name for packages
* skipped because their plugin name was already in the list
* @returns Promise that resolves to true if any plugins were added to the list, false otherwise
*/
async function findNpmPlugins(pluginList: Map<string, string>): Promise<boolean> {
let dependencies;
try {
const rootPackageJson = JSON.parse(await fs.readFile(path.resolve("package.json"), { encoding: "utf8" }));
dependencies = rootPackageJson.dependencies;
} catch (err: any) {
if (err.code === "ENOENT") {
// Skip if package.json doesn't exist
return false;
}
throw err;
}
if (!dependencies) {
async function findNpmPlugins(
pluginList: Map<string, string>,
skippedNpmPlugins?: Map<string, string>,
): Promise<boolean> {
const rootPackageJson = await readPackageJson(process.cwd(), false);
if (!rootPackageJson) {
return false;
}
const dependencies = installedDependencies(rootPackageJson);
let hasChanged = 0;
const pendingPackages = Object.entries(dependencies).map(([packageName, packageVersion]) => ({
packageName,
packageVersion,
parentPath: process.cwd(),
isRootDependency: true,
}));
const corePackages = new Set([
"@clusterio/controller",
"@clusterio/ctl",
"@clusterio/host",
"@clusterio/lib",
"@clusterio/web_ui",
]);
const visitedPackages = new Set<string>();
while (pendingPackages.length) {
const { packageName, packageVersion, parentPath, isRootDependency } = pendingPackages.shift()!;
if (visitedPackages.has(packageName)) {
const { plugins, nonPluginRoots } = await discoverNpmPluginPackages(dependencies);
for (const { packageName, packageVersion, packagePath } of plugins) {
const pluginName = getPluginName(packagePath);
const listedPath = pluginList.get(pluginName);
if (listedPath === packageName) {
continue;
}
const npmPackagePath = resolvePackagePath(packageName, parentPath);
if (!npmPackagePath) {
continue;
}
visitedPackages.add(packageName);
if (await checkPackageJson(npmPackagePath)) {
const pluginName = getPluginName(npmPackagePath);
if (pluginList.has(pluginName)) {
logger.warn(
`${pluginName} provided by ${packageName}@${packageVersion} is already in the plugin list, ` +
`but with a different path - using ${pluginList.get(pluginName)}`
);
} else {
pluginList.set(pluginName, packageName);
logger.info(`Added ${pluginName} from node modules`);
hasChanged += 1;
}
const packageJson = JSON.parse(
await fs.readFile(path.join(npmPackagePath, "package.json"), { encoding: "utf8" })
if (listedPath !== undefined && !await pluginPathExists(listedPath)) {
logger.warn(`${pluginName} at ${listedPath} no longer exists, using ${packageName}`);
pluginList.set(pluginName, packageName);
hasChanged += 1;
} else if (pluginList.has(pluginName)) {
logger.warn(
`${pluginName} provided by ${packageName}@${packageVersion} is already in the plugin list, ` +
`but with a different path - using ${pluginList.get(pluginName)}`
);
for (const [dependencyName, dependencyVersion] of Object.entries(packageJson.dependencies ?? {})) {
pendingPackages.push({
packageName: dependencyName,
packageVersion: dependencyVersion,
parentPath: npmPackagePath,
isRootDependency: false,
});
}
} else if (isRootDependency && !corePackages.has(packageName)) {
skippedNpmPlugins?.set(pluginName, await realDir(packagePath));
} else {
pluginList.set(pluginName, packageName);
logger.info(`Added ${pluginName} from node modules`);
hasChanged += 1;
}
}
for (const { packageName, packageVersion } of nonPluginRoots) {
if (!corePackages.has(packageName)) {
logger.warn(`${packageName}@${packageVersion} is not a clusterio-plugin`);
}
}
@@ -176,9 +294,15 @@ async function findNpmPlugins(pluginList: Map<string, string>): Promise<boolean>
*
* @param pluginListPath - Path to the plugin list JSON file
* @param findLocal - Whether to look for local plugins in plugins/ and external_plugins/ directories
* @param skippedNpmPlugins - Filled with the package directory by plugin name for npm packages
* skipped because another plugin has the same name, for loadPluginInfos
* @returns Map of plugin name to plugin path
*/
export async function loadPluginList(pluginListPath: string, findLocal = true): Promise<Map<string, string>> {
export async function loadPluginList(
pluginListPath: string,
findLocal = true,
skippedNpmPlugins?: Map<string, string>,
): Promise<Map<string, string>> {
let pluginList = new Map<string, string>();
// Try to load existing plugin list
@@ -201,7 +325,7 @@ export async function loadPluginList(pluginListPath: string, findLocal = true):
}
// Discover NPM plugins
hasChanges = await findNpmPlugins(pluginList) || hasChanges;
hasChanges = await findNpmPlugins(pluginList, skippedNpmPlugins) || hasChanges;
if (hasChanges) {
await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, "\t"));
@@ -209,24 +333,3 @@ export async function loadPluginList(pluginListPath: string, findLocal = true):
return pluginList;
}
/**
* Helper function to check if a package.json file exists and contains the clusterio-plugin keyword
* @param pluginPath - Path to the plugin directory
* @returns Promise that resolves to true if package.json exists and has clusterio-plugin keyword
*/
async function checkPackageJson(pluginPath: string): Promise<boolean> {
const packageJsonPath = path.join(pluginPath, "package.json");
// Read and parse package.json
let content;
try {
content = await fs.readFile(packageJsonPath, { encoding: "utf8" });
} catch (err: any) {
return false;
}
const packageJson = JSON.parse(content);
// Check if package has the clusterio-plugin keyword
return packageJson.keywords?.includes("clusterio-plugin");
}
+159
View File
@@ -0,0 +1,159 @@
import { logger } from "./logging.js";
import { RequestError } from "./errors.js";
import { PluginSearchRequest, PluginSearchResult } from "./data/messages_plugin.js";
const registryTimeoutMs = 10e3;
// One name segment, optionally after an @scope/ segment. Not starting with . or - keeps npm from
// reading the spec as a path, a git repository or a command line option.
// https://docs.npmjs.com/cli/v11/configuring-npm/package-json#name
const packageNamePattern = /^(?:@[a-zA-Z0-9][a-zA-Z0-9\-._~]*\/)?[a-zA-Z0-9][a-zA-Z0-9\-._~]*$/;
/**
* Split a package spec into its name and optional version, as in name@version. The name itself
* may start with @ for scoped packages.
*/
export function parsePackageSpec(spec: string) {
const versionSeparator = spec.lastIndexOf("@");
const [name, version] = versionSeparator > 0
? [spec.slice(0, versionSeparator), spec.slice(versionSeparator + 1)]
: [spec, undefined];
if (name.length > 214 || !packageNamePattern.test(name)) {
throw new RequestError(`Invalid plugin name: ${spec}`);
}
if (version !== undefined && !/^[a-zA-Z0-9][a-zA-Z0-9\-.+]*$/.test(version)) {
throw new RequestError(`Invalid plugin version: ${version}`);
}
return { name, version };
}
/** Throws unless the package, and version if given, is published on npm, returns the spec to install */
export async function checkRegistry(name: string, version?: string) {
const packageName = encodeURI(name);
const spec = version !== undefined ? `${packageName}@${version}` : packageName;
const registryPath = version !== undefined ? `${packageName}/${encodeURIComponent(version)}` : packageName;
const npmRequest = await fetch(`https://registry.npmjs.com/${registryPath}`, {
method: "HEAD",
signal: AbortSignal.timeout(registryTimeoutMs),
});
if (!npmRequest.ok) {
throw new RequestError(`Unknown plugin: ${spec}`);
}
return spec;
}
const latestVersionCacheMs = 5 * 60e3;
const latestVersionCacheMaxSize = 1000;
const latestVersionConcurrency = 8;
/** Latest version of each package by name, as found on npm, with when it was looked up */
export type LatestVersionCache = Map<string, { version?: string, fetchedAt: number }>;
/**
* Look up the latest published version of each package on npm, packages that can't be
* found are left out. Lookups are kept in the given cache for a few minutes.
*/
export async function fetchLatestVersions(packages: string[], cache: LatestVersionCache) {
if (packages.length > 500) {
throw new RequestError("Too many packages");
}
const names = [...new Set(packages.map(packageName => {
const { name, version: pinned } = parsePackageSpec(packageName);
if (pinned !== undefined) {
throw new RequestError(`Invalid plugin name: ${packageName}`);
}
return name;
}))];
const now = Date.now();
for (const [name, cached] of cache) {
if (now - cached.fetchedAt > latestVersionCacheMs) {
cache.delete(name);
}
}
const versions: Record<string, string> = {};
const queue = [...names];
async function lookUp(name: string) {
let cached = cache.get(name);
if (!cached) {
let response;
try {
response = await fetch(`https://registry.npmjs.com/${name}/latest`, {
signal: AbortSignal.timeout(registryTimeoutMs),
});
} catch (err: any) {
logger.warn(`Failed to look up latest version of ${name}: ${err.message}`);
return;
}
if (!response.ok && response.status !== 404) {
// Rate limits and outages are retried on the next lookup instead of cached
logger.warn(`Failed to look up latest version of ${name}: ${response.status}`);
return;
}
const version = response.ok ? (await response.json() as { version?: string }).version : undefined;
cached = { version, fetchedAt: now };
for (const oldest of cache.keys()) {
if (cache.size < latestVersionCacheMaxSize) {
break;
}
cache.delete(oldest);
}
cache.set(name, cached);
}
if (cached.version) {
versions[name] = cached.version;
}
}
await Promise.all(Array.from({ length: latestVersionConcurrency }, async () => {
for (let name = queue.shift(); name !== undefined; name = queue.shift()) {
await lookUp(name);
}
}));
return versions;
}
const npmSearchMaxSize = 250; // Limit imposed by the registry
export async function searchPlugins(query: string, page: number, pageSize: number) {
if (query.length > 214) {
throw new RequestError("Search query too long");
}
if (page < 1 || pageSize < 1 || pageSize > npmSearchMaxSize) {
throw new RequestError("Invalid page or page size");
}
const url = new URL("https://registry.npmjs.com/-/v1/search");
url.searchParams.set("text", `keywords:clusterio-plugin ${query}`.trim());
url.searchParams.set("size", String(pageSize));
url.searchParams.set("from", String((page - 1) * pageSize));
const response = await fetch(url, { signal: AbortSignal.timeout(registryTimeoutMs) });
if (!response.ok) {
throw new RequestError(`npm registry search failed: ${response.status} ${response.statusText}`);
}
const json = await response.json() as {
total: number,
objects: {
package: {
name: string,
version: string,
description?: string | null,
date?: string | null,
publisher?: { username?: string | null } | null,
links?: { npm?: string | null, homepage?: string | null, repository?: string | null } | null,
},
}[],
};
return new PluginSearchRequest.Response(
json.total,
json.objects.map(({ package: pkg }) => new PluginSearchResult(
// The registry sends null for some missing fields
pkg.name, pkg.version, pkg.description ?? undefined, pkg.publisher?.username ?? undefined,
pkg.date ?? undefined, pkg.links?.homepage ?? undefined, pkg.links?.repository ?? undefined,
pkg.links?.npm ?? undefined,
)),
);
}
+5
View File
@@ -583,6 +583,11 @@ export const corePermissions = [
title: "Install plugin",
description: "Remotely install a plugin if the target allows remote installs of plugins.",
},
{
name: "core.plugin.search",
title: "Search plugins",
description: "Search the npm registry for plugins via the controller.",
},
{
name: "core.debug.dump_ws",
+5
View File
@@ -82,6 +82,11 @@ export type PluginNodeEnvInfo = PluginDeclaration & {
* package.
*/
npmPackage?: string;
/**
* Directory of an npm package providing a plugin with the same name
* that was skipped on startup as this plugin was already in the list.
*/
skippedNpmPackage?: string;
};
export type PluginWebpackEnvInfo = PluginDeclaration & {
+103 -2
View File
@@ -13,6 +13,10 @@ import { loadPluginEntrypoint, loadPluginClass } from "./loadPlugin.js";
import { Config } from "./config/classes.js";
import { ControllerConfig, HostConfig, InstanceConfig } from "./config/definitions.js";
import Role from "./data/Role.js";
import { PluginDetails } from "./data/messages_plugin.js";
import {
discoverNpmPluginPackages, findNonRegistryDirs, installedDependencies, readPackageJson, realDir, resolvePackagePath,
} from "./load_plugin_list.js";
/**
@@ -24,10 +28,13 @@ import Role from "./data/Role.js";
*
* @param pluginList -
* Mapping of plugin name to require path for the plugins to load.
* @param skippedNpmPlugins -
* Directory of the npm package skipped by plugin name, from loadPluginList.
* @returns Array of plugin info modules.
*/
export async function loadPluginInfos(pluginList: Map<string, string>) {
export async function loadPluginInfos(pluginList: Map<string, string>, skippedNpmPlugins?: Map<string, string>) {
let plugins: libPlugin.PluginNodeEnvInfo[] = [];
const nonRegistryDirs = await findNonRegistryDirs();
for (let [pluginName, pluginPath] of pluginList) {
let pluginInfo: libPlugin.PluginNodeEnvInfo;
let pluginPackage: { name?: string, version: string, main?: string, private?: boolean };
@@ -87,12 +94,106 @@ export async function loadPluginInfos(pluginList: Map<string, string>) {
pluginInfo.packagePath = packagePath;
pluginInfo.requirePath = pluginPath;
pluginInfo.version = pluginPackage.version;
pluginInfo.npmPackage = !pluginPackage.private && pluginPath === pluginPackage.name ? pluginPath : undefined;
pluginInfo.npmPackage = !pluginPackage.private && pluginPath === pluginPackage.name
&& !nonRegistryDirs.has(await fs.realpath(path.dirname(packagePath))) ? pluginPath : undefined;
pluginInfo.skippedNpmPackage = skippedNpmPlugins?.get(pluginName);
plugins.push(pluginInfo);
}
return plugins;
}
/**
* Lists the plugins on this machine along with the versions currently installed on disk,
* including npm plugins installed since startup that will only be loaded after a restart.
*
* @param pluginInfos - Plugins discovered when this machine started
* @param isLoaded - Whether the plugin is loaded
* @param isEnabled - Whether the plugin is enabled
* @param hasEntrypoint - Whether the plugin has an entrypoint this machine loads
* @returns Details of every plugin on this machine
*/
export async function listPluginDetails(
pluginInfos: libPlugin.PluginNodeEnvInfo[],
isLoaded: (pluginInfo: libPlugin.PluginNodeEnvInfo) => boolean,
isEnabled: (pluginInfo: libPlugin.PluginNodeEnvInfo) => boolean,
hasEntrypoint: (pluginInfo: libPlugin.PluginNodeEnvInfo) => boolean = () => true,
): Promise<PluginDetails[]> {
const plugins: PluginDetails[] = [];
const dependencies = installedDependencies(await readPackageJson(process.cwd(), true));
// By directory, as a plugin loaded from a path may also be an npm dependency
const knownDirs = new Set<string>();
// Packages skipped on startup by directory, for plugins still installed
const takenDirs = new Set<string>();
const { plugins: npmPlugins, nonRegistryDirs } = await discoverNpmPluginPackages(dependencies, true);
for (const pluginInfo of pluginInfos) {
const dir = await realDir(path.dirname(pluginInfo.packagePath));
knownDirs.add(dir);
// Without an entrypoint for this machine the plugin has no load_plugin field here
const loadable = hasEntrypoint(pluginInfo);
const plugin = PluginDetails.fromNodeEnvInfo(
pluginInfo, isLoaded(pluginInfo), loadable && isEnabled(pluginInfo),
);
if (!loadable) {
plugin.noEntrypoint = true;
}
if (Object.hasOwn(dependencies, pluginInfo.requirePath)) {
plugin.dependencyName = pluginInfo.requirePath;
}
if (nonRegistryDirs.has(dir)) {
plugin.npmPackage = undefined;
}
let packageJson;
try {
packageJson = await readPackageJson(path.dirname(pluginInfo.packagePath), false);
if (packageJson === undefined) {
// Uninstalled since startup, it won't be found on the next restart
plugin.removed = true;
}
} catch {
// Ignored: npm is still writing it
}
if (packageJson) {
plugin.installedVersion = packageJson.version;
if (packageJson.name !== plugin.npmPackage) {
// Replaced by an alias since startup
plugin.npmPackage = undefined;
}
}
if (!plugin.removed && pluginInfo.skippedNpmPackage !== undefined) {
takenDirs.add(pluginInfo.skippedNpmPackage);
}
plugins.push(plugin);
}
for (const { packageName, packagePath, packageJson } of npmPlugins) {
const dir = await realDir(packagePath);
if (knownDirs.has(dir)) {
continue;
}
// Startup loads it by package name from the root, which doesn't reach nested packages
const rootPath = resolvePackagePath(packageName, process.cwd(), true);
if (!rootPath || await realDir(rootPath) !== dir) {
continue;
}
if (typeof packageJson.version !== "string") {
continue;
}
// Same rule as loadPluginInfos, aliases, private and non-registry packages can't be installed by this name
const npmPackage = !packageJson.private && packageJson.name === packageName && !nonRegistryDirs.has(dir)
? packageName : undefined;
const dependencyName = Object.hasOwn(dependencies, packageName) ? packageName : undefined;
const plugin = PluginDetails.fromPackageJson(packageName, packageJson, npmPackage, dependencyName);
if (takenDirs.has(dir)) {
// Skipped on startup as its plugin name is taken, will be again
plugin.pendingInstall = undefined;
plugin.nameTaken = true;
}
plugins.push(plugin);
}
return plugins;
}
/**
* Load the migration entrypoints of plugins
*
+79 -24
View File
@@ -4,6 +4,9 @@ import { exec } from "child_process";
import { logger } from "./logging.js";
import { RequestError } from "./errors.js";
import type { PluginNodeEnvInfo } from "./plugin.js";
import { isNonRegistryPackage } from "./load_plugin_list.js";
import { LockFile, LockFileExistsError, LockFileRecoveryError } from "./LockFile.js";
import { checkRegistry, parsePackageSpec } from "./npm_registry.js";
const execAsync = util.promisify(exec);
function isDev() {
@@ -12,10 +15,55 @@ function isDev() {
return import.meta.dirname.split(path.sep).at(-5) === "packages"; // opposed to "@clusterio"
}
async function logExec(cmd: string) {
logger.audit(`RCE | ${cmd}`);
if (!isDev()) {
await execAsync(cmd);
const operationInProgressMessage = "Another package operation is already in progress on this machine";
let operationInProgress = false;
/**
* Run an npm command, at most one at a time per installation. npm runs in the working directory,
* which a standalone controller and host share, so a lock file there guards against operations
* from other processes and other copies of lib. The flag makes concurrent calls through this copy
* reject without racing a release of the lock.
*
* @param cmd - Command to run
* @param check - Called while holding the lock before running the command, may throw to cancel it
*/
async function logExec(cmd: string, check?: () => Promise<void>) {
// Checked and set before the first await so concurrent calls in this process can't interleave
if (operationInProgress) {
throw new RequestError(operationInProgressMessage);
}
operationInProgress = true;
const lockPath = path.resolve("npm-operation.lock");
// Signal listeners would change how the process exits. If only this process is killed npm keeps
// running unlocked, Ctrl+C, systemd and containers stop it too.
const lock = new LockFile(lockPath, false);
try {
try {
await lock.acquire();
} catch (err) {
if (err instanceof LockFileRecoveryError) {
throw new RequestError(err.message);
}
if (!(err instanceof LockFileExistsError)) {
throw err;
}
const holder = await lock.holderPid();
throw new RequestError(
`${operationInProgressMessage}${holder ? ` (${lockPath} is held by process ${holder})` : ""}`
);
}
try {
await check?.();
logger.audit(`RCE | ${cmd}`);
if (!isDev()) {
await execAsync(cmd);
}
} finally {
await lock.release();
}
} finally {
operationInProgress = false;
}
}
@@ -23,16 +71,13 @@ export async function updatePackage(...names: string[]) {
return logExec(`npm update --save ${names.join(" ")}`);
}
export async function installPackage(name: string) {
return logExec(`npm install --save ${name}`);
export async function installPackage(name: string, check?: () => Promise<void>) {
return logExec(`npm install --save ${name}`, check);
}
export async function handlePluginUpdate(pluginName: string, pluginInfos: PluginNodeEnvInfo[]) {
if (!pluginInfos.some(plugin => plugin.npmPackage === pluginName)) {
throw new RequestError(`Plugin ${pluginName} is not installed on this machine`);
}
return await updatePackage(pluginName);
/** Whether an npm operation is running in this installation, from this or another process */
export async function packageOperationRunning() {
return operationInProgress || !await new LockFile(path.resolve("npm-operation.lock")).isStale();
}
export async function handleUpdateAll(corePackage: string, pluginInfos: PluginNodeEnvInfo[]) {
@@ -42,20 +87,30 @@ export async function handleUpdateAll(corePackage: string, pluginInfos: PluginNo
return await updatePackage(corePackage, ...pluginPackages);
}
export async function handlePluginInstall(pluginName: string) {
if (pluginName.length > 214 || /[^a-zA-Z0-9\-_.+@\/]/.test(pluginName)) {
// https://docs.npmjs.com/cli/v11/configuring-npm/package-json#name
// https://www.npmjs.com/package/validate-npm-package-name
throw new RequestError(`Invalid plugin name: ${pluginName}`);
/** Throws if the dependency by this name is a path, git, url or alias that installing by name would replace */
async function checkNotLocalDependency(name: string) {
if (await isNonRegistryPackage(name)) {
throw new RequestError(`${name} is not installed from npm on this machine`);
}
}
const packageName = encodeURI(pluginName);
const npmRequest = await fetch(`https://registry.npmjs.com/${packageName}`, {
method: "HEAD",
});
if (!npmRequest.ok) {
throw new RequestError(`Unknown plugin: ${packageName}`);
export async function handlePluginUpdate(pluginName: string, pluginInfos: PluginNodeEnvInfo[]) {
// Optionally pinned to a version as name@version to update past the range in package.json
const { name, version } = parsePackageSpec(pluginName);
if (!pluginInfos.some(plugin => plugin.npmPackage === name)) {
throw new RequestError(`Plugin ${name} is not installed on this machine`);
}
return await installPackage(packageName);
if (version === undefined) {
return await updatePackage(name);
}
// Checked under the lock as another operation may change where it's installed from
return await installPackage(await checkRegistry(name, version), () => checkNotLocalDependency(name));
}
/** Install a plugin, optionally pinned to a version as name@version, returns the package name */
export async function handlePluginInstall(pluginName: string) {
const { name, version } = parsePackageSpec(pluginName);
await installPackage(await checkRegistry(name, version), () => checkNotLocalDependency(name));
return name;
}
+4 -2
View File
@@ -2,7 +2,7 @@ import fs from "node:fs/promises";
import os from "os";
import util from "util";
import { Gauge } from "./prometheus.js";
import { SystemInfo } from "./data/index.js";
import { PluginDetails, SystemInfo } from "./data/index.js";
function filterCpuModel() {
const model = os.cpus()[0].model;
@@ -125,7 +125,8 @@ export async function gatherSystemInfo(
id: SystemInfo["id"],
canRestart: boolean,
restartRequired: boolean,
startedAtMs: number
startedAtMs: number,
plugins?: PluginDetails[],
) {
const cpus = os.cpus();
const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys);
@@ -160,5 +161,6 @@ export async function gatherSystemInfo(
startedAtMs,
now,
false,
plugins,
);
}