mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-18 07:33:12 +09:00
Merge pull request #1010 from bbassie/feature/plugin-browser
Plugin management through web interface
This commit is contained in:
1 parent
e36a76acc5
commit
ffe82f81d8
48 files changed
+4275
-536
No files matched your search
+142
-31
@@ -1,5 +1,5 @@
|
||||
import { FileHandle, open, unlink, readFile } from "node:fs/promises";
|
||||
import { unlinkSync } from "node:fs";
|
||||
import fs, { type FileHandle } from "node:fs/promises";
|
||||
import { readFileSync, unlinkSync } from "node:fs";
|
||||
import assert from "assert";
|
||||
|
||||
/**
|
||||
@@ -24,23 +24,99 @@ export class LockFileBusyError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Error thrown when a process exited while clearing a stale lock, which needs the recovery file removed by hand.
|
||||
*/
|
||||
export class LockFileRecoveryError extends Error {
|
||||
constructor(
|
||||
readonly recoveryPath: string
|
||||
) {
|
||||
super(`Lock file recovery was interrupted, remove ${recoveryPath} to allow the lock to be acquired`);
|
||||
this.name = "LockFileRecoveryError";
|
||||
}
|
||||
}
|
||||
|
||||
type LockState = "idle" | "acquiring" | "acquired" | "releasing";
|
||||
type LockSnapshot = { content: string, ino: bigint, mtimeNs: bigint };
|
||||
|
||||
/**
|
||||
* Cross-platform lock file utility for Node.js.
|
||||
*/
|
||||
export class LockFile {
|
||||
/**
|
||||
* Content written to the lock file. The start time tells this process apart from an earlier one with
|
||||
* the same pid, and is the same for every copy of lib loaded in this process.
|
||||
*/
|
||||
private static readonly content = `${process.pid} ${performance.timeOrigin}\n`;
|
||||
// A lock file without a valid pid is left alone for this long, as it may still be being written
|
||||
private static readonly writeGraceMs = 10e3;
|
||||
|
||||
private handle: FileHandle | null = null;
|
||||
private state: LockState = "idle";
|
||||
|
||||
/**
|
||||
* @param filePath - Path of the lock file
|
||||
* @param handleSignals -
|
||||
* Release the lock on SIGINT, SIGTERM, SIGHUP and uncaught exceptions. Listening to these
|
||||
* changes how the process exits, so without it the lock is only removed on exit.
|
||||
*/
|
||||
constructor(
|
||||
private readonly filePath: string
|
||||
private readonly filePath: string,
|
||||
private readonly handleSignals = true,
|
||||
) {}
|
||||
|
||||
/** Content and identity of a lock file, read through one handle so both belong to the same file */
|
||||
private static async readLock(filePath: string): Promise<LockSnapshot | undefined> {
|
||||
let handle;
|
||||
try {
|
||||
handle = await fs.open(filePath, "r");
|
||||
} catch (err: any) {
|
||||
if (err.code === "ENOENT") {
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
try {
|
||||
const stat = await handle.stat({ bigint: true });
|
||||
return { content: await handle.readFile("utf8"), ino: stat.ino, mtimeNs: stat.mtimeNs };
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
private static isSameLock(a: LockSnapshot, b: LockSnapshot) {
|
||||
return a.content === b.content && a.ino === b.ino && a.mtimeNs === b.mtimeNs;
|
||||
}
|
||||
|
||||
private static lockPid(lock: LockSnapshot) {
|
||||
const pid = Number(lock.content.split(" ")[0]);
|
||||
return Number.isInteger(pid) && pid > 0 ? pid : undefined;
|
||||
}
|
||||
|
||||
/** True if the lock names a process that is still running, or may still be written to */
|
||||
private static lockIsLive(lock: LockSnapshot) {
|
||||
const pid = LockFile.lockPid(lock);
|
||||
if (pid === undefined) {
|
||||
return Date.now() - Number(lock.mtimeNs / 1000000n) < LockFile.writeGraceMs;
|
||||
}
|
||||
if (pid === process.pid) {
|
||||
// Held by this process, or left by an earlier process with the same pid
|
||||
return lock.content === LockFile.content;
|
||||
}
|
||||
try {
|
||||
// Signal 0 only checks that the process exists
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch (err: any) {
|
||||
return err.code !== "ESRCH";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire the lock file.
|
||||
* Throws LockFileExistsError if already locked.
|
||||
* Throws LockFileBusyError if not in idle state.
|
||||
* Throws LockFileRecoveryError if clearing a stale lock was interrupted.
|
||||
*/
|
||||
async acquire(): Promise<void> {
|
||||
if (this.state !== "idle") {
|
||||
@@ -49,13 +125,10 @@ export class LockFile {
|
||||
|
||||
this.state = "acquiring";
|
||||
|
||||
if (await this.isStale()) {
|
||||
await unlink(this.filePath).catch(() => {}); // ignore errors, may already be removed
|
||||
}
|
||||
|
||||
try {
|
||||
this.handle = await open(this.filePath, "wx");
|
||||
await this.handle.write(`${process.pid}\n`);
|
||||
await this.removeStale();
|
||||
this.handle = await fs.open(this.filePath, "wx");
|
||||
await this.handle.write(LockFile.content);
|
||||
this.attachExitHandlers();
|
||||
this.state = "acquired";
|
||||
} catch (err: any) {
|
||||
@@ -81,7 +154,9 @@ export class LockFile {
|
||||
|
||||
// Ignored: file may have already been removed by another process
|
||||
try { await this.handle?.close(); } catch {}
|
||||
try { await unlink(this.filePath); } catch {}
|
||||
if (await this.isHeldFile()) {
|
||||
await fs.unlink(this.filePath).catch(() => {});
|
||||
}
|
||||
|
||||
this.handle = null;
|
||||
this.state = "idle";
|
||||
@@ -89,19 +164,20 @@ export class LockFile {
|
||||
|
||||
/**
|
||||
* Check whether the lock is currently held by this process.
|
||||
* True only if the lock is acquired and the PID in the file
|
||||
* matches the current process.
|
||||
* True only if the lock is acquired and the lock file was written by this process.
|
||||
*/
|
||||
async isHeld(): Promise<boolean> {
|
||||
if (this.state !== "acquired") {
|
||||
return false;
|
||||
}
|
||||
return await this.isHeldFile();
|
||||
}
|
||||
|
||||
private async isHeldFile() {
|
||||
try {
|
||||
const pidInLock = Number(await readFile(this.filePath, "utf8"));
|
||||
return pidInLock === process.pid;
|
||||
return (await LockFile.readLock(this.filePath))?.content === LockFile.content;
|
||||
} catch {
|
||||
// Ignored: file may not exist or be unreadable, so lock is not held
|
||||
// Ignored: file may be unreadable, so lock is not held
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -110,14 +186,45 @@ export class LockFile {
|
||||
* Determine whether an existing lock is stale.
|
||||
*/
|
||||
async isStale(): Promise<boolean> {
|
||||
const lock = await LockFile.readLock(this.filePath);
|
||||
return lock === undefined || !LockFile.lockIsLive(lock);
|
||||
}
|
||||
|
||||
/** Pid of the process holding the lock, if it has written it */
|
||||
async holderPid(): Promise<number | undefined> {
|
||||
const lock = await LockFile.readLock(this.filePath);
|
||||
return lock && LockFile.lockPid(lock);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove a stale lock file if it is still the same file. Clearing is done while holding a separate
|
||||
* recovery lock, so a lock another process took after the stale check is never removed.
|
||||
*/
|
||||
private async removeStale(): Promise<void> {
|
||||
const stale = await LockFile.readLock(this.filePath);
|
||||
if (!stale || LockFile.lockIsLive(stale)) {
|
||||
return;
|
||||
}
|
||||
const recoveryPath = `${this.filePath}.recovery`;
|
||||
try {
|
||||
const pidInLock = Number(await readFile(this.filePath, "utf8"));
|
||||
if (!Number.isFinite(pidInLock)) { return true; }
|
||||
// signal 0 is a special case to test if a process exists
|
||||
process.kill(pidInLock, 0);
|
||||
return false;
|
||||
} catch {
|
||||
return true;
|
||||
await fs.writeFile(recoveryPath, LockFile.content, { flag: "wx" });
|
||||
} catch (err: any) {
|
||||
if (err.code !== "EEXIST") {
|
||||
throw err;
|
||||
}
|
||||
const holder = await LockFile.readLock(recoveryPath);
|
||||
if (holder && !LockFile.lockIsLive(holder)) {
|
||||
throw new LockFileRecoveryError(recoveryPath);
|
||||
}
|
||||
return; // Another process is clearing it
|
||||
}
|
||||
try {
|
||||
const current = await LockFile.readLock(this.filePath);
|
||||
if (current && LockFile.isSameLock(current, stale)) {
|
||||
await fs.unlink(this.filePath);
|
||||
}
|
||||
} finally {
|
||||
await fs.unlink(recoveryPath).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -126,10 +233,12 @@ export class LockFile {
|
||||
*/
|
||||
private attachExitHandlers(): void {
|
||||
assert(this.state === "acquiring");
|
||||
process.on("SIGINT", this.handleExit);
|
||||
process.on("SIGTERM", this.handleExit);
|
||||
process.on("SIGHUP", this.handleExit);
|
||||
process.on("uncaughtException", this.handleExit);
|
||||
if (this.handleSignals) {
|
||||
process.on("SIGINT", this.handleExit);
|
||||
process.on("SIGTERM", this.handleExit);
|
||||
process.on("SIGHUP", this.handleExit);
|
||||
process.on("uncaughtException", this.handleExit);
|
||||
}
|
||||
// eslint-disable-next-line node/no-sync
|
||||
process.on("exit", this.handleExitSync);
|
||||
}
|
||||
@@ -137,7 +246,7 @@ export class LockFile {
|
||||
/**
|
||||
* Detach process handlers, called when transitioning to "idle"
|
||||
*/
|
||||
private async detachExitHandlers(): Promise<void> {
|
||||
private detachExitHandlers(): void {
|
||||
assert(this.state === "releasing");
|
||||
process.off("SIGINT", this.handleExit);
|
||||
process.off("SIGTERM", this.handleExit);
|
||||
@@ -147,17 +256,19 @@ export class LockFile {
|
||||
process.off("exit", this.handleExitSync);
|
||||
}
|
||||
|
||||
// Event handler for async signals / exceptions
|
||||
// Event handler for async signals / exceptions
|
||||
private handleExit = async (): Promise<void> => {
|
||||
await this.release();
|
||||
};
|
||||
|
||||
// Event handler for synchronous process exit
|
||||
// Event handler for synchronous process exit, readLock can't be used as it is async
|
||||
private handleExitSync = (): void => {
|
||||
try {
|
||||
unlinkSync(this.filePath);
|
||||
if (readFileSync(this.filePath, "utf8") === LockFile.content) {
|
||||
unlinkSync(this.filePath);
|
||||
}
|
||||
} catch (err) {
|
||||
// Ignored: file may have already been removed by another process
|
||||
// Ignored: file may have already been removed by another process
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { JsonString, StringEnum, StringKey, plainJson } from "./composites.js";
|
||||
import { type LogEntry, levels } from "../logging.js";
|
||||
import { ControllerConfig, HostConfig } from "../config/index.js";
|
||||
import { ExternalFactorioVersionSchema, LatestReleasesSchema } from "../external/index.js";
|
||||
import { PluginDetails } from "./messages_plugin.js";
|
||||
|
||||
const LogEntrySchema = Type.Object({
|
||||
"level": StringKey(levels),
|
||||
@@ -298,6 +299,8 @@ export class SystemInfo {
|
||||
/** Millisecond Unix timestamp this entry was last updated at */
|
||||
public updatedAtMs: number,
|
||||
public isDeleted: boolean,
|
||||
/** Plugins on the machine when last collected, kept while it is offline */
|
||||
public plugins?: PluginDetails[],
|
||||
) { }
|
||||
|
||||
static jsonSchema = Type.Object({
|
||||
@@ -318,6 +321,7 @@ export class SystemInfo {
|
||||
"processStartedAtMs": Type.Number(),
|
||||
"updatedAtMs": Type.Number(),
|
||||
"isDeleted": Type.Boolean(),
|
||||
"plugins": Type.Optional(Type.Array(PluginDetails.jsonSchema)),
|
||||
});
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
@@ -339,6 +343,7 @@ export class SystemInfo {
|
||||
json.processStartedAtMs,
|
||||
json.updatedAtMs,
|
||||
json.isDeleted,
|
||||
json.plugins?.map(plugin => PluginDetails.fromJSON(plugin)),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -14,6 +14,18 @@ export class PluginDetails {
|
||||
public enabled: boolean,
|
||||
public description?: string,
|
||||
public npmPackage?: string,
|
||||
/** Version installed on disk, differs from version when an update is waiting for a restart */
|
||||
public installedVersion?: string,
|
||||
/** Installed after the machine started, will be loaded on the next restart */
|
||||
public pendingInstall?: boolean,
|
||||
/** Uninstalled after the machine started, will not be loaded on the next restart */
|
||||
public removed?: boolean,
|
||||
/** Name in the root package.json dependencies, differs from npmPackage for aliases */
|
||||
public dependencyName?: string,
|
||||
/** Has no entrypoint for this machine, so it is never loaded here */
|
||||
public noEntrypoint?: boolean,
|
||||
/** Not loaded as another package provides a plugin with the same name */
|
||||
public nameTaken?: boolean,
|
||||
) {}
|
||||
|
||||
static jsonSchema = Type.Object({
|
||||
@@ -24,12 +36,19 @@ export class PluginDetails {
|
||||
enabled: Type.Boolean(),
|
||||
description: Type.Optional(Type.String()),
|
||||
npmPackage: Type.Optional(Type.String()),
|
||||
installedVersion: Type.Optional(Type.String()),
|
||||
pendingInstall: Type.Optional(Type.Boolean()),
|
||||
removed: Type.Optional(Type.Boolean()),
|
||||
dependencyName: Type.Optional(Type.String()),
|
||||
noEntrypoint: Type.Optional(Type.Boolean()),
|
||||
nameTaken: Type.Optional(Type.Boolean()),
|
||||
});
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
return new this(
|
||||
json.name, json.title, json.version, json.loaded, json.enabled,
|
||||
json.description, json.npmPackage
|
||||
json.description, json.npmPackage, json.installedVersion, json.pendingInstall, json.removed,
|
||||
json.dependencyName, json.noEntrypoint, json.nameTaken,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -39,6 +58,15 @@ export class PluginDetails {
|
||||
pluginInfo.description, pluginInfo.npmPackage,
|
||||
);
|
||||
}
|
||||
|
||||
/** Plugin package installed since startup, it is loaded on the next restart */
|
||||
static fromPackageJson(packageName: string, packageJson: any, npmPackage?: string, dependencyName?: string) {
|
||||
const description = typeof packageJson.description === "string" ? packageJson.description : undefined;
|
||||
return new this(
|
||||
packageName, packageName, packageJson.version, false, false,
|
||||
description, npmPackage, packageJson.version, true, undefined, dependencyName,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class PluginListRequest {
|
||||
@@ -94,6 +122,114 @@ export class PluginInstallRequest {
|
||||
}
|
||||
}
|
||||
|
||||
/* Plugin package listed on the npm registry */
|
||||
export class PluginSearchResult {
|
||||
constructor(
|
||||
public name: string,
|
||||
public version: string,
|
||||
public description?: string,
|
||||
public publisher?: string,
|
||||
public date?: string,
|
||||
public homepage?: string,
|
||||
public repository?: string,
|
||||
public npm?: string,
|
||||
) {}
|
||||
|
||||
static jsonSchema = Type.Object({
|
||||
name: Type.String(),
|
||||
version: Type.String(),
|
||||
description: Type.Optional(Type.String()),
|
||||
publisher: Type.Optional(Type.String()),
|
||||
date: Type.Optional(Type.String()),
|
||||
homepage: Type.Optional(Type.String()),
|
||||
repository: Type.Optional(Type.String()),
|
||||
npm: Type.Optional(Type.String()),
|
||||
});
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
return new this(
|
||||
json.name, json.version, json.description, json.publisher,
|
||||
json.date, json.homepage, json.repository, json.npm,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export class PluginSearchRequest {
|
||||
declare ["constructor"]: typeof PluginSearchRequest;
|
||||
static type = "request" as const;
|
||||
static src = "control" as const;
|
||||
static dst = "controller" as const;
|
||||
static permission = "core.plugin.search";
|
||||
|
||||
constructor(
|
||||
public query: string = "",
|
||||
public page: number = 1,
|
||||
public pageSize: number = 20,
|
||||
) {}
|
||||
|
||||
static jsonSchema = Type.Object({
|
||||
query: Type.String(),
|
||||
page: Type.Integer(),
|
||||
pageSize: Type.Integer(),
|
||||
});
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
return new this(json.query, json.page, json.pageSize);
|
||||
}
|
||||
|
||||
static Response = class Response {
|
||||
constructor(
|
||||
public total: number,
|
||||
public results: PluginSearchResult[],
|
||||
) {}
|
||||
|
||||
static jsonSchema = Type.Object({
|
||||
total: Type.Integer(),
|
||||
results: Type.Array(PluginSearchResult.jsonSchema),
|
||||
});
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
return new this(json.total, json.results.map(r => PluginSearchResult.fromJSON(r)));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/* Latest version of each package published on npm, used to offer plugin updates */
|
||||
export class PluginLatestVersionsRequest {
|
||||
declare ["constructor"]: typeof PluginLatestVersionsRequest;
|
||||
static type = "request" as const;
|
||||
static src = "control" as const;
|
||||
static dst = "controller" as const;
|
||||
static permission = "core.plugin.list";
|
||||
|
||||
constructor(
|
||||
public packages: string[],
|
||||
) {}
|
||||
|
||||
static jsonSchema = Type.Object({
|
||||
packages: Type.Array(Type.String()),
|
||||
});
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
return new this(json.packages);
|
||||
}
|
||||
|
||||
static Response = class Response {
|
||||
constructor(
|
||||
/** Package name to its latest version, packages not found on npm are left out */
|
||||
public versions: Record<string, string>,
|
||||
) {}
|
||||
|
||||
static jsonSchema = Type.Object({
|
||||
versions: Type.Record(Type.String(), Type.String()),
|
||||
});
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
return new this(json.versions);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export class UpdateAllRequest {
|
||||
declare ["constructor"]: typeof UpdateAllRequest;
|
||||
static type = "request" as const;
|
||||
|
||||
@@ -321,6 +321,57 @@ export function redactUrl(url: URL | string) {
|
||||
return isPath ? parsed.pathname + parsed.search + parsed.hash : parsed.toString();
|
||||
}
|
||||
|
||||
const gitProtocols = new Set(["git:", "git+http:", "git+https:", "git+ssh:", "ssh:"]);
|
||||
const repositoryShorthands = new Map([
|
||||
["github:", "https://github.com/"],
|
||||
["gitlab:", "https://gitlab.com/"],
|
||||
["bitbucket:", "https://bitbucket.org/"],
|
||||
]);
|
||||
|
||||
/**
|
||||
* Convert a link from untrusted metadata, like a package's homepage or repository, into a URL
|
||||
* that is safe to use as the href of a link
|
||||
*
|
||||
* Only http and https URLs are returned, so links like `javascript:` can't run code when
|
||||
* clicked. Accepts the formats npm uses for repositories: a string or an object with a `url`
|
||||
* property, `git+https://`, `git://` and `git+ssh://` URLs, `git@host:path` and shorthands
|
||||
* like `github:owner/repo`, which are converted to https URLs. Credentials are removed.
|
||||
*
|
||||
* @param link - URL string or object with a `url` property.
|
||||
* @returns http(s) URL, or undefined if the link can't be converted to one.
|
||||
*/
|
||||
export function safeExternalUrl(link: unknown): string | undefined {
|
||||
const value = typeof link === "object" && link !== null ? (link as { url?: unknown }).url : link;
|
||||
if (typeof value !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
let url = value.trim();
|
||||
const scpLike = /^[\w.-]+@([\w.-]+):(?!\/)(.+)$/.exec(url);
|
||||
if (scpLike) {
|
||||
url = `ssh://${scpLike[1]}/${scpLike[2]}`;
|
||||
}
|
||||
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
const shorthand = repositoryShorthands.get(parsed.protocol);
|
||||
if (shorthand) {
|
||||
parsed = new URL(url.slice(parsed.protocol.length).replace(/^\/+/, ""), shorthand);
|
||||
} else if (gitProtocols.has(parsed.protocol)) {
|
||||
// The protocol setter can't change a URL into an https URL, so parse it again
|
||||
parsed = new URL(`https://${parsed.host}${parsed.pathname.replace(/\.git$/, "")}`);
|
||||
}
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
return undefined;
|
||||
}
|
||||
parsed.username = "";
|
||||
parsed.password = "";
|
||||
return parsed.href;
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy of parsed query parameters with credentials replaced by `REDACTED`
|
||||
*
|
||||
|
||||
@@ -147,5 +147,7 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
|
||||
plugin.PluginListRequest,
|
||||
plugin.PluginUpdateRequest,
|
||||
plugin.PluginInstallRequest,
|
||||
plugin.PluginSearchRequest,
|
||||
plugin.PluginLatestVersionsRequest,
|
||||
plugin.UpdateAllRequest,
|
||||
];
|
||||
@@ -5,6 +5,96 @@ import { logger } from "./logging.js";
|
||||
import * as libFileOps from "./file_ops.js";
|
||||
import vm from "vm";
|
||||
|
||||
const corePackages = new Set([
|
||||
"@clusterio/controller",
|
||||
"@clusterio/ctl",
|
||||
"@clusterio/host",
|
||||
"@clusterio/lib",
|
||||
"@clusterio/web_ui",
|
||||
]);
|
||||
|
||||
function isPluginPackageJson(packageJson: any): boolean {
|
||||
return Boolean(packageJson?.keywords?.includes?.("clusterio-plugin"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the package.json in a package directory.
|
||||
* @param packagePath - Directory of the package
|
||||
* @param tolerant - Return undefined instead of throwing if the file can't be parsed, as npm may be writing it
|
||||
* @returns The parsed package.json, or undefined if it does not exist
|
||||
*/
|
||||
export async function readPackageJson(packagePath: string, tolerant: boolean): Promise<any> {
|
||||
let content;
|
||||
try {
|
||||
content = await fs.readFile(path.join(packagePath, "package.json"), "utf8");
|
||||
} catch (err: any) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(content);
|
||||
} catch (err: any) {
|
||||
if (tolerant) {
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Dependencies npm installs, an optionalDependencies entry wins over a dependencies entry of the same name */
|
||||
export function installedDependencies(packageJson: any): Record<string, unknown> {
|
||||
return { ...packageJson?.dependencies, ...packageJson?.optionalDependencies };
|
||||
}
|
||||
|
||||
/** Whether a dependency spec installs from the registry, as opposed to a path, git, url, tarball or alias */
|
||||
export function isRegistrySpec(spec: unknown) {
|
||||
return typeof spec === "string" && !/[:/\\]/.test(spec) && !spec.startsWith(".")
|
||||
&& !/\.(?:tgz|tar\.gz|tar)$/i.test(spec);
|
||||
}
|
||||
|
||||
/** Real path of a directory, used to compare packages as npm and pnpm install them behind symlinks */
|
||||
export async function realDir(dir: string) {
|
||||
try {
|
||||
return await fs.realpath(dir);
|
||||
} catch {
|
||||
// Removed since, nothing else can be at this path
|
||||
return path.resolve(dir);
|
||||
}
|
||||
}
|
||||
|
||||
/** Directory of a package as require resolves it from fromPath, or undefined if not found */
|
||||
export function resolvePackagePath(packageName: string, fromPath: string, tolerant: boolean): string | undefined {
|
||||
const packageRequire = createRequire(path.join(fromPath, "package.json"));
|
||||
try {
|
||||
return path.dirname(packageRequire.resolve(`${packageName}/package.json`));
|
||||
} catch (err: any) {
|
||||
// A package.json npm is still writing fails with ERR_INVALID_PACKAGE_CONFIG
|
||||
if (tolerant || err.code === "MODULE_NOT_FOUND" || err.code === "ERR_PACKAGE_PATH_NOT_EXPORTED") {
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function pluginPathExists(pluginPath: string) {
|
||||
if (!path.isAbsolute(pluginPath)) {
|
||||
return resolvePackagePath(pluginPath, process.cwd(), true) !== undefined;
|
||||
}
|
||||
try {
|
||||
await fs.access(path.join(pluginPath, "package.json"));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
function getPluginName(requireSpec: string) {
|
||||
const context = vm.createContext({ require: require, pluginInfo: null });
|
||||
const code = `pluginInfo = require(${JSON.stringify(requireSpec)}).plugin;`;
|
||||
vm.runInContext(code, context);
|
||||
return context.pluginInfo.name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches for local plugins in the plugins/ and external_plugins/ directories
|
||||
* and adds them to the plugin list if they contain a package.json file.
|
||||
@@ -49,7 +139,7 @@ async function findLocalPlugins(pluginList: Map<string, string>): Promise<boolea
|
||||
const stats = await fs.stat(pluginPath);
|
||||
if (stats.isDirectory()) {
|
||||
// Check for package.json and plugin keyword
|
||||
if (await checkPackageJson(pluginPath)) {
|
||||
if (isPluginPackageJson(await readPackageJson(pluginPath, false))) {
|
||||
pluginList.set(pluginName, path.resolve(pluginPath));
|
||||
logger.info(`Added ${pluginName} from ${pluginFolder}`);
|
||||
hasChanged += 1;
|
||||
@@ -67,24 +157,85 @@ async function findLocalPlugins(pluginList: Map<string, string>): Promise<boolea
|
||||
return hasChanged > 0;
|
||||
}
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
function getPluginName(requireSpec: string) {
|
||||
const context = vm.createContext({ require: require, pluginInfo: null });
|
||||
const code = `pluginInfo = require(${JSON.stringify(requireSpec)}).plugin;`;
|
||||
vm.runInContext(code, context);
|
||||
return context.pluginInfo.name;
|
||||
type NpmPluginPackage = {
|
||||
packageName: string,
|
||||
packageVersion: unknown,
|
||||
packagePath: string,
|
||||
packageJson: any,
|
||||
};
|
||||
|
||||
/**
|
||||
* Walk the dependencies of the root package.json and of every plugin package found, without
|
||||
* traversing packages that aren't plugins. Does not execute any plugin code.
|
||||
*
|
||||
* @param dependencies - Dependencies of the root package.json
|
||||
* @param tolerant - Skip packages with a package.json that can't be parsed instead of throwing,
|
||||
* for use while npm may be rewriting them
|
||||
* @returns Plugin packages found, and root dependencies that are not plugins
|
||||
*/
|
||||
export async function discoverNpmPluginPackages(dependencies: Record<string, unknown>, tolerant = false) {
|
||||
const plugins: NpmPluginPackage[] = [];
|
||||
const nonPluginRoots: { packageName: string, packageVersion: unknown }[] = [];
|
||||
const pendingPackages = Object.entries(dependencies).map(([packageName, packageVersion]) => ({
|
||||
packageName,
|
||||
packageVersion,
|
||||
parentPath: process.cwd(),
|
||||
isRootDependency: true,
|
||||
}));
|
||||
const visitedPackages = new Set<string>();
|
||||
// Directories some dependent installs from a path, git, url, tarball or alias
|
||||
const nonRegistryDirs = new Set<string>();
|
||||
|
||||
while (pendingPackages.length) {
|
||||
const { packageName, packageVersion, parentPath, isRootDependency } = pendingPackages.shift()!;
|
||||
const packagePath = resolvePackagePath(packageName, parentPath, tolerant);
|
||||
if (!packagePath) {
|
||||
continue;
|
||||
}
|
||||
if (!isRegistrySpec(packageVersion)) {
|
||||
nonRegistryDirs.add(await realDir(packagePath));
|
||||
}
|
||||
if (visitedPackages.has(packageName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
visitedPackages.add(packageName);
|
||||
|
||||
const packageJson = await readPackageJson(packagePath, tolerant);
|
||||
if (packageJson && isPluginPackageJson(packageJson)) {
|
||||
plugins.push({ packageName, packageVersion, packagePath, packageJson });
|
||||
for (const [dependencyName, dependencyVersion] of Object.entries(installedDependencies(packageJson))) {
|
||||
pendingPackages.push({
|
||||
packageName: dependencyName,
|
||||
packageVersion: dependencyVersion,
|
||||
parentPath: packagePath,
|
||||
isRootDependency: false,
|
||||
});
|
||||
}
|
||||
} else if (isRootDependency) {
|
||||
nonPluginRoots.push({ packageName, packageVersion });
|
||||
}
|
||||
}
|
||||
return { plugins, nonPluginRoots, nonRegistryDirs };
|
||||
}
|
||||
|
||||
function resolvePackagePath(packageName: string, fromPath: string): string | undefined {
|
||||
const packageRequire = createRequire(path.join(fromPath, "package.json"));
|
||||
try {
|
||||
return path.dirname(packageRequire.resolve(`${packageName}/package.json`));
|
||||
} catch (err: any) {
|
||||
if (err.code === "MODULE_NOT_FOUND" || err.code === "ERR_PACKAGE_PATH_NOT_EXPORTED") {
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
/** Real paths of plugin packages some dependent installs from a path, git, url, tarball or alias */
|
||||
export async function findNonRegistryDirs() {
|
||||
const dependencies = installedDependencies(await readPackageJson(process.cwd(), true));
|
||||
return (await discoverNpmPluginPackages(dependencies, true)).nonRegistryDirs;
|
||||
}
|
||||
|
||||
/** Whether npm install name would replace a plugin package not installed from the registry */
|
||||
export async function isNonRegistryPackage(name: string) {
|
||||
const dependencies = installedDependencies(await readPackageJson(process.cwd(), true));
|
||||
if (Object.hasOwn(dependencies, name) && !isRegistrySpec(dependencies[name])) {
|
||||
return true;
|
||||
}
|
||||
const packagePath = resolvePackagePath(name, process.cwd(), true);
|
||||
if (!packagePath) {
|
||||
return false;
|
||||
}
|
||||
return (await findNonRegistryDirs()).has(await realDir(packagePath));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -92,79 +243,46 @@ function resolvePackagePath(packageName: string, fromPath: string): string | und
|
||||
* Adds discovered plugins to the plugin list.
|
||||
*
|
||||
* @param pluginList - Map of plugin names to their file paths
|
||||
* @param skippedNpmPlugins - Filled with the package directory by plugin name for packages
|
||||
* skipped because their plugin name was already in the list
|
||||
* @returns Promise that resolves to true if any plugins were added to the list, false otherwise
|
||||
*/
|
||||
async function findNpmPlugins(pluginList: Map<string, string>): Promise<boolean> {
|
||||
let dependencies;
|
||||
try {
|
||||
const rootPackageJson = JSON.parse(await fs.readFile(path.resolve("package.json"), { encoding: "utf8" }));
|
||||
dependencies = rootPackageJson.dependencies;
|
||||
} catch (err: any) {
|
||||
if (err.code === "ENOENT") {
|
||||
// Skip if package.json doesn't exist
|
||||
return false;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (!dependencies) {
|
||||
async function findNpmPlugins(
|
||||
pluginList: Map<string, string>,
|
||||
skippedNpmPlugins?: Map<string, string>,
|
||||
): Promise<boolean> {
|
||||
const rootPackageJson = await readPackageJson(process.cwd(), false);
|
||||
if (!rootPackageJson) {
|
||||
return false;
|
||||
}
|
||||
const dependencies = installedDependencies(rootPackageJson);
|
||||
|
||||
let hasChanged = 0;
|
||||
const pendingPackages = Object.entries(dependencies).map(([packageName, packageVersion]) => ({
|
||||
packageName,
|
||||
packageVersion,
|
||||
parentPath: process.cwd(),
|
||||
isRootDependency: true,
|
||||
}));
|
||||
const corePackages = new Set([
|
||||
"@clusterio/controller",
|
||||
"@clusterio/ctl",
|
||||
"@clusterio/host",
|
||||
"@clusterio/lib",
|
||||
"@clusterio/web_ui",
|
||||
]);
|
||||
const visitedPackages = new Set<string>();
|
||||
|
||||
while (pendingPackages.length) {
|
||||
const { packageName, packageVersion, parentPath, isRootDependency } = pendingPackages.shift()!;
|
||||
if (visitedPackages.has(packageName)) {
|
||||
const { plugins, nonPluginRoots } = await discoverNpmPluginPackages(dependencies);
|
||||
for (const { packageName, packageVersion, packagePath } of plugins) {
|
||||
const pluginName = getPluginName(packagePath);
|
||||
const listedPath = pluginList.get(pluginName);
|
||||
if (listedPath === packageName) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const npmPackagePath = resolvePackagePath(packageName, parentPath);
|
||||
if (!npmPackagePath) {
|
||||
continue;
|
||||
}
|
||||
|
||||
visitedPackages.add(packageName);
|
||||
|
||||
if (await checkPackageJson(npmPackagePath)) {
|
||||
const pluginName = getPluginName(npmPackagePath);
|
||||
if (pluginList.has(pluginName)) {
|
||||
logger.warn(
|
||||
`${pluginName} provided by ${packageName}@${packageVersion} is already in the plugin list, ` +
|
||||
`but with a different path - using ${pluginList.get(pluginName)}`
|
||||
);
|
||||
} else {
|
||||
pluginList.set(pluginName, packageName);
|
||||
logger.info(`Added ${pluginName} from node modules`);
|
||||
hasChanged += 1;
|
||||
}
|
||||
|
||||
const packageJson = JSON.parse(
|
||||
await fs.readFile(path.join(npmPackagePath, "package.json"), { encoding: "utf8" })
|
||||
if (listedPath !== undefined && !await pluginPathExists(listedPath)) {
|
||||
logger.warn(`${pluginName} at ${listedPath} no longer exists, using ${packageName}`);
|
||||
pluginList.set(pluginName, packageName);
|
||||
hasChanged += 1;
|
||||
} else if (pluginList.has(pluginName)) {
|
||||
logger.warn(
|
||||
`${pluginName} provided by ${packageName}@${packageVersion} is already in the plugin list, ` +
|
||||
`but with a different path - using ${pluginList.get(pluginName)}`
|
||||
);
|
||||
for (const [dependencyName, dependencyVersion] of Object.entries(packageJson.dependencies ?? {})) {
|
||||
pendingPackages.push({
|
||||
packageName: dependencyName,
|
||||
packageVersion: dependencyVersion,
|
||||
parentPath: npmPackagePath,
|
||||
isRootDependency: false,
|
||||
});
|
||||
}
|
||||
} else if (isRootDependency && !corePackages.has(packageName)) {
|
||||
skippedNpmPlugins?.set(pluginName, await realDir(packagePath));
|
||||
} else {
|
||||
pluginList.set(pluginName, packageName);
|
||||
logger.info(`Added ${pluginName} from node modules`);
|
||||
hasChanged += 1;
|
||||
}
|
||||
}
|
||||
for (const { packageName, packageVersion } of nonPluginRoots) {
|
||||
if (!corePackages.has(packageName)) {
|
||||
logger.warn(`${packageName}@${packageVersion} is not a clusterio-plugin`);
|
||||
}
|
||||
}
|
||||
@@ -176,9 +294,15 @@ async function findNpmPlugins(pluginList: Map<string, string>): Promise<boolean>
|
||||
*
|
||||
* @param pluginListPath - Path to the plugin list JSON file
|
||||
* @param findLocal - Whether to look for local plugins in plugins/ and external_plugins/ directories
|
||||
* @param skippedNpmPlugins - Filled with the package directory by plugin name for npm packages
|
||||
* skipped because another plugin has the same name, for loadPluginInfos
|
||||
* @returns Map of plugin name to plugin path
|
||||
*/
|
||||
export async function loadPluginList(pluginListPath: string, findLocal = true): Promise<Map<string, string>> {
|
||||
export async function loadPluginList(
|
||||
pluginListPath: string,
|
||||
findLocal = true,
|
||||
skippedNpmPlugins?: Map<string, string>,
|
||||
): Promise<Map<string, string>> {
|
||||
let pluginList = new Map<string, string>();
|
||||
|
||||
// Try to load existing plugin list
|
||||
@@ -201,7 +325,7 @@ export async function loadPluginList(pluginListPath: string, findLocal = true):
|
||||
}
|
||||
|
||||
// Discover NPM plugins
|
||||
hasChanges = await findNpmPlugins(pluginList) || hasChanges;
|
||||
hasChanges = await findNpmPlugins(pluginList, skippedNpmPlugins) || hasChanges;
|
||||
|
||||
if (hasChanges) {
|
||||
await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, "\t"));
|
||||
@@ -209,24 +333,3 @@ export async function loadPluginList(pluginListPath: string, findLocal = true):
|
||||
|
||||
return pluginList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to check if a package.json file exists and contains the clusterio-plugin keyword
|
||||
* @param pluginPath - Path to the plugin directory
|
||||
* @returns Promise that resolves to true if package.json exists and has clusterio-plugin keyword
|
||||
*/
|
||||
async function checkPackageJson(pluginPath: string): Promise<boolean> {
|
||||
const packageJsonPath = path.join(pluginPath, "package.json");
|
||||
|
||||
// Read and parse package.json
|
||||
let content;
|
||||
try {
|
||||
content = await fs.readFile(packageJsonPath, { encoding: "utf8" });
|
||||
} catch (err: any) {
|
||||
return false;
|
||||
}
|
||||
const packageJson = JSON.parse(content);
|
||||
|
||||
// Check if package has the clusterio-plugin keyword
|
||||
return packageJson.keywords?.includes("clusterio-plugin");
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import { logger } from "./logging.js";
|
||||
import { RequestError } from "./errors.js";
|
||||
import { PluginSearchRequest, PluginSearchResult } from "./data/messages_plugin.js";
|
||||
|
||||
const registryTimeoutMs = 10e3;
|
||||
|
||||
// One name segment, optionally after an @scope/ segment. Not starting with . or - keeps npm from
|
||||
// reading the spec as a path, a git repository or a command line option.
|
||||
// https://docs.npmjs.com/cli/v11/configuring-npm/package-json#name
|
||||
const packageNamePattern = /^(?:@[a-zA-Z0-9][a-zA-Z0-9\-._~]*\/)?[a-zA-Z0-9][a-zA-Z0-9\-._~]*$/;
|
||||
|
||||
/**
|
||||
* Split a package spec into its name and optional version, as in name@version. The name itself
|
||||
* may start with @ for scoped packages.
|
||||
*/
|
||||
export function parsePackageSpec(spec: string) {
|
||||
const versionSeparator = spec.lastIndexOf("@");
|
||||
const [name, version] = versionSeparator > 0
|
||||
? [spec.slice(0, versionSeparator), spec.slice(versionSeparator + 1)]
|
||||
: [spec, undefined];
|
||||
if (name.length > 214 || !packageNamePattern.test(name)) {
|
||||
throw new RequestError(`Invalid plugin name: ${spec}`);
|
||||
}
|
||||
if (version !== undefined && !/^[a-zA-Z0-9][a-zA-Z0-9\-.+]*$/.test(version)) {
|
||||
throw new RequestError(`Invalid plugin version: ${version}`);
|
||||
}
|
||||
return { name, version };
|
||||
}
|
||||
|
||||
/** Throws unless the package, and version if given, is published on npm, returns the spec to install */
|
||||
export async function checkRegistry(name: string, version?: string) {
|
||||
const packageName = encodeURI(name);
|
||||
const spec = version !== undefined ? `${packageName}@${version}` : packageName;
|
||||
const registryPath = version !== undefined ? `${packageName}/${encodeURIComponent(version)}` : packageName;
|
||||
const npmRequest = await fetch(`https://registry.npmjs.com/${registryPath}`, {
|
||||
method: "HEAD",
|
||||
signal: AbortSignal.timeout(registryTimeoutMs),
|
||||
});
|
||||
if (!npmRequest.ok) {
|
||||
throw new RequestError(`Unknown plugin: ${spec}`);
|
||||
}
|
||||
return spec;
|
||||
}
|
||||
|
||||
const latestVersionCacheMs = 5 * 60e3;
|
||||
const latestVersionCacheMaxSize = 1000;
|
||||
const latestVersionConcurrency = 8;
|
||||
|
||||
/** Latest version of each package by name, as found on npm, with when it was looked up */
|
||||
export type LatestVersionCache = Map<string, { version?: string, fetchedAt: number }>;
|
||||
|
||||
/**
|
||||
* Look up the latest published version of each package on npm, packages that can't be
|
||||
* found are left out. Lookups are kept in the given cache for a few minutes.
|
||||
*/
|
||||
export async function fetchLatestVersions(packages: string[], cache: LatestVersionCache) {
|
||||
if (packages.length > 500) {
|
||||
throw new RequestError("Too many packages");
|
||||
}
|
||||
const names = [...new Set(packages.map(packageName => {
|
||||
const { name, version: pinned } = parsePackageSpec(packageName);
|
||||
if (pinned !== undefined) {
|
||||
throw new RequestError(`Invalid plugin name: ${packageName}`);
|
||||
}
|
||||
return name;
|
||||
}))];
|
||||
|
||||
const now = Date.now();
|
||||
for (const [name, cached] of cache) {
|
||||
if (now - cached.fetchedAt > latestVersionCacheMs) {
|
||||
cache.delete(name);
|
||||
}
|
||||
}
|
||||
|
||||
const versions: Record<string, string> = {};
|
||||
const queue = [...names];
|
||||
async function lookUp(name: string) {
|
||||
let cached = cache.get(name);
|
||||
if (!cached) {
|
||||
let response;
|
||||
try {
|
||||
response = await fetch(`https://registry.npmjs.com/${name}/latest`, {
|
||||
signal: AbortSignal.timeout(registryTimeoutMs),
|
||||
});
|
||||
} catch (err: any) {
|
||||
logger.warn(`Failed to look up latest version of ${name}: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
if (!response.ok && response.status !== 404) {
|
||||
// Rate limits and outages are retried on the next lookup instead of cached
|
||||
logger.warn(`Failed to look up latest version of ${name}: ${response.status}`);
|
||||
return;
|
||||
}
|
||||
const version = response.ok ? (await response.json() as { version?: string }).version : undefined;
|
||||
cached = { version, fetchedAt: now };
|
||||
for (const oldest of cache.keys()) {
|
||||
if (cache.size < latestVersionCacheMaxSize) {
|
||||
break;
|
||||
}
|
||||
cache.delete(oldest);
|
||||
}
|
||||
cache.set(name, cached);
|
||||
}
|
||||
if (cached.version) {
|
||||
versions[name] = cached.version;
|
||||
}
|
||||
}
|
||||
await Promise.all(Array.from({ length: latestVersionConcurrency }, async () => {
|
||||
for (let name = queue.shift(); name !== undefined; name = queue.shift()) {
|
||||
await lookUp(name);
|
||||
}
|
||||
}));
|
||||
return versions;
|
||||
}
|
||||
|
||||
const npmSearchMaxSize = 250; // Limit imposed by the registry
|
||||
|
||||
export async function searchPlugins(query: string, page: number, pageSize: number) {
|
||||
if (query.length > 214) {
|
||||
throw new RequestError("Search query too long");
|
||||
}
|
||||
if (page < 1 || pageSize < 1 || pageSize > npmSearchMaxSize) {
|
||||
throw new RequestError("Invalid page or page size");
|
||||
}
|
||||
|
||||
const url = new URL("https://registry.npmjs.com/-/v1/search");
|
||||
url.searchParams.set("text", `keywords:clusterio-plugin ${query}`.trim());
|
||||
url.searchParams.set("size", String(pageSize));
|
||||
url.searchParams.set("from", String((page - 1) * pageSize));
|
||||
|
||||
const response = await fetch(url, { signal: AbortSignal.timeout(registryTimeoutMs) });
|
||||
if (!response.ok) {
|
||||
throw new RequestError(`npm registry search failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const json = await response.json() as {
|
||||
total: number,
|
||||
objects: {
|
||||
package: {
|
||||
name: string,
|
||||
version: string,
|
||||
description?: string | null,
|
||||
date?: string | null,
|
||||
publisher?: { username?: string | null } | null,
|
||||
links?: { npm?: string | null, homepage?: string | null, repository?: string | null } | null,
|
||||
},
|
||||
}[],
|
||||
};
|
||||
|
||||
return new PluginSearchRequest.Response(
|
||||
json.total,
|
||||
json.objects.map(({ package: pkg }) => new PluginSearchResult(
|
||||
// The registry sends null for some missing fields
|
||||
pkg.name, pkg.version, pkg.description ?? undefined, pkg.publisher?.username ?? undefined,
|
||||
pkg.date ?? undefined, pkg.links?.homepage ?? undefined, pkg.links?.repository ?? undefined,
|
||||
pkg.links?.npm ?? undefined,
|
||||
)),
|
||||
);
|
||||
}
|
||||
@@ -583,6 +583,11 @@ export const corePermissions = [
|
||||
title: "Install plugin",
|
||||
description: "Remotely install a plugin if the target allows remote installs of plugins.",
|
||||
},
|
||||
{
|
||||
name: "core.plugin.search",
|
||||
title: "Search plugins",
|
||||
description: "Search the npm registry for plugins via the controller.",
|
||||
},
|
||||
|
||||
{
|
||||
name: "core.debug.dump_ws",
|
||||
|
||||
@@ -82,6 +82,11 @@ export type PluginNodeEnvInfo = PluginDeclaration & {
|
||||
* package.
|
||||
*/
|
||||
npmPackage?: string;
|
||||
/**
|
||||
* Directory of an npm package providing a plugin with the same name
|
||||
* that was skipped on startup as this plugin was already in the list.
|
||||
*/
|
||||
skippedNpmPackage?: string;
|
||||
};
|
||||
|
||||
export type PluginWebpackEnvInfo = PluginDeclaration & {
|
||||
|
||||
@@ -13,6 +13,10 @@ import { loadPluginEntrypoint, loadPluginClass } from "./loadPlugin.js";
|
||||
import { Config } from "./config/classes.js";
|
||||
import { ControllerConfig, HostConfig, InstanceConfig } from "./config/definitions.js";
|
||||
import Role from "./data/Role.js";
|
||||
import { PluginDetails } from "./data/messages_plugin.js";
|
||||
import {
|
||||
discoverNpmPluginPackages, findNonRegistryDirs, installedDependencies, readPackageJson, realDir, resolvePackagePath,
|
||||
} from "./load_plugin_list.js";
|
||||
|
||||
|
||||
/**
|
||||
@@ -24,10 +28,13 @@ import Role from "./data/Role.js";
|
||||
*
|
||||
* @param pluginList -
|
||||
* Mapping of plugin name to require path for the plugins to load.
|
||||
* @param skippedNpmPlugins -
|
||||
* Directory of the npm package skipped by plugin name, from loadPluginList.
|
||||
* @returns Array of plugin info modules.
|
||||
*/
|
||||
export async function loadPluginInfos(pluginList: Map<string, string>) {
|
||||
export async function loadPluginInfos(pluginList: Map<string, string>, skippedNpmPlugins?: Map<string, string>) {
|
||||
let plugins: libPlugin.PluginNodeEnvInfo[] = [];
|
||||
const nonRegistryDirs = await findNonRegistryDirs();
|
||||
for (let [pluginName, pluginPath] of pluginList) {
|
||||
let pluginInfo: libPlugin.PluginNodeEnvInfo;
|
||||
let pluginPackage: { name?: string, version: string, main?: string, private?: boolean };
|
||||
@@ -87,12 +94,106 @@ export async function loadPluginInfos(pluginList: Map<string, string>) {
|
||||
pluginInfo.packagePath = packagePath;
|
||||
pluginInfo.requirePath = pluginPath;
|
||||
pluginInfo.version = pluginPackage.version;
|
||||
pluginInfo.npmPackage = !pluginPackage.private && pluginPath === pluginPackage.name ? pluginPath : undefined;
|
||||
pluginInfo.npmPackage = !pluginPackage.private && pluginPath === pluginPackage.name
|
||||
&& !nonRegistryDirs.has(await fs.realpath(path.dirname(packagePath))) ? pluginPath : undefined;
|
||||
pluginInfo.skippedNpmPackage = skippedNpmPlugins?.get(pluginName);
|
||||
plugins.push(pluginInfo);
|
||||
}
|
||||
return plugins;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists the plugins on this machine along with the versions currently installed on disk,
|
||||
* including npm plugins installed since startup that will only be loaded after a restart.
|
||||
*
|
||||
* @param pluginInfos - Plugins discovered when this machine started
|
||||
* @param isLoaded - Whether the plugin is loaded
|
||||
* @param isEnabled - Whether the plugin is enabled
|
||||
* @param hasEntrypoint - Whether the plugin has an entrypoint this machine loads
|
||||
* @returns Details of every plugin on this machine
|
||||
*/
|
||||
export async function listPluginDetails(
|
||||
pluginInfos: libPlugin.PluginNodeEnvInfo[],
|
||||
isLoaded: (pluginInfo: libPlugin.PluginNodeEnvInfo) => boolean,
|
||||
isEnabled: (pluginInfo: libPlugin.PluginNodeEnvInfo) => boolean,
|
||||
hasEntrypoint: (pluginInfo: libPlugin.PluginNodeEnvInfo) => boolean = () => true,
|
||||
): Promise<PluginDetails[]> {
|
||||
const plugins: PluginDetails[] = [];
|
||||
const dependencies = installedDependencies(await readPackageJson(process.cwd(), true));
|
||||
// By directory, as a plugin loaded from a path may also be an npm dependency
|
||||
const knownDirs = new Set<string>();
|
||||
// Packages skipped on startup by directory, for plugins still installed
|
||||
const takenDirs = new Set<string>();
|
||||
const { plugins: npmPlugins, nonRegistryDirs } = await discoverNpmPluginPackages(dependencies, true);
|
||||
for (const pluginInfo of pluginInfos) {
|
||||
const dir = await realDir(path.dirname(pluginInfo.packagePath));
|
||||
knownDirs.add(dir);
|
||||
// Without an entrypoint for this machine the plugin has no load_plugin field here
|
||||
const loadable = hasEntrypoint(pluginInfo);
|
||||
const plugin = PluginDetails.fromNodeEnvInfo(
|
||||
pluginInfo, isLoaded(pluginInfo), loadable && isEnabled(pluginInfo),
|
||||
);
|
||||
if (!loadable) {
|
||||
plugin.noEntrypoint = true;
|
||||
}
|
||||
if (Object.hasOwn(dependencies, pluginInfo.requirePath)) {
|
||||
plugin.dependencyName = pluginInfo.requirePath;
|
||||
}
|
||||
if (nonRegistryDirs.has(dir)) {
|
||||
plugin.npmPackage = undefined;
|
||||
}
|
||||
let packageJson;
|
||||
try {
|
||||
packageJson = await readPackageJson(path.dirname(pluginInfo.packagePath), false);
|
||||
if (packageJson === undefined) {
|
||||
// Uninstalled since startup, it won't be found on the next restart
|
||||
plugin.removed = true;
|
||||
}
|
||||
} catch {
|
||||
// Ignored: npm is still writing it
|
||||
}
|
||||
if (packageJson) {
|
||||
plugin.installedVersion = packageJson.version;
|
||||
if (packageJson.name !== plugin.npmPackage) {
|
||||
// Replaced by an alias since startup
|
||||
plugin.npmPackage = undefined;
|
||||
}
|
||||
}
|
||||
if (!plugin.removed && pluginInfo.skippedNpmPackage !== undefined) {
|
||||
takenDirs.add(pluginInfo.skippedNpmPackage);
|
||||
}
|
||||
plugins.push(plugin);
|
||||
}
|
||||
|
||||
for (const { packageName, packagePath, packageJson } of npmPlugins) {
|
||||
const dir = await realDir(packagePath);
|
||||
if (knownDirs.has(dir)) {
|
||||
continue;
|
||||
}
|
||||
// Startup loads it by package name from the root, which doesn't reach nested packages
|
||||
const rootPath = resolvePackagePath(packageName, process.cwd(), true);
|
||||
if (!rootPath || await realDir(rootPath) !== dir) {
|
||||
continue;
|
||||
}
|
||||
if (typeof packageJson.version !== "string") {
|
||||
continue;
|
||||
}
|
||||
// Same rule as loadPluginInfos, aliases, private and non-registry packages can't be installed by this name
|
||||
const npmPackage = !packageJson.private && packageJson.name === packageName && !nonRegistryDirs.has(dir)
|
||||
? packageName : undefined;
|
||||
const dependencyName = Object.hasOwn(dependencies, packageName) ? packageName : undefined;
|
||||
const plugin = PluginDetails.fromPackageJson(packageName, packageJson, npmPackage, dependencyName);
|
||||
if (takenDirs.has(dir)) {
|
||||
// Skipped on startup as its plugin name is taken, will be again
|
||||
plugin.pendingInstall = undefined;
|
||||
plugin.nameTaken = true;
|
||||
}
|
||||
plugins.push(plugin);
|
||||
}
|
||||
|
||||
return plugins;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load the migration entrypoints of plugins
|
||||
*
|
||||
|
||||
+79
-24
@@ -4,6 +4,9 @@ import { exec } from "child_process";
|
||||
import { logger } from "./logging.js";
|
||||
import { RequestError } from "./errors.js";
|
||||
import type { PluginNodeEnvInfo } from "./plugin.js";
|
||||
import { isNonRegistryPackage } from "./load_plugin_list.js";
|
||||
import { LockFile, LockFileExistsError, LockFileRecoveryError } from "./LockFile.js";
|
||||
import { checkRegistry, parsePackageSpec } from "./npm_registry.js";
|
||||
const execAsync = util.promisify(exec);
|
||||
|
||||
function isDev() {
|
||||
@@ -12,10 +15,55 @@ function isDev() {
|
||||
return import.meta.dirname.split(path.sep).at(-5) === "packages"; // opposed to "@clusterio"
|
||||
}
|
||||
|
||||
async function logExec(cmd: string) {
|
||||
logger.audit(`RCE | ${cmd}`);
|
||||
if (!isDev()) {
|
||||
await execAsync(cmd);
|
||||
const operationInProgressMessage = "Another package operation is already in progress on this machine";
|
||||
|
||||
let operationInProgress = false;
|
||||
|
||||
/**
|
||||
* Run an npm command, at most one at a time per installation. npm runs in the working directory,
|
||||
* which a standalone controller and host share, so a lock file there guards against operations
|
||||
* from other processes and other copies of lib. The flag makes concurrent calls through this copy
|
||||
* reject without racing a release of the lock.
|
||||
*
|
||||
* @param cmd - Command to run
|
||||
* @param check - Called while holding the lock before running the command, may throw to cancel it
|
||||
*/
|
||||
async function logExec(cmd: string, check?: () => Promise<void>) {
|
||||
// Checked and set before the first await so concurrent calls in this process can't interleave
|
||||
if (operationInProgress) {
|
||||
throw new RequestError(operationInProgressMessage);
|
||||
}
|
||||
operationInProgress = true;
|
||||
const lockPath = path.resolve("npm-operation.lock");
|
||||
// Signal listeners would change how the process exits. If only this process is killed npm keeps
|
||||
// running unlocked, Ctrl+C, systemd and containers stop it too.
|
||||
const lock = new LockFile(lockPath, false);
|
||||
try {
|
||||
try {
|
||||
await lock.acquire();
|
||||
} catch (err) {
|
||||
if (err instanceof LockFileRecoveryError) {
|
||||
throw new RequestError(err.message);
|
||||
}
|
||||
if (!(err instanceof LockFileExistsError)) {
|
||||
throw err;
|
||||
}
|
||||
const holder = await lock.holderPid();
|
||||
throw new RequestError(
|
||||
`${operationInProgressMessage}${holder ? ` (${lockPath} is held by process ${holder})` : ""}`
|
||||
);
|
||||
}
|
||||
try {
|
||||
await check?.();
|
||||
logger.audit(`RCE | ${cmd}`);
|
||||
if (!isDev()) {
|
||||
await execAsync(cmd);
|
||||
}
|
||||
} finally {
|
||||
await lock.release();
|
||||
}
|
||||
} finally {
|
||||
operationInProgress = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,16 +71,13 @@ export async function updatePackage(...names: string[]) {
|
||||
return logExec(`npm update --save ${names.join(" ")}`);
|
||||
}
|
||||
|
||||
export async function installPackage(name: string) {
|
||||
return logExec(`npm install --save ${name}`);
|
||||
export async function installPackage(name: string, check?: () => Promise<void>) {
|
||||
return logExec(`npm install --save ${name}`, check);
|
||||
}
|
||||
|
||||
export async function handlePluginUpdate(pluginName: string, pluginInfos: PluginNodeEnvInfo[]) {
|
||||
if (!pluginInfos.some(plugin => plugin.npmPackage === pluginName)) {
|
||||
throw new RequestError(`Plugin ${pluginName} is not installed on this machine`);
|
||||
}
|
||||
|
||||
return await updatePackage(pluginName);
|
||||
/** Whether an npm operation is running in this installation, from this or another process */
|
||||
export async function packageOperationRunning() {
|
||||
return operationInProgress || !await new LockFile(path.resolve("npm-operation.lock")).isStale();
|
||||
}
|
||||
|
||||
export async function handleUpdateAll(corePackage: string, pluginInfos: PluginNodeEnvInfo[]) {
|
||||
@@ -42,20 +87,30 @@ export async function handleUpdateAll(corePackage: string, pluginInfos: PluginNo
|
||||
return await updatePackage(corePackage, ...pluginPackages);
|
||||
}
|
||||
|
||||
export async function handlePluginInstall(pluginName: string) {
|
||||
if (pluginName.length > 214 || /[^a-zA-Z0-9\-_.+@\/]/.test(pluginName)) {
|
||||
// https://docs.npmjs.com/cli/v11/configuring-npm/package-json#name
|
||||
// https://www.npmjs.com/package/validate-npm-package-name
|
||||
throw new RequestError(`Invalid plugin name: ${pluginName}`);
|
||||
/** Throws if the dependency by this name is a path, git, url or alias that installing by name would replace */
|
||||
async function checkNotLocalDependency(name: string) {
|
||||
if (await isNonRegistryPackage(name)) {
|
||||
throw new RequestError(`${name} is not installed from npm on this machine`);
|
||||
}
|
||||
}
|
||||
|
||||
const packageName = encodeURI(pluginName);
|
||||
const npmRequest = await fetch(`https://registry.npmjs.com/${packageName}`, {
|
||||
method: "HEAD",
|
||||
});
|
||||
if (!npmRequest.ok) {
|
||||
throw new RequestError(`Unknown plugin: ${packageName}`);
|
||||
export async function handlePluginUpdate(pluginName: string, pluginInfos: PluginNodeEnvInfo[]) {
|
||||
// Optionally pinned to a version as name@version to update past the range in package.json
|
||||
const { name, version } = parsePackageSpec(pluginName);
|
||||
if (!pluginInfos.some(plugin => plugin.npmPackage === name)) {
|
||||
throw new RequestError(`Plugin ${name} is not installed on this machine`);
|
||||
}
|
||||
|
||||
return await installPackage(packageName);
|
||||
if (version === undefined) {
|
||||
return await updatePackage(name);
|
||||
}
|
||||
// Checked under the lock as another operation may change where it's installed from
|
||||
return await installPackage(await checkRegistry(name, version), () => checkNotLocalDependency(name));
|
||||
}
|
||||
|
||||
/** Install a plugin, optionally pinned to a version as name@version, returns the package name */
|
||||
export async function handlePluginInstall(pluginName: string) {
|
||||
const { name, version } = parsePackageSpec(pluginName);
|
||||
await installPackage(await checkRegistry(name, version), () => checkNotLocalDependency(name));
|
||||
return name;
|
||||
}
|
||||
@@ -2,7 +2,7 @@ import fs from "node:fs/promises";
|
||||
import os from "os";
|
||||
import util from "util";
|
||||
import { Gauge } from "./prometheus.js";
|
||||
import { SystemInfo } from "./data/index.js";
|
||||
import { PluginDetails, SystemInfo } from "./data/index.js";
|
||||
|
||||
function filterCpuModel() {
|
||||
const model = os.cpus()[0].model;
|
||||
@@ -125,7 +125,8 @@ export async function gatherSystemInfo(
|
||||
id: SystemInfo["id"],
|
||||
canRestart: boolean,
|
||||
restartRequired: boolean,
|
||||
startedAtMs: number
|
||||
startedAtMs: number,
|
||||
plugins?: PluginDetails[],
|
||||
) {
|
||||
const cpus = os.cpus();
|
||||
const currentTotalCpuMs = cpus.map(({ times }) => times.user + times.idle + times.irq + times.nice + times.sys);
|
||||
@@ -160,5 +161,6 @@ export async function gatherSystemInfo(
|
||||
startedAtMs,
|
||||
now,
|
||||
false,
|
||||
plugins,
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user