mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-19 07:33:12 +09:00
Merge pull request #1010 from bbassie/feature/plugin-browser
Plugin management through web interface
This commit is contained in:
1 parent
e36a76acc5
commit
ffe82f81d8
48 files changed
+4275
-536
No files matched your search
@@ -5,6 +5,96 @@ import { logger } from "./logging.js";
|
||||
import * as libFileOps from "./file_ops.js";
|
||||
import vm from "vm";
|
||||
|
||||
const corePackages = new Set([
|
||||
"@clusterio/controller",
|
||||
"@clusterio/ctl",
|
||||
"@clusterio/host",
|
||||
"@clusterio/lib",
|
||||
"@clusterio/web_ui",
|
||||
]);
|
||||
|
||||
function isPluginPackageJson(packageJson: any): boolean {
|
||||
return Boolean(packageJson?.keywords?.includes?.("clusterio-plugin"));
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the package.json in a package directory.
|
||||
* @param packagePath - Directory of the package
|
||||
* @param tolerant - Return undefined instead of throwing if the file can't be parsed, as npm may be writing it
|
||||
* @returns The parsed package.json, or undefined if it does not exist
|
||||
*/
|
||||
export async function readPackageJson(packagePath: string, tolerant: boolean): Promise<any> {
|
||||
let content;
|
||||
try {
|
||||
content = await fs.readFile(path.join(packagePath, "package.json"), "utf8");
|
||||
} catch (err: any) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(content);
|
||||
} catch (err: any) {
|
||||
if (tolerant) {
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/** Dependencies npm installs, an optionalDependencies entry wins over a dependencies entry of the same name */
|
||||
export function installedDependencies(packageJson: any): Record<string, unknown> {
|
||||
return { ...packageJson?.dependencies, ...packageJson?.optionalDependencies };
|
||||
}
|
||||
|
||||
/** Whether a dependency spec installs from the registry, as opposed to a path, git, url, tarball or alias */
|
||||
export function isRegistrySpec(spec: unknown) {
|
||||
return typeof spec === "string" && !/[:/\\]/.test(spec) && !spec.startsWith(".")
|
||||
&& !/\.(?:tgz|tar\.gz|tar)$/i.test(spec);
|
||||
}
|
||||
|
||||
/** Real path of a directory, used to compare packages as npm and pnpm install them behind symlinks */
|
||||
export async function realDir(dir: string) {
|
||||
try {
|
||||
return await fs.realpath(dir);
|
||||
} catch {
|
||||
// Removed since, nothing else can be at this path
|
||||
return path.resolve(dir);
|
||||
}
|
||||
}
|
||||
|
||||
/** Directory of a package as require resolves it from fromPath, or undefined if not found */
|
||||
export function resolvePackagePath(packageName: string, fromPath: string, tolerant: boolean): string | undefined {
|
||||
const packageRequire = createRequire(path.join(fromPath, "package.json"));
|
||||
try {
|
||||
return path.dirname(packageRequire.resolve(`${packageName}/package.json`));
|
||||
} catch (err: any) {
|
||||
// A package.json npm is still writing fails with ERR_INVALID_PACKAGE_CONFIG
|
||||
if (tolerant || err.code === "MODULE_NOT_FOUND" || err.code === "ERR_PACKAGE_PATH_NOT_EXPORTED") {
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async function pluginPathExists(pluginPath: string) {
|
||||
if (!path.isAbsolute(pluginPath)) {
|
||||
return resolvePackagePath(pluginPath, process.cwd(), true) !== undefined;
|
||||
}
|
||||
try {
|
||||
await fs.access(path.join(pluginPath, "package.json"));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
function getPluginName(requireSpec: string) {
|
||||
const context = vm.createContext({ require: require, pluginInfo: null });
|
||||
const code = `pluginInfo = require(${JSON.stringify(requireSpec)}).plugin;`;
|
||||
vm.runInContext(code, context);
|
||||
return context.pluginInfo.name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Searches for local plugins in the plugins/ and external_plugins/ directories
|
||||
* and adds them to the plugin list if they contain a package.json file.
|
||||
@@ -49,7 +139,7 @@ async function findLocalPlugins(pluginList: Map<string, string>): Promise<boolea
|
||||
const stats = await fs.stat(pluginPath);
|
||||
if (stats.isDirectory()) {
|
||||
// Check for package.json and plugin keyword
|
||||
if (await checkPackageJson(pluginPath)) {
|
||||
if (isPluginPackageJson(await readPackageJson(pluginPath, false))) {
|
||||
pluginList.set(pluginName, path.resolve(pluginPath));
|
||||
logger.info(`Added ${pluginName} from ${pluginFolder}`);
|
||||
hasChanged += 1;
|
||||
@@ -67,24 +157,85 @@ async function findLocalPlugins(pluginList: Map<string, string>): Promise<boolea
|
||||
return hasChanged > 0;
|
||||
}
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
function getPluginName(requireSpec: string) {
|
||||
const context = vm.createContext({ require: require, pluginInfo: null });
|
||||
const code = `pluginInfo = require(${JSON.stringify(requireSpec)}).plugin;`;
|
||||
vm.runInContext(code, context);
|
||||
return context.pluginInfo.name;
|
||||
type NpmPluginPackage = {
|
||||
packageName: string,
|
||||
packageVersion: unknown,
|
||||
packagePath: string,
|
||||
packageJson: any,
|
||||
};
|
||||
|
||||
/**
|
||||
* Walk the dependencies of the root package.json and of every plugin package found, without
|
||||
* traversing packages that aren't plugins. Does not execute any plugin code.
|
||||
*
|
||||
* @param dependencies - Dependencies of the root package.json
|
||||
* @param tolerant - Skip packages with a package.json that can't be parsed instead of throwing,
|
||||
* for use while npm may be rewriting them
|
||||
* @returns Plugin packages found, and root dependencies that are not plugins
|
||||
*/
|
||||
export async function discoverNpmPluginPackages(dependencies: Record<string, unknown>, tolerant = false) {
|
||||
const plugins: NpmPluginPackage[] = [];
|
||||
const nonPluginRoots: { packageName: string, packageVersion: unknown }[] = [];
|
||||
const pendingPackages = Object.entries(dependencies).map(([packageName, packageVersion]) => ({
|
||||
packageName,
|
||||
packageVersion,
|
||||
parentPath: process.cwd(),
|
||||
isRootDependency: true,
|
||||
}));
|
||||
const visitedPackages = new Set<string>();
|
||||
// Directories some dependent installs from a path, git, url, tarball or alias
|
||||
const nonRegistryDirs = new Set<string>();
|
||||
|
||||
while (pendingPackages.length) {
|
||||
const { packageName, packageVersion, parentPath, isRootDependency } = pendingPackages.shift()!;
|
||||
const packagePath = resolvePackagePath(packageName, parentPath, tolerant);
|
||||
if (!packagePath) {
|
||||
continue;
|
||||
}
|
||||
if (!isRegistrySpec(packageVersion)) {
|
||||
nonRegistryDirs.add(await realDir(packagePath));
|
||||
}
|
||||
if (visitedPackages.has(packageName)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
visitedPackages.add(packageName);
|
||||
|
||||
const packageJson = await readPackageJson(packagePath, tolerant);
|
||||
if (packageJson && isPluginPackageJson(packageJson)) {
|
||||
plugins.push({ packageName, packageVersion, packagePath, packageJson });
|
||||
for (const [dependencyName, dependencyVersion] of Object.entries(installedDependencies(packageJson))) {
|
||||
pendingPackages.push({
|
||||
packageName: dependencyName,
|
||||
packageVersion: dependencyVersion,
|
||||
parentPath: packagePath,
|
||||
isRootDependency: false,
|
||||
});
|
||||
}
|
||||
} else if (isRootDependency) {
|
||||
nonPluginRoots.push({ packageName, packageVersion });
|
||||
}
|
||||
}
|
||||
return { plugins, nonPluginRoots, nonRegistryDirs };
|
||||
}
|
||||
|
||||
function resolvePackagePath(packageName: string, fromPath: string): string | undefined {
|
||||
const packageRequire = createRequire(path.join(fromPath, "package.json"));
|
||||
try {
|
||||
return path.dirname(packageRequire.resolve(`${packageName}/package.json`));
|
||||
} catch (err: any) {
|
||||
if (err.code === "MODULE_NOT_FOUND" || err.code === "ERR_PACKAGE_PATH_NOT_EXPORTED") {
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
/** Real paths of plugin packages some dependent installs from a path, git, url, tarball or alias */
|
||||
export async function findNonRegistryDirs() {
|
||||
const dependencies = installedDependencies(await readPackageJson(process.cwd(), true));
|
||||
return (await discoverNpmPluginPackages(dependencies, true)).nonRegistryDirs;
|
||||
}
|
||||
|
||||
/** Whether npm install name would replace a plugin package not installed from the registry */
|
||||
export async function isNonRegistryPackage(name: string) {
|
||||
const dependencies = installedDependencies(await readPackageJson(process.cwd(), true));
|
||||
if (Object.hasOwn(dependencies, name) && !isRegistrySpec(dependencies[name])) {
|
||||
return true;
|
||||
}
|
||||
const packagePath = resolvePackagePath(name, process.cwd(), true);
|
||||
if (!packagePath) {
|
||||
return false;
|
||||
}
|
||||
return (await findNonRegistryDirs()).has(await realDir(packagePath));
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -92,79 +243,46 @@ function resolvePackagePath(packageName: string, fromPath: string): string | und
|
||||
* Adds discovered plugins to the plugin list.
|
||||
*
|
||||
* @param pluginList - Map of plugin names to their file paths
|
||||
* @param skippedNpmPlugins - Filled with the package directory by plugin name for packages
|
||||
* skipped because their plugin name was already in the list
|
||||
* @returns Promise that resolves to true if any plugins were added to the list, false otherwise
|
||||
*/
|
||||
async function findNpmPlugins(pluginList: Map<string, string>): Promise<boolean> {
|
||||
let dependencies;
|
||||
try {
|
||||
const rootPackageJson = JSON.parse(await fs.readFile(path.resolve("package.json"), { encoding: "utf8" }));
|
||||
dependencies = rootPackageJson.dependencies;
|
||||
} catch (err: any) {
|
||||
if (err.code === "ENOENT") {
|
||||
// Skip if package.json doesn't exist
|
||||
return false;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (!dependencies) {
|
||||
async function findNpmPlugins(
|
||||
pluginList: Map<string, string>,
|
||||
skippedNpmPlugins?: Map<string, string>,
|
||||
): Promise<boolean> {
|
||||
const rootPackageJson = await readPackageJson(process.cwd(), false);
|
||||
if (!rootPackageJson) {
|
||||
return false;
|
||||
}
|
||||
const dependencies = installedDependencies(rootPackageJson);
|
||||
|
||||
let hasChanged = 0;
|
||||
const pendingPackages = Object.entries(dependencies).map(([packageName, packageVersion]) => ({
|
||||
packageName,
|
||||
packageVersion,
|
||||
parentPath: process.cwd(),
|
||||
isRootDependency: true,
|
||||
}));
|
||||
const corePackages = new Set([
|
||||
"@clusterio/controller",
|
||||
"@clusterio/ctl",
|
||||
"@clusterio/host",
|
||||
"@clusterio/lib",
|
||||
"@clusterio/web_ui",
|
||||
]);
|
||||
const visitedPackages = new Set<string>();
|
||||
|
||||
while (pendingPackages.length) {
|
||||
const { packageName, packageVersion, parentPath, isRootDependency } = pendingPackages.shift()!;
|
||||
if (visitedPackages.has(packageName)) {
|
||||
const { plugins, nonPluginRoots } = await discoverNpmPluginPackages(dependencies);
|
||||
for (const { packageName, packageVersion, packagePath } of plugins) {
|
||||
const pluginName = getPluginName(packagePath);
|
||||
const listedPath = pluginList.get(pluginName);
|
||||
if (listedPath === packageName) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const npmPackagePath = resolvePackagePath(packageName, parentPath);
|
||||
if (!npmPackagePath) {
|
||||
continue;
|
||||
}
|
||||
|
||||
visitedPackages.add(packageName);
|
||||
|
||||
if (await checkPackageJson(npmPackagePath)) {
|
||||
const pluginName = getPluginName(npmPackagePath);
|
||||
if (pluginList.has(pluginName)) {
|
||||
logger.warn(
|
||||
`${pluginName} provided by ${packageName}@${packageVersion} is already in the plugin list, ` +
|
||||
`but with a different path - using ${pluginList.get(pluginName)}`
|
||||
);
|
||||
} else {
|
||||
pluginList.set(pluginName, packageName);
|
||||
logger.info(`Added ${pluginName} from node modules`);
|
||||
hasChanged += 1;
|
||||
}
|
||||
|
||||
const packageJson = JSON.parse(
|
||||
await fs.readFile(path.join(npmPackagePath, "package.json"), { encoding: "utf8" })
|
||||
if (listedPath !== undefined && !await pluginPathExists(listedPath)) {
|
||||
logger.warn(`${pluginName} at ${listedPath} no longer exists, using ${packageName}`);
|
||||
pluginList.set(pluginName, packageName);
|
||||
hasChanged += 1;
|
||||
} else if (pluginList.has(pluginName)) {
|
||||
logger.warn(
|
||||
`${pluginName} provided by ${packageName}@${packageVersion} is already in the plugin list, ` +
|
||||
`but with a different path - using ${pluginList.get(pluginName)}`
|
||||
);
|
||||
for (const [dependencyName, dependencyVersion] of Object.entries(packageJson.dependencies ?? {})) {
|
||||
pendingPackages.push({
|
||||
packageName: dependencyName,
|
||||
packageVersion: dependencyVersion,
|
||||
parentPath: npmPackagePath,
|
||||
isRootDependency: false,
|
||||
});
|
||||
}
|
||||
} else if (isRootDependency && !corePackages.has(packageName)) {
|
||||
skippedNpmPlugins?.set(pluginName, await realDir(packagePath));
|
||||
} else {
|
||||
pluginList.set(pluginName, packageName);
|
||||
logger.info(`Added ${pluginName} from node modules`);
|
||||
hasChanged += 1;
|
||||
}
|
||||
}
|
||||
for (const { packageName, packageVersion } of nonPluginRoots) {
|
||||
if (!corePackages.has(packageName)) {
|
||||
logger.warn(`${packageName}@${packageVersion} is not a clusterio-plugin`);
|
||||
}
|
||||
}
|
||||
@@ -176,9 +294,15 @@ async function findNpmPlugins(pluginList: Map<string, string>): Promise<boolean>
|
||||
*
|
||||
* @param pluginListPath - Path to the plugin list JSON file
|
||||
* @param findLocal - Whether to look for local plugins in plugins/ and external_plugins/ directories
|
||||
* @param skippedNpmPlugins - Filled with the package directory by plugin name for npm packages
|
||||
* skipped because another plugin has the same name, for loadPluginInfos
|
||||
* @returns Map of plugin name to plugin path
|
||||
*/
|
||||
export async function loadPluginList(pluginListPath: string, findLocal = true): Promise<Map<string, string>> {
|
||||
export async function loadPluginList(
|
||||
pluginListPath: string,
|
||||
findLocal = true,
|
||||
skippedNpmPlugins?: Map<string, string>,
|
||||
): Promise<Map<string, string>> {
|
||||
let pluginList = new Map<string, string>();
|
||||
|
||||
// Try to load existing plugin list
|
||||
@@ -201,7 +325,7 @@ export async function loadPluginList(pluginListPath: string, findLocal = true):
|
||||
}
|
||||
|
||||
// Discover NPM plugins
|
||||
hasChanges = await findNpmPlugins(pluginList) || hasChanges;
|
||||
hasChanges = await findNpmPlugins(pluginList, skippedNpmPlugins) || hasChanges;
|
||||
|
||||
if (hasChanges) {
|
||||
await libFileOps.safeOutputFile(pluginListPath, JSON.stringify([...pluginList], null, "\t"));
|
||||
@@ -209,24 +333,3 @@ export async function loadPluginList(pluginListPath: string, findLocal = true):
|
||||
|
||||
return pluginList;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper function to check if a package.json file exists and contains the clusterio-plugin keyword
|
||||
* @param pluginPath - Path to the plugin directory
|
||||
* @returns Promise that resolves to true if package.json exists and has clusterio-plugin keyword
|
||||
*/
|
||||
async function checkPackageJson(pluginPath: string): Promise<boolean> {
|
||||
const packageJsonPath = path.join(pluginPath, "package.json");
|
||||
|
||||
// Read and parse package.json
|
||||
let content;
|
||||
try {
|
||||
content = await fs.readFile(packageJsonPath, { encoding: "utf8" });
|
||||
} catch (err: any) {
|
||||
return false;
|
||||
}
|
||||
const packageJson = JSON.parse(content);
|
||||
|
||||
// Check if package has the clusterio-plugin keyword
|
||||
return packageJson.keywords?.includes("clusterio-plugin");
|
||||
}
|
||||
Reference in new issue
Block a user