mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-19 23:33:13 +09:00
Remove fs-extra and klaw (#882)
These packages are obsoleted by the promises API to the built-in Node fs module. The fs-extra package is also causing problems with broken APIs that don't behave the same as Node's fs module, and throwing errors without code properties on them.
This commit is contained in:
1 parent
130dca4261
commit
2e48b728dd
74 files changed
+570
-529
No files matched your search
@@ -1,5 +1,5 @@
|
||||
import { FileHandle, open, unlink, readFile } from "fs/promises";
|
||||
import { unlinkSync } from "fs";
|
||||
import { FileHandle, open, unlink, readFile } from "node:fs/promises";
|
||||
import { unlinkSync } from "node:fs";
|
||||
import assert from "assert";
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import fs from "fs-extra";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "path";
|
||||
import events from "events";
|
||||
import { Static } from "@sinclair/typebox";
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Configuration classes
|
||||
import { Type, Static } from "@sinclair/typebox";
|
||||
import * as fs from "fs-extra";
|
||||
import * as fs from "node:fs/promises";
|
||||
import events from "events";
|
||||
|
||||
import isDeepStrictEqual from "../is_deep_strict_equal";
|
||||
@@ -446,7 +446,7 @@ export class Config<
|
||||
* @returns Instance of this config
|
||||
*/
|
||||
static async fromFile(location: ConfigLocation, filepath: string) {
|
||||
const fields = await fs.readJson(filepath, { encoding: "utf8" });
|
||||
const fields = JSON.parse(await fs.readFile(filepath, { encoding: "utf8" }));
|
||||
return this.fromJSON(fields, location, filepath);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import fs from "fs-extra";
|
||||
import fs from "node:fs/promises";
|
||||
import JSZip from "jszip";
|
||||
import { Type, Static } from "@sinclair/typebox";
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* @module lib/database
|
||||
* @author Hornwitser
|
||||
*/
|
||||
import fs from "fs-extra";
|
||||
import fs from "node:fs/promises";
|
||||
|
||||
import * as libFileOps from "./file_ops";
|
||||
import { basicType } from "./helpers";
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import path from "path";
|
||||
import fs from "fs-extra";
|
||||
import fs from "node:fs/promises";
|
||||
import { EventEmitter } from "events";
|
||||
|
||||
import { safeOutputFile } from "./file_ops";
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import events from "events";
|
||||
import fs from "fs-extra";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "path";
|
||||
import stream from "stream";
|
||||
|
||||
@@ -46,7 +46,7 @@ export async function getNewestFile(directory: string, filter = (_name: string)
|
||||
* @returns The size in bytes of all files in the directory.
|
||||
*/
|
||||
export async function directorySize(directory: string) {
|
||||
let dirEntries: fs.Dirent[];
|
||||
let dirEntries;
|
||||
try {
|
||||
dirEntries = await fs.readdir(directory, { withFileTypes: true });
|
||||
} catch (err: any) {
|
||||
@@ -91,8 +91,12 @@ export async function findUnusedName(directory: string, name: string, extension
|
||||
}
|
||||
|
||||
while (true) {
|
||||
if (!await fs.pathExists(path.join(directory, `${name}${extension}`))) {
|
||||
return `${name}${extension}`;
|
||||
try {
|
||||
await fs.access(path.join(directory, `${name}${extension}`));
|
||||
} catch (err: any) {
|
||||
if (err.code === "ENOENT") {
|
||||
return `${name}${extension}`;
|
||||
}
|
||||
}
|
||||
|
||||
let match = /^(.*?)(-(\d+))?$/.exec(name)!;
|
||||
@@ -125,12 +129,13 @@ export async function getTempFile(prefix = "tmp.", suffix = "", tmpdir = "./") {
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely write data to a file
|
||||
* Safely overwrite file data
|
||||
*
|
||||
* Same as fs-extra.outputFile except the data is written to a temporary
|
||||
* file that's renamed over the target file. The name of the temporary file
|
||||
* is the same as the target file with the suffix `.tmp` added before the
|
||||
* extension.
|
||||
* Writes the passed data to a temporary file and rename it over the target
|
||||
* file. If the operation fails because the parent directory does not exist, it
|
||||
* attempts to create the directory, including any ancestors and then retries
|
||||
* the operation. The name of the temporary file is the same as the target file
|
||||
* with the suffix `.tmp` added before the extension.
|
||||
*
|
||||
* If the operation fails it may leave behind the temporary file. This
|
||||
* should not be too much of an issue as the next time the same file is
|
||||
@@ -138,26 +143,32 @@ export async function getTempFile(prefix = "tmp.", suffix = "", tmpdir = "./") {
|
||||
*
|
||||
* @param file - Path to file to write.
|
||||
* @param data - Content to write.
|
||||
* @param options - see fs.writeFile, `flag` must not be set.
|
||||
* @param options - see fs.writeFile, `flag` and `flush` may not be used.
|
||||
*/
|
||||
export async function safeOutputFile(file: string, data: string | Buffer, options: fs.WriteFileOptions ={}) {
|
||||
export async function safeOutputFile(
|
||||
file: string,
|
||||
data: string | Buffer,
|
||||
options?: Omit<
|
||||
Extract<Parameters<typeof fs.writeFile>[2], { flush?: boolean }>,
|
||||
"flag" | "flush"
|
||||
>,
|
||||
) {
|
||||
let { dir, name, ext } = path.parse(file);
|
||||
if (dir && !await fs.pathExists(dir)) {
|
||||
await fs.mkdirs(dir);
|
||||
}
|
||||
let temporary = path.join(dir, `${name}.tmp${ext}`);
|
||||
let fd = await fs.open(temporary, "w");
|
||||
try {
|
||||
await fs.writeFile(fd, data, options);
|
||||
await fs.fsync(fd);
|
||||
} finally {
|
||||
await fs.close(fd);
|
||||
await fs.writeFile(temporary, data, { ...options, flush: true });
|
||||
} catch (err: any) {
|
||||
if (err.code === "ENOENT") {
|
||||
// Try creating the folder and then retry the operation.
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await fs.writeFile(temporary, data, { ...options, flush: true });
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
await fs.rename(temporary, file);
|
||||
}
|
||||
|
||||
|
||||
// Reserved names by almost all filesystems
|
||||
const badNames = [".", ".."];
|
||||
|
||||
@@ -246,16 +257,14 @@ export function cleanFilename(name: string) {
|
||||
* @param filePath - The target file to create a temporary stream for.
|
||||
* @returns The path to the temporary file that was opened and the write stream to it.
|
||||
*/
|
||||
export async function createTempWriteStream(filePath: string): Promise<[string, fs.WriteStream]> {
|
||||
export async function createTempWriteStream(filePath: string) {
|
||||
const { dir, name: fileName, ext } = path.parse(filePath);
|
||||
let increment = 1;
|
||||
let writeStream;
|
||||
let tempFilePath = path.format({ dir, name: `${fileName}.tmp`, ext });
|
||||
while (true) {
|
||||
try {
|
||||
writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true });
|
||||
// replace "open" event with "ready" after removing fs-extra
|
||||
await events.once(writeStream, "open");
|
||||
writeStream = (await fs.open(tempFilePath, "wx")).createWriteStream({ flush: true });
|
||||
break;
|
||||
} catch (err: any) {
|
||||
if (err.code === "EEXIST") {
|
||||
@@ -267,7 +276,7 @@ export async function createTempWriteStream(filePath: string): Promise<[string,
|
||||
}
|
||||
}
|
||||
|
||||
return [tempFilePath, writeStream];
|
||||
return [tempFilePath, writeStream] as const;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -291,9 +300,7 @@ export async function downloadFile(
|
||||
let tempFilePath;
|
||||
if (overwriteMode === "error") {
|
||||
tempFilePath = filePath;
|
||||
writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true });
|
||||
// replace "open" event with "ready" after removing fs-extra
|
||||
await events.once(writeStream, "open");
|
||||
writeStream = (await fs.open(tempFilePath, "wx")).createWriteStream({ flush: true });
|
||||
} else {
|
||||
[tempFilePath, writeStream] = await createTempWriteStream(filePath);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
* @module lib/hash
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import fs from "fs-extra";
|
||||
import fs from "node:fs/promises";
|
||||
|
||||
|
||||
/**
|
||||
@@ -35,6 +35,6 @@ export function hashStream(stream: NodeJS.ReadableStream): Promise<string> {
|
||||
* @param path - Path to the file to hash.
|
||||
* @returns hash of the file given.
|
||||
*/
|
||||
export function hashFile(path: string) {
|
||||
return hashStream(fs.createReadStream(path));
|
||||
export async function hashFile(path: string) {
|
||||
return hashStream((await fs.open(path)).createReadStream());
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import fs from "fs-extra";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "path";
|
||||
import { logger } from "./logging";
|
||||
import * as libFileOps from "./file_ops";
|
||||
@@ -174,12 +174,15 @@ export async function loadPluginList(pluginListPath: string, findLocal = true):
|
||||
*/
|
||||
async function checkPackageJson(pluginPath: string): Promise<boolean> {
|
||||
const packageJsonPath = path.join(pluginPath, "package.json");
|
||||
if (!await fs.exists(packageJsonPath)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Read and parse package.json
|
||||
const packageJson = JSON.parse(await fs.readFile(packageJsonPath, { encoding: "utf8" }));
|
||||
let content;
|
||||
try {
|
||||
content = await fs.readFile(packageJsonPath, { encoding: "utf8" });
|
||||
} catch (err: any) {
|
||||
return false;
|
||||
}
|
||||
const packageJson = JSON.parse(content);
|
||||
|
||||
// Check if package has the clusterio-plugin keyword
|
||||
return packageJson.keywords?.includes("clusterio-plugin");
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
* @author Hornwitser
|
||||
* @module lib/logging_utils
|
||||
*/
|
||||
import fs from "fs-extra";
|
||||
import fs from "node:fs/promises";
|
||||
import type { ReadStream } from "node:fs";
|
||||
import path from "path";
|
||||
import stream from "stream";
|
||||
import util from "util";
|
||||
@@ -320,7 +321,7 @@ export class LogIndex {
|
||||
async indexFile(file: string) {
|
||||
let filePath = path.join(this.logDirectory, file);
|
||||
let lineStream = new libStream.LineSplitter({ readableObjectMode: true });
|
||||
let fileStream = fs.createReadStream(filePath);
|
||||
let fileStream = (await fs.open(filePath)).createReadStream();
|
||||
fileStream.pipe(lineStream);
|
||||
let entry = {
|
||||
levels: new Set() as Set<keyof typeof levels>,
|
||||
@@ -478,11 +479,11 @@ export async function queryLog(logDirectory: string, filter: QueryLogFilter, ind
|
||||
continue;
|
||||
}
|
||||
let filePath = path.join(logDirectory, file);
|
||||
let fileStream: fs.ReadStream;
|
||||
let fileStream: ReadStream;
|
||||
let lineStream: libStream.LineSplitter;
|
||||
if (filter.order === "asc") {
|
||||
lineStream = new libStream.LineSplitter({ readableObjectMode: true });
|
||||
fileStream = fs.createReadStream(filePath);
|
||||
fileStream = (await fs.open(filePath)).createReadStream();
|
||||
} else {
|
||||
lineStream = new libStream.ReverseLineSplitter({ readableObjectMode: true });
|
||||
fileStream = await libStream.createReverseReadStream(filePath);
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* @module lib/plugin_loader
|
||||
*/
|
||||
import path from "path";
|
||||
import fs from "fs/promises";
|
||||
import fs from "node:fs/promises";
|
||||
import * as libErrors from "./errors";
|
||||
import * as libPlugin from "./plugin";
|
||||
import { logger } from "./logging";
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import assert from "assert";
|
||||
import fs from "fs-extra";
|
||||
import fs from "node:fs"; // Need the non-promise api for reverse stream
|
||||
import stream from "stream";
|
||||
import util from "node:util";
|
||||
|
||||
const fsOpen = util.promisify(fs.open);
|
||||
const fsFstat = util.promisify(fs.fstat);
|
||||
|
||||
|
||||
/**
|
||||
@@ -13,7 +17,7 @@ import stream from "stream";
|
||||
* ```ts
|
||||
* let lineStream = new lib.LineSplitter({ readableObjectMode: true });
|
||||
* lineStream.on("data", line => { console.log(line.toString("utf8")); });
|
||||
* let fileStream = fs.createReadStream(path);
|
||||
* let fileStream = (await fs.open(path)).createReadStream();
|
||||
* fileStream.pipe(lineStream);
|
||||
* ```
|
||||
*/
|
||||
@@ -141,8 +145,8 @@ export async function createReverseReadStream(path: string, options?: ReadStream
|
||||
if (typeof options === "string") {
|
||||
options = { encoding: options };
|
||||
}
|
||||
const fileFd = await fs.open(path, "r");
|
||||
const fileSize = (await fs.fstat(fileFd)).size;
|
||||
const fileFd = await fsOpen(path, "r");
|
||||
const fileSize = (await fsFstat(fileFd)).size;
|
||||
let filePosition = fileSize;
|
||||
const reverseFs = {
|
||||
read(
|
||||
@@ -162,8 +166,5 @@ export async function createReverseReadStream(path: string, options?: ReadStream
|
||||
open() { assert(false); },
|
||||
close(fd: number, callback: () => void) { return fs.close(fd, callback); },
|
||||
};
|
||||
return fs.createReadStream(
|
||||
"",
|
||||
{ ...options, fd: fileFd, fs: reverseFs } as ReadStreamOptions, // type specification is missing fs prop.
|
||||
);
|
||||
return fs.createReadStream("", { ...options, fd: fileFd, fs: reverseFs });
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import fs from "fs-extra";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "os";
|
||||
import util from "util";
|
||||
import { Gauge } from "./prometheus";
|
||||
@@ -92,14 +92,7 @@ export const systemDiskCapacityBytes = new Gauge(
|
||||
"Size of the filesystem of the current working directory Node.js runs on",
|
||||
{
|
||||
callback: async function(collector) {
|
||||
// statfs was added in Node.js v18.15.0 and may not be present.
|
||||
// TODO: remove this check once minimum supported Node.js >= v18.15.0.
|
||||
if (!fs.statfs) {
|
||||
collector.set(0);
|
||||
return;
|
||||
}
|
||||
const statFsAsync = util.promisify(fs.statfs);
|
||||
const stats = await statFsAsync(".");
|
||||
const stats = await fs.statfs(".");
|
||||
collector.set(stats.blocks * stats.bsize);
|
||||
},
|
||||
},
|
||||
@@ -110,13 +103,7 @@ export const systemDiskAvailableBytes = new Gauge(
|
||||
"Available space on the filesystem of the current working directory Node.js runs on",
|
||||
{
|
||||
callback: async function(collector) {
|
||||
// TODO: remove this check once minimum supported Node.js >= v18.15.0.
|
||||
if (!fs.statfs) {
|
||||
collector.set(0);
|
||||
return;
|
||||
}
|
||||
const statFsAsync = util.promisify(fs.statfs);
|
||||
const stats = await statFsAsync(".");
|
||||
const stats = await fs.statfs(".");
|
||||
collector.set(stats.bavail * stats.bsize);
|
||||
},
|
||||
},
|
||||
@@ -148,14 +135,9 @@ export async function gatherSystemInfo(
|
||||
const cpuUsage = minZip(deltaTotalCpuMs, deltaIdleCpuMs).map(([total, idle]) => (total - idle) / total);
|
||||
previousTotalCpuMs = currentTotalCpuMs;
|
||||
previousIdleCpuMs = currentIdleCpuMs;
|
||||
let diskCapacity = 0;
|
||||
let diskAvailable = 0;
|
||||
if (fs.statfs) { // TODO: remove this check once minimum supported Node.js >= v18.15.0.
|
||||
const statFsAsync = util.promisify(fs.statfs);
|
||||
const stats = await statFsAsync(".");
|
||||
diskCapacity = stats.blocks * stats.bsize;
|
||||
diskAvailable = stats.bavail * stats.bsize;
|
||||
}
|
||||
const stats = await fs.statfs(".");
|
||||
let diskCapacity = stats.blocks * stats.bsize;
|
||||
let diskAvailable = stats.bavail * stats.bsize;
|
||||
|
||||
const now = Date.now();
|
||||
return new SystemInfo(
|
||||
|
||||
Reference in new issue
Block a user