factorio-clusterio/packages/lib/src/file_ops.ts

/**
 * @module lib/file_ops
 */
import crypto from "crypto";
import events from "events";
import fs from "node:fs/promises";
import path from "path";
import stream from "stream";

import { redactUrl } from "./helpers.js";


/**
 * Returns the newest file in a directory
 *
 * @param directory - The directory to look for the newest file in.
 * @param filter -
 *     Optional function to filter out files to ignore.  Should return true
 *     if the file is to be considered.
 * @returns
 *     Name of the file with the newest timestamp or null if the
 *     directory contains no files.
 */
export async function getNewestFile(directory: string, filter = (_name: string) => true) {
	let newestTime = new Date(0);
	let newestFile: string | undefined;
	for (let entry of await fs.readdir(directory, { withFileTypes: true })) {
		if (entry.isFile()) {
			let stat = await fs.stat(path.join(directory, entry.name));
			if (filter(entry.name) && stat.mtime > newestTime) {
				newestTime = stat.mtime;
				newestFile = entry.name;
			}
		}
	}

	return newestFile;
}

/**
 * Returns the total size of all files in a directory
 *
 * Sums up the file size of all files in the given directory if it exists.
 * Error reading the size of files are ignored, and error reading the
 * directory will result in 0 being returned.
 *
 * @param directory - The directory to sum files in.
 * @returns The size in bytes of all files in the directory.
 */
export async function directorySize(directory: string) {
	let dirEntries;
	try {
		dirEntries = await fs.readdir(directory, { withFileTypes: true });
	} catch (err: any) {
		if (err.code === "ENOENT") {
			return 0;
		}
		throw err;
	}
	let statTasks: Promise<number>[] = [];
	for (let entry of dirEntries) {
		if (entry.isFile()) {
			statTasks.push(fs
				.stat(path.join(directory, entry.name))
				.then(stat => stat.size, _ => 0)
			);
		}
	}
	return (await Promise.all(statTasks)).reduce((a, v) => a + v, 0);
}

/**
 * Modifies name in case it already exisist at the given directory
 *
 * Checks the directory passed if it already contains a file or folder with
 * the given name, if it does not returns name unmodified, otherwise it
 * returns a modified name that does not exist in the directory.
 *
 * Warning: this function should not be relied upon to be accurate for
 * security sensitive applications.  The selection process is inherently
 * racy and a file or folder may have been created in the folder by the time
 * this function returns.
 *
 * @param directory - directory to check in.
 * @param name - file name to check for, may have extension.
 * @param extension - dot extension used for the file name.
 * @returns modified name with extension that likely does
 *     not exist in the folder
 */
export async function findUnusedName(directory: string, name: string, extension = "") {
	if (extension && name.endsWith(extension)) {
		name = name.slice(0, -extension.length);
	}

	while (true) {
		try {
			await fs.access(path.join(directory, `${name}${extension}`));
		} catch (err: any) {
			if (err.code === "ENOENT") {
				return `${name}${extension}`;
			}
		}

		let match = /^(.*?)(-(\d+))?$/.exec(name)!;
		if (!match[2]) {
			name = `${match[1]}-2`;
		} else {
			name = `${match[1]}-${Number.parseInt(match[3], 10) + 1}`;
		}
	}
}

/**
 * Generate collision resistante temporary file name
 *
 * Warning: this function should not be relied upon to be accurate for
 * security sensitive applications.  The selection process is inherently
 * racy and a file or folder may have been created in the folder by the time
 * this function returns.
 *
 * @param prefix - Prefix for file
 * @param suffix - Suffix for file
 * @param tmpdir - Directory for temp file
 * @returns Temporary file name
 */
export async function getTempFile(prefix = "tmp.", suffix = "", tmpdir = "./") {
	let fileName = path.join(prefix + crypto.randomBytes(16).toString("hex") + suffix);
	let freeFile = await findUnusedName(tmpdir, fileName);
	let fullPath = path.join(tmpdir, freeFile);
	return fullPath;
}

/**
 * Safely overwrite file data
 *
 * Writes the passed data to a temporary file and rename it over the target
 * file.  If the operation fails because the parent directory does not exist, it
 * attempts to create the directory, including any ancestors and then retries
 * the operation.  The name of the temporary file is the same as the target file
 * with a random tag and the suffix `.tmp` added before the extension, so
 * concurrent writes to the same file do not interfere with each other.
 *
 * If the operation fails the temporary file is removed.  Should the process
 * die in the middle of writing the temporary file it is left behind.
 *
 * @param file - Path to file to write.
 * @param data - Content to write.
 * @param options - see fs.writeFile, `flag` and `flush` may not be used.
 */
export async function safeOutputFile(
	file: string,
	data: string | Buffer,
	options?: Omit<
		Extract<Parameters<typeof fs.writeFile>[2], { flush?: boolean }>,
		"flag" | "flush"
	>,
) {
	let { dir, name, ext } = path.parse(file);
	let temporary = path.join(dir, `${name}.${crypto.randomBytes(8).toString("hex")}.tmp${ext}`);
	try {
		await fs.writeFile(temporary, data, { ...options, flag: "wx", flush: true });
	} catch (err: any) {
		if (err.code === "ENOENT") {
			// Try creating the folder and then retry the operation.
			await fs.mkdir(dir, { recursive: true });
			await fs.writeFile(temporary, data, { ...options, flag: "wx", flush: true });
		} else {
			throw err;
		}
	}
	try {
		await fs.rename(temporary, file);
	} catch (err: any) {
		await fs.unlink(temporary).catch(() => {});
		throw err;
	}
}

// Reserved names by almost all filesystems
const badNames = [".", ".."];

// Reserved namespaces in Windows
const oneToNine = [1, 2, 3, 4, 5, 6, 7, 8, 9];
const badWinNamespaces = [
	"CON", "PRN", "AUX", "NUL",
	...oneToNine.map(n => `COM${n}`),
	...oneToNine.map(n => `LPT${n}`),
];

/**
 * Check if a string is a valid file name
 *
 * @param name - Name to check
 * @throws Error if the name is unsuitable.
 */
export function checkFilename(name: string) {
	// All of these are bad in Windows only, except for /, . and ..
	// See: https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
	const badChars = /[<>:"\/\\|?*\x00-\x1f]/g;
	const badEnd = /[. ]$/;

	if (typeof name !== "string") {
		throw new Error("must be a string");
	}

	if (name === "") {
		throw new Error("cannot be empty");
	}

	if (badChars.test(name)) {
		throw new Error('cannot contain <>:"\\/|=* or control characters');
	}

	if (badNames.includes(name)) {
		throw new Error(
			`cannot be named ${name}`
		);
	}

	if (badWinNamespaces.includes(name.toUpperCase().split(".")[0])) {
		throw new Error(
			"cannot be named any of CON PRN AUX NUL COM1-9 and LPT1-9"
		);
	}

	if (badEnd.test(name)) {
		throw new Error("cannot end with . or space");
	}
}

/**
 * Clean up string to be suitable for use as filename
 *
 * @param name - Arbitrary name string
 * @returns Filename suitable to use in the filesystem
 */
export function cleanFilename(name: string) {
	// copied from checkFilename due to RegExp with global flag containing state.
	const badChars = /[<>:"\/\\|?*\x00-\x1f]/g;
	const badEnd = /[. ]$/;

	if (typeof name !== "string") {
		throw new Error("name must be a string");
	}

	if (name === "" || badNames.includes(name.toUpperCase())) {
		name += "_";
	}

	if (badWinNamespaces.includes(name.toUpperCase().split(".")[0])) {
		name = [`${name.split(".")[0]}_`, ...name.split(".").slice(1)].join(".");
	}

	name = name.replace(badChars, "_");
	name = name.replace(badEnd, "_");

	return name;
}

/**
 * Creates a write stream to a guaranteed newly created temporary file in the
 * same folder as the target `filePath`.
 *
 * @param filePath - The target file to create a temporary stream for.
 * @returns The path to the temporary file that was opened and the write stream to it.
 */
export async function createTempWriteStream(filePath: string) {
	const { dir, name: fileName, ext } = path.parse(filePath);
	let increment = 1;
	let writeStream;
	let tempFilePath = path.format({ dir, name: `${fileName}.tmp`, ext });
	while (true) {
		try {
			writeStream = (await fs.open(tempFilePath, "wx")).createWriteStream({ flush: true });
			break;
		} catch (err: any) {
			if (err.code === "EEXIST") {
				increment += 1;
				tempFilePath = path.format({ dir, name: `${fileName}-${increment}.tmp`, ext });
			} else {
				throw err;
			}
		}
	}

	return [tempFilePath, writeStream] as const;
}

/**
 * Discard a partially written download
 *
 * Destroys the write stream and removes the file it was writing to, so a
 * download which failed part way through does not leave a stray temporary
 * file and an open file handle behind.  Safe to call whether or not the
 * stream has already closed.
 *
 * @param writeStream - stream the download was written to.
 * @param tempFilePath - path the stream was writing to.
 */
async function discardPartialDownload(writeStream: stream.Writable, tempFilePath: string) {
	if (!writeStream.closed) {
		writeStream.destroy();
		await events.once(writeStream, "close");
	}
	try {
		await fs.unlink(tempFilePath);
	} catch (err: any) {
		// Nothing to discard if the file never made it to disk.
		if (err.code !== "ENOENT") {
			throw err;
		}
	}
}

/**
 * Download a file by fetching the given url and streaming the body content to it.
 *
 * @param url - The resource to download
 * @param filePath - Path to the file store the resource data in.
 * @param overwriteMode - What to do if the target filePath already exists. Use
 *     overwrite to clobber it, use rename to write to a differently named file,
 *     and error to throw an EEXIST error.
 * @returns file path that was written to. May be different from `filePath` if
 * overwriteMode is set to rename.
 */
export async function downloadFile(
	url: URL | string,
	filePath: string,
	overwriteMode: "overwrite" | "rename" | "error",
) {
	// Prepare file to stream content to.
	let writeStream;
	let tempFilePath;
	if (overwriteMode === "error") {
		tempFilePath = filePath;
		writeStream = (await fs.open(tempFilePath, "wx")).createWriteStream({ flush: true });
	} else {
		[tempFilePath, writeStream] = await createTempWriteStream(filePath);
	}

	// Fetch stream bytes.  Anything that goes wrong from here until the
	// body has been written leaves a temporary file that needs to be removed
	try {
		const response = await fetch(url);
		if (!response.ok || !response.body) {
			throw new Error(`Failed to download ${redactUrl(url)}: ${response.status} ${response.statusText}`);
		}
		const writer = stream.Writable.toWeb(writeStream);
		await response.body.pipeTo(writer);

	} catch (err) {
		await discardPartialDownload(writeStream, tempFilePath);
		throw err;
	}

	let targetFilePath = filePath;
	if (overwriteMode === "overwrite") {
		await fs.rename(tempFilePath, targetFilePath);
	} else if (overwriteMode === "rename") {
		let increment = 1;
		let { dir, name, ext } = path.parse(targetFilePath);
		while (true) {
			try {
				await fs.writeFile(targetFilePath, Buffer.alloc(0), { flag: "wx" });
				break;
			} catch (err: any) {
				if (err.code === "EEXIST") {
					increment += 1;
					targetFilePath = path.format({ dir, name: `${name}-${increment}`, ext });
				} else {
					throw err;
				}
			}
		}
		await fs.rename(tempFilePath, targetFilePath);
	}
	return targetFilePath;
}