Refactor file write streams (#877)

A lot of code is repeated in various different ways in order to handle
streaming content to files without overwriting them, whether it's from
web resources or local streams.

Refactor the handling of these streams to all use the same utility
function for creating a write stream, and for the places downloading
urls use the same downloadFile function.
This commit is contained in:
Hornwitser authored and GitHub committed 2026-03-19 21:55:52 +01:00
1 parent 7701010fee
commit 86eefab9f5
9 files changed
+328 -312

No files matched your search

+7 -23
View File
@@ -553,30 +553,16 @@ async function uploadMod(req: Request, res: Response) {
}
const modsDirectory = req.app.locals.controller.config.get("controller.mods_directory");
let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
const filePath = path.join(modsDirectory, filename);
const [tempFilePath, writeStream] = await lib.createTempWriteStream(filePath);
let writeStream;
try {
while (true) {
try {
writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" });
await events.once(writeStream, "open");
break;
} catch (err: any) {
if (err.code === "EEXIST") {
tempFilename = await lib.findUnusedName(modsDirectory, tempFilename, ".tmp.zip");
} else {
throw err;
}
}
}
stream.pipe(writeStream);
await finished(writeStream);
const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename));
const modInfo = await lib.ModInfo.fromModFile(tempFilePath);
checkModName(modInfo.filename);
await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename));
await fs.rename(tempFilePath, path.join(modsDirectory, modInfo.filename));
req.app.locals.controller.modStore.addMod(modInfo);
mods.push(modInfo.toJSON());
@@ -586,15 +572,13 @@ async function uploadMod(req: Request, res: Response) {
stream.resume();
// Attempt to clean up.
if (writeStream) {
writeStream.destroy();
}
writeStream.destroy();
try {
await fs.unlink(path.join(modsDirectory, tempFilename));
await fs.unlink(tempFilePath);
} catch (unlinkErr: any) {
if (unlinkErr.code !== "ENOENT") {
logger.error(`Error removing ${tempFilename}: ${unlinkErr.message}`);
logger.error(`Error removing ${tempFilePath}: ${unlinkErr.message}`);
}
}
}
+1 -25
View File
@@ -417,31 +417,7 @@ instanceSaveCommands.add(new lib.Command({
let url = new URL(control.config.get("control.controller_url")!);
url.pathname += `api/stream/${streamId}`;
let response = await fetch(url);
if (!response.body) {
throw new lib.CommandError("Got empty response from server.");
}
let writeStream;
let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip");
while (true) {
try {
writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
await events.once(writeStream, "open");
break;
} catch (err: any) {
if (err.code === "EEXIST") {
tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip");
} else {
throw err;
}
}
}
stream.Readable.fromWeb(response.body).pipe(writeStream);
await finished(writeStream);
let filename = await lib.findUnusedName(".", args.save, ".zip");
await fs.rename(tempFilename, filename);
const filename = await lib.downloadFile(url, args.save, "rename");
logger.info(`Downloaded ${args.save}${args.save === filename ? "" : ` as ${filename}`}`);
},
+1 -23
View File
@@ -194,30 +194,8 @@ modCommands.add(new lib.Command({
let url = new URL(control.config.get("control.controller_url")!);
url.pathname += `api/stream/${streamId}`;
let response = await fetch(url);
if (!response.body) {
throw new lib.CommandError("Got empty response from server");
}
let writeStream;
let filename = `${args.name}_${args.modVersion}.zip`;
let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
while (true) {
try {
writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
await events.once(writeStream, "open");
break;
} catch (err: any) {
if (err.code === "EEXIST") {
tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip");
} else {
throw err;
}
}
}
stream.Readable.fromWeb(response.body).pipe(writeStream);
await finished(writeStream);
await fs.rename(tempFilename, filename);
await lib.downloadFile(url, filename, "overwrite");
logger.info(`Downloaded ${filename}`);
},
+3 -38
View File
@@ -591,19 +591,10 @@ export default class Host extends lib.Link {
let url = new URL(this.config.get("host.controller_url"));
url.pathname += `api/stream/${streamId}`;
let response = await fetch(url);
if (!response.body) {
throw new Error(`Empty body downloading stream ${streamId} from controller`);
}
const file = lib.ModInfo.filename(mod.name, mod.version);
const filePath = path.join(this.modStore.modsDirectory, file);
const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip");
const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" });
await events.once(writeStream, "open");
stream.Readable.fromWeb(response.body).pipe(writeStream);
await finished(writeStream);
await fs.rename(tempFilePath, filePath);
await lib.downloadFile(url, filePath, "overwrite");
const modInfo = await this.modStore.loadFile(file);
if (mod.sha1 && mod.sha1 !== modInfo.sha1) {
@@ -1024,35 +1015,9 @@ export default class Host extends lib.Link {
let url = new URL(this.config.get("host.controller_url"));
url.pathname += `api/stream/${streamId}`;
let response = await fetch(url);
if (response.status !== 200 || !response.body) {
let body = Buffer.from(await response.arrayBuffer());
throw new lib.RequestError(`Stream returned ${response.status}: ${body.toString()}`);
}
let savesDir = path.join(instance.path, "saves");
let tempFilename = name.replace(/(\.zip)?$/, ".tmp.zip");
let writeStream: NodeJS.WritableStream;
while (true) {
try {
writeStream = fs.createWriteStream(path.join(savesDir, tempFilename), { flags: "wx" });
await events.once(writeStream, "open");
break;
} catch (err: any) {
if (err.code === "EEXIST") {
tempFilename = await lib.findUnusedName(savesDir, tempFilename, ".tmp.zip");
} else {
throw err;
}
}
}
stream.Readable.fromWeb(response.body).pipe(writeStream);
await finished(writeStream);
name = await lib.findUnusedName(savesDir, name, ".zip");
await fs.rename(path.join(savesDir, tempFilename), path.join(savesDir, name));
const savedPath = await lib.downloadFile(url, path.join(savesDir, name), "rename");
name = path.basename(savedPath);
await this.sendSaveListUpdate(instanceId, savesDir);
return name;
}
+3 -5
View File
@@ -431,15 +431,13 @@ export async function patch(savePath: string, modules: SaveModule[]) {
root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
// Write back the save
let tempSavePath = savePath.replace(/(\.zip)?$/, ".tmp.zip");
let stream = zip.generateNodeStream({ compression: "DEFLATE" });
let fd = await fs.open(tempSavePath, "w");
const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
try {
let pipe = stream.pipe(fs.createWriteStream("", { fd, autoClose: false }));
let pipe = stream.pipe(writeStream);
await events.once(pipe, "finish");
await fs.fsync(fd);
} finally {
await fs.close(fd);
writeStream.destroy();
}
await fs.rename(tempSavePath, savePath);
}
+4 -43
View File
@@ -3,8 +3,7 @@ import path from "path";
import events from "events";
import { Static } from "@sinclair/typebox";
import { logger } from "./logging";
import { safeOutputFile } from "./file_ops";
import { Writable } from "stream";
import { downloadFile, safeOutputFile } from "./file_ops";
import { ModPortalReleaseSchema, ModPortalDetailsSchema, ModNameVersionPair } from "./data/messages_mod";
import { ModInfo, ModPack, FullVersion, ApiVersion, ModVersionEquality } from "./data";
@@ -155,33 +154,6 @@ export default class ModStore extends events.EventEmitter<ModStoreEvents> {
return new this(modsDirectory, files);
}
/**
* Stream a url's content into a file.
*
* @param url - Web path to stream from.
* @param filePath - File path to stream to.
*/
private static async downloadFile(url: string | URL, filePath: string) {
try {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
}
if (!response.body) {
throw new Error("Response body is missing.");
}
const fileStream = fs.createWriteStream(filePath);
const writer = Writable.toWeb(fileStream);
await response.body.pipeTo(writer);
} catch (error) {
logger.error("Error downloading file:", error);
throw error;
}
}
/**
* Download a matching version from a selection of available versions.
*
@@ -202,27 +174,16 @@ export default class ModStore extends events.EventEmitter<ModStoreEvents> {
}
try {
const url = new URL(`https://mods.factorio.com/${selectedRelease.download_url}`);
const url = new URL(selectedRelease.download_url, "https://mods.factorio.com/");
url.searchParams.set("username", username);
url.searchParams.set("token", token);
const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
const finalFileName = `${this.modsDirectory}/${selectedRelease.file_name}`;
const filePath = path.join(this.modsDirectory, selectedRelease.file_name);
await ModStore.downloadFile(url, tempFileName);
await fs.rename(tempFileName, finalFileName);
await downloadFile(url, filePath, "overwrite");
await this.loadFile(selectedRelease.file_name);
} catch (err: any) {
logger.error(`Failed to download or load mod ${selectedRelease.file_name} (${version}): ${err.message}`);
// Attempt to clean up temp file if it exists
try {
const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
await fs.unlink(tempFileName);
} catch (unlinkErr: any) {
if (unlinkErr.code !== "ENOENT") { // Ignore if file doesn't exist
logger.warn(`Failed to clean up temp file ${selectedRelease.file_name}.tmp: ${unlinkErr.message}`);
}
}
}
}
+98 -1
View File
@@ -1,9 +1,12 @@
/**
* @module lib/file_ops
*/
import crypto from "crypto";
import events from "events";
import fs from "fs-extra";
import path from "path";
import crypto from "crypto"; // needed for getTempFile
import stream from "stream";
/**
* Returns the newest file in a directory
@@ -235,3 +238,97 @@ export function cleanFilename(name: string) {
return name;
}
/**
* Creates a write stream to a guaranteed newly created temporary file in the
* same folder as the target `filePath`.
*
* @param filePath - The target file to create a temporary stream for.
* @returns The path to the temporary file that was opened and the write stream to it.
*/
export async function createTempWriteStream(filePath: string): Promise<[string, fs.WriteStream]> {
const { dir, name: fileName, ext } = path.parse(filePath);
let increment = 1;
let writeStream;
let tempFilePath = path.format({ dir, name: `${fileName}.tmp`, ext });
while (true) {
try {
writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true });
// replace "open" event with "ready" after removing fs-extra
await events.once(writeStream, "open");
break;
} catch (err: any) {
if (err.code === "EEXIST") {
increment += 1;
tempFilePath = path.format({ dir, name: `${fileName}-${increment}.tmp`, ext });
} else {
throw err;
}
}
}
return [tempFilePath, writeStream];
}
/**
* Download a file by fetching the given url and streaming the body content to it.
*
* @param url - The resource to download
* @param filePath - Path to the file store the resource data in.
* @param overwriteMode - What to do if the target filePath already exists. Use
* overwrite to clobber it, use rename to write to a differently named file,
* and error to throw an EEXIST error.
* @returns file path that was written to. May be different from `filePath` if
* overwriteMode is set to rename.
*/
export async function downloadFile(
url: URL | string,
filePath: string,
overwriteMode: "overwrite" | "rename" | "error",
) {
// Prepare file to stream content to.
let writeStream;
let tempFilePath;
if (overwriteMode === "error") {
tempFilePath = filePath;
writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true });
// replace "open" event with "ready" after removing fs-extra
await events.once(writeStream, "open");
} else {
[tempFilePath, writeStream] = await createTempWriteStream(filePath);
}
// Fetch stream bytes
const response = await fetch(url);
if (!response.ok || !response.body) {
writeStream.end();
await events.once(writeStream, "close");
await fs.unlink(tempFilePath);
throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`);
}
const writer = stream.Writable.toWeb(writeStream);
await response.body.pipeTo(writer);
let targetFilePath = filePath;
if (overwriteMode === "overwrite") {
await fs.rename(tempFilePath, targetFilePath);
} else if (overwriteMode === "rename") {
let increment = 1;
let { dir, name, ext } = path.parse(targetFilePath);
while (true) {
try {
await fs.writeFile(targetFilePath, Buffer.alloc(0), { flag: "wx" });
break;
} catch (err: any) {
if (err.code === "EEXIST") {
increment += 1;
targetFilePath = path.format({ dir, name: `${name}-${increment}`, ext });
} else {
throw err;
}
}
}
await fs.rename(tempFilePath, targetFilePath);
}
return targetFilePath;
}