mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-19 23:33:13 +09:00
Refactor file write streams (#877)
A lot of code is repeated in various different ways in order to handle streaming content to files without overwriting them, whether it's from web resources or local streams. Refactor the handling of these streams to all use the same utility function for creating a write stream, and for the places downloading urls use the same downloadFile function.
This commit is contained in:
1 parent
7701010fee
commit
86eefab9f5
9 files changed
+328
-312
No files matched your search
@@ -553,30 +553,16 @@ async function uploadMod(req: Request, res: Response) {
|
||||
}
|
||||
|
||||
const modsDirectory = req.app.locals.controller.config.get("controller.mods_directory");
|
||||
let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
|
||||
const filePath = path.join(modsDirectory, filename);
|
||||
const [tempFilePath, writeStream] = await lib.createTempWriteStream(filePath);
|
||||
|
||||
let writeStream;
|
||||
try {
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" });
|
||||
await events.once(writeStream, "open");
|
||||
break;
|
||||
} catch (err: any) {
|
||||
if (err.code === "EEXIST") {
|
||||
tempFilename = await lib.findUnusedName(modsDirectory, tempFilename, ".tmp.zip");
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
stream.pipe(writeStream);
|
||||
await finished(writeStream);
|
||||
|
||||
const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename));
|
||||
const modInfo = await lib.ModInfo.fromModFile(tempFilePath);
|
||||
checkModName(modInfo.filename);
|
||||
await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename));
|
||||
await fs.rename(tempFilePath, path.join(modsDirectory, modInfo.filename));
|
||||
req.app.locals.controller.modStore.addMod(modInfo);
|
||||
mods.push(modInfo.toJSON());
|
||||
|
||||
@@ -586,15 +572,13 @@ async function uploadMod(req: Request, res: Response) {
|
||||
stream.resume();
|
||||
|
||||
// Attempt to clean up.
|
||||
if (writeStream) {
|
||||
writeStream.destroy();
|
||||
}
|
||||
writeStream.destroy();
|
||||
|
||||
try {
|
||||
await fs.unlink(path.join(modsDirectory, tempFilename));
|
||||
await fs.unlink(tempFilePath);
|
||||
} catch (unlinkErr: any) {
|
||||
if (unlinkErr.code !== "ENOENT") {
|
||||
logger.error(`Error removing ${tempFilename}: ${unlinkErr.message}`);
|
||||
logger.error(`Error removing ${tempFilePath}: ${unlinkErr.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -417,31 +417,7 @@ instanceSaveCommands.add(new lib.Command({
|
||||
|
||||
let url = new URL(control.config.get("control.controller_url")!);
|
||||
url.pathname += `api/stream/${streamId}`;
|
||||
let response = await fetch(url);
|
||||
if (!response.body) {
|
||||
throw new lib.CommandError("Got empty response from server.");
|
||||
}
|
||||
|
||||
let writeStream;
|
||||
let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip");
|
||||
while (true) {
|
||||
try {
|
||||
writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
|
||||
await events.once(writeStream, "open");
|
||||
break;
|
||||
} catch (err: any) {
|
||||
if (err.code === "EEXIST") {
|
||||
tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip");
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
stream.Readable.fromWeb(response.body).pipe(writeStream);
|
||||
await finished(writeStream);
|
||||
|
||||
let filename = await lib.findUnusedName(".", args.save, ".zip");
|
||||
await fs.rename(tempFilename, filename);
|
||||
const filename = await lib.downloadFile(url, args.save, "rename");
|
||||
|
||||
logger.info(`Downloaded ${args.save}${args.save === filename ? "" : ` as ${filename}`}`);
|
||||
},
|
||||
|
||||
@@ -194,30 +194,8 @@ modCommands.add(new lib.Command({
|
||||
|
||||
let url = new URL(control.config.get("control.controller_url")!);
|
||||
url.pathname += `api/stream/${streamId}`;
|
||||
let response = await fetch(url);
|
||||
if (!response.body) {
|
||||
throw new lib.CommandError("Got empty response from server");
|
||||
}
|
||||
|
||||
let writeStream;
|
||||
let filename = `${args.name}_${args.modVersion}.zip`;
|
||||
let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
|
||||
while (true) {
|
||||
try {
|
||||
writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
|
||||
await events.once(writeStream, "open");
|
||||
break;
|
||||
} catch (err: any) {
|
||||
if (err.code === "EEXIST") {
|
||||
tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip");
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
stream.Readable.fromWeb(response.body).pipe(writeStream);
|
||||
await finished(writeStream);
|
||||
await fs.rename(tempFilename, filename);
|
||||
await lib.downloadFile(url, filename, "overwrite");
|
||||
|
||||
logger.info(`Downloaded ${filename}`);
|
||||
},
|
||||
|
||||
@@ -591,19 +591,10 @@ export default class Host extends lib.Link {
|
||||
|
||||
let url = new URL(this.config.get("host.controller_url"));
|
||||
url.pathname += `api/stream/${streamId}`;
|
||||
let response = await fetch(url);
|
||||
if (!response.body) {
|
||||
throw new Error(`Empty body downloading stream ${streamId} from controller`);
|
||||
}
|
||||
|
||||
const file = lib.ModInfo.filename(mod.name, mod.version);
|
||||
const filePath = path.join(this.modStore.modsDirectory, file);
|
||||
const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip");
|
||||
const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" });
|
||||
await events.once(writeStream, "open");
|
||||
stream.Readable.fromWeb(response.body).pipe(writeStream);
|
||||
await finished(writeStream);
|
||||
await fs.rename(tempFilePath, filePath);
|
||||
await lib.downloadFile(url, filePath, "overwrite");
|
||||
|
||||
const modInfo = await this.modStore.loadFile(file);
|
||||
if (mod.sha1 && mod.sha1 !== modInfo.sha1) {
|
||||
@@ -1024,35 +1015,9 @@ export default class Host extends lib.Link {
|
||||
|
||||
let url = new URL(this.config.get("host.controller_url"));
|
||||
url.pathname += `api/stream/${streamId}`;
|
||||
let response = await fetch(url);
|
||||
|
||||
if (response.status !== 200 || !response.body) {
|
||||
let body = Buffer.from(await response.arrayBuffer());
|
||||
throw new lib.RequestError(`Stream returned ${response.status}: ${body.toString()}`);
|
||||
}
|
||||
|
||||
let savesDir = path.join(instance.path, "saves");
|
||||
let tempFilename = name.replace(/(\.zip)?$/, ".tmp.zip");
|
||||
let writeStream: NodeJS.WritableStream;
|
||||
while (true) {
|
||||
try {
|
||||
writeStream = fs.createWriteStream(path.join(savesDir, tempFilename), { flags: "wx" });
|
||||
await events.once(writeStream, "open");
|
||||
break;
|
||||
} catch (err: any) {
|
||||
if (err.code === "EEXIST") {
|
||||
tempFilename = await lib.findUnusedName(savesDir, tempFilename, ".tmp.zip");
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
stream.Readable.fromWeb(response.body).pipe(writeStream);
|
||||
await finished(writeStream);
|
||||
|
||||
name = await lib.findUnusedName(savesDir, name, ".zip");
|
||||
await fs.rename(path.join(savesDir, tempFilename), path.join(savesDir, name));
|
||||
|
||||
const savedPath = await lib.downloadFile(url, path.join(savesDir, name), "rename");
|
||||
name = path.basename(savedPath);
|
||||
await this.sendSaveListUpdate(instanceId, savesDir);
|
||||
return name;
|
||||
}
|
||||
|
||||
@@ -431,15 +431,13 @@ export async function patch(savePath: string, modules: SaveModule[]) {
|
||||
root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
|
||||
|
||||
// Write back the save
|
||||
let tempSavePath = savePath.replace(/(\.zip)?$/, ".tmp.zip");
|
||||
let stream = zip.generateNodeStream({ compression: "DEFLATE" });
|
||||
let fd = await fs.open(tempSavePath, "w");
|
||||
const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
|
||||
try {
|
||||
let pipe = stream.pipe(fs.createWriteStream("", { fd, autoClose: false }));
|
||||
let pipe = stream.pipe(writeStream);
|
||||
await events.once(pipe, "finish");
|
||||
await fs.fsync(fd);
|
||||
} finally {
|
||||
await fs.close(fd);
|
||||
writeStream.destroy();
|
||||
}
|
||||
await fs.rename(tempSavePath, savePath);
|
||||
}
|
||||
|
||||
@@ -3,8 +3,7 @@ import path from "path";
|
||||
import events from "events";
|
||||
import { Static } from "@sinclair/typebox";
|
||||
import { logger } from "./logging";
|
||||
import { safeOutputFile } from "./file_ops";
|
||||
import { Writable } from "stream";
|
||||
import { downloadFile, safeOutputFile } from "./file_ops";
|
||||
import { ModPortalReleaseSchema, ModPortalDetailsSchema, ModNameVersionPair } from "./data/messages_mod";
|
||||
import { ModInfo, ModPack, FullVersion, ApiVersion, ModVersionEquality } from "./data";
|
||||
|
||||
@@ -155,33 +154,6 @@ export default class ModStore extends events.EventEmitter<ModStoreEvents> {
|
||||
return new this(modsDirectory, files);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a url's content into a file.
|
||||
*
|
||||
* @param url - Web path to stream from.
|
||||
* @param filePath - File path to stream to.
|
||||
*/
|
||||
private static async downloadFile(url: string | URL, filePath: string) {
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
if (!response.body) {
|
||||
throw new Error("Response body is missing.");
|
||||
}
|
||||
|
||||
const fileStream = fs.createWriteStream(filePath);
|
||||
const writer = Writable.toWeb(fileStream);
|
||||
await response.body.pipeTo(writer);
|
||||
} catch (error) {
|
||||
logger.error("Error downloading file:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a matching version from a selection of available versions.
|
||||
*
|
||||
@@ -202,27 +174,16 @@ export default class ModStore extends events.EventEmitter<ModStoreEvents> {
|
||||
}
|
||||
|
||||
try {
|
||||
const url = new URL(`https://mods.factorio.com/${selectedRelease.download_url}`);
|
||||
const url = new URL(selectedRelease.download_url, "https://mods.factorio.com/");
|
||||
url.searchParams.set("username", username);
|
||||
url.searchParams.set("token", token);
|
||||
const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
|
||||
const finalFileName = `${this.modsDirectory}/${selectedRelease.file_name}`;
|
||||
const filePath = path.join(this.modsDirectory, selectedRelease.file_name);
|
||||
|
||||
await ModStore.downloadFile(url, tempFileName);
|
||||
await fs.rename(tempFileName, finalFileName);
|
||||
await downloadFile(url, filePath, "overwrite");
|
||||
await this.loadFile(selectedRelease.file_name);
|
||||
|
||||
} catch (err: any) {
|
||||
logger.error(`Failed to download or load mod ${selectedRelease.file_name} (${version}): ${err.message}`);
|
||||
// Attempt to clean up temp file if it exists
|
||||
try {
|
||||
const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
|
||||
await fs.unlink(tempFileName);
|
||||
} catch (unlinkErr: any) {
|
||||
if (unlinkErr.code !== "ENOENT") { // Ignore if file doesn't exist
|
||||
logger.warn(`Failed to clean up temp file ${selectedRelease.file_name}.tmp: ${unlinkErr.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
/**
|
||||
* @module lib/file_ops
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import events from "events";
|
||||
import fs from "fs-extra";
|
||||
import path from "path";
|
||||
import crypto from "crypto"; // needed for getTempFile
|
||||
import stream from "stream";
|
||||
|
||||
|
||||
/**
|
||||
* Returns the newest file in a directory
|
||||
@@ -235,3 +238,97 @@ export function cleanFilename(name: string) {
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a write stream to a guaranteed newly created temporary file in the
|
||||
* same folder as the target `filePath`.
|
||||
*
|
||||
* @param filePath - The target file to create a temporary stream for.
|
||||
* @returns The path to the temporary file that was opened and the write stream to it.
|
||||
*/
|
||||
export async function createTempWriteStream(filePath: string): Promise<[string, fs.WriteStream]> {
|
||||
const { dir, name: fileName, ext } = path.parse(filePath);
|
||||
let increment = 1;
|
||||
let writeStream;
|
||||
let tempFilePath = path.format({ dir, name: `${fileName}.tmp`, ext });
|
||||
while (true) {
|
||||
try {
|
||||
writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true });
|
||||
// replace "open" event with "ready" after removing fs-extra
|
||||
await events.once(writeStream, "open");
|
||||
break;
|
||||
} catch (err: any) {
|
||||
if (err.code === "EEXIST") {
|
||||
increment += 1;
|
||||
tempFilePath = path.format({ dir, name: `${fileName}-${increment}.tmp`, ext });
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [tempFilePath, writeStream];
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a file by fetching the given url and streaming the body content to it.
|
||||
*
|
||||
* @param url - The resource to download
|
||||
* @param filePath - Path to the file store the resource data in.
|
||||
* @param overwriteMode - What to do if the target filePath already exists. Use
|
||||
* overwrite to clobber it, use rename to write to a differently named file,
|
||||
* and error to throw an EEXIST error.
|
||||
* @returns file path that was written to. May be different from `filePath` if
|
||||
* overwriteMode is set to rename.
|
||||
*/
|
||||
export async function downloadFile(
|
||||
url: URL | string,
|
||||
filePath: string,
|
||||
overwriteMode: "overwrite" | "rename" | "error",
|
||||
) {
|
||||
// Prepare file to stream content to.
|
||||
let writeStream;
|
||||
let tempFilePath;
|
||||
if (overwriteMode === "error") {
|
||||
tempFilePath = filePath;
|
||||
writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true });
|
||||
// replace "open" event with "ready" after removing fs-extra
|
||||
await events.once(writeStream, "open");
|
||||
} else {
|
||||
[tempFilePath, writeStream] = await createTempWriteStream(filePath);
|
||||
}
|
||||
|
||||
// Fetch stream bytes
|
||||
const response = await fetch(url);
|
||||
if (!response.ok || !response.body) {
|
||||
writeStream.end();
|
||||
await events.once(writeStream, "close");
|
||||
await fs.unlink(tempFilePath);
|
||||
throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
const writer = stream.Writable.toWeb(writeStream);
|
||||
await response.body.pipeTo(writer);
|
||||
|
||||
let targetFilePath = filePath;
|
||||
if (overwriteMode === "overwrite") {
|
||||
await fs.rename(tempFilePath, targetFilePath);
|
||||
} else if (overwriteMode === "rename") {
|
||||
let increment = 1;
|
||||
let { dir, name, ext } = path.parse(targetFilePath);
|
||||
while (true) {
|
||||
try {
|
||||
await fs.writeFile(targetFilePath, Buffer.alloc(0), { flag: "wx" });
|
||||
break;
|
||||
} catch (err: any) {
|
||||
if (err.code === "EEXIST") {
|
||||
increment += 1;
|
||||
targetFilePath = path.format({ dir, name: `${name}-${increment}`, ext });
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
await fs.rename(tempFilePath, targetFilePath);
|
||||
}
|
||||
return targetFilePath;
|
||||
}
|
||||
Reference in new issue
Block a user