From 86eefab9f5e7c1880cd82acf7f9566a20a496f84 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 19 Mar 2026 21:55:52 +0100 Subject: [PATCH] Refactor file write streams (#877) A lot of code is repeated in various different ways in order to handle streaming content to files without overwriting them, whether it's from web resources or local streams. Refactor the handling of these streams to all use the same utility function for creating a write stream, and for the places downloading urls use the same downloadFile function. --- packages/controller/src/routes.ts | 30 +---- packages/ctl/src/commands_instance.ts | 26 +--- packages/ctl/src/commands_mod.ts | 24 +--- packages/host/src/Host.ts | 41 +----- packages/host/src/patch.ts | 8 +- packages/lib/src/ModStore.ts | 47 +------ packages/lib/src/file_ops.ts | 99 +++++++++++++- test/lib/ModStore.test.js | 183 ++++---------------------- test/lib/file_ops.js | 182 +++++++++++++++++++++++++ 9 files changed, 328 insertions(+), 312 deletions(-) diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index b9fba89b..433414e6 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -553,30 +553,16 @@ async function uploadMod(req: Request, res: Response) { } const modsDirectory = req.app.locals.controller.config.get("controller.mods_directory"); - let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); + const filePath = path.join(modsDirectory, filename); + const [tempFilePath, writeStream] = await lib.createTempWriteStream(filePath); - let writeStream; try { - - while (true) { - try { - writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" }); - await events.once(writeStream, "open"); - break; - } catch (err: any) { - if (err.code === "EEXIST") { - tempFilename = await lib.findUnusedName(modsDirectory, tempFilename, ".tmp.zip"); - } else { - throw err; - } - } - } stream.pipe(writeStream); await finished(writeStream); - const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename)); + const modInfo = await lib.ModInfo.fromModFile(tempFilePath); checkModName(modInfo.filename); - await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename)); + await fs.rename(tempFilePath, path.join(modsDirectory, modInfo.filename)); req.app.locals.controller.modStore.addMod(modInfo); mods.push(modInfo.toJSON()); @@ -586,15 +572,13 @@ async function uploadMod(req: Request, res: Response) { stream.resume(); // Attempt to clean up. - if (writeStream) { - writeStream.destroy(); - } + writeStream.destroy(); try { - await fs.unlink(path.join(modsDirectory, tempFilename)); + await fs.unlink(tempFilePath); } catch (unlinkErr: any) { if (unlinkErr.code !== "ENOENT") { - logger.error(`Error removing ${tempFilename}: ${unlinkErr.message}`); + logger.error(`Error removing ${tempFilePath}: ${unlinkErr.message}`); } } } diff --git a/packages/ctl/src/commands_instance.ts b/packages/ctl/src/commands_instance.ts index d7bfacda..b938ff97 100644 --- a/packages/ctl/src/commands_instance.ts +++ b/packages/ctl/src/commands_instance.ts @@ -417,31 +417,7 @@ instanceSaveCommands.add(new lib.Command({ let url = new URL(control.config.get("control.controller_url")!); url.pathname += `api/stream/${streamId}`; - let response = await fetch(url); - if (!response.body) { - throw new lib.CommandError("Got empty response from server."); - } - - let writeStream; - let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip"); - while (true) { - try { - writeStream = fs.createWriteStream(tempFilename, { flags: "wx" }); - await events.once(writeStream, "open"); - break; - } catch (err: any) { - if (err.code === "EEXIST") { - tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip"); - } else { - throw err; - } - } - } - stream.Readable.fromWeb(response.body).pipe(writeStream); - await finished(writeStream); - - let filename = await lib.findUnusedName(".", args.save, ".zip"); - await fs.rename(tempFilename, filename); + const filename = await lib.downloadFile(url, args.save, "rename"); logger.info(`Downloaded ${args.save}${args.save === filename ? "" : ` as ${filename}`}`); }, diff --git a/packages/ctl/src/commands_mod.ts b/packages/ctl/src/commands_mod.ts index a7e9f11e..5be260a4 100644 --- a/packages/ctl/src/commands_mod.ts +++ b/packages/ctl/src/commands_mod.ts @@ -194,30 +194,8 @@ modCommands.add(new lib.Command({ let url = new URL(control.config.get("control.controller_url")!); url.pathname += `api/stream/${streamId}`; - let response = await fetch(url); - if (!response.body) { - throw new lib.CommandError("Got empty response from server"); - } - - let writeStream; let filename = `${args.name}_${args.modVersion}.zip`; - let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); - while (true) { - try { - writeStream = fs.createWriteStream(tempFilename, { flags: "wx" }); - await events.once(writeStream, "open"); - break; - } catch (err: any) { - if (err.code === "EEXIST") { - tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip"); - } else { - throw err; - } - } - } - stream.Readable.fromWeb(response.body).pipe(writeStream); - await finished(writeStream); - await fs.rename(tempFilename, filename); + await lib.downloadFile(url, filename, "overwrite"); logger.info(`Downloaded ${filename}`); }, diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index c3f2466e..61792496 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -591,19 +591,10 @@ export default class Host extends lib.Link { let url = new URL(this.config.get("host.controller_url")); url.pathname += `api/stream/${streamId}`; - let response = await fetch(url); - if (!response.body) { - throw new Error(`Empty body downloading stream ${streamId} from controller`); - } const file = lib.ModInfo.filename(mod.name, mod.version); const filePath = path.join(this.modStore.modsDirectory, file); - const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip"); - const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" }); - await events.once(writeStream, "open"); - stream.Readable.fromWeb(response.body).pipe(writeStream); - await finished(writeStream); - await fs.rename(tempFilePath, filePath); + await lib.downloadFile(url, filePath, "overwrite"); const modInfo = await this.modStore.loadFile(file); if (mod.sha1 && mod.sha1 !== modInfo.sha1) { @@ -1024,35 +1015,9 @@ export default class Host extends lib.Link { let url = new URL(this.config.get("host.controller_url")); url.pathname += `api/stream/${streamId}`; - let response = await fetch(url); - - if (response.status !== 200 || !response.body) { - let body = Buffer.from(await response.arrayBuffer()); - throw new lib.RequestError(`Stream returned ${response.status}: ${body.toString()}`); - } - let savesDir = path.join(instance.path, "saves"); - let tempFilename = name.replace(/(\.zip)?$/, ".tmp.zip"); - let writeStream: NodeJS.WritableStream; - while (true) { - try { - writeStream = fs.createWriteStream(path.join(savesDir, tempFilename), { flags: "wx" }); - await events.once(writeStream, "open"); - break; - } catch (err: any) { - if (err.code === "EEXIST") { - tempFilename = await lib.findUnusedName(savesDir, tempFilename, ".tmp.zip"); - } else { - throw err; - } - } - } - stream.Readable.fromWeb(response.body).pipe(writeStream); - await finished(writeStream); - - name = await lib.findUnusedName(savesDir, name, ".zip"); - await fs.rename(path.join(savesDir, tempFilename), path.join(savesDir, name)); - + const savedPath = await lib.downloadFile(url, path.join(savesDir, name), "rename"); + name = path.basename(savedPath); await this.sendSaveListUpdate(instanceId, savesDir); return name; } diff --git a/packages/host/src/patch.ts b/packages/host/src/patch.ts index b84d2350..b4bb1dd0 100644 --- a/packages/host/src/patch.ts +++ b/packages/host/src/patch.ts @@ -431,15 +431,13 @@ export async function patch(savePath: string, modules: SaveModule[]) { root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t")); // Write back the save - let tempSavePath = savePath.replace(/(\.zip)?$/, ".tmp.zip"); let stream = zip.generateNodeStream({ compression: "DEFLATE" }); - let fd = await fs.open(tempSavePath, "w"); + const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath); try { - let pipe = stream.pipe(fs.createWriteStream("", { fd, autoClose: false })); + let pipe = stream.pipe(writeStream); await events.once(pipe, "finish"); - await fs.fsync(fd); } finally { - await fs.close(fd); + writeStream.destroy(); } await fs.rename(tempSavePath, savePath); } diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 13a94be6..ac36df4e 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -3,8 +3,7 @@ import path from "path"; import events from "events"; import { Static } from "@sinclair/typebox"; import { logger } from "./logging"; -import { safeOutputFile } from "./file_ops"; -import { Writable } from "stream"; +import { downloadFile, safeOutputFile } from "./file_ops"; import { ModPortalReleaseSchema, ModPortalDetailsSchema, ModNameVersionPair } from "./data/messages_mod"; import { ModInfo, ModPack, FullVersion, ApiVersion, ModVersionEquality } from "./data"; @@ -155,33 +154,6 @@ export default class ModStore extends events.EventEmitter { return new this(modsDirectory, files); } - /** - * Stream a url's content into a file. - * - * @param url - Web path to stream from. - * @param filePath - File path to stream to. - */ - private static async downloadFile(url: string | URL, filePath: string) { - try { - const response = await fetch(url); - - if (!response.ok) { - throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); - } - - if (!response.body) { - throw new Error("Response body is missing."); - } - - const fileStream = fs.createWriteStream(filePath); - const writer = Writable.toWeb(fileStream); - await response.body.pipeTo(writer); - } catch (error) { - logger.error("Error downloading file:", error); - throw error; - } - } - /** * Download a matching version from a selection of available versions. * @@ -202,27 +174,16 @@ export default class ModStore extends events.EventEmitter { } try { - const url = new URL(`https://mods.factorio.com/${selectedRelease.download_url}`); + const url = new URL(selectedRelease.download_url, "https://mods.factorio.com/"); url.searchParams.set("username", username); url.searchParams.set("token", token); - const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`; - const finalFileName = `${this.modsDirectory}/${selectedRelease.file_name}`; + const filePath = path.join(this.modsDirectory, selectedRelease.file_name); - await ModStore.downloadFile(url, tempFileName); - await fs.rename(tempFileName, finalFileName); + await downloadFile(url, filePath, "overwrite"); await this.loadFile(selectedRelease.file_name); } catch (err: any) { logger.error(`Failed to download or load mod ${selectedRelease.file_name} (${version}): ${err.message}`); - // Attempt to clean up temp file if it exists - try { - const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`; - await fs.unlink(tempFileName); - } catch (unlinkErr: any) { - if (unlinkErr.code !== "ENOENT") { // Ignore if file doesn't exist - logger.warn(`Failed to clean up temp file ${selectedRelease.file_name}.tmp: ${unlinkErr.message}`); - } - } } } diff --git a/packages/lib/src/file_ops.ts b/packages/lib/src/file_ops.ts index 95d30714..228abc1a 100644 --- a/packages/lib/src/file_ops.ts +++ b/packages/lib/src/file_ops.ts @@ -1,9 +1,12 @@ /** * @module lib/file_ops */ +import crypto from "crypto"; +import events from "events"; import fs from "fs-extra"; import path from "path"; -import crypto from "crypto"; // needed for getTempFile +import stream from "stream"; + /** * Returns the newest file in a directory @@ -235,3 +238,97 @@ export function cleanFilename(name: string) { return name; } + +/** + * Creates a write stream to a guaranteed newly created temporary file in the + * same folder as the target `filePath`. + * + * @param filePath - The target file to create a temporary stream for. + * @returns The path to the temporary file that was opened and the write stream to it. + */ +export async function createTempWriteStream(filePath: string): Promise<[string, fs.WriteStream]> { + const { dir, name: fileName, ext } = path.parse(filePath); + let increment = 1; + let writeStream; + let tempFilePath = path.format({ dir, name: `${fileName}.tmp`, ext }); + while (true) { + try { + writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true }); + // replace "open" event with "ready" after removing fs-extra + await events.once(writeStream, "open"); + break; + } catch (err: any) { + if (err.code === "EEXIST") { + increment += 1; + tempFilePath = path.format({ dir, name: `${fileName}-${increment}.tmp`, ext }); + } else { + throw err; + } + } + } + + return [tempFilePath, writeStream]; +} + +/** + * Download a file by fetching the given url and streaming the body content to it. + * + * @param url - The resource to download + * @param filePath - Path to the file store the resource data in. + * @param overwriteMode - What to do if the target filePath already exists. Use + * overwrite to clobber it, use rename to write to a differently named file, + * and error to throw an EEXIST error. + * @returns file path that was written to. May be different from `filePath` if + * overwriteMode is set to rename. + */ +export async function downloadFile( + url: URL | string, + filePath: string, + overwriteMode: "overwrite" | "rename" | "error", +) { + // Prepare file to stream content to. + let writeStream; + let tempFilePath; + if (overwriteMode === "error") { + tempFilePath = filePath; + writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true }); + // replace "open" event with "ready" after removing fs-extra + await events.once(writeStream, "open"); + } else { + [tempFilePath, writeStream] = await createTempWriteStream(filePath); + } + + // Fetch stream bytes + const response = await fetch(url); + if (!response.ok || !response.body) { + writeStream.end(); + await events.once(writeStream, "close"); + await fs.unlink(tempFilePath); + throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`); + } + const writer = stream.Writable.toWeb(writeStream); + await response.body.pipeTo(writer); + + let targetFilePath = filePath; + if (overwriteMode === "overwrite") { + await fs.rename(tempFilePath, targetFilePath); + } else if (overwriteMode === "rename") { + let increment = 1; + let { dir, name, ext } = path.parse(targetFilePath); + while (true) { + try { + await fs.writeFile(targetFilePath, Buffer.alloc(0), { flag: "wx" }); + break; + } catch (err: any) { + if (err.code === "EEXIST") { + increment += 1; + targetFilePath = path.format({ dir, name: `${name}-${increment}`, ext }); + } else { + throw err; + } + } + } + await fs.rename(tempFilePath, targetFilePath); + } + return targetFilePath; +} diff --git a/test/lib/ModStore.test.js b/test/lib/ModStore.test.js index b1d733b1..6f188c8b 100644 --- a/test/lib/ModStore.test.js +++ b/test/lib/ModStore.test.js @@ -2,6 +2,7 @@ const assert = require("assert").strict; const path = require("path"); const fs = require("fs-extra"); +const stream = require("node:stream"); const JSZip = require("jszip"); // Added for creating mock zips // Capture native fetch before any potential mocks are applied @@ -12,8 +13,7 @@ const { ModStore, ModInfo, ModVersionEquality } = require("@clusterio/lib"); // const MODS_DIR = path.join("temp", "test", "mod_store", "mods"); const CACHE_FILE = path.join(MODS_DIR, "mod-info-cache.json"); -// Helper function to create a mock mod zip file -async function createMockModZip(filePath, name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) { +function createMockModJSZip(name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) { const zip = new JSZip(); const infoJson = { name: name, @@ -27,7 +27,12 @@ async function createMockModZip(filePath, name, version, factorioVersion = "1.1" zip.file(`${name}/info.json`, JSON.stringify(infoJson, null, 4)); zip.file(`${name}/dummy.lua`, "-- Mock Lua file"); - await fs.ensureDir(path.dirname(filePath)); + return zip; +} + +// Helper function to create a mock mod zip file +async function createMockModZip(filePath, name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) { + const zip = createMockModJSZip(name, version, factorioVersion, dependencies); const buffer = await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }); await fs.writeFile(filePath, buffer); } @@ -174,79 +179,19 @@ describe("lib/ModStore", function () { // - Error handling for corrupt zips }); - describe("downloadFile (static)", function () { - const downloadUrl = "http://example.com/download.zip"; - const downloadPath = path.join(MODS_DIR, "downloaded.zip"); - const fileContent = "mock file content"; - - beforeEach(function () { - // Reset fetch mock specifically for downloadFile tests - global.fetch = async (url, options) => { - if (url.toString() === downloadUrl) { - // Simulate a readable stream for the response body - const { Readable } = require("stream"); - const stream = Readable.from(fileContent); - return { - ok: true, - status: 200, - statusText: "OK", - body: Readable.toWeb(stream), // Convert Node stream to Web ReadableStream - }; - } - return { - ok: false, - status: 404, - statusText: "Not Found", - body: null, - }; - }; - }); - - it("should download content from URL to the specified file path", async function () { - await ModStore.downloadFile(downloadUrl, downloadPath); - const writtenContent = await fs.readFile(downloadPath, "utf8"); - assert.equal(writtenContent, fileContent); - }); - - it("should throw if fetch response is not ok", async function () { - await assert.rejects( - ModStore.downloadFile("http://example.com/not-found", downloadPath), - /Failed to fetch .* 404 Not Found/ - ); - }); - - it("should throw if fetch response body is missing", async function () { - global.fetch = async (url, options) => ({ - ok: true, - status: 200, - statusText: "OK", - body: null, // Simulate missing body - }); - await assert.rejects( - ModStore.downloadFile(downloadUrl, downloadPath), - /Response body is missing/ - ); - }); - - // Note: Testing the stream piping error handling might require more complex stream mocking. - }); - describe("downloadMods", function () { let modStore; const username = "testuser"; const token = "testtoken"; const factorioVersion = "1.1"; - // Store original downloadFile and fetch, restore them after tests - let originalDownloadFile; + // Store fetch and restore it after tests let originalFetch; before(function () { - originalDownloadFile = ModStore.downloadFile; originalFetch = global.fetch; }); after(function () { - ModStore.downloadFile = originalDownloadFile; global.fetch = originalFetch; }); @@ -255,17 +200,6 @@ describe("lib/ModStore", function () { await fs.emptyDir(MODS_DIR); // Clear any existing files in the temp dir modStore = await ModStore.fromDirectory(MODS_DIR); // Initialize using fromDirectory - // Replace downloadFile with a mock that creates the file directly - ModStore.downloadFile = async (url, filePath) => { - const finalFilename = path.basename(filePath).replace(/\.tmp$/, ""); - const match = finalFilename.match(/^(.*)_(\d+\.\d+\.\d+)\.zip$/); - if (!match) { - throw new Error(`Mock downloadFile could not parse filename: ${finalFilename}`); - } - const [, name, version] = match; - await createMockModZip(filePath, name, version, factorioVersion); - }; - // Restore fetch before each test in this suite to avoid pollution global.fetch = originalFetch; }); @@ -304,6 +238,21 @@ describe("lib/ModStore", function () { }); return { ok: true, status: 200, json: async () => ({ results: results, pagination: {} }) }; } + if (urlString.startsWith("https://mods.factorio.com/download/")) { + const [, name, version] = /\/([^\/]+)\/([^\/]+)$/.exec(url.pathname); + const requestedMod = modsToDownload.find(mod => ( + mod.name === name + && mod.version.version === version + )); + if (!requestedMod) { + return { ok: false, status: 404, statusText: `Test Mock 404, no such mod: ${urlString}` }; + } + if (name === "bad-download-mod") { + return { ok: false, status: 503, statusText: "Test Mock download error" }; + } + const zip = createMockModJSZip(name, version); + return { ok: true, status: 200, body: stream.Readable.toWeb(zip.generateNodeStream()) }; + } // Fallback for unhandled fetches in this test return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` }; }; @@ -418,56 +367,10 @@ describe("lib/ModStore", function () { { name: "another-good-mod", version: ModVersionEquality.fromString("1.0.0") }, // Success ]; - // Mock ModStore.downloadFile to fail for the specific mod - const originalDSF = ModStore.downloadFile; - ModStore.downloadFile = async (url, filePath) => { - if (filePath.includes("bad-download-mod")) { - throw new Error("Simulated download failure"); - } else { - // Use original mock logic for successful downloads - const finalFilename = path.basename(filePath).replace(/\.tmp$/, ""); - const match = finalFilename.match(/^(.*)_(\d+\.\d+\.\d+)\.zip$/); - if (!match) { - throw new Error(`Mock downloadFile could not parse filename: ${finalFilename}`); - } - const [, name, version] = match; - await createMockModZip(filePath, name, version, factorioVersion); - } - }; - // Define fetch mock specific to this test (for the API call) - global.fetch = async (url, options) => { - const urlString = url.toString(); - if (urlString.startsWith("https://mods.factorio.com/api/mods") && options?.method === "POST") { - const body = options.body; - let requestedModNames = []; - if (body instanceof URLSearchParams) { - requestedModNames = (body.get("namelist") || "").split(",").filter(Boolean); - } - const results = requestedModNames.map(name => { - const requestedMod = modsToDownload.find(mod => mod.name === name); - if (!requestedMod) { - throw new Error(`[Test Mock ERROR] No version for ${name}`); - } - const requestedVersion = requestedMod.version.version; - return { - name: name, - latest_release: { - download_url: `/download/${name}/${requestedVersion}`, - file_name: `${name}_${requestedVersion}.zip`, - info_json: { factorio_version: factorioVersion }, - version: requestedVersion, - sha1: `sha1_${name}_${requestedVersion}`, - }, - }; - }); - return { ok: true, status: 200, json: async () => ({ results: results, pagination: {} }) }; - } - return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` }; - }; + global.fetch = createMockPortalApiFetch(modsToDownload); await modStore.downloadMods(modsToDownload, username, token, factorioVersion); - ModStore.downloadFile = originalDSF; // Restore downloadFile mock assert.equal(modStore.files.size, 2, "Only successfully downloaded mods should be added"); assert(modStore.files.has("good-mod_1.0.0.zip")); @@ -482,43 +385,15 @@ describe("lib/ModStore", function () { { name: "chunk-mod-3", version: ModVersionEquality.fromString("1.0.0") }, { name: "chunk-mod-4", version: ModVersionEquality.fromString("1.0.0") }, ]; - let postCalls = 0; + let fetchCalls = []; // Define fetch mock specific to this test - const testFetch = async (url, options) => { - const urlString = url.toString(); - if (urlString.startsWith("https://mods.factorio.com/api/mods") && options?.method === "POST") { - postCalls += 1; // Track POST calls - const body = options.body; - let requestedModNames = []; - if (body instanceof URLSearchParams) { - requestedModNames = (body.get("namelist") || "").split(",").filter(Boolean); - } - const results = requestedModNames.map(name => { - const requestedMod = modsToDownload.find(mod => mod.name === name); - if (!requestedMod) { - throw new Error(`[Test Mock ERROR] No version for ${name}`); - } - const requestedVersion = requestedMod.version.version; - return { - name: name, - latest_release: { - download_url: `/download/${name}/${requestedVersion}`, - file_name: `${name}_${requestedVersion}.zip`, - info_json: { factorio_version: factorioVersion }, - version: requestedVersion, - sha1: `sha1_${name}_${requestedVersion}`, - }, - }; - }); - return { ok: true, status: 200, json: async () => ({ results: results, pagination: {} }) }; - } - return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` }; - }; - global.fetch = testFetch; + global.fetch = createMockPortalApiFetch(modsToDownload, fetchCalls); await modStore.downloadMods(modsToDownload, username, token, factorioVersion); + let postCalls = fetchCalls.reduce((a, v) => a + (v.options?.method === "POST"), 0); + // Assuming chunkSize is 500, 4 mods should be 1 chunk/POST call assert.equal(postCalls, 1, "Should make 1 POST call for 4 mods"); assert.equal(modStore.files.size, modsToDownload.length, "All chunked mods should be downloaded"); diff --git a/test/lib/file_ops.js b/test/lib/file_ops.js index 8d4dcc2f..a4e9048d 100644 --- a/test/lib/file_ops.js +++ b/test/lib/file_ops.js @@ -1,7 +1,9 @@ "use strict"; const assert = require("assert").strict; const fs = require("fs-extra"); +const http = require("node:http"); const path = require("path"); +const util = require("node:util"); const lib = require("@clusterio/lib"); @@ -218,4 +220,184 @@ describe("lib/file_ops", function() { }); }); + describe("downloadFile()", function () { + const baseUrl = new URL("http://localhost/"); + const downloadDir = path.join("temp", "test", "downloadFile"); + const fileContent = "mock file content"; + let onStream; + let server; + + before(async function () { + await fs.emptyDir(downloadDir); + server = http.createServer(); + server.unref(); + server.on("request", (req, res) => { + if (req.url === "/simple-file") { + res.writeHead(200, { + "Content-Length": Buffer.byteLength(fileContent), + "Content-Type": "text/plain", + }).end(fileContent); + } else if (req.url === "/empty-file") { + res.writeHead(200, { + "Content-Length": 0, + "Content-Type": "text/plain", + }).end(); + } else if (req.url === "/no-content") { + res.writeHead(204).end(); + } else if (req.url === "/stream") { + res.writeHead(200, { + "Content-Type": "text/plain", + }); + res.write("streaming\n"); + onStream(res); + } else { + const body = "not found"; + res.writeHead(404, { + "Content-Length": Buffer.byteLength(body), + "Content-Type": "text/plain", + }).end(body); + } + }); + await util.promisify(server.listen.bind(server))(); + baseUrl.port = server.address().port; + }); + after(function() { + server.close(); + server.closeAllConnections(); + }); + + it("should download content from URL to the specified file path", async function () { + for (const mode of ["overwrite", "rename", "error"]) { + const downloadUrl = new URL("/simple-file", baseUrl); + const downloadPath = path.join(downloadDir, `simple-file-${mode}.txt`); + const newDownloadPath = await lib.downloadFile(downloadUrl, downloadPath, mode); + const writtenContent = await fs.readFile(downloadPath, "utf8"); + assert.equal(writtenContent, fileContent); + assert.equal(downloadPath, newDownloadPath); + } + }); + + it("should throw if fetch response is not ok", async function () { + const downloadPath = path.join(downloadDir, "not-found.txt"); + await assert.rejects( + lib.downloadFile(new URL("/not-found", baseUrl), downloadPath, "overwrite"), + /Failed to download .* 404 Not Found/ + ); + await assert.rejects( + fs.readFile(downloadPath), + { code: "ENOENT" }, + ); + }); + + it("should throw if fetch response body is missing", async function () { + const downloadPath = path.join(downloadDir, "not-content.txt"); + await assert.rejects( + lib.downloadFile(new URL("/no-content", baseUrl), downloadPath, "overwrite"), + /Failed to download .* 204 No Content/ + ); + await assert.rejects( + fs.readFile(downloadPath), + { code: "ENOENT" }, + ); + }); + + it("should throw if file exists and overwriteMode is error", async function () { + const downloadPath = path.join(downloadDir, "exists.txt"); + await fs.outputFile(downloadPath, ""); + await assert.rejects( + lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "error"), + { code: "EEXIST" }, + ); + }); + + it("should throw if target directory does not exist", async function () { + const downloadPath = path.join(downloadDir, "does-not-exist", "simple.txt"); + for (const mode of ["overwrite", "rename", "error"]) { + await assert.rejects( + lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, mode), + { code: "ENOENT" }, + ); + } + }); + + it("should write a new file if it exists and overwriteMode is rename", async function () { + const downloadPath = path.join(downloadDir, "exists.txt"); + await fs.outputFile(downloadPath, ""); + const newDownloadPath = await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "rename"); + assert.equal(newDownloadPath, path.join(downloadDir, "exists-2.txt")); + const writtenContent = await fs.readFile(newDownloadPath, "utf8"); + assert.equal(writtenContent, fileContent); + }); + + it("should write a new file if many files exists and overwriteMode is rename", async function () { + const downloadPath = path.join(downloadDir, "contended.txt"); + const additionalPaths = [ + path.join(downloadDir, "contended-2.txt"), + path.join(downloadDir, "contended-3.txt"), + path.join(downloadDir, "contended-4.txt"), + ]; + await fs.outputFile(downloadPath, ""); + for (const additionalPath of additionalPaths) { + await fs.outputFile(additionalPath, ""); + } + const newDownloadPath = await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "rename"); + assert.equal(newDownloadPath, path.join(downloadDir, "contended-5.txt")); + const writtenContent = await fs.readFile(newDownloadPath, "utf8"); + assert.equal(writtenContent, fileContent); + }); + + it("should overwrite existing file if overwriteMode is overwrite", async function () { + const downloadPath = path.join(downloadDir, "overwrite.txt"); + await fs.outputFile(downloadPath, "original content"); + const newDownloadPath = await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "overwrite"); + assert.equal(newDownloadPath, downloadPath); + const writtenContent = await fs.readFile(newDownloadPath, "utf8"); + assert.equal(writtenContent, fileContent); + }); + + it("should not overwrite existing temp files", async function () { + const tempPath = path.join(downloadDir, "temp-exists.tmp.txt"); + await fs.outputFile(tempPath, "original content"); + const downloadPath = path.join(downloadDir, "temp-exists.txt"); + await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "overwrite"); + const writtenContent = await fs.readFile(tempPath, "utf8"); + assert.equal(writtenContent, "original content"); + }); + + it("should throw if overwriteMode is rename and directory disappeared", async function () { + if (process.platform === "win32") { + // Can't remove the directory while the stream is running on Windows. + this.skip(); + } + function stageWaiter() { + let stage; + let waitForStage = new Promise(resolve => { stage = resolve; }); + return [stage, waitForStage]; + } + + const downloadSubdir = path.join(downloadDir, "subdir"); + const downloadPath = path.join(downloadSubdir, "test.txt"); + await fs.ensureDir(downloadSubdir); + + const [stage1, waitForStage1] = stageWaiter(); + const [stage2, waitForStage2] = stageWaiter(); + + onStream = async (res) => { + stage1(); + await waitForStage2; + res.end("done"); + }; + const download = lib.downloadFile(new URL("/stream", baseUrl), downloadPath, "rename"); + download.catch(() => {}); + await waitForStage1; + + await fs.remove(downloadSubdir); + stage2(); + + await assert.rejects( + download, + { code: "ENOENT" }, + ); + }); + }); });