diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index b9fba89b..433414e6 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -553,30 +553,16 @@ async function uploadMod(req: Request, res: Response) { } const modsDirectory = req.app.locals.controller.config.get("controller.mods_directory"); - let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); + const filePath = path.join(modsDirectory, filename); + const [tempFilePath, writeStream] = await lib.createTempWriteStream(filePath); - let writeStream; try { - - while (true) { - try { - writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" }); - await events.once(writeStream, "open"); - break; - } catch (err: any) { - if (err.code === "EEXIST") { - tempFilename = await lib.findUnusedName(modsDirectory, tempFilename, ".tmp.zip"); - } else { - throw err; - } - } - } stream.pipe(writeStream); await finished(writeStream); - const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename)); + const modInfo = await lib.ModInfo.fromModFile(tempFilePath); checkModName(modInfo.filename); - await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename)); + await fs.rename(tempFilePath, path.join(modsDirectory, modInfo.filename)); req.app.locals.controller.modStore.addMod(modInfo); mods.push(modInfo.toJSON()); @@ -586,15 +572,13 @@ async function uploadMod(req: Request, res: Response) { stream.resume(); // Attempt to clean up. - if (writeStream) { - writeStream.destroy(); - } + writeStream.destroy(); try { - await fs.unlink(path.join(modsDirectory, tempFilename)); + await fs.unlink(tempFilePath); } catch (unlinkErr: any) { if (unlinkErr.code !== "ENOENT") { - logger.error(`Error removing ${tempFilename}: ${unlinkErr.message}`); + logger.error(`Error removing ${tempFilePath}: ${unlinkErr.message}`); } } } diff --git a/packages/ctl/src/commands_instance.ts b/packages/ctl/src/commands_instance.ts index d7bfacda..b938ff97 100644 --- a/packages/ctl/src/commands_instance.ts +++ b/packages/ctl/src/commands_instance.ts @@ -417,31 +417,7 @@ instanceSaveCommands.add(new lib.Command({ let url = new URL(control.config.get("control.controller_url")!); url.pathname += `api/stream/${streamId}`; - let response = await fetch(url); - if (!response.body) { - throw new lib.CommandError("Got empty response from server."); - } - - let writeStream; - let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip"); - while (true) { - try { - writeStream = fs.createWriteStream(tempFilename, { flags: "wx" }); - await events.once(writeStream, "open"); - break; - } catch (err: any) { - if (err.code === "EEXIST") { - tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip"); - } else { - throw err; - } - } - } - stream.Readable.fromWeb(response.body).pipe(writeStream); - await finished(writeStream); - - let filename = await lib.findUnusedName(".", args.save, ".zip"); - await fs.rename(tempFilename, filename); + const filename = await lib.downloadFile(url, args.save, "rename"); logger.info(`Downloaded ${args.save}${args.save === filename ? "" : ` as ${filename}`}`); }, diff --git a/packages/ctl/src/commands_mod.ts b/packages/ctl/src/commands_mod.ts index a7e9f11e..5be260a4 100644 --- a/packages/ctl/src/commands_mod.ts +++ b/packages/ctl/src/commands_mod.ts @@ -194,30 +194,8 @@ modCommands.add(new lib.Command({ let url = new URL(control.config.get("control.controller_url")!); url.pathname += `api/stream/${streamId}`; - let response = await fetch(url); - if (!response.body) { - throw new lib.CommandError("Got empty response from server"); - } - - let writeStream; let filename = `${args.name}_${args.modVersion}.zip`; - let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip"); - while (true) { - try { - writeStream = fs.createWriteStream(tempFilename, { flags: "wx" }); - await events.once(writeStream, "open"); - break; - } catch (err: any) { - if (err.code === "EEXIST") { - tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip"); - } else { - throw err; - } - } - } - stream.Readable.fromWeb(response.body).pipe(writeStream); - await finished(writeStream); - await fs.rename(tempFilename, filename); + await lib.downloadFile(url, filename, "overwrite"); logger.info(`Downloaded ${filename}`); }, diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index c3f2466e..61792496 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -591,19 +591,10 @@ export default class Host extends lib.Link { let url = new URL(this.config.get("host.controller_url")); url.pathname += `api/stream/${streamId}`; - let response = await fetch(url); - if (!response.body) { - throw new Error(`Empty body downloading stream ${streamId} from controller`); - } const file = lib.ModInfo.filename(mod.name, mod.version); const filePath = path.join(this.modStore.modsDirectory, file); - const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip"); - const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" }); - await events.once(writeStream, "open"); - stream.Readable.fromWeb(response.body).pipe(writeStream); - await finished(writeStream); - await fs.rename(tempFilePath, filePath); + await lib.downloadFile(url, filePath, "overwrite"); const modInfo = await this.modStore.loadFile(file); if (mod.sha1 && mod.sha1 !== modInfo.sha1) { @@ -1024,35 +1015,9 @@ export default class Host extends lib.Link { let url = new URL(this.config.get("host.controller_url")); url.pathname += `api/stream/${streamId}`; - let response = await fetch(url); - - if (response.status !== 200 || !response.body) { - let body = Buffer.from(await response.arrayBuffer()); - throw new lib.RequestError(`Stream returned ${response.status}: ${body.toString()}`); - } - let savesDir = path.join(instance.path, "saves"); - let tempFilename = name.replace(/(\.zip)?$/, ".tmp.zip"); - let writeStream: NodeJS.WritableStream; - while (true) { - try { - writeStream = fs.createWriteStream(path.join(savesDir, tempFilename), { flags: "wx" }); - await events.once(writeStream, "open"); - break; - } catch (err: any) { - if (err.code === "EEXIST") { - tempFilename = await lib.findUnusedName(savesDir, tempFilename, ".tmp.zip"); - } else { - throw err; - } - } - } - stream.Readable.fromWeb(response.body).pipe(writeStream); - await finished(writeStream); - - name = await lib.findUnusedName(savesDir, name, ".zip"); - await fs.rename(path.join(savesDir, tempFilename), path.join(savesDir, name)); - + const savedPath = await lib.downloadFile(url, path.join(savesDir, name), "rename"); + name = path.basename(savedPath); await this.sendSaveListUpdate(instanceId, savesDir); return name; } diff --git a/packages/host/src/patch.ts b/packages/host/src/patch.ts index b84d2350..b4bb1dd0 100644 --- a/packages/host/src/patch.ts +++ b/packages/host/src/patch.ts @@ -431,15 +431,13 @@ export async function patch(savePath: string, modules: SaveModule[]) { root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t")); // Write back the save - let tempSavePath = savePath.replace(/(\.zip)?$/, ".tmp.zip"); let stream = zip.generateNodeStream({ compression: "DEFLATE" }); - let fd = await fs.open(tempSavePath, "w"); + const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath); try { - let pipe = stream.pipe(fs.createWriteStream("", { fd, autoClose: false })); + let pipe = stream.pipe(writeStream); await events.once(pipe, "finish"); - await fs.fsync(fd); } finally { - await fs.close(fd); + writeStream.destroy(); } await fs.rename(tempSavePath, savePath); } diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 13a94be6..ac36df4e 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -3,8 +3,7 @@ import path from "path"; import events from "events"; import { Static } from "@sinclair/typebox"; import { logger } from "./logging"; -import { safeOutputFile } from "./file_ops"; -import { Writable } from "stream"; +import { downloadFile, safeOutputFile } from "./file_ops"; import { ModPortalReleaseSchema, ModPortalDetailsSchema, ModNameVersionPair } from "./data/messages_mod"; import { ModInfo, ModPack, FullVersion, ApiVersion, ModVersionEquality } from "./data"; @@ -155,33 +154,6 @@ export default class ModStore extends events.EventEmitter { return new this(modsDirectory, files); } - /** - * Stream a url's content into a file. - * - * @param url - Web path to stream from. - * @param filePath - File path to stream to. - */ - private static async downloadFile(url: string | URL, filePath: string) { - try { - const response = await fetch(url); - - if (!response.ok) { - throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`); - } - - if (!response.body) { - throw new Error("Response body is missing."); - } - - const fileStream = fs.createWriteStream(filePath); - const writer = Writable.toWeb(fileStream); - await response.body.pipeTo(writer); - } catch (error) { - logger.error("Error downloading file:", error); - throw error; - } - } - /** * Download a matching version from a selection of available versions. * @@ -202,27 +174,16 @@ export default class ModStore extends events.EventEmitter { } try { - const url = new URL(`https://mods.factorio.com/${selectedRelease.download_url}`); + const url = new URL(selectedRelease.download_url, "https://mods.factorio.com/"); url.searchParams.set("username", username); url.searchParams.set("token", token); - const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`; - const finalFileName = `${this.modsDirectory}/${selectedRelease.file_name}`; + const filePath = path.join(this.modsDirectory, selectedRelease.file_name); - await ModStore.downloadFile(url, tempFileName); - await fs.rename(tempFileName, finalFileName); + await downloadFile(url, filePath, "overwrite"); await this.loadFile(selectedRelease.file_name); } catch (err: any) { logger.error(`Failed to download or load mod ${selectedRelease.file_name} (${version}): ${err.message}`); - // Attempt to clean up temp file if it exists - try { - const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`; - await fs.unlink(tempFileName); - } catch (unlinkErr: any) { - if (unlinkErr.code !== "ENOENT") { // Ignore if file doesn't exist - logger.warn(`Failed to clean up temp file ${selectedRelease.file_name}.tmp: ${unlinkErr.message}`); - } - } } } diff --git a/packages/lib/src/file_ops.ts b/packages/lib/src/file_ops.ts index 95d30714..228abc1a 100644 --- a/packages/lib/src/file_ops.ts +++ b/packages/lib/src/file_ops.ts @@ -1,9 +1,12 @@ /** * @module lib/file_ops */ +import crypto from "crypto"; +import events from "events"; import fs from "fs-extra"; import path from "path"; -import crypto from "crypto"; // needed for getTempFile +import stream from "stream"; + /** * Returns the newest file in a directory @@ -235,3 +238,97 @@ export function cleanFilename(name: string) { return name; } + +/** + * Creates a write stream to a guaranteed newly created temporary file in the + * same folder as the target `filePath`. + * + * @param filePath - The target file to create a temporary stream for. + * @returns The path to the temporary file that was opened and the write stream to it. + */ +export async function createTempWriteStream(filePath: string): Promise<[string, fs.WriteStream]> { + const { dir, name: fileName, ext } = path.parse(filePath); + let increment = 1; + let writeStream; + let tempFilePath = path.format({ dir, name: `${fileName}.tmp`, ext }); + while (true) { + try { + writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true }); + // replace "open" event with "ready" after removing fs-extra + await events.once(writeStream, "open"); + break; + } catch (err: any) { + if (err.code === "EEXIST") { + increment += 1; + tempFilePath = path.format({ dir, name: `${fileName}-${increment}.tmp`, ext }); + } else { + throw err; + } + } + } + + return [tempFilePath, writeStream]; +} + +/** + * Download a file by fetching the given url and streaming the body content to it. + * + * @param url - The resource to download + * @param filePath - Path to the file store the resource data in. + * @param overwriteMode - What to do if the target filePath already exists. Use + * overwrite to clobber it, use rename to write to a differently named file, + * and error to throw an EEXIST error. + * @returns file path that was written to. May be different from `filePath` if + * overwriteMode is set to rename. + */ +export async function downloadFile( + url: URL | string, + filePath: string, + overwriteMode: "overwrite" | "rename" | "error", +) { + // Prepare file to stream content to. + let writeStream; + let tempFilePath; + if (overwriteMode === "error") { + tempFilePath = filePath; + writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true }); + // replace "open" event with "ready" after removing fs-extra + await events.once(writeStream, "open"); + } else { + [tempFilePath, writeStream] = await createTempWriteStream(filePath); + } + + // Fetch stream bytes + const response = await fetch(url); + if (!response.ok || !response.body) { + writeStream.end(); + await events.once(writeStream, "close"); + await fs.unlink(tempFilePath); + throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`); + } + const writer = stream.Writable.toWeb(writeStream); + await response.body.pipeTo(writer); + + let targetFilePath = filePath; + if (overwriteMode === "overwrite") { + await fs.rename(tempFilePath, targetFilePath); + } else if (overwriteMode === "rename") { + let increment = 1; + let { dir, name, ext } = path.parse(targetFilePath); + while (true) { + try { + await fs.writeFile(targetFilePath, Buffer.alloc(0), { flag: "wx" }); + break; + } catch (err: any) { + if (err.code === "EEXIST") { + increment += 1; + targetFilePath = path.format({ dir, name: `${name}-${increment}`, ext }); + } else { + throw err; + } + } + } + await fs.rename(tempFilePath, targetFilePath); + } + return targetFilePath; +} diff --git a/test/lib/ModStore.test.js b/test/lib/ModStore.test.js index b1d733b1..6f188c8b 100644 --- a/test/lib/ModStore.test.js +++ b/test/lib/ModStore.test.js @@ -2,6 +2,7 @@ const assert = require("assert").strict; const path = require("path"); const fs = require("fs-extra"); +const stream = require("node:stream"); const JSZip = require("jszip"); // Added for creating mock zips // Capture native fetch before any potential mocks are applied @@ -12,8 +13,7 @@ const { ModStore, ModInfo, ModVersionEquality } = require("@clusterio/lib"); // const MODS_DIR = path.join("temp", "test", "mod_store", "mods"); const CACHE_FILE = path.join(MODS_DIR, "mod-info-cache.json"); -// Helper function to create a mock mod zip file -async function createMockModZip(filePath, name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) { +function createMockModJSZip(name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) { const zip = new JSZip(); const infoJson = { name: name, @@ -27,7 +27,12 @@ async function createMockModZip(filePath, name, version, factorioVersion = "1.1" zip.file(`${name}/info.json`, JSON.stringify(infoJson, null, 4)); zip.file(`${name}/dummy.lua`, "-- Mock Lua file"); - await fs.ensureDir(path.dirname(filePath)); + return zip; +} + +// Helper function to create a mock mod zip file +async function createMockModZip(filePath, name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) { + const zip = createMockModJSZip(name, version, factorioVersion, dependencies); const buffer = await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }); await fs.writeFile(filePath, buffer); } @@ -174,79 +179,19 @@ describe("lib/ModStore", function () { // - Error handling for corrupt zips }); - describe("downloadFile (static)", function () { - const downloadUrl = "http://example.com/download.zip"; - const downloadPath = path.join(MODS_DIR, "downloaded.zip"); - const fileContent = "mock file content"; - - beforeEach(function () { - // Reset fetch mock specifically for downloadFile tests - global.fetch = async (url, options) => { - if (url.toString() === downloadUrl) { - // Simulate a readable stream for the response body - const { Readable } = require("stream"); - const stream = Readable.from(fileContent); - return { - ok: true, - status: 200, - statusText: "OK", - body: Readable.toWeb(stream), // Convert Node stream to Web ReadableStream - }; - } - return { - ok: false, - status: 404, - statusText: "Not Found", - body: null, - }; - }; - }); - - it("should download content from URL to the specified file path", async function () { - await ModStore.downloadFile(downloadUrl, downloadPath); - const writtenContent = await fs.readFile(downloadPath, "utf8"); - assert.equal(writtenContent, fileContent); - }); - - it("should throw if fetch response is not ok", async function () { - await assert.rejects( - ModStore.downloadFile("http://example.com/not-found", downloadPath), - /Failed to fetch .* 404 Not Found/ - ); - }); - - it("should throw if fetch response body is missing", async function () { - global.fetch = async (url, options) => ({ - ok: true, - status: 200, - statusText: "OK", - body: null, // Simulate missing body - }); - await assert.rejects( - ModStore.downloadFile(downloadUrl, downloadPath), - /Response body is missing/ - ); - }); - - // Note: Testing the stream piping error handling might require more complex stream mocking. - }); - describe("downloadMods", function () { let modStore; const username = "testuser"; const token = "testtoken"; const factorioVersion = "1.1"; - // Store original downloadFile and fetch, restore them after tests - let originalDownloadFile; + // Store fetch and restore it after tests let originalFetch; before(function () { - originalDownloadFile = ModStore.downloadFile; originalFetch = global.fetch; }); after(function () { - ModStore.downloadFile = originalDownloadFile; global.fetch = originalFetch; }); @@ -255,17 +200,6 @@ describe("lib/ModStore", function () { await fs.emptyDir(MODS_DIR); // Clear any existing files in the temp dir modStore = await ModStore.fromDirectory(MODS_DIR); // Initialize using fromDirectory - // Replace downloadFile with a mock that creates the file directly - ModStore.downloadFile = async (url, filePath) => { - const finalFilename = path.basename(filePath).replace(/\.tmp$/, ""); - const match = finalFilename.match(/^(.*)_(\d+\.\d+\.\d+)\.zip$/); - if (!match) { - throw new Error(`Mock downloadFile could not parse filename: ${finalFilename}`); - } - const [, name, version] = match; - await createMockModZip(filePath, name, version, factorioVersion); - }; - // Restore fetch before each test in this suite to avoid pollution global.fetch = originalFetch; }); @@ -304,6 +238,21 @@ describe("lib/ModStore", function () { }); return { ok: true, status: 200, json: async () => ({ results: results, pagination: {} }) }; } + if (urlString.startsWith("https://mods.factorio.com/download/")) { + const [, name, version] = /\/([^\/]+)\/([^\/]+)$/.exec(url.pathname); + const requestedMod = modsToDownload.find(mod => ( + mod.name === name + && mod.version.version === version + )); + if (!requestedMod) { + return { ok: false, status: 404, statusText: `Test Mock 404, no such mod: ${urlString}` }; + } + if (name === "bad-download-mod") { + return { ok: false, status: 503, statusText: "Test Mock download error" }; + } + const zip = createMockModJSZip(name, version); + return { ok: true, status: 200, body: stream.Readable.toWeb(zip.generateNodeStream()) }; + } // Fallback for unhandled fetches in this test return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` }; }; @@ -418,56 +367,10 @@ describe("lib/ModStore", function () { { name: "another-good-mod", version: ModVersionEquality.fromString("1.0.0") }, // Success ]; - // Mock ModStore.downloadFile to fail for the specific mod - const originalDSF = ModStore.downloadFile; - ModStore.downloadFile = async (url, filePath) => { - if (filePath.includes("bad-download-mod")) { - throw new Error("Simulated download failure"); - } else { - // Use original mock logic for successful downloads - const finalFilename = path.basename(filePath).replace(/\.tmp$/, ""); - const match = finalFilename.match(/^(.*)_(\d+\.\d+\.\d+)\.zip$/); - if (!match) { - throw new Error(`Mock downloadFile could not parse filename: ${finalFilename}`); - } - const [, name, version] = match; - await createMockModZip(filePath, name, version, factorioVersion); - } - }; - // Define fetch mock specific to this test (for the API call) - global.fetch = async (url, options) => { - const urlString = url.toString(); - if (urlString.startsWith("https://mods.factorio.com/api/mods") && options?.method === "POST") { - const body = options.body; - let requestedModNames = []; - if (body instanceof URLSearchParams) { - requestedModNames = (body.get("namelist") || "").split(",").filter(Boolean); - } - const results = requestedModNames.map(name => { - const requestedMod = modsToDownload.find(mod => mod.name === name); - if (!requestedMod) { - throw new Error(`[Test Mock ERROR] No version for ${name}`); - } - const requestedVersion = requestedMod.version.version; - return { - name: name, - latest_release: { - download_url: `/download/${name}/${requestedVersion}`, - file_name: `${name}_${requestedVersion}.zip`, - info_json: { factorio_version: factorioVersion }, - version: requestedVersion, - sha1: `sha1_${name}_${requestedVersion}`, - }, - }; - }); - return { ok: true, status: 200, json: async () => ({ results: results, pagination: {} }) }; - } - return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` }; - }; + global.fetch = createMockPortalApiFetch(modsToDownload); await modStore.downloadMods(modsToDownload, username, token, factorioVersion); - ModStore.downloadFile = originalDSF; // Restore downloadFile mock assert.equal(modStore.files.size, 2, "Only successfully downloaded mods should be added"); assert(modStore.files.has("good-mod_1.0.0.zip")); @@ -482,43 +385,15 @@ describe("lib/ModStore", function () { { name: "chunk-mod-3", version: ModVersionEquality.fromString("1.0.0") }, { name: "chunk-mod-4", version: ModVersionEquality.fromString("1.0.0") }, ]; - let postCalls = 0; + let fetchCalls = []; // Define fetch mock specific to this test - const testFetch = async (url, options) => { - const urlString = url.toString(); - if (urlString.startsWith("https://mods.factorio.com/api/mods") && options?.method === "POST") { - postCalls += 1; // Track POST calls - const body = options.body; - let requestedModNames = []; - if (body instanceof URLSearchParams) { - requestedModNames = (body.get("namelist") || "").split(",").filter(Boolean); - } - const results = requestedModNames.map(name => { - const requestedMod = modsToDownload.find(mod => mod.name === name); - if (!requestedMod) { - throw new Error(`[Test Mock ERROR] No version for ${name}`); - } - const requestedVersion = requestedMod.version.version; - return { - name: name, - latest_release: { - download_url: `/download/${name}/${requestedVersion}`, - file_name: `${name}_${requestedVersion}.zip`, - info_json: { factorio_version: factorioVersion }, - version: requestedVersion, - sha1: `sha1_${name}_${requestedVersion}`, - }, - }; - }); - return { ok: true, status: 200, json: async () => ({ results: results, pagination: {} }) }; - } - return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` }; - }; - global.fetch = testFetch; + global.fetch = createMockPortalApiFetch(modsToDownload, fetchCalls); await modStore.downloadMods(modsToDownload, username, token, factorioVersion); + let postCalls = fetchCalls.reduce((a, v) => a + (v.options?.method === "POST"), 0); + // Assuming chunkSize is 500, 4 mods should be 1 chunk/POST call assert.equal(postCalls, 1, "Should make 1 POST call for 4 mods"); assert.equal(modStore.files.size, modsToDownload.length, "All chunked mods should be downloaded"); diff --git a/test/lib/file_ops.js b/test/lib/file_ops.js index 8d4dcc2f..a4e9048d 100644 --- a/test/lib/file_ops.js +++ b/test/lib/file_ops.js @@ -1,7 +1,9 @@ "use strict"; const assert = require("assert").strict; const fs = require("fs-extra"); +const http = require("node:http"); const path = require("path"); +const util = require("node:util"); const lib = require("@clusterio/lib"); @@ -218,4 +220,184 @@ describe("lib/file_ops", function() { }); }); + describe("downloadFile()", function () { + const baseUrl = new URL("http://localhost/"); + const downloadDir = path.join("temp", "test", "downloadFile"); + const fileContent = "mock file content"; + let onStream; + let server; + + before(async function () { + await fs.emptyDir(downloadDir); + server = http.createServer(); + server.unref(); + server.on("request", (req, res) => { + if (req.url === "/simple-file") { + res.writeHead(200, { + "Content-Length": Buffer.byteLength(fileContent), + "Content-Type": "text/plain", + }).end(fileContent); + } else if (req.url === "/empty-file") { + res.writeHead(200, { + "Content-Length": 0, + "Content-Type": "text/plain", + }).end(); + } else if (req.url === "/no-content") { + res.writeHead(204).end(); + } else if (req.url === "/stream") { + res.writeHead(200, { + "Content-Type": "text/plain", + }); + res.write("streaming\n"); + onStream(res); + } else { + const body = "not found"; + res.writeHead(404, { + "Content-Length": Buffer.byteLength(body), + "Content-Type": "text/plain", + }).end(body); + } + }); + await util.promisify(server.listen.bind(server))(); + baseUrl.port = server.address().port; + }); + after(function() { + server.close(); + server.closeAllConnections(); + }); + + it("should download content from URL to the specified file path", async function () { + for (const mode of ["overwrite", "rename", "error"]) { + const downloadUrl = new URL("/simple-file", baseUrl); + const downloadPath = path.join(downloadDir, `simple-file-${mode}.txt`); + const newDownloadPath = await lib.downloadFile(downloadUrl, downloadPath, mode); + const writtenContent = await fs.readFile(downloadPath, "utf8"); + assert.equal(writtenContent, fileContent); + assert.equal(downloadPath, newDownloadPath); + } + }); + + it("should throw if fetch response is not ok", async function () { + const downloadPath = path.join(downloadDir, "not-found.txt"); + await assert.rejects( + lib.downloadFile(new URL("/not-found", baseUrl), downloadPath, "overwrite"), + /Failed to download .* 404 Not Found/ + ); + await assert.rejects( + fs.readFile(downloadPath), + { code: "ENOENT" }, + ); + }); + + it("should throw if fetch response body is missing", async function () { + const downloadPath = path.join(downloadDir, "not-content.txt"); + await assert.rejects( + lib.downloadFile(new URL("/no-content", baseUrl), downloadPath, "overwrite"), + /Failed to download .* 204 No Content/ + ); + await assert.rejects( + fs.readFile(downloadPath), + { code: "ENOENT" }, + ); + }); + + it("should throw if file exists and overwriteMode is error", async function () { + const downloadPath = path.join(downloadDir, "exists.txt"); + await fs.outputFile(downloadPath, ""); + await assert.rejects( + lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "error"), + { code: "EEXIST" }, + ); + }); + + it("should throw if target directory does not exist", async function () { + const downloadPath = path.join(downloadDir, "does-not-exist", "simple.txt"); + for (const mode of ["overwrite", "rename", "error"]) { + await assert.rejects( + lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, mode), + { code: "ENOENT" }, + ); + } + }); + + it("should write a new file if it exists and overwriteMode is rename", async function () { + const downloadPath = path.join(downloadDir, "exists.txt"); + await fs.outputFile(downloadPath, ""); + const newDownloadPath = await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "rename"); + assert.equal(newDownloadPath, path.join(downloadDir, "exists-2.txt")); + const writtenContent = await fs.readFile(newDownloadPath, "utf8"); + assert.equal(writtenContent, fileContent); + }); + + it("should write a new file if many files exists and overwriteMode is rename", async function () { + const downloadPath = path.join(downloadDir, "contended.txt"); + const additionalPaths = [ + path.join(downloadDir, "contended-2.txt"), + path.join(downloadDir, "contended-3.txt"), + path.join(downloadDir, "contended-4.txt"), + ]; + await fs.outputFile(downloadPath, ""); + for (const additionalPath of additionalPaths) { + await fs.outputFile(additionalPath, ""); + } + const newDownloadPath = await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "rename"); + assert.equal(newDownloadPath, path.join(downloadDir, "contended-5.txt")); + const writtenContent = await fs.readFile(newDownloadPath, "utf8"); + assert.equal(writtenContent, fileContent); + }); + + it("should overwrite existing file if overwriteMode is overwrite", async function () { + const downloadPath = path.join(downloadDir, "overwrite.txt"); + await fs.outputFile(downloadPath, "original content"); + const newDownloadPath = await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "overwrite"); + assert.equal(newDownloadPath, downloadPath); + const writtenContent = await fs.readFile(newDownloadPath, "utf8"); + assert.equal(writtenContent, fileContent); + }); + + it("should not overwrite existing temp files", async function () { + const tempPath = path.join(downloadDir, "temp-exists.tmp.txt"); + await fs.outputFile(tempPath, "original content"); + const downloadPath = path.join(downloadDir, "temp-exists.txt"); + await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "overwrite"); + const writtenContent = await fs.readFile(tempPath, "utf8"); + assert.equal(writtenContent, "original content"); + }); + + it("should throw if overwriteMode is rename and directory disappeared", async function () { + if (process.platform === "win32") { + // Can't remove the directory while the stream is running on Windows. + this.skip(); + } + function stageWaiter() { + let stage; + let waitForStage = new Promise(resolve => { stage = resolve; }); + return [stage, waitForStage]; + } + + const downloadSubdir = path.join(downloadDir, "subdir"); + const downloadPath = path.join(downloadSubdir, "test.txt"); + await fs.ensureDir(downloadSubdir); + + const [stage1, waitForStage1] = stageWaiter(); + const [stage2, waitForStage2] = stageWaiter(); + + onStream = async (res) => { + stage1(); + await waitForStage2; + res.end("done"); + }; + const download = lib.downloadFile(new URL("/stream", baseUrl), downloadPath, "rename"); + download.catch(() => {}); + await waitForStage1; + + await fs.remove(downloadSubdir); + stage2(); + + await assert.rejects( + download, + { code: "ENOENT" }, + ); + }); + }); });