aperx/factorio-clusterio
1
0 Fork 0
mirror of https://github.com/clusterio/clusterio.git synced 2026-11-13 23:14:53 +09:00
Code Issues Packages Projects Releases Wiki Activity

Refactor file write streams (#877)

Browse Source 
A lot of code is repeated in various different ways in order to handle
streaming content to files without overwriting them, whether it's from
web resources or local streams.

Refactor the handling of these streams to all use the same utility
function for creating a write stream, and for the places downloading
urls use the same downloadFile function.
This commit is contained in:
Hornwitser authored and GitHub committed 2026-03-19 21:55:52 +01:00
1 parent 7701010fee
commit 86eefab9f5
9 files changed
+328 -312
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/controller/src/routes.ts
+7 -23
View File
@@ -553,30 +553,16 @@ async function uploadMod(req: Request, res: Response) {
}
const modsDirectory = req.app.locals.controller.config.get("controller.mods_directory");
let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
const filePath = path.join(modsDirectory, filename);
const [tempFilePath, writeStream] = await lib.createTempWriteStream(filePath);
let writeStream;
try {
while (true) {
try {
writeStream = fs.createWriteStream(path.join(modsDirectory, tempFilename), { flags: "wx" });
await events.once(writeStream, "open");
break;
} catch (err: any) {
if (err.code === "EEXIST") {
tempFilename = await lib.findUnusedName(modsDirectory, tempFilename, ".tmp.zip");
} else {
throw err;
}
}
}
stream.pipe(writeStream);
await finished(writeStream);
const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename));
const modInfo = await lib.ModInfo.fromModFile(tempFilePath);
checkModName(modInfo.filename);
await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename));
await fs.rename(tempFilePath, path.join(modsDirectory, modInfo.filename));
req.app.locals.controller.modStore.addMod(modInfo);
mods.push(modInfo.toJSON());
@@ -586,15 +572,13 @@ async function uploadMod(req: Request, res: Response) {
stream.resume();
// Attempt to clean up.
if (writeStream) {
writeStream.destroy();
}
writeStream.destroy();
try {
await fs.unlink(path.join(modsDirectory, tempFilename));
await fs.unlink(tempFilePath);
} catch (unlinkErr: any) {
if (unlinkErr.code !== "ENOENT") {
logger.error(`Error removing ${tempFilename}: ${unlinkErr.message}`);
logger.error(`Error removing ${tempFilePath}: ${unlinkErr.message}`);
}
}
}
packages/ctl/src/commands_instance.ts
+1 -25
View File
@@ -417,31 +417,7 @@ instanceSaveCommands.add(new lib.Command({
let url = new URL(control.config.get("control.controller_url")!);
url.pathname += `api/stream/${streamId}`;
let response = await fetch(url);
if (!response.body) {
throw new lib.CommandError("Got empty response from server.");
}
let writeStream;
let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip");
while (true) {
try {
writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
await events.once(writeStream, "open");
break;
} catch (err: any) {
if (err.code === "EEXIST") {
tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip");
} else {
throw err;
}
}
}
stream.Readable.fromWeb(response.body).pipe(writeStream);
await finished(writeStream);
let filename = await lib.findUnusedName(".", args.save, ".zip");
await fs.rename(tempFilename, filename);
const filename = await lib.downloadFile(url, args.save, "rename");
logger.info(`Downloaded ${args.save}${args.save === filename ? "" : ` as ${filename}`}`);
},
packages/ctl/src/commands_mod.ts
+1 -23
View File
@@ -194,30 +194,8 @@ modCommands.add(new lib.Command({
let url = new URL(control.config.get("control.controller_url")!);
url.pathname += `api/stream/${streamId}`;
let response = await fetch(url);
if (!response.body) {
throw new lib.CommandError("Got empty response from server");
}
let writeStream;
let filename = `${args.name}_${args.modVersion}.zip`;
let tempFilename = filename.replace(/(\.zip)?$/, ".tmp.zip");
while (true) {
try {
writeStream = fs.createWriteStream(tempFilename, { flags: "wx" });
await events.once(writeStream, "open");
break;
} catch (err: any) {
if (err.code === "EEXIST") {
tempFilename = await lib.findUnusedName(".", tempFilename, ".tmp.zip");
} else {
throw err;
}
}
}
stream.Readable.fromWeb(response.body).pipe(writeStream);
await finished(writeStream);
await fs.rename(tempFilename, filename);
await lib.downloadFile(url, filename, "overwrite");
logger.info(`Downloaded ${filename}`);
},
packages/host/src/Host.ts
+3 -38
View File
@@ -591,19 +591,10 @@ export default class Host extends lib.Link {
let url = new URL(this.config.get("host.controller_url"));
url.pathname += `api/stream/${streamId}`;
let response = await fetch(url);
if (!response.body) {
throw new Error(`Empty body downloading stream ${streamId} from controller`);
}
const file = lib.ModInfo.filename(mod.name, mod.version);
const filePath = path.join(this.modStore.modsDirectory, file);
const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip");
const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" });
await events.once(writeStream, "open");
stream.Readable.fromWeb(response.body).pipe(writeStream);
await finished(writeStream);
await fs.rename(tempFilePath, filePath);
await lib.downloadFile(url, filePath, "overwrite");
const modInfo = await this.modStore.loadFile(file);
if (mod.sha1 && mod.sha1 !== modInfo.sha1) {
@@ -1024,35 +1015,9 @@ export default class Host extends lib.Link {
let url = new URL(this.config.get("host.controller_url"));
url.pathname += `api/stream/${streamId}`;
let response = await fetch(url);
if (response.status !== 200 || !response.body) {
let body = Buffer.from(await response.arrayBuffer());
throw new lib.RequestError(`Stream returned ${response.status}: ${body.toString()}`);
}
let savesDir = path.join(instance.path, "saves");
let tempFilename = name.replace(/(\.zip)?$/, ".tmp.zip");
let writeStream: NodeJS.WritableStream;
while (true) {
try {
writeStream = fs.createWriteStream(path.join(savesDir, tempFilename), { flags: "wx" });
await events.once(writeStream, "open");
break;
} catch (err: any) {
if (err.code === "EEXIST") {
tempFilename = await lib.findUnusedName(savesDir, tempFilename, ".tmp.zip");
} else {
throw err;
}
}
}
stream.Readable.fromWeb(response.body).pipe(writeStream);
await finished(writeStream);
name = await lib.findUnusedName(savesDir, name, ".zip");
await fs.rename(path.join(savesDir, tempFilename), path.join(savesDir, name));
const savedPath = await lib.downloadFile(url, path.join(savesDir, name), "rename");
name = path.basename(savedPath);
await this.sendSaveListUpdate(instanceId, savesDir);
return name;
}
packages/host/src/patch.ts
+3 -5
View File
@@ -431,15 +431,13 @@ export async function patch(savePath: string, modules: SaveModule[]) {
root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
// Write back the save
let tempSavePath = savePath.replace(/(\.zip)?$/, ".tmp.zip");
let stream = zip.generateNodeStream({ compression: "DEFLATE" });
let fd = await fs.open(tempSavePath, "w");
const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
try {
let pipe = stream.pipe(fs.createWriteStream("", { fd, autoClose: false }));
let pipe = stream.pipe(writeStream);
await events.once(pipe, "finish");
await fs.fsync(fd);
} finally {
await fs.close(fd);
writeStream.destroy();
}
await fs.rename(tempSavePath, savePath);
}
packages/lib/src/ModStore.ts
+4 -43
View File
@@ -3,8 +3,7 @@ import path from "path";
import events from "events";
import { Static } from "@sinclair/typebox";
import { logger } from "./logging";
import { safeOutputFile } from "./file_ops";
import { Writable } from "stream";
import { downloadFile, safeOutputFile } from "./file_ops";
import { ModPortalReleaseSchema, ModPortalDetailsSchema, ModNameVersionPair } from "./data/messages_mod";
import { ModInfo, ModPack, FullVersion, ApiVersion, ModVersionEquality } from "./data";
@@ -155,33 +154,6 @@ export default class ModStore extends events.EventEmitter<ModStoreEvents> {
return new this(modsDirectory, files);
}
/**
* Stream a url's content into a file.
*
* @param url - Web path to stream from.
* @param filePath - File path to stream to.
*/
private static async downloadFile(url: string | URL, filePath: string) {
try {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
}
if (!response.body) {
throw new Error("Response body is missing.");
}
const fileStream = fs.createWriteStream(filePath);
const writer = Writable.toWeb(fileStream);
await response.body.pipeTo(writer);
} catch (error) {
logger.error("Error downloading file:", error);
throw error;
}
}
/**
* Download a matching version from a selection of available versions.
*
@@ -202,27 +174,16 @@ export default class ModStore extends events.EventEmitter<ModStoreEvents> {
}
try {
const url = new URL(`https://mods.factorio.com/${selectedRelease.download_url}`);
const url = new URL(selectedRelease.download_url, "https://mods.factorio.com/");
url.searchParams.set("username", username);
url.searchParams.set("token", token);
const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
const finalFileName = `${this.modsDirectory}/${selectedRelease.file_name}`;
const filePath = path.join(this.modsDirectory, selectedRelease.file_name);
await ModStore.downloadFile(url, tempFileName);
await fs.rename(tempFileName, finalFileName);
await downloadFile(url, filePath, "overwrite");
await this.loadFile(selectedRelease.file_name);
} catch (err: any) {
logger.error(`Failed to download or load mod ${selectedRelease.file_name} (${version}): ${err.message}`);
// Attempt to clean up temp file if it exists
try {
const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
await fs.unlink(tempFileName);
} catch (unlinkErr: any) {
if (unlinkErr.code !== "ENOENT") { // Ignore if file doesn't exist
logger.warn(`Failed to clean up temp file ${selectedRelease.file_name}.tmp: ${unlinkErr.message}`);
}
}
}
}
packages/lib/src/file_ops.ts
+98 -1
View File
@@ -1,9 +1,12 @@
/**
* @module lib/file_ops
*/
import crypto from "crypto";
import events from "events";
import fs from "fs-extra";
import path from "path";
import crypto from "crypto"; // needed for getTempFile
import stream from "stream";
/**
* Returns the newest file in a directory
@@ -235,3 +238,97 @@ export function cleanFilename(name: string) {
return name;
}
/**
* Creates a write stream to a guaranteed newly created temporary file in the
* same folder as the target `filePath`.
*
* @param filePath - The target file to create a temporary stream for.
* @returns The path to the temporary file that was opened and the write stream to it.
*/
export async function createTempWriteStream(filePath: string): Promise<[string, fs.WriteStream]> {
const { dir, name: fileName, ext } = path.parse(filePath);
let increment = 1;
let writeStream;
let tempFilePath = path.format({ dir, name: `${fileName}.tmp`, ext });
while (true) {
try {
writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true });
// replace "open" event with "ready" after removing fs-extra
await events.once(writeStream, "open");
break;
} catch (err: any) {
if (err.code === "EEXIST") {
increment += 1;
tempFilePath = path.format({ dir, name: `${fileName}-${increment}.tmp`, ext });
} else {
throw err;
}
}
}
return [tempFilePath, writeStream];
}
/**
* Download a file by fetching the given url and streaming the body content to it.
*
* @param url - The resource to download
* @param filePath - Path to the file store the resource data in.
* @param overwriteMode - What to do if the target filePath already exists. Use
* overwrite to clobber it, use rename to write to a differently named file,
* and error to throw an EEXIST error.
* @returns file path that was written to. May be different from `filePath` if
* overwriteMode is set to rename.
*/
export async function downloadFile(
url: URL | string,
filePath: string,
overwriteMode: "overwrite" | "rename" | "error",
) {
// Prepare file to stream content to.
let writeStream;
let tempFilePath;
if (overwriteMode === "error") {
tempFilePath = filePath;
writeStream = fs.createWriteStream(tempFilePath, { flags: "wx", flush: true });
// replace "open" event with "ready" after removing fs-extra
await events.once(writeStream, "open");
} else {
[tempFilePath, writeStream] = await createTempWriteStream(filePath);
}
// Fetch stream bytes
const response = await fetch(url);
if (!response.ok || !response.body) {
writeStream.end();
await events.once(writeStream, "close");
await fs.unlink(tempFilePath);
throw new Error(`Failed to download ${url}: ${response.status} ${response.statusText}`);
}
const writer = stream.Writable.toWeb(writeStream);
await response.body.pipeTo(writer);
let targetFilePath = filePath;
if (overwriteMode === "overwrite") {
await fs.rename(tempFilePath, targetFilePath);
} else if (overwriteMode === "rename") {
let increment = 1;
let { dir, name, ext } = path.parse(targetFilePath);
while (true) {
try {
await fs.writeFile(targetFilePath, Buffer.alloc(0), { flag: "wx" });
break;
} catch (err: any) {
if (err.code === "EEXIST") {
increment += 1;
targetFilePath = path.format({ dir, name: `${name}-${increment}`, ext });
} else {
throw err;
}
}
}
await fs.rename(tempFilePath, targetFilePath);
}
return targetFilePath;
}
test/lib/ModStore.test.js
+29 -154
View File
@@ -2,6 +2,7 @@
const assert = require("assert").strict;
const path = require("path");
const fs = require("fs-extra");
const stream = require("node:stream");
const JSZip = require("jszip"); // Added for creating mock zips
// Capture native fetch before any potential mocks are applied
@@ -12,8 +13,7 @@ const { ModStore, ModInfo, ModVersionEquality } = require("@clusterio/lib"); //
const MODS_DIR = path.join("temp", "test", "mod_store", "mods");
const CACHE_FILE = path.join(MODS_DIR, "mod-info-cache.json");
// Helper function to create a mock mod zip file
async function createMockModZip(filePath, name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) {
function createMockModJSZip(name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) {
const zip = new JSZip();
const infoJson = {
name: name,
@@ -27,7 +27,12 @@ async function createMockModZip(filePath, name, version, factorioVersion = "1.1"
zip.file(`${name}/info.json`, JSON.stringify(infoJson, null, 4));
zip.file(`${name}/dummy.lua`, "-- Mock Lua file");
await fs.ensureDir(path.dirname(filePath));
return zip;
}
// Helper function to create a mock mod zip file
async function createMockModZip(filePath, name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) {
const zip = createMockModJSZip(name, version, factorioVersion, dependencies);
const buffer = await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" });
await fs.writeFile(filePath, buffer);
}
@@ -174,79 +179,19 @@ describe("lib/ModStore", function () {
// - Error handling for corrupt zips
});
describe("downloadFile (static)", function () {
const downloadUrl = "http://example.com/download.zip";
const downloadPath = path.join(MODS_DIR, "downloaded.zip");
const fileContent = "mock file content";
beforeEach(function () {
// Reset fetch mock specifically for downloadFile tests
global.fetch = async (url, options) => {
if (url.toString() === downloadUrl) {
// Simulate a readable stream for the response body
const { Readable } = require("stream");
const stream = Readable.from(fileContent);
return {
ok: true,
status: 200,
statusText: "OK",
body: Readable.toWeb(stream), // Convert Node stream to Web ReadableStream
};
}
return {
ok: false,
status: 404,
statusText: "Not Found",
body: null,
};
};
});
it("should download content from URL to the specified file path", async function () {
await ModStore.downloadFile(downloadUrl, downloadPath);
const writtenContent = await fs.readFile(downloadPath, "utf8");
assert.equal(writtenContent, fileContent);
});
it("should throw if fetch response is not ok", async function () {
await assert.rejects(
ModStore.downloadFile("http://example.com/not-found", downloadPath),
/Failed to fetch .* 404 Not Found/
);
});
it("should throw if fetch response body is missing", async function () {
global.fetch = async (url, options) => ({
ok: true,
status: 200,
statusText: "OK",
body: null, // Simulate missing body
});
await assert.rejects(
ModStore.downloadFile(downloadUrl, downloadPath),
/Response body is missing/
);
});
// Note: Testing the stream piping error handling might require more complex stream mocking.
});
describe("downloadMods", function () {
let modStore;
const username = "testuser";
const token = "testtoken";
const factorioVersion = "1.1";
// Store original downloadFile and fetch, restore them after tests
let originalDownloadFile;
// Store fetch and restore it after tests
let originalFetch;
before(function () {
originalDownloadFile = ModStore.downloadFile;
originalFetch = global.fetch;
});
after(function () {
ModStore.downloadFile = originalDownloadFile;
global.fetch = originalFetch;
});
@@ -255,17 +200,6 @@ describe("lib/ModStore", function () {
await fs.emptyDir(MODS_DIR); // Clear any existing files in the temp dir
modStore = await ModStore.fromDirectory(MODS_DIR); // Initialize using fromDirectory
// Replace downloadFile with a mock that creates the file directly
ModStore.downloadFile = async (url, filePath) => {
const finalFilename = path.basename(filePath).replace(/\.tmp$/, "");
const match = finalFilename.match(/^(.*)_(\d+\.\d+\.\d+)\.zip$/);
if (!match) {
throw new Error(`Mock downloadFile could not parse filename: ${finalFilename}`);
}
const [, name, version] = match;
await createMockModZip(filePath, name, version, factorioVersion);
};
// Restore fetch before each test in this suite to avoid pollution
global.fetch = originalFetch;
});
@@ -304,6 +238,21 @@ describe("lib/ModStore", function () {
});
return { ok: true, status: 200, json: async () => ({ results: results, pagination: {} }) };
}
if (urlString.startsWith("https://mods.factorio.com/download/")) {
const [, name, version] = /\/([^\/]+)\/([^\/]+)$/.exec(url.pathname);
const requestedMod = modsToDownload.find(mod => (
mod.name === name
&& mod.version.version === version
));
if (!requestedMod) {
return { ok: false, status: 404, statusText: `Test Mock 404, no such mod: ${urlString}` };
}
if (name === "bad-download-mod") {
return { ok: false, status: 503, statusText: "Test Mock download error" };
}
const zip = createMockModJSZip(name, version);
return { ok: true, status: 200, body: stream.Readable.toWeb(zip.generateNodeStream()) };
}
// Fallback for unhandled fetches in this test
return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` };
};
@@ -418,56 +367,10 @@ describe("lib/ModStore", function () {
{ name: "another-good-mod", version: ModVersionEquality.fromString("1.0.0") }, // Success
];
// Mock ModStore.downloadFile to fail for the specific mod
const originalDSF = ModStore.downloadFile;
ModStore.downloadFile = async (url, filePath) => {
if (filePath.includes("bad-download-mod")) {
throw new Error("Simulated download failure");
} else {
// Use original mock logic for successful downloads
const finalFilename = path.basename(filePath).replace(/\.tmp$/, "");
const match = finalFilename.match(/^(.*)_(\d+\.\d+\.\d+)\.zip$/);
if (!match) {
throw new Error(`Mock downloadFile could not parse filename: ${finalFilename}`);
}
const [, name, version] = match;
await createMockModZip(filePath, name, version, factorioVersion);
}
};
// Define fetch mock specific to this test (for the API call)
global.fetch = async (url, options) => {
const urlString = url.toString();
if (urlString.startsWith("https://mods.factorio.com/api/mods") && options?.method === "POST") {
const body = options.body;
let requestedModNames = [];
if (body instanceof URLSearchParams) {
requestedModNames = (body.get("namelist") || "").split(",").filter(Boolean);
}
const results = requestedModNames.map(name => {
const requestedMod = modsToDownload.find(mod => mod.name === name);
if (!requestedMod) {
throw new Error(`[Test Mock ERROR] No version for ${name}`);
}
const requestedVersion = requestedMod.version.version;
return {
name: name,
latest_release: {
download_url: `/download/${name}/${requestedVersion}`,
file_name: `${name}_${requestedVersion}.zip`,
info_json: { factorio_version: factorioVersion },
version: requestedVersion,
sha1: `sha1_${name}_${requestedVersion}`,
},
};
});
return { ok: true, status: 200, json: async () => ({ results: results, pagination: {} }) };
}
return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` };
};
global.fetch = createMockPortalApiFetch(modsToDownload);
await modStore.downloadMods(modsToDownload, username, token, factorioVersion);
ModStore.downloadFile = originalDSF; // Restore downloadFile mock
assert.equal(modStore.files.size, 2, "Only successfully downloaded mods should be added");
assert(modStore.files.has("good-mod_1.0.0.zip"));
@@ -482,43 +385,15 @@ describe("lib/ModStore", function () {
{ name: "chunk-mod-3", version: ModVersionEquality.fromString("1.0.0") },
{ name: "chunk-mod-4", version: ModVersionEquality.fromString("1.0.0") },
];
let postCalls = 0;
let fetchCalls = [];
// Define fetch mock specific to this test
const testFetch = async (url, options) => {
const urlString = url.toString();
if (urlString.startsWith("https://mods.factorio.com/api/mods") && options?.method === "POST") {
postCalls += 1; // Track POST calls
const body = options.body;
let requestedModNames = [];
if (body instanceof URLSearchParams) {
requestedModNames = (body.get("namelist") || "").split(",").filter(Boolean);
}
const results = requestedModNames.map(name => {
const requestedMod = modsToDownload.find(mod => mod.name === name);
if (!requestedMod) {
throw new Error(`[Test Mock ERROR] No version for ${name}`);
}
const requestedVersion = requestedMod.version.version;
return {
name: name,
latest_release: {
download_url: `/download/${name}/${requestedVersion}`,
file_name: `${name}_${requestedVersion}.zip`,
info_json: { factorio_version: factorioVersion },
version: requestedVersion,
sha1: `sha1_${name}_${requestedVersion}`,
},
};
});
return { ok: true, status: 200, json: async () => ({ results: results, pagination: {} }) };
}
return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` };
};
global.fetch = testFetch;
global.fetch = createMockPortalApiFetch(modsToDownload, fetchCalls);
await modStore.downloadMods(modsToDownload, username, token, factorioVersion);
let postCalls = fetchCalls.reduce((a, v) => a + (v.options?.method === "POST"), 0);
// Assuming chunkSize is 500, 4 mods should be 1 chunk/POST call
assert.equal(postCalls, 1, "Should make 1 POST call for 4 mods");
assert.equal(modStore.files.size, modsToDownload.length, "All chunked mods should be downloaded");
test/lib/file_ops.js
+182
View File
@@ -1,7 +1,9 @@
"use strict";
const assert = require("assert").strict;
const fs = require("fs-extra");
const http = require("node:http");
const path = require("path");
const util = require("node:util");
const lib = require("@clusterio/lib");
@@ -218,4 +220,184 @@ describe("lib/file_ops", function() {
});
});
describe("downloadFile()", function () {
const baseUrl = new URL("http://localhost/");
const downloadDir = path.join("temp", "test", "downloadFile");
const fileContent = "mock file content";
let onStream;
let server;
before(async function () {
await fs.emptyDir(downloadDir);
server = http.createServer();
server.unref();
server.on("request", (req, res) => {
if (req.url === "/simple-file") {
res.writeHead(200, {
"Content-Length": Buffer.byteLength(fileContent),
"Content-Type": "text/plain",
}).end(fileContent);
} else if (req.url === "/empty-file") {
res.writeHead(200, {
"Content-Length": 0,
"Content-Type": "text/plain",
}).end();
} else if (req.url === "/no-content") {
res.writeHead(204).end();
} else if (req.url === "/stream") {
res.writeHead(200, {
"Content-Type": "text/plain",
});
res.write("streaming\n");
onStream(res);
} else {
const body = "not found";
res.writeHead(404, {
"Content-Length": Buffer.byteLength(body),
"Content-Type": "text/plain",
}).end(body);
}
});
await util.promisify(server.listen.bind(server))();
baseUrl.port = server.address().port;
});
after(function() {
server.close();
server.closeAllConnections();
});
it("should download content from URL to the specified file path", async function () {
for (const mode of ["overwrite", "rename", "error"]) {
const downloadUrl = new URL("/simple-file", baseUrl);
const downloadPath = path.join(downloadDir, `simple-file-${mode}.txt`);
const newDownloadPath = await lib.downloadFile(downloadUrl, downloadPath, mode);
const writtenContent = await fs.readFile(downloadPath, "utf8");
assert.equal(writtenContent, fileContent);
assert.equal(downloadPath, newDownloadPath);
}
});
it("should throw if fetch response is not ok", async function () {
const downloadPath = path.join(downloadDir, "not-found.txt");
await assert.rejects(
lib.downloadFile(new URL("/not-found", baseUrl), downloadPath, "overwrite"),
/Failed to download .* 404 Not Found/
);
await assert.rejects(
fs.readFile(downloadPath),
{ code: "ENOENT" },
);
});
it("should throw if fetch response body is missing", async function () {
const downloadPath = path.join(downloadDir, "not-content.txt");
await assert.rejects(
lib.downloadFile(new URL("/no-content", baseUrl), downloadPath, "overwrite"),
/Failed to download .* 204 No Content/
);
await assert.rejects(
fs.readFile(downloadPath),
{ code: "ENOENT" },
);
});
it("should throw if file exists and overwriteMode is error", async function () {
const downloadPath = path.join(downloadDir, "exists.txt");
await fs.outputFile(downloadPath, "");
await assert.rejects(
lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "error"),
{ code: "EEXIST" },
);
});
it("should throw if target directory does not exist", async function () {
const downloadPath = path.join(downloadDir, "does-not-exist", "simple.txt");
for (const mode of ["overwrite", "rename", "error"]) {
await assert.rejects(
lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, mode),
{ code: "ENOENT" },
);
}
});
it("should write a new file if it exists and overwriteMode is rename", async function () {
const downloadPath = path.join(downloadDir, "exists.txt");
await fs.outputFile(downloadPath, "");
const newDownloadPath = await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "rename");
assert.equal(newDownloadPath, path.join(downloadDir, "exists-2.txt"));
const writtenContent = await fs.readFile(newDownloadPath, "utf8");
assert.equal(writtenContent, fileContent);
});
it("should write a new file if many files exists and overwriteMode is rename", async function () {
const downloadPath = path.join(downloadDir, "contended.txt");
const additionalPaths = [
path.join(downloadDir, "contended-2.txt"),
path.join(downloadDir, "contended-3.txt"),
path.join(downloadDir, "contended-4.txt"),
];
await fs.outputFile(downloadPath, "");
for (const additionalPath of additionalPaths) {
await fs.outputFile(additionalPath, "");
}
const newDownloadPath = await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "rename");
assert.equal(newDownloadPath, path.join(downloadDir, "contended-5.txt"));
const writtenContent = await fs.readFile(newDownloadPath, "utf8");
assert.equal(writtenContent, fileContent);
});
it("should overwrite existing file if overwriteMode is overwrite", async function () {
const downloadPath = path.join(downloadDir, "overwrite.txt");
await fs.outputFile(downloadPath, "original content");
const newDownloadPath = await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "overwrite");
assert.equal(newDownloadPath, downloadPath);
const writtenContent = await fs.readFile(newDownloadPath, "utf8");
assert.equal(writtenContent, fileContent);
});
it("should not overwrite existing temp files", async function () {
const tempPath = path.join(downloadDir, "temp-exists.tmp.txt");
await fs.outputFile(tempPath, "original content");
const downloadPath = path.join(downloadDir, "temp-exists.txt");
await lib.downloadFile(new URL("/simple-file", baseUrl), downloadPath, "overwrite");
const writtenContent = await fs.readFile(tempPath, "utf8");
assert.equal(writtenContent, "original content");
});
it("should throw if overwriteMode is rename and directory disappeared", async function () {
if (process.platform === "win32") {
// Can't remove the directory while the stream is running on Windows.
this.skip();
}
function stageWaiter() {
let stage;
let waitForStage = new Promise(resolve => { stage = resolve; });
return [stage, waitForStage];
}
const downloadSubdir = path.join(downloadDir, "subdir");
const downloadPath = path.join(downloadSubdir, "test.txt");
await fs.ensureDir(downloadSubdir);
const [stage1, waitForStage1] = stageWaiter();
const [stage2, waitForStage2] = stageWaiter();
onStream = async (res) => {
stage1();
await waitForStage2;
res.end("done");
};
const download = lib.downloadFile(new URL("/stream", baseUrl), downloadPath, "rename");
download.catch(() => {});
await waitForStage1;
await fs.remove(downloadSubdir);
stage2();
await assert.rejects(
download,
{ code: "ENOENT" },
);
});
});
});