Replace jszip with yauzl and yazl (#1011)

* Replace jszip with yauzl and yazl

jszip loads a whole archive into memory to read it and buffers the
output when writing.  Saves and mods can be several hundred megabytes,
so read zips with yauzl, which only reads the central directory and
streams entries on demand, and write them with yazl, which streams the
output.

lib gains a ZipArchive wrapper around yauzl with the lookup helpers the
callers need, findRoot() now takes it instead of a JSZip.  Saves are
patched by streaming every kept entry into a new zip, mod info.json is
read straight from the file, exports are streamed to the controller and
Factorio zip downloads go through a temp file instead of a buffer.

Patching now inflates and deflates each entry again where jszip reused
the compressed data, on a 336 MB synthetic save that is 7 s instead of
0.7 s, with peak memory going from 416 MB to 151 MB.

Closes #1002

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Copy save entries without recompressing them

yazl inflates and deflates every entry it is given, so patching a save
took ten times longer than with jszip, which reused the compressed
data.  Add a ZipWriter to lib that streams entries out as they are
added and copies entries from a ZipArchive byte for byte, with ZIP64
structures when sizes or offsets need them, and use it in patch().

On a 336 MB synthetic save patching now takes 0.4 s at 143 MB peak RSS
against 0.75 s at 417 MB with jszip.  Factorio loads the result.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Handle save write errors before the patch loop awaits them

A write error on the temporary save, such as a full disk, rejected the
pipeline promise while addEntry() or addBuffer() was still pending. Nothing
was listening yet, so the host's unhandledRejection handler exited the
process instead of failing the instance start.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Keep the export zip cache local to each export

The cache was module level, so when two instances on one host exported at
once, the first to finish closed archives the other was still reading.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Start writing zips before adding files from disk

yazl reads files passed to addFile() as soon as they are added. An
unreadable file emitted an error while the directory traversal was still
running, before the output pipeline existed, and the process exited on the
uncaught error. build_mod and zipDirectory now start the pipeline first and
reject instead. build_mod also removes the partial zip on failure.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
BastiaanandClaude Opus 5.5 authored and GitHub committed 2026-10-17 21:44:19 +03:00
1 parent e810020db5
commit c28e617b22
23 files changed
+976 -298

No files matched your search

+2 -2
View File
@@ -1,6 +1,7 @@
import fs from "node:fs/promises";
import path from "path";
import pidusage from "pidusage";
import stream from "node:stream";
import util from "util";
import type { Static } from "@sinclair/typebox";
import { exec } from "child_process";
@@ -1401,13 +1402,12 @@ end`.replace(/\r?\n/g, " ");
await this.syncMods();
let zip = await exportData(this.server);
let content = await zip.generateAsync({ type: "nodebuffer" });
let url = new URL(this._host.config.get("host.controller_url"));
url.pathname += "api/upload-export";
url.searchParams.set("mod_pack_id", String(this.activeModPack.id));
let response = await fetch(url, {
method: "PUT",
body: content,
body: stream.Readable.toWeb(lib.zipOutputStream(zip)),
duplex: "half",
headers: {
"Content-Type": "application/zip",
+39 -21
View File
@@ -1,8 +1,9 @@
// Implementation of the create-scenario command
import fs from "node:fs/promises";
import os from "node:os";
import JSZip from "jszip";
import path from "path";
import { pipeline } from "node:stream/promises";
import yazl from "yazl";
import * as lib from "@clusterio/lib";
import { logger } from "@clusterio/lib";
@@ -61,33 +62,50 @@ export async function createScenario(
async function zipDirectory(dirPath: string, zipPath: string) {
const root = path.basename(path.resolve(dirPath));
const zip = new JSZip();
let dirs: [string, string][] = [[dirPath, root]];
while (dirs.length) {
let [dir, relativeDir] = dirs.pop()!;
for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
let fsPath = path.join(dir, entry.name);
let relativePath = path.posix.join(relativeDir, entry.name);
if (entry.isFile()) {
zip.file(relativePath, await fs.readFile(fsPath));
} else if (entry.isDirectory()) {
dirs.push([fsPath, relativePath]);
const zip = new yazl.ZipFile();
const output = lib.zipOutputStream(zip);
const fd = await fs.open(zipPath, "w");
// addFile() can fail before the traversal is done, so start writing first.
const writing = pipeline(output, fd.createWriteStream());
writing.catch(() => {});
try {
let dirs: [string, string][] = [[dirPath, root]];
while (dirs.length) {
let [dir, relativeDir] = dirs.pop()!;
for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
let fsPath = path.join(dir, entry.name);
let relativePath = path.posix.join(relativeDir, entry.name);
if (entry.isFile()) {
zip.addFile(fsPath, relativePath);
} else if (entry.isDirectory()) {
dirs.push([fsPath, relativePath]);
}
}
}
zip.end();
} catch (err) {
output.destroy();
await writing.catch(() => {});
throw err;
}
await fs.writeFile(zipPath, await zip.generateAsync({ type: "nodebuffer" }));
await writing;
}
async function extractZip(zipPath: string, dirPath: string) {
const zip = await JSZip.loadAsync(await fs.readFile(zipPath));
const root = lib.findRoot(zip);
for (let file of Object.values(zip.files)) {
if (file.dir || !file.name.startsWith(`${root}/`)) {
continue;
const zip = await lib.ZipArchive.fromFile(zipPath);
try {
const root = lib.findRoot(zip);
for (let entry of zip.entries.values()) {
if (entry.fileName.endsWith("/") || !entry.fileName.startsWith(`${root}/`)) {
continue;
}
const fsPath = path.join(dirPath, entry.fileName.slice(root.length + 1));
await fs.mkdir(path.dirname(fsPath), { recursive: true });
const fd = await fs.open(fsPath, "w");
await pipeline(await zip.openStream(entry), fd.createWriteStream());
}
const fsPath = path.join(dirPath, file.name.slice(root.length + 1));
await fs.mkdir(path.dirname(fsPath), { recursive: true });
await fs.writeFile(fsPath, await file.async("nodebuffer"));
} finally {
zip.close();
}
}
+80 -40
View File
@@ -2,7 +2,7 @@
import fs from "node:fs/promises";
import path from "path";
import { Jimp, JimpMime, type JimpInstance } from "jimp";
import JSZip from "jszip";
import yazl from "yazl";
import * as lib from "@clusterio/lib";
import * as libBuildMod from "@clusterio/lib/dist/node/build_mod.js";
@@ -64,21 +64,38 @@ async function generateExportMod(server: FactorioServer) {
});
}
let zipCache = new Map<string, JSZip>();
async function loadZip(server: FactorioServer, modVersions: Map<string, string>, mod: string) {
interface ModZip {
zip: lib.ZipArchive;
root: string;
}
type ZipCache = Map<string, ModZip>;
async function loadZip(server: FactorioServer, modVersions: Map<string, string>, zipCache: ZipCache, mod: string) {
let modVersion = modVersions.get(mod);
if (!modVersion) {
throw new Error(`Got path for unknown mod ${mod}`);
}
let zipPath = server.writePath("mods", `${mod}_${modVersion}.zip`);
let zip = zipCache.get(zipPath);
if (!zip) {
zip = await JSZip.loadAsync(await fs.readFile(zipPath));
zipCache.set(zipPath, zip);
let modZip = zipCache.get(zipPath);
if (!modZip) {
const zip = await lib.ZipArchive.fromFile(zipPath);
try {
modZip = { zip, root: lib.findRoot(zip) };
} catch (err) {
zip.close();
throw err;
}
zipCache.set(zipPath, modZip);
}
return zip.folder(lib.findRoot(zip))!;
return modZip;
}
function closeZipCache(zipCache: ZipCache) {
for (const { zip } of zipCache.values()) {
zip.close();
}
zipCache.clear();
}
/**
@@ -86,11 +103,17 @@ async function loadZip(server: FactorioServer, modVersions: Map<string, string>,
*
* @param server - The server to load the file from.
* @param modVersions - Mapping of mod to version used.
* @param zipCache - Mod zips opened by this export.
* @param modPath - Factorio style path to the file to load.
* @returns The content of the file or null if not found.
* @internal
*/
async function loadFile(server: FactorioServer, modVersions: Map<string, string>, modPath: string) {
async function loadFile(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
modPath: string,
) {
let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
if (!match) {
server._logger.warn(`Skipping icon with bad mod path: ${JSON.stringify(modPath)}`);
@@ -111,9 +134,9 @@ async function loadFile(server: FactorioServer, modVersions: Map<string, string>
}
}
let zip;
let modZip;
try {
zip = await loadZip(server, modVersions, mod);
modZip = await loadZip(server, modVersions, zipCache, mod);
} catch (err: any) {
if (err.code === "ENOENT") {
return null;
@@ -121,12 +144,7 @@ async function loadFile(server: FactorioServer, modVersions: Map<string, string>
throw err;
}
let file = zip.file(filePath);
if (!file) {
return null;
}
return await file.async("nodebuffer");
return await modZip.zip.readFile(path.posix.join(modZip.root, filePath));
}
type Image = JimpInstance;
@@ -134,6 +152,7 @@ type IconCache = Map<string, Image | null>;
async function loadIcon(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
iconPath: string,
iconSize: number,
iconMipmaps: number,
@@ -141,7 +160,7 @@ async function loadIcon(
) {
let icon = iconCache.get(iconPath);
if (icon === undefined) {
let fileContent = await loadFile(server, modVersions, iconPath);
let fileContent = await loadFile(server, modVersions, zipCache, iconPath);
if (fileContent) {
icon = await Jimp.fromBuffer(fileContent) as Image;
icon.crop({ x: 0, y: 0, w: iconSize, h: iconSize });
@@ -157,11 +176,12 @@ async function loadIcon(
async function loadSimpleIcon(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
item: SimpleIconSpecification,
size: number,
iconCache: IconCache,
): Promise<Image | null> {
let icon = await loadIcon(server, modVersions, item.icon, item.icon_size ?? 64, 0, iconCache);
let icon = await loadIcon(server, modVersions, zipCache, item.icon, item.icon_size ?? 64, 0, iconCache);
if (icon) {
let iconScale = size / (item.icon_size ?? 64);
if (iconScale !== 1) {
@@ -175,6 +195,7 @@ async function loadSimpleIcon(
async function loadLayeredIcon(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
item: LayeredIconSpecification,
size: number,
iconCache: IconCache,
@@ -190,7 +211,9 @@ async function loadLayeredIcon(
for (let layer of item.icons) {
let layerSize = (layer.icon_size || item.icon_size) ?? 64;
let iconLayer = await loadIcon(server, modVersions, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache);
let iconLayer = await loadIcon(
server, modVersions, zipCache, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache,
);
if (!iconLayer) {
continue;
@@ -308,6 +331,7 @@ async function growSheet(state: SheetState) {
async function exportIcon(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
state: SheetState,
prototype: PrototypeWithIcon,
) {
@@ -318,7 +342,7 @@ async function exportIcon(
if (prototype.icons) {
icon = await loadLayeredIcon(
server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache,
server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache,
);
iconPos = state.pos;
iconPath = prototype.icons[0].icon;
@@ -328,7 +352,7 @@ async function exportIcon(
iconPath = prototype.icon;
if (iconPos === undefined) {
icon = await loadSimpleIcon(
server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache,
server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache,
);
if (icon) {
iconPos = state.pos;
@@ -364,6 +388,7 @@ async function exportIcon(
*
* @param server - The server to load icons from.
* @param modVersions - Mapping of mod name to version.
* @param zipCache - Mod zips opened by this export.
* @param prototypes - Array of prototypes with icon to export for this base type.
* @param baseName - name of base type (used in metadata).
* @param state - Shared spritesheet state (mutated in place).
@@ -373,6 +398,7 @@ async function exportIcon(
async function exportIcons(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
prototypes: PrototypeWithIcon[],
baseName: string,
state: SheetState,
@@ -381,7 +407,7 @@ async function exportIcons(
state.metadata[baseName] ??= metadata;
let count = 0;
for (let prototype of prototypes) {
const icon = await exportIcon(server, modVersions, state, prototype);
const icon = await exportIcon(server, modVersions, zipCache, state, prototype);
if (icon) {
count += 1;
}
@@ -404,6 +430,7 @@ async function exportIcons(
*
* @param server - The server to export the locale from.
* @param modVersions - Mapping of mod name to version to export locale from.
* @param zipCache - Mod zips opened by this export.
* @param modOrder - Load order of the mods.
* @param languageCode - Language to export locale for.
* @returns merged locale information
@@ -412,6 +439,7 @@ async function exportIcons(
async function exportLocale(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
modOrder: string[],
languageCode: string
) {
@@ -445,18 +473,21 @@ async function exportLocale(
continue;
}
let zip;
let modZip;
try {
zip = await loadZip(server, modVersions, mod);
modZip = await loadZip(server, modVersions, zipCache, mod);
} catch (err: any) {
if (err.code === "ENOENT") {
continue;
}
throw err;
}
for (let file of zip.file(new RegExp(`locale\\/${languageCode}\\/.*\\.cfg`))) {
let content = await file.async("nodebuffer");
mergeLocale(lib.parse(content.toString("utf8")));
const localePath = `${path.posix.join(modZip.root, "locale", languageCode)}/`;
for (const entry of modZip.zip.entries.values()) {
if (entry.fileName.startsWith(localePath) && entry.fileName.endsWith(".cfg")) {
let content = await modZip.zip.readEntry(entry);
mergeLocale(lib.parse(content.toString("utf8")));
}
}
}
@@ -470,6 +501,16 @@ async function exportLocale(
* @returns zip file with exported data.
*/
export async function exportData(server: FactorioServer) {
const zipCache: ZipCache = new Map();
try {
return await buildExport(server, zipCache);
} finally {
// Free up the zip files loaded during the export.
closeZipCache(zipCache);
}
}
async function buildExport(server: FactorioServer, zipCache: ZipCache) {
await generateExportMod(server);
let settings: lib.ExportSettings = {};
@@ -549,16 +590,14 @@ export async function exportData(server: FactorioServer) {
}
}
let locale = await exportLocale(server, modVersions, modOrder, "en");
let locale = await exportLocale(server, modVersions, zipCache, modOrder, "en");
// Free up the memory used by zip files loaded during the export.
zipCache.clear();
let zip = new JSZip();
zip.file("export/settings.json", JSON.stringify(settings satisfies lib.ExportSettings));
zip.file("export/prototypes.json", JSON.stringify(prototypes satisfies lib.ExportPrototypes));
zip.file("export/locale.json", JSON.stringify([...locale.entries()] satisfies lib.ExportLocale));
zip.file("export/defines.json", JSON.stringify(defines satisfies lib.ExportDefines));
let zip = new yazl.ZipFile();
const addJson = (name: string, value: unknown) => zip.addBuffer(Buffer.from(JSON.stringify(value)), name);
addJson("export/settings.json", settings satisfies lib.ExportSettings);
addJson("export/prototypes.json", prototypes satisfies lib.ExportPrototypes);
addJson("export/locale.json", [...locale.entries()] satisfies lib.ExportLocale);
addJson("export/defines.json", defines satisfies lib.ExportDefines);
// Build a single unified spritesheet across all prototypes with icons.
const state = await createSheetState();
@@ -572,15 +611,16 @@ export async function exportData(server: FactorioServer) {
if (prototypesWithIcons.length === 0) {
continue;
}
const count = await exportIcons(server, modVersions, prototypesWithIcons, baseName, state);
const count = await exportIcons(server, modVersions, zipCache, prototypesWithIcons, baseName, state);
server._logger.info(`Exported ${count} ${baseName} icons`);
}
// Crop sheet to actual used height and write single spritesheet + metadata.
const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
state.sheet.crop({ x: 0, y: 0, w: SHEET_WIDTH, h: Math.max(usedRows * SHEET_ICON_SIZE, 1) });
zip.file("export/spritesheet.png", await state.sheet.getBuffer(JimpMime.png));
zip.file("export/metadata.json", JSON.stringify(state.metadata satisfies lib.ExportMetadata));
zip.addBuffer(await state.sheet.getBuffer(JimpMime.png), "export/spritesheet.png");
addJson("export/metadata.json", state.metadata satisfies lib.ExportMetadata);
zip.end();
server._logger.info(`Export complete: ${state.pos} icons on ${usedRows} row(s)`);
+105 -77
View File
@@ -1,10 +1,9 @@
// Library for patching Factorio saves with scenario code.
import events from "events";
import fs from "node:fs/promises";
import JSZip from "jszip";
import path from "path";
import semver from "semver";
import { pipeline } from "node:stream/promises";
import { Type, Static } from "@sinclair/typebox";
import * as lib from "@clusterio/lib";
@@ -72,18 +71,16 @@ export class SaveModule {
};
}
static async fromSave(json: Static<typeof SaveModule.jsonSchema>, root: JSZip) {
static async fromSave(json: Static<typeof SaveModule.jsonSchema>, zip: lib.ZipArchive, root: string) {
const module = new this(lib.ModuleInfo.fromJSON(json));
module.files = new Map(await Promise.all(json.files
.map(filename => ({filename, file: root.file(filename)}))
.filter(({filename, file}) => {
if (file === null) {
lib.logger.warn(`Missing file ${filename} in save`);
}
return file !== null;
})
.map(async ({filename, file}) => [filename, await file!.async("nodebuffer")] as const)
));
for (const filename of json.files) {
const content = await zip.readFile(path.posix.join(root, filename));
if (content === null) {
lib.logger.warn(`Missing file ${filename} in save`);
continue;
}
module.files.set(filename, content);
}
return module;
}
@@ -205,7 +202,7 @@ export class PatchInfo {
};
}
static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, root: JSZip) {
static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, zip: lib.ZipArchive, root: string) {
if (json.version === undefined) {
interface ScenarioInfoV0 {
name: string;
@@ -229,7 +226,7 @@ export class PatchInfo {
return new this(
json.patch_number,
lib.ModuleInfo.fromJSON(json.scenario),
await Promise.all(json.modules.map(m => SaveModule.fromSave(m, root))),
await Promise.all(json.modules.map(m => SaveModule.fromSave(m, zip, root))),
);
}
}
@@ -396,80 +393,111 @@ const knownScenarios: Record<string, lib.ModuleInfo> = {
* @param modules - Description of the modules to patch.
*/
export async function patch(savePath: string, modules: SaveModule[]) {
let zip = await JSZip.loadAsync(await fs.readFile(savePath));
let root = zip.folder(lib.findRoot(zip))!;
const zip = await lib.ZipArchive.fromFile(savePath);
let tempSavePath: string;
try {
const root = lib.findRoot(zip);
const rootPath = (relativePath: string) => path.posix.join(root, relativePath);
let patchInfoFile = root.file("clusterio.json");
let patchInfo: PatchInfo;
if (patchInfoFile !== null) {
let content = await patchInfoFile.async("string");
patchInfo = await PatchInfo.fromSave(JSON.parse(content), root);
if (patchInfo.version > PatchInfo.currentVersion) {
throw new Error(
`Save patch version ${patchInfo.version} is newer than the patch version this ` +
`version of Clusterio can load (${PatchInfo.currentVersion})`
);
}
// Files to add or replace in the save, relative to the root
const files = new Map<string, Buffer>();
let patchInfo: PatchInfo;
const patchInfoContent = await zip.readFile(rootPath("clusterio.json"));
if (patchInfoContent !== null) {
patchInfo = await PatchInfo.fromSave(JSON.parse(patchInfoContent.toString("utf8")), zip, root);
if (patchInfo.version > PatchInfo.currentVersion) {
throw new Error(
`Save patch version ${patchInfo.version} is newer than the patch version this ` +
`version of Clusterio can load (${PatchInfo.currentVersion})`
);
}
// No info file present, try to detect if it's a known compatible scenario.
} else {
let controlFile = root.file("control.lua");
if (!controlFile) {
throw new Error("Unable to patch save, missing control.lua file.");
}
let controlStream = controlFile.nodeStream("nodebuffer");
let controlHash = await lib.hashStream(controlStream);
if (controlHash in knownScenarios) {
patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
root.file("scenario.lua", controlFile.nodeStream("nodebuffer"));
// No info file present, try to detect if it's a known compatible scenario.
} else {
throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
}
}
const controlFile = zip.file(rootPath("control.lua"));
if (!controlFile) {
throw new Error("Unable to patch save, missing control.lua file.");
}
const controlHash = await lib.hashStream(await zip.openStream(controlFile));
// Increment patch number
patchInfo.patchNumber = patchInfo.patchNumber + 1;
// Remove any existing modules from the save
if (patchInfo.version === 0) {
for (let file of root.file(/^modules\//)) {
zip.remove(file.name);
}
} else {
for (let module of patchInfo.modules) {
for (let filepath of module.files.keys()) {
const file = root.file(filepath);
if (file !== null) {
zip.remove(file.name);
}
if (controlHash in knownScenarios) {
patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
files.set("scenario.lua", await zip.readEntry(controlFile));
} else {
throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
}
}
patchInfo.modules = [];
}
reorderDependencies(modules);
// Increment patch number
patchInfo.patchNumber = patchInfo.patchNumber + 1;
// Add the modules to the save.
for (let module of modules) {
for (let [relativePath, contents] of module.files) {
root.file(relativePath, contents);
// Remove any existing modules from the save
const removed = new Set<string>();
if (patchInfo.version === 0) {
const modulesPath = rootPath("modules/");
for (const name of zip.entries.keys()) {
if (name.startsWith(modulesPath)) {
removed.add(name);
}
}
} else {
for (const module of patchInfo.modules) {
for (const filepath of module.files.keys()) {
removed.add(rootPath(filepath));
}
}
patchInfo.modules = [];
}
patchInfo.modules.push(module);
}
// Add loading code and patch info
root.file("control.lua", generateLoader(patchInfo));
root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
reorderDependencies(modules);
// Write back the save
let stream = zip.generateNodeStream({ compression: "DEFLATE" });
const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
try {
let pipe = stream.pipe(writeStream);
await events.once(pipe, "finish");
// Add the modules to the save.
for (const module of modules) {
for (const [relativePath, contents] of module.files) {
files.set(relativePath, contents);
}
patchInfo.modules.push(module);
}
// Add loading code and patch info
files.set("control.lua", Buffer.from(generateLoader(patchInfo), "utf8"));
files.set("clusterio.json", Buffer.from(JSON.stringify(patchInfo, null, "\t"), "utf8"));
// Write back the save, keeping the order of the entries in the original
const added = new Map([...files].map(([relativePath, contents]) => [rootPath(relativePath), contents]));
const output = new lib.ZipWriter();
let writeStream;
[tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
const writing = pipeline(output.outputStream, writeStream);
// Awaited below, but a write error can reject it before then.
writing.catch(() => {});
try {
for (const entry of zip.entries.values()) {
const name = entry.fileName;
if (removed.has(name)) {
continue;
}
const contents = added.get(name);
if (contents !== undefined) {
await output.addBuffer(contents, name);
added.delete(name);
} else {
await output.addEntry(zip, entry);
}
}
for (const [name, contents] of added) {
await output.addBuffer(contents, name);
}
await output.end();
await writing;
} catch (err) {
output.outputStream.destroy();
await writing.catch(() => {});
await fs.rm(tempSavePath, { force: true });
throw err;
}
} finally {
writeStream.destroy();
zip.close();
}
await fs.rename(tempSavePath, savePath);
}
+33 -22
View File
@@ -1,9 +1,11 @@
// Factorio Server interface
import fs from "node:fs/promises";
import { type FSWatcher, watch as fsWatch, writeFileSync } from "node:fs";
import { type FSWatcher, createWriteStream, watch as fsWatch, writeFileSync } from "node:fs";
import child_process from "child_process";
import path from "path";
import JSZip from "jszip";
import stream from "node:stream";
import { pipeline } from "node:stream/promises";
import type { ReadableStream } from "node:stream/web";
import events from "events";
import util from "util";
import crypto from "crypto";
@@ -112,29 +114,38 @@ async function downloadAndExtractZip(url: string, targetDir: string) {
throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
}
// Load the ZIP with JSZip
const buffer = Buffer.from(await res.arrayBuffer());
const zip = await JSZip.loadAsync(buffer);
if (!res.body) {
throw new Error(`Failed to fetch ${url}: empty response`);
}
// Extract each file/folder
await fs.mkdir(targetDir, { recursive: true });
await Promise.all(
Object.values(zip.files).map(async (entry) => {
const parts = entry.name.split("/").filter(Boolean);
const strippedPath = parts.slice(1).join("/");
if (!strippedPath) {
return; // This copies the behaviour of tar --strip-components 1
}
// Write the download to a temp file, zip files need random access to read
const tmpFilePath = `${targetDir}.tmp`;
await fs.writeFile(tmpFilePath, stream.Readable.fromWeb(res.body as ReadableStream));
try {
const zip = await lib.ZipArchive.fromFile(tmpFilePath);
try {
await fs.mkdir(targetDir, { recursive: true });
for (const entry of zip.entries.values()) {
const parts = entry.fileName.split("/").filter(Boolean);
const strippedPath = parts.slice(1).join("/");
if (!strippedPath) {
continue; // This copies the behaviour of tar --strip-components 1
}
const entryPath = path.join(targetDir, strippedPath);
if (entry.dir) {
await fs.mkdir(entryPath, { recursive: true });
} else {
await fs.mkdir(path.dirname(entryPath), { recursive: true });
await fs.writeFile(entryPath, await entry.async("nodebuffer"));
const entryPath = path.join(targetDir, strippedPath);
if (entry.fileName.endsWith("/")) {
await fs.mkdir(entryPath, { recursive: true });
} else {
await fs.mkdir(path.dirname(entryPath), { recursive: true });
await pipeline(await zip.openStream(entry), createWriteStream(entryPath));
}
}
})
);
} finally {
zip.close();
}
} finally {
await fs.rm(tmpFilePath);
}
}
/**