Replace jszip with yauzl and yazl (#1011)

* Replace jszip with yauzl and yazl

jszip loads a whole archive into memory to read it and buffers the
output when writing.  Saves and mods can be several hundred megabytes,
so read zips with yauzl, which only reads the central directory and
streams entries on demand, and write them with yazl, which streams the
output.

lib gains a ZipArchive wrapper around yauzl with the lookup helpers the
callers need, findRoot() now takes it instead of a JSZip.  Saves are
patched by streaming every kept entry into a new zip, mod info.json is
read straight from the file, exports are streamed to the controller and
Factorio zip downloads go through a temp file instead of a buffer.

Patching now inflates and deflates each entry again where jszip reused
the compressed data, on a 336 MB synthetic save that is 7 s instead of
0.7 s, with peak memory going from 416 MB to 151 MB.

Closes #1002

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Copy save entries without recompressing them

yazl inflates and deflates every entry it is given, so patching a save
took ten times longer than with jszip, which reused the compressed
data.  Add a ZipWriter to lib that streams entries out as they are
added and copies entries from a ZipArchive byte for byte, with ZIP64
structures when sizes or offsets need them, and use it in patch().

On a 336 MB synthetic save patching now takes 0.4 s at 143 MB peak RSS
against 0.75 s at 417 MB with jszip.  Factorio loads the result.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Handle save write errors before the patch loop awaits them

A write error on the temporary save, such as a full disk, rejected the
pipeline promise while addEntry() or addBuffer() was still pending. Nothing
was listening yet, so the host's unhandledRejection handler exited the
process instead of failing the instance start.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Keep the export zip cache local to each export

The cache was module level, so when two instances on one host exported at
once, the first to finish closed archives the other was still reading.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Start writing zips before adding files from disk

yazl reads files passed to addFile() as soon as they are added. An
unreadable file emitted an error while the directory traversal was still
running, before the output pipeline existed, and the process exited on the
uncaught error. build_mod and zipDirectory now start the pipeline first and
reject instead. build_mod also removes the partial zip on failure.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
BastiaanandClaude Opus 5.5 authored and GitHub committed 2026-10-17 21:44:19 +03:00
1 parent e810020db5
commit c28e617b22
23 files changed
+976 -298

No files matched your search

+105 -77
View File
@@ -1,10 +1,9 @@
// Library for patching Factorio saves with scenario code.
import events from "events";
import fs from "node:fs/promises";
import JSZip from "jszip";
import path from "path";
import semver from "semver";
import { pipeline } from "node:stream/promises";
import { Type, Static } from "@sinclair/typebox";
import * as lib from "@clusterio/lib";
@@ -72,18 +71,16 @@ export class SaveModule {
};
}
static async fromSave(json: Static<typeof SaveModule.jsonSchema>, root: JSZip) {
static async fromSave(json: Static<typeof SaveModule.jsonSchema>, zip: lib.ZipArchive, root: string) {
const module = new this(lib.ModuleInfo.fromJSON(json));
module.files = new Map(await Promise.all(json.files
.map(filename => ({filename, file: root.file(filename)}))
.filter(({filename, file}) => {
if (file === null) {
lib.logger.warn(`Missing file ${filename} in save`);
}
return file !== null;
})
.map(async ({filename, file}) => [filename, await file!.async("nodebuffer")] as const)
));
for (const filename of json.files) {
const content = await zip.readFile(path.posix.join(root, filename));
if (content === null) {
lib.logger.warn(`Missing file ${filename} in save`);
continue;
}
module.files.set(filename, content);
}
return module;
}
@@ -205,7 +202,7 @@ export class PatchInfo {
};
}
static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, root: JSZip) {
static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, zip: lib.ZipArchive, root: string) {
if (json.version === undefined) {
interface ScenarioInfoV0 {
name: string;
@@ -229,7 +226,7 @@ export class PatchInfo {
return new this(
json.patch_number,
lib.ModuleInfo.fromJSON(json.scenario),
await Promise.all(json.modules.map(m => SaveModule.fromSave(m, root))),
await Promise.all(json.modules.map(m => SaveModule.fromSave(m, zip, root))),
);
}
}
@@ -396,80 +393,111 @@ const knownScenarios: Record<string, lib.ModuleInfo> = {
* @param modules - Description of the modules to patch.
*/
export async function patch(savePath: string, modules: SaveModule[]) {
let zip = await JSZip.loadAsync(await fs.readFile(savePath));
let root = zip.folder(lib.findRoot(zip))!;
const zip = await lib.ZipArchive.fromFile(savePath);
let tempSavePath: string;
try {
const root = lib.findRoot(zip);
const rootPath = (relativePath: string) => path.posix.join(root, relativePath);
let patchInfoFile = root.file("clusterio.json");
let patchInfo: PatchInfo;
if (patchInfoFile !== null) {
let content = await patchInfoFile.async("string");
patchInfo = await PatchInfo.fromSave(JSON.parse(content), root);
if (patchInfo.version > PatchInfo.currentVersion) {
throw new Error(
`Save patch version ${patchInfo.version} is newer than the patch version this ` +
`version of Clusterio can load (${PatchInfo.currentVersion})`
);
}
// Files to add or replace in the save, relative to the root
const files = new Map<string, Buffer>();
let patchInfo: PatchInfo;
const patchInfoContent = await zip.readFile(rootPath("clusterio.json"));
if (patchInfoContent !== null) {
patchInfo = await PatchInfo.fromSave(JSON.parse(patchInfoContent.toString("utf8")), zip, root);
if (patchInfo.version > PatchInfo.currentVersion) {
throw new Error(
`Save patch version ${patchInfo.version} is newer than the patch version this ` +
`version of Clusterio can load (${PatchInfo.currentVersion})`
);
}
// No info file present, try to detect if it's a known compatible scenario.
} else {
let controlFile = root.file("control.lua");
if (!controlFile) {
throw new Error("Unable to patch save, missing control.lua file.");
}
let controlStream = controlFile.nodeStream("nodebuffer");
let controlHash = await lib.hashStream(controlStream);
if (controlHash in knownScenarios) {
patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
root.file("scenario.lua", controlFile.nodeStream("nodebuffer"));
// No info file present, try to detect if it's a known compatible scenario.
} else {
throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
}
}
const controlFile = zip.file(rootPath("control.lua"));
if (!controlFile) {
throw new Error("Unable to patch save, missing control.lua file.");
}
const controlHash = await lib.hashStream(await zip.openStream(controlFile));
// Increment patch number
patchInfo.patchNumber = patchInfo.patchNumber + 1;
// Remove any existing modules from the save
if (patchInfo.version === 0) {
for (let file of root.file(/^modules\//)) {
zip.remove(file.name);
}
} else {
for (let module of patchInfo.modules) {
for (let filepath of module.files.keys()) {
const file = root.file(filepath);
if (file !== null) {
zip.remove(file.name);
}
if (controlHash in knownScenarios) {
patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
files.set("scenario.lua", await zip.readEntry(controlFile));
} else {
throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
}
}
patchInfo.modules = [];
}
reorderDependencies(modules);
// Increment patch number
patchInfo.patchNumber = patchInfo.patchNumber + 1;
// Add the modules to the save.
for (let module of modules) {
for (let [relativePath, contents] of module.files) {
root.file(relativePath, contents);
// Remove any existing modules from the save
const removed = new Set<string>();
if (patchInfo.version === 0) {
const modulesPath = rootPath("modules/");
for (const name of zip.entries.keys()) {
if (name.startsWith(modulesPath)) {
removed.add(name);
}
}
} else {
for (const module of patchInfo.modules) {
for (const filepath of module.files.keys()) {
removed.add(rootPath(filepath));
}
}
patchInfo.modules = [];
}
patchInfo.modules.push(module);
}
// Add loading code and patch info
root.file("control.lua", generateLoader(patchInfo));
root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
reorderDependencies(modules);
// Write back the save
let stream = zip.generateNodeStream({ compression: "DEFLATE" });
const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
try {
let pipe = stream.pipe(writeStream);
await events.once(pipe, "finish");
// Add the modules to the save.
for (const module of modules) {
for (const [relativePath, contents] of module.files) {
files.set(relativePath, contents);
}
patchInfo.modules.push(module);
}
// Add loading code and patch info
files.set("control.lua", Buffer.from(generateLoader(patchInfo), "utf8"));
files.set("clusterio.json", Buffer.from(JSON.stringify(patchInfo, null, "\t"), "utf8"));
// Write back the save, keeping the order of the entries in the original
const added = new Map([...files].map(([relativePath, contents]) => [rootPath(relativePath), contents]));
const output = new lib.ZipWriter();
let writeStream;
[tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
const writing = pipeline(output.outputStream, writeStream);
// Awaited below, but a write error can reject it before then.
writing.catch(() => {});
try {
for (const entry of zip.entries.values()) {
const name = entry.fileName;
if (removed.has(name)) {
continue;
}
const contents = added.get(name);
if (contents !== undefined) {
await output.addBuffer(contents, name);
added.delete(name);
} else {
await output.addEntry(zip, entry);
}
}
for (const [name, contents] of added) {
await output.addBuffer(contents, name);
}
await output.end();
await writing;
} catch (err) {
output.outputStream.destroy();
await writing.catch(() => {});
await fs.rm(tempSavePath, { force: true });
throw err;
}
} finally {
writeStream.destroy();
zip.close();
}
await fs.rename(tempSavePath, savePath);
}