mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-18 15:33:12 +09:00
Replace jszip with yauzl and yazl (#1011)
* Replace jszip with yauzl and yazl jszip loads a whole archive into memory to read it and buffers the output when writing. Saves and mods can be several hundred megabytes, so read zips with yauzl, which only reads the central directory and streams entries on demand, and write them with yazl, which streams the output. lib gains a ZipArchive wrapper around yauzl with the lookup helpers the callers need, findRoot() now takes it instead of a JSZip. Saves are patched by streaming every kept entry into a new zip, mod info.json is read straight from the file, exports are streamed to the controller and Factorio zip downloads go through a temp file instead of a buffer. Patching now inflates and deflates each entry again where jszip reused the compressed data, on a 336 MB synthetic save that is 7 s instead of 0.7 s, with peak memory going from 416 MB to 151 MB. Closes #1002 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Copy save entries without recompressing them yazl inflates and deflates every entry it is given, so patching a save took ten times longer than with jszip, which reused the compressed data. Add a ZipWriter to lib that streams entries out as they are added and copies entries from a ZipArchive byte for byte, with ZIP64 structures when sizes or offsets need them, and use it in patch(). On a 336 MB synthetic save patching now takes 0.4 s at 143 MB peak RSS against 0.75 s at 417 MB with jszip. Factorio loads the result. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Handle save write errors before the patch loop awaits them A write error on the temporary save, such as a full disk, rejected the pipeline promise while addEntry() or addBuffer() was still pending. Nothing was listening yet, so the host's unhandledRejection handler exited the process instead of failing the instance start. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Keep the export zip cache local to each export The cache was module level, so when two instances on one host exported at once, the first to finish closed archives the other was still reading. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Start writing zips before adding files from disk yazl reads files passed to addFile() as soon as they are added. An unreadable file emitted an error while the directory traversal was still running, before the output pipeline existed, and the process exited on the uncaught error. build_mod and zipDirectory now start the pipeline first and reject instead. build_mod also removes the partial zip on failure. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
e810020db5
commit
c28e617b22
23 files changed
+976
-298
No files matched your search
+105
-77
@@ -1,10 +1,9 @@
|
||||
// Library for patching Factorio saves with scenario code.
|
||||
|
||||
import events from "events";
|
||||
import fs from "node:fs/promises";
|
||||
import JSZip from "jszip";
|
||||
import path from "path";
|
||||
import semver from "semver";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { Type, Static } from "@sinclair/typebox";
|
||||
|
||||
import * as lib from "@clusterio/lib";
|
||||
@@ -72,18 +71,16 @@ export class SaveModule {
|
||||
};
|
||||
}
|
||||
|
||||
static async fromSave(json: Static<typeof SaveModule.jsonSchema>, root: JSZip) {
|
||||
static async fromSave(json: Static<typeof SaveModule.jsonSchema>, zip: lib.ZipArchive, root: string) {
|
||||
const module = new this(lib.ModuleInfo.fromJSON(json));
|
||||
module.files = new Map(await Promise.all(json.files
|
||||
.map(filename => ({filename, file: root.file(filename)}))
|
||||
.filter(({filename, file}) => {
|
||||
if (file === null) {
|
||||
lib.logger.warn(`Missing file ${filename} in save`);
|
||||
}
|
||||
return file !== null;
|
||||
})
|
||||
.map(async ({filename, file}) => [filename, await file!.async("nodebuffer")] as const)
|
||||
));
|
||||
for (const filename of json.files) {
|
||||
const content = await zip.readFile(path.posix.join(root, filename));
|
||||
if (content === null) {
|
||||
lib.logger.warn(`Missing file ${filename} in save`);
|
||||
continue;
|
||||
}
|
||||
module.files.set(filename, content);
|
||||
}
|
||||
return module;
|
||||
}
|
||||
|
||||
@@ -205,7 +202,7 @@ export class PatchInfo {
|
||||
};
|
||||
}
|
||||
|
||||
static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, root: JSZip) {
|
||||
static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, zip: lib.ZipArchive, root: string) {
|
||||
if (json.version === undefined) {
|
||||
interface ScenarioInfoV0 {
|
||||
name: string;
|
||||
@@ -229,7 +226,7 @@ export class PatchInfo {
|
||||
return new this(
|
||||
json.patch_number,
|
||||
lib.ModuleInfo.fromJSON(json.scenario),
|
||||
await Promise.all(json.modules.map(m => SaveModule.fromSave(m, root))),
|
||||
await Promise.all(json.modules.map(m => SaveModule.fromSave(m, zip, root))),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -396,80 +393,111 @@ const knownScenarios: Record<string, lib.ModuleInfo> = {
|
||||
* @param modules - Description of the modules to patch.
|
||||
*/
|
||||
export async function patch(savePath: string, modules: SaveModule[]) {
|
||||
let zip = await JSZip.loadAsync(await fs.readFile(savePath));
|
||||
let root = zip.folder(lib.findRoot(zip))!;
|
||||
const zip = await lib.ZipArchive.fromFile(savePath);
|
||||
let tempSavePath: string;
|
||||
try {
|
||||
const root = lib.findRoot(zip);
|
||||
const rootPath = (relativePath: string) => path.posix.join(root, relativePath);
|
||||
|
||||
let patchInfoFile = root.file("clusterio.json");
|
||||
let patchInfo: PatchInfo;
|
||||
if (patchInfoFile !== null) {
|
||||
let content = await patchInfoFile.async("string");
|
||||
patchInfo = await PatchInfo.fromSave(JSON.parse(content), root);
|
||||
if (patchInfo.version > PatchInfo.currentVersion) {
|
||||
throw new Error(
|
||||
`Save patch version ${patchInfo.version} is newer than the patch version this ` +
|
||||
`version of Clusterio can load (${PatchInfo.currentVersion})`
|
||||
);
|
||||
}
|
||||
// Files to add or replace in the save, relative to the root
|
||||
const files = new Map<string, Buffer>();
|
||||
let patchInfo: PatchInfo;
|
||||
const patchInfoContent = await zip.readFile(rootPath("clusterio.json"));
|
||||
if (patchInfoContent !== null) {
|
||||
patchInfo = await PatchInfo.fromSave(JSON.parse(patchInfoContent.toString("utf8")), zip, root);
|
||||
if (patchInfo.version > PatchInfo.currentVersion) {
|
||||
throw new Error(
|
||||
`Save patch version ${patchInfo.version} is newer than the patch version this ` +
|
||||
`version of Clusterio can load (${PatchInfo.currentVersion})`
|
||||
);
|
||||
}
|
||||
|
||||
// No info file present, try to detect if it's a known compatible scenario.
|
||||
} else {
|
||||
let controlFile = root.file("control.lua");
|
||||
if (!controlFile) {
|
||||
throw new Error("Unable to patch save, missing control.lua file.");
|
||||
}
|
||||
let controlStream = controlFile.nodeStream("nodebuffer");
|
||||
let controlHash = await lib.hashStream(controlStream);
|
||||
|
||||
if (controlHash in knownScenarios) {
|
||||
patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
|
||||
root.file("scenario.lua", controlFile.nodeStream("nodebuffer"));
|
||||
// No info file present, try to detect if it's a known compatible scenario.
|
||||
} else {
|
||||
throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
|
||||
}
|
||||
}
|
||||
const controlFile = zip.file(rootPath("control.lua"));
|
||||
if (!controlFile) {
|
||||
throw new Error("Unable to patch save, missing control.lua file.");
|
||||
}
|
||||
const controlHash = await lib.hashStream(await zip.openStream(controlFile));
|
||||
|
||||
// Increment patch number
|
||||
patchInfo.patchNumber = patchInfo.patchNumber + 1;
|
||||
|
||||
// Remove any existing modules from the save
|
||||
if (patchInfo.version === 0) {
|
||||
for (let file of root.file(/^modules\//)) {
|
||||
zip.remove(file.name);
|
||||
}
|
||||
} else {
|
||||
for (let module of patchInfo.modules) {
|
||||
for (let filepath of module.files.keys()) {
|
||||
const file = root.file(filepath);
|
||||
if (file !== null) {
|
||||
zip.remove(file.name);
|
||||
}
|
||||
if (controlHash in knownScenarios) {
|
||||
patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
|
||||
files.set("scenario.lua", await zip.readEntry(controlFile));
|
||||
} else {
|
||||
throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
|
||||
}
|
||||
}
|
||||
patchInfo.modules = [];
|
||||
}
|
||||
|
||||
reorderDependencies(modules);
|
||||
// Increment patch number
|
||||
patchInfo.patchNumber = patchInfo.patchNumber + 1;
|
||||
|
||||
// Add the modules to the save.
|
||||
for (let module of modules) {
|
||||
for (let [relativePath, contents] of module.files) {
|
||||
root.file(relativePath, contents);
|
||||
// Remove any existing modules from the save
|
||||
const removed = new Set<string>();
|
||||
if (patchInfo.version === 0) {
|
||||
const modulesPath = rootPath("modules/");
|
||||
for (const name of zip.entries.keys()) {
|
||||
if (name.startsWith(modulesPath)) {
|
||||
removed.add(name);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const module of patchInfo.modules) {
|
||||
for (const filepath of module.files.keys()) {
|
||||
removed.add(rootPath(filepath));
|
||||
}
|
||||
}
|
||||
patchInfo.modules = [];
|
||||
}
|
||||
patchInfo.modules.push(module);
|
||||
}
|
||||
|
||||
// Add loading code and patch info
|
||||
root.file("control.lua", generateLoader(patchInfo));
|
||||
root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
|
||||
reorderDependencies(modules);
|
||||
|
||||
// Write back the save
|
||||
let stream = zip.generateNodeStream({ compression: "DEFLATE" });
|
||||
const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
|
||||
try {
|
||||
let pipe = stream.pipe(writeStream);
|
||||
await events.once(pipe, "finish");
|
||||
// Add the modules to the save.
|
||||
for (const module of modules) {
|
||||
for (const [relativePath, contents] of module.files) {
|
||||
files.set(relativePath, contents);
|
||||
}
|
||||
patchInfo.modules.push(module);
|
||||
}
|
||||
|
||||
// Add loading code and patch info
|
||||
files.set("control.lua", Buffer.from(generateLoader(patchInfo), "utf8"));
|
||||
files.set("clusterio.json", Buffer.from(JSON.stringify(patchInfo, null, "\t"), "utf8"));
|
||||
|
||||
// Write back the save, keeping the order of the entries in the original
|
||||
const added = new Map([...files].map(([relativePath, contents]) => [rootPath(relativePath), contents]));
|
||||
const output = new lib.ZipWriter();
|
||||
let writeStream;
|
||||
[tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
|
||||
const writing = pipeline(output.outputStream, writeStream);
|
||||
// Awaited below, but a write error can reject it before then.
|
||||
writing.catch(() => {});
|
||||
try {
|
||||
for (const entry of zip.entries.values()) {
|
||||
const name = entry.fileName;
|
||||
if (removed.has(name)) {
|
||||
continue;
|
||||
}
|
||||
const contents = added.get(name);
|
||||
if (contents !== undefined) {
|
||||
await output.addBuffer(contents, name);
|
||||
added.delete(name);
|
||||
} else {
|
||||
await output.addEntry(zip, entry);
|
||||
}
|
||||
}
|
||||
for (const [name, contents] of added) {
|
||||
await output.addBuffer(contents, name);
|
||||
}
|
||||
await output.end();
|
||||
await writing;
|
||||
} catch (err) {
|
||||
output.outputStream.destroy();
|
||||
await writing.catch(() => {});
|
||||
await fs.rm(tempSavePath, { force: true });
|
||||
throw err;
|
||||
}
|
||||
} finally {
|
||||
writeStream.destroy();
|
||||
zip.close();
|
||||
}
|
||||
await fs.rename(tempSavePath, savePath);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user