Files
factorio-clusterio/test/host/patch.js
T
BastiaanandClaude Opus 5.5 c28e617b22 Replace jszip with yauzl and yazl (#1011)
* Replace jszip with yauzl and yazl

jszip loads a whole archive into memory to read it and buffers the
output when writing.  Saves and mods can be several hundred megabytes,
so read zips with yauzl, which only reads the central directory and
streams entries on demand, and write them with yazl, which streams the
output.

lib gains a ZipArchive wrapper around yauzl with the lookup helpers the
callers need, findRoot() now takes it instead of a JSZip.  Saves are
patched by streaming every kept entry into a new zip, mod info.json is
read straight from the file, exports are streamed to the controller and
Factorio zip downloads go through a temp file instead of a buffer.

Patching now inflates and deflates each entry again where jszip reused
the compressed data, on a 336 MB synthetic save that is 7 s instead of
0.7 s, with peak memory going from 416 MB to 151 MB.

Closes #1002

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Copy save entries without recompressing them

yazl inflates and deflates every entry it is given, so patching a save
took ten times longer than with jszip, which reused the compressed
data.  Add a ZipWriter to lib that streams entries out as they are
added and copies entries from a ZipArchive byte for byte, with ZIP64
structures when sizes or offsets need them, and use it in patch().

On a 336 MB synthetic save patching now takes 0.4 s at 143 MB peak RSS
against 0.75 s at 417 MB with jszip.  Factorio loads the result.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Handle save write errors before the patch loop awaits them

A write error on the temporary save, such as a full disk, rejected the
pipeline promise while addEntry() or addBuffer() was still pending. Nothing
was listening yet, so the host's unhandledRejection handler exited the
process instead of failing the instance start.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Keep the export zip cache local to each export

The cache was module level, so when two instances on one host exported at
once, the first to finish closed archives the other was still reading.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Start writing zips before adding files from disk

yazl reads files passed to addFile() as soon as they are added. An
unreadable file emitted an error while the directory traversal was still
running, before the output pipeline existed, and the process exited on the
uncaught error. build_mod and zipDirectory now start the pipeline first and
reject instead. build_mod also removes the partial zip on failure.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-17 21:44:19 +03:00

275 lines
9.4 KiB
JavaScript

import assert from "node:assert/strict";
import fs from "node:fs/promises";
import path from "node:path";
import * as lib from "@clusterio/lib";
import * as patch from "@clusterio/host/dist/node/src/patch.js";
import { createZipBuffer } from "../common.js";
describe("host/patch", function() {
describe("class SaveModule", function() {
describe("static moduleFilePath()", function() {
it("Should remap locale files", function() {
const mappings = [
["locale/en/foo.cfg", "locale/en/test-foo.cfg"],
["locale/en/foo.txt", "locale/en/test-foo.txt"],
["locale/en/foo.tmp.txt", "locale/en/test-foo.tmp.txt"],
["locale/en/foo", "locale/en/test-foo"],
["locale/en.cfg", "locale/en/test.cfg"],
["locale/en.txt", "locale/en/test.txt"],
["locale/en.tmp.txt", "locale/en/test.tmp.txt"],
["locale/en", "locale/en/test"],
["locale/en/bar/foo.cfg", "locale/en/test-bar/foo.cfg"],
];
for (let [input, expected] of mappings) {
assert.equal(patch.SaveModule.moduleFilePath(input, "test"), expected);
}
});
});
describe("loadFiles()", function() {
const testModule = new patch.SaveModule(new lib.ModuleInfo("test", "1.0.0"));
const referenceFiles = new Map([
["modules/test/test.lua", "-- test\n"],
["modules/test/module_exports.lua", "-- module_exports\n"],
["locale/en/test.cfg", "module-test=A Test\n"],
["locale/en/test-locale.cfg", "module-test-locale=Test Locale\n"],
["modules/test.lua", "return require(\"modules/test/module_exports\")"],
]);
before(async function() {
await testModule.loadFiles("test/file/modules/test");
});
it("Should read the contents of all files in a module", function() {
assert.equal(testModule.files.size, referenceFiles.size);
for (let [relativePath, contents] of referenceFiles) {
if (!testModule.files.has(relativePath)) {
assert.fail(`Module is missing file: ${relativePath}`);
} else {
assert.equal(testModule.files.get(relativePath).toString("utf-8"), contents);
}
}
});
});
});
describe("generateLoader()", function() {
let reference = [
"-- Auto generated scenario module loader created by Clusterio",
"-- Modifications to this file will be lost when loaded in Clusterio",
"clusterio_patch_number = 1",
"",
'local event_handler = require("event_handler")',
"",
"-- Scenario modules",
'event_handler.add_lib(require("foo"))',
"",
"-- Clusterio modules",
'event_handler.add_lib(require("modules/spam/bar"))',
'require("modules/spam/spam")',
"",
].join("\n");
let patchInfo = new patch.PatchInfo(
1,
new lib.ModuleInfo("foo", "0.0.0", ["foo"]),
[
new patch.SaveModule(
new lib.ModuleInfo(
"spam",
"0.0.0",
["bar.lua"],
["spam.lua"],
),
new Map([
["bar.lua", Buffer.from("")],
["spam.lua", Buffer.from("")],
["excluded.lua", Buffer.from("")],
]),
),
],
);
it("should produce output matching the reference", function() {
assert.equal(patch._generateLoader(patchInfo), reference);
});
});
function module({ name, version, dependencies = {} }) {
const dependencyMap = new Map(Object.entries(dependencies));
return new patch.SaveModule(new lib.ModuleInfo(name, version, [], [], dependencyMap));
}
describe("reorderDependencies()", function() {
it("should reorder to satisfy simple dependency", function() {
let modules = [
module({ name: "a", version: "1.0.0", dependencies: {"b": "*"} }),
module({ name: "b", version: "1.0.0", dependencies: {} }),
];
patch._reorderDependencies(modules);
assert(modules[0].info.name === "b", "Dependency was not reorederd");
});
it("should throw on invalid version", function() {
let modules = [
module({ name: "a", version: "foo", dependencies: {} }),
];
assert.throws(
() => patch._reorderDependencies(modules),
new Error("Invalid version 'foo' for module a")
);
});
it("should throw on invalid version range", function() {
let modules = [
module({ name: "a", version: "1.0.0", dependencies: {"b": "invalid"} }),
];
assert.throws(
() => patch._reorderDependencies(modules),
new Error("Invalid version range 'invalid' for dependency b on module a")
);
});
it("should throw on missing dependency", function() {
let modules = [
module({ name: "a", version: "1.0.0", dependencies: {"b": "*"} }),
];
assert.throws(
() => patch._reorderDependencies(modules),
new Error("Missing dependency b for module a")
);
modules = [
module({ name: "a", version: "1.0.0", dependencies: {"b": "*"} }),
module({ name: "b", version: "1.0.0", dependencies: {"c": "*"} }),
module({ name: "c", version: "1.0.0", dependencies: {"d": "*"} }),
];
assert.throws(
() => patch._reorderDependencies(modules),
new Error("Missing dependency d for module c")
);
});
it("should throw on outdated dependency", function() {
let modules = [
module({ name: "a", version: "1.0.0", dependencies: {} }),
module({ name: "b", version: "1.0.0", dependencies: {"a": ">=2"} }),
];
assert.throws(
() => patch._reorderDependencies(modules),
new Error("Module b requires a >=2")
);
});
it("should throw on dependency loops", function() {
let modules = [
module({ name: "a", version: "1.0.0", dependencies: {"b": "*"} }),
module({ name: "b", version: "1.0.0", dependencies: {"a": "*"} }),
];
assert.throws(
() => patch._reorderDependencies(modules),
new Error("Module dependency loop detected: a -> b -> a")
);
modules = [
module({ name: "d", version: "1.0.0", dependencies: {"b": "*"} }),
module({ name: "a", version: "1.0.0", dependencies: {"b": "*"} }),
module({ name: "b", version: "1.0.0", dependencies: {"c": "*"} }),
module({ name: "c", version: "1.0.0", dependencies: {"a": "*"} }),
];
assert.throws(
() => patch._reorderDependencies(modules),
new Error("Module dependency loop detected: b -> c -> a -> b")
);
});
it("should reorder correctly for multiple dependencies", function() {
function validate(modules) {
let present = new Set();
for (let { info } of modules) {
for (let dependency of info.dependencies.keys()) {
if (!present.has(dependency)) {
assert.fail(`${info.name} depends on ${dependency}, but was ordered before it`);
}
}
present.add(info.name);
}
}
let names = ["a", "b", "c"];
let successCount = 0;
const bitCount = names.length * (names.length - 1);
for (let i=0; i < 2**bitCount; i++) {
let value = i;
let bits = [];
for (let j=0; j < bitCount; j++) {
bits.push(value & 1);
value >>= 1;
}
let modules = [];
for (let j=0; j < names.length; j++) {
let name = names[j];
let dependencies = {};
for (let k=0; k < names.length - 1; k++) {
if (bits[k + j * (names.length - 1)]) {
dependencies[names[k + (k >= j)]] = "*";
}
}
modules.push(module({ name, version: "1.0.0", dependencies }));
}
let success = false;
try {
patch._reorderDependencies(modules);
success = true;
} catch (err) { }
if (success) {
validate(modules);
successCount += 1;
}
}
// Deriving the formula for the number of graphs without
// dependencies loops is left as an excercise for the reader.
assert.equal(successCount, 25, "Incorrect number of successful orderings");
});
});
describe("loadModules()", function() {
it("should load the bundled clusterio module", async function() {
const modules = await patch.loadModules([]);
assert.deepEqual([...modules.keys()], ["clusterio"]);
assert(modules.get("clusterio").files.has("modules/clusterio/api.lua"));
});
it("should load modules from plugins", async function() {
const pluginInfos = await lib.loadPluginInfos(
new Map([["research_sync", path.resolve("plugins/research_sync")]])
);
const modules = await patch.loadModules(pluginInfos);
assert.deepEqual([...modules.keys()], ["research_sync", "clusterio"]);
assert(modules.get("research_sync").files.has("modules/research_sync/sync.lua"));
});
it("should skip plugins without a module", async function() {
const pluginInfos = await lib.loadPluginInfos(
new Map([["test_plugin", path.resolve("test/file/test_plugin")]])
);
const modules = await patch.loadModules(pluginInfos);
assert.deepEqual([...modules.keys()], ["clusterio"]);
});
it("should throw on a plugin module named like a bundled module", async function() {
const pluginDir = path.join("temp", "test", "clusterio_plugin");
await fs.mkdir(path.join(pluginDir, "module"), { recursive: true });
await fs.writeFile(path.join(pluginDir, "package.json"), "{}");
await fs.writeFile(path.join(pluginDir, "module", "module.json"), "{}");
await assert.rejects(
patch.loadModules([{
name: "clusterio", version: "1.0.0", packagePath: path.resolve(pluginDir, "package.json"),
}]),
new Error("Module with name clusterio already exists in a plugin")
);
});
});
describe("patch()", function() {
it("should throw on unknown scenario", async function() {
let zipPath = path.join("temp", "test", "patch.zip");
await fs.writeFile(zipPath, await createZipBuffer([["world/control.lua", "-- unknown\n"]]));
await assert.rejects(
patch.patch(zipPath, []),
new Error("Unable to patch save, unknown scenario (3acc3be3861144e55604f5ac2f2555071885ebc4)")
);
});
});
});