aperx/factorio-clusterio
1
0 Fork 0
mirror of https://github.com/clusterio/clusterio.git synced 2026-11-11 15:04:52 +09:00
Code Issues Packages Projects Releases Wiki Activity

Replace jszip with yauzl and yazl (#1011)

Browse Source 
* Replace jszip with yauzl and yazl

jszip loads a whole archive into memory to read it and buffers the
output when writing.  Saves and mods can be several hundred megabytes,
so read zips with yauzl, which only reads the central directory and
streams entries on demand, and write them with yazl, which streams the
output.

lib gains a ZipArchive wrapper around yauzl with the lookup helpers the
callers need, findRoot() now takes it instead of a JSZip.  Saves are
patched by streaming every kept entry into a new zip, mod info.json is
read straight from the file, exports are streamed to the controller and
Factorio zip downloads go through a temp file instead of a buffer.

Patching now inflates and deflates each entry again where jszip reused
the compressed data, on a 336 MB synthetic save that is 7 s instead of
0.7 s, with peak memory going from 416 MB to 151 MB.

Closes #1002

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Copy save entries without recompressing them

yazl inflates and deflates every entry it is given, so patching a save
took ten times longer than with jszip, which reused the compressed
data.  Add a ZipWriter to lib that streams entries out as they are
added and copies entries from a ZipArchive byte for byte, with ZIP64
structures when sizes or offsets need them, and use it in patch().

On a 336 MB synthetic save patching now takes 0.4 s at 143 MB peak RSS
against 0.75 s at 417 MB with jszip.  Factorio loads the result.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Handle save write errors before the patch loop awaits them

A write error on the temporary save, such as a full disk, rejected the
pipeline promise while addEntry() or addBuffer() was still pending. Nothing
was listening yet, so the host's unhandledRejection handler exited the
process instead of failing the instance start.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Keep the export zip cache local to each export

The cache was module level, so when two instances on one host exported at
once, the first to finish closed archives the other was still reading.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Start writing zips before adding files from disk

yazl reads files passed to addFile() as soon as they are added. An
unreadable file emitted an error while the directory traversal was still
running, before the output pipeline existed, and the process exited on the
uncaught error. build_mod and zipDirectory now start the pipeline first and
reject instead. build_mod also removes the partial zip on failure.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
BastiaanandClaude Opus 5.5 authored and GitHub committed 2026-10-17 21:44:19 +03:00
1 parent e810020db5
commit c28e617b22
23 files changed
+976 -298
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

package.json
+3 -2
View File
@@ -57,10 +57,11 @@
"form-data": "^4.0.6",
"globals": "^17.12.0",
"jsonwebtoken": "catalog:",
"jszip": "catalog:",
"mocha": "^11.8.0",
"typedoc": "^0.28.20",
"typescript": "catalog:",
"yargs": "catalog:"
"yargs": "catalog:",
"yauzl": "catalog:",
"yazl": "catalog:"
}
}
packages/controller/package.json
-1
View File
@@ -43,7 +43,6 @@
"express": "catalog:",
"finalhandler": "^2.1.1",
"jsonwebtoken": "catalog:",
"jszip": "catalog:",
"semver": "catalog:",
"winston": "catalog:",
"winston-daily-rotate-file": "catalog:",
packages/controller/src/routes.ts
+5 -7
View File
@@ -5,7 +5,6 @@ import busboy from "busboy";
import crypto from "crypto";
import events from "events";
import fs from "node:fs/promises";
import JSZip from "jszip";
import jwt from "jsonwebtoken";
import path from "path";
import nodeStream from "stream";
@@ -266,9 +265,7 @@ async function uploadExport(req: Request, res: Response) {
for await (let chunk of req) {
data.push(chunk);
}
let buffer: Buffer | null = Buffer.concat(data);
let zip = await JSZip.loadAsync(buffer);
buffer = null;
let zip = await lib.ZipArchive.fromBuffer(Buffer.concat(data));
// This is hardcoded to prevent path expansion attacks
let exportFiles = [
@@ -288,14 +285,15 @@ async function uploadExport(req: Request, res: Response) {
continue;
}
let content = await zip.readEntry(file);
if (filePath === "export/settings.json") {
settingPrototypes = JSON.parse(await file.async("text"));
settingPrototypes = JSON.parse(content.toString("utf8"));
}
let { name, ext } = path.posix.parse(filePath);
let hash = await lib.hashStream(file.nodeStream());
let hash = await lib.hashStream(await zip.openStream(file));
assets[name] = `${name}.${hash}${ext}`;
await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer"));
await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), content);
}
modPack.exportManifest = new lib.ExportManifest(assets, new Date().toISOString());
packages/host/package.json
+3 -2
View File
@@ -40,14 +40,14 @@
"@clusterio/lib": "workspace:^",
"@sinclair/typebox": "catalog:",
"jimp": "^1.6.1",
"jszip": "catalog:",
"pidusage": "^4.0.1",
"rcon-client": "^4.2.5",
"semver": "catalog:",
"set-blocking": "catalog:",
"winston": "catalog:",
"winston-daily-rotate-file": "catalog:",
"yargs": "catalog:"
"yargs": "catalog:",
"yazl": "catalog:"
},
"devDependencies": {
"@types/node": "catalog:",
@@ -55,6 +55,7 @@
"@types/semver": "catalog:",
"@types/set-blocking": "catalog:",
"@types/yargs": "catalog:",
"@types/yazl": "catalog:",
"typescript": "catalog:"
}
}
packages/host/src/Instance.ts
+2 -2
View File
@@ -1,6 +1,7 @@
import fs from "node:fs/promises";
import path from "path";
import pidusage from "pidusage";
import stream from "node:stream";
import util from "util";
import type { Static } from "@sinclair/typebox";
import { exec } from "child_process";
@@ -1401,13 +1402,12 @@ end`.replace(/\r?\n/g, " ");
await this.syncMods();
let zip = await exportData(this.server);
let content = await zip.generateAsync({ type: "nodebuffer" });
let url = new URL(this._host.config.get("host.controller_url"));
url.pathname += "api/upload-export";
url.searchParams.set("mod_pack_id", String(this.activeModPack.id));
let response = await fetch(url, {
method: "PUT",
body: content,
body: stream.Readable.toWeb(lib.zipOutputStream(zip)),
duplex: "half",
headers: {
"Content-Type": "application/zip",
packages/host/src/create_scenario.ts
+39 -21
View File
@@ -1,8 +1,9 @@
// Implementation of the create-scenario command
import fs from "node:fs/promises";
import os from "node:os";
import JSZip from "jszip";
import path from "path";
import { pipeline } from "node:stream/promises";
import yazl from "yazl";
import * as lib from "@clusterio/lib";
import { logger } from "@clusterio/lib";
@@ -61,33 +62,50 @@ export async function createScenario(
async function zipDirectory(dirPath: string, zipPath: string) {
const root = path.basename(path.resolve(dirPath));
const zip = new JSZip();
let dirs: [string, string][] = [[dirPath, root]];
while (dirs.length) {
let [dir, relativeDir] = dirs.pop()!;
for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
let fsPath = path.join(dir, entry.name);
let relativePath = path.posix.join(relativeDir, entry.name);
if (entry.isFile()) {
zip.file(relativePath, await fs.readFile(fsPath));
} else if (entry.isDirectory()) {
dirs.push([fsPath, relativePath]);
const zip = new yazl.ZipFile();
const output = lib.zipOutputStream(zip);
const fd = await fs.open(zipPath, "w");
// addFile() can fail before the traversal is done, so start writing first.
const writing = pipeline(output, fd.createWriteStream());
writing.catch(() => {});
try {
let dirs: [string, string][] = [[dirPath, root]];
while (dirs.length) {
let [dir, relativeDir] = dirs.pop()!;
for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
let fsPath = path.join(dir, entry.name);
let relativePath = path.posix.join(relativeDir, entry.name);
if (entry.isFile()) {
zip.addFile(fsPath, relativePath);
} else if (entry.isDirectory()) {
dirs.push([fsPath, relativePath]);
}
}
}
zip.end();
} catch (err) {
output.destroy();
await writing.catch(() => {});
throw err;
}
await fs.writeFile(zipPath, await zip.generateAsync({ type: "nodebuffer" }));
await writing;
}
async function extractZip(zipPath: string, dirPath: string) {
const zip = await JSZip.loadAsync(await fs.readFile(zipPath));
const root = lib.findRoot(zip);
for (let file of Object.values(zip.files)) {
if (file.dir || !file.name.startsWith(`${root}/`)) {
continue;
const zip = await lib.ZipArchive.fromFile(zipPath);
try {
const root = lib.findRoot(zip);
for (let entry of zip.entries.values()) {
if (entry.fileName.endsWith("/") || !entry.fileName.startsWith(`${root}/`)) {
continue;
}
const fsPath = path.join(dirPath, entry.fileName.slice(root.length + 1));
await fs.mkdir(path.dirname(fsPath), { recursive: true });
const fd = await fs.open(fsPath, "w");
await pipeline(await zip.openStream(entry), fd.createWriteStream());
}
const fsPath = path.join(dirPath, file.name.slice(root.length + 1));
await fs.mkdir(path.dirname(fsPath), { recursive: true });
await fs.writeFile(fsPath, await file.async("nodebuffer"));
} finally {
zip.close();
}
}
packages/host/src/export.ts
+80 -40
View File
@@ -2,7 +2,7 @@
import fs from "node:fs/promises";
import path from "path";
import { Jimp, JimpMime, type JimpInstance } from "jimp";
import JSZip from "jszip";
import yazl from "yazl";
import * as lib from "@clusterio/lib";
import * as libBuildMod from "@clusterio/lib/dist/node/build_mod.js";
@@ -64,21 +64,38 @@ async function generateExportMod(server: FactorioServer) {
});
}
let zipCache = new Map<string, JSZip>();
async function loadZip(server: FactorioServer, modVersions: Map<string, string>, mod: string) {
interface ModZip {
zip: lib.ZipArchive;
root: string;
}
type ZipCache = Map<string, ModZip>;
async function loadZip(server: FactorioServer, modVersions: Map<string, string>, zipCache: ZipCache, mod: string) {
let modVersion = modVersions.get(mod);
if (!modVersion) {
throw new Error(`Got path for unknown mod ${mod}`);
}
let zipPath = server.writePath("mods", `${mod}_${modVersion}.zip`);
let zip = zipCache.get(zipPath);
if (!zip) {
zip = await JSZip.loadAsync(await fs.readFile(zipPath));
zipCache.set(zipPath, zip);
let modZip = zipCache.get(zipPath);
if (!modZip) {
const zip = await lib.ZipArchive.fromFile(zipPath);
try {
modZip = { zip, root: lib.findRoot(zip) };
} catch (err) {
zip.close();
throw err;
}
zipCache.set(zipPath, modZip);
}
return zip.folder(lib.findRoot(zip))!;
return modZip;
}
function closeZipCache(zipCache: ZipCache) {
for (const { zip } of zipCache.values()) {
zip.close();
}
zipCache.clear();
}
/**
@@ -86,11 +103,17 @@ async function loadZip(server: FactorioServer, modVersions: Map<string, string>,
*
* @param server - The server to load the file from.
* @param modVersions - Mapping of mod to version used.
* @param zipCache - Mod zips opened by this export.
* @param modPath - Factorio style path to the file to load.
* @returns The content of the file or null if not found.
* @internal
*/
async function loadFile(server: FactorioServer, modVersions: Map<string, string>, modPath: string) {
async function loadFile(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
modPath: string,
) {
let match = /^__([^\/]+)__\/(.*)$/.exec(modPath);
if (!match) {
server._logger.warn(`Skipping icon with bad mod path: ${JSON.stringify(modPath)}`);
@@ -111,9 +134,9 @@ async function loadFile(server: FactorioServer, modVersions: Map<string, string>
}
}
let zip;
let modZip;
try {
zip = await loadZip(server, modVersions, mod);
modZip = await loadZip(server, modVersions, zipCache, mod);
} catch (err: any) {
if (err.code === "ENOENT") {
return null;
@@ -121,12 +144,7 @@ async function loadFile(server: FactorioServer, modVersions: Map<string, string>
throw err;
}
let file = zip.file(filePath);
if (!file) {
return null;
}
return await file.async("nodebuffer");
return await modZip.zip.readFile(path.posix.join(modZip.root, filePath));
}
type Image = JimpInstance;
@@ -134,6 +152,7 @@ type IconCache = Map<string, Image | null>;
async function loadIcon(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
iconPath: string,
iconSize: number,
iconMipmaps: number,
@@ -141,7 +160,7 @@ async function loadIcon(
) {
let icon = iconCache.get(iconPath);
if (icon === undefined) {
let fileContent = await loadFile(server, modVersions, iconPath);
let fileContent = await loadFile(server, modVersions, zipCache, iconPath);
if (fileContent) {
icon = await Jimp.fromBuffer(fileContent) as Image;
icon.crop({ x: 0, y: 0, w: iconSize, h: iconSize });
@@ -157,11 +176,12 @@ async function loadIcon(
async function loadSimpleIcon(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
item: SimpleIconSpecification,
size: number,
iconCache: IconCache,
): Promise<Image | null> {
let icon = await loadIcon(server, modVersions, item.icon, item.icon_size ?? 64, 0, iconCache);
let icon = await loadIcon(server, modVersions, zipCache, item.icon, item.icon_size ?? 64, 0, iconCache);
if (icon) {
let iconScale = size / (item.icon_size ?? 64);
if (iconScale !== 1) {
@@ -175,6 +195,7 @@ async function loadSimpleIcon(
async function loadLayeredIcon(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
item: LayeredIconSpecification,
size: number,
iconCache: IconCache,
@@ -190,7 +211,9 @@ async function loadLayeredIcon(
for (let layer of item.icons) {
let layerSize = (layer.icon_size || item.icon_size) ?? 64;
let iconLayer = await loadIcon(server, modVersions, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache);
let iconLayer = await loadIcon(
server, modVersions, zipCache, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache,
);
if (!iconLayer) {
continue;
@@ -308,6 +331,7 @@ async function growSheet(state: SheetState) {
async function exportIcon(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
state: SheetState,
prototype: PrototypeWithIcon,
) {
@@ -318,7 +342,7 @@ async function exportIcon(
if (prototype.icons) {
icon = await loadLayeredIcon(
server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache,
server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache,
);
iconPos = state.pos;
iconPath = prototype.icons[0].icon;
@@ -328,7 +352,7 @@ async function exportIcon(
iconPath = prototype.icon;
if (iconPos === undefined) {
icon = await loadSimpleIcon(
server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache,
server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache,
);
if (icon) {
iconPos = state.pos;
@@ -364,6 +388,7 @@ async function exportIcon(
*
* @param server - The server to load icons from.
* @param modVersions - Mapping of mod name to version.
* @param zipCache - Mod zips opened by this export.
* @param prototypes - Array of prototypes with icon to export for this base type.
* @param baseName - name of base type (used in metadata).
* @param state - Shared spritesheet state (mutated in place).
@@ -373,6 +398,7 @@ async function exportIcon(
async function exportIcons(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
prototypes: PrototypeWithIcon[],
baseName: string,
state: SheetState,
@@ -381,7 +407,7 @@ async function exportIcons(
state.metadata[baseName] ??= metadata;
let count = 0;
for (let prototype of prototypes) {
const icon = await exportIcon(server, modVersions, state, prototype);
const icon = await exportIcon(server, modVersions, zipCache, state, prototype);
if (icon) {
count += 1;
}
@@ -404,6 +430,7 @@ async function exportIcons(
*
* @param server - The server to export the locale from.
* @param modVersions - Mapping of mod name to version to export locale from.
* @param zipCache - Mod zips opened by this export.
* @param modOrder - Load order of the mods.
* @param languageCode - Language to export locale for.
* @returns merged locale information
@@ -412,6 +439,7 @@ async function exportIcons(
async function exportLocale(
server: FactorioServer,
modVersions: Map<string, string>,
zipCache: ZipCache,
modOrder: string[],
languageCode: string
) {
@@ -445,18 +473,21 @@ async function exportLocale(
continue;
}
let zip;
let modZip;
try {
zip = await loadZip(server, modVersions, mod);
modZip = await loadZip(server, modVersions, zipCache, mod);
} catch (err: any) {
if (err.code === "ENOENT") {
continue;
}
throw err;
}
for (let file of zip.file(new RegExp(`locale\\/${languageCode}\\/.*\\.cfg`))) {
let content = await file.async("nodebuffer");
mergeLocale(lib.parse(content.toString("utf8")));
const localePath = `${path.posix.join(modZip.root, "locale", languageCode)}/`;
for (const entry of modZip.zip.entries.values()) {
if (entry.fileName.startsWith(localePath) && entry.fileName.endsWith(".cfg")) {
let content = await modZip.zip.readEntry(entry);
mergeLocale(lib.parse(content.toString("utf8")));
}
}
}
@@ -470,6 +501,16 @@ async function exportLocale(
* @returns zip file with exported data.
*/
export async function exportData(server: FactorioServer) {
const zipCache: ZipCache = new Map();
try {
return await buildExport(server, zipCache);
} finally {
// Free up the zip files loaded during the export.
closeZipCache(zipCache);
}
}
async function buildExport(server: FactorioServer, zipCache: ZipCache) {
await generateExportMod(server);
let settings: lib.ExportSettings = {};
@@ -549,16 +590,14 @@ export async function exportData(server: FactorioServer) {
}
}
let locale = await exportLocale(server, modVersions, modOrder, "en");
let locale = await exportLocale(server, modVersions, zipCache, modOrder, "en");
// Free up the memory used by zip files loaded during the export.
zipCache.clear();
let zip = new JSZip();
zip.file("export/settings.json", JSON.stringify(settings satisfies lib.ExportSettings));
zip.file("export/prototypes.json", JSON.stringify(prototypes satisfies lib.ExportPrototypes));
zip.file("export/locale.json", JSON.stringify([...locale.entries()] satisfies lib.ExportLocale));
zip.file("export/defines.json", JSON.stringify(defines satisfies lib.ExportDefines));
let zip = new yazl.ZipFile();
const addJson = (name: string, value: unknown) => zip.addBuffer(Buffer.from(JSON.stringify(value)), name);
addJson("export/settings.json", settings satisfies lib.ExportSettings);
addJson("export/prototypes.json", prototypes satisfies lib.ExportPrototypes);
addJson("export/locale.json", [...locale.entries()] satisfies lib.ExportLocale);
addJson("export/defines.json", defines satisfies lib.ExportDefines);
// Build a single unified spritesheet across all prototypes with icons.
const state = await createSheetState();
@@ -572,15 +611,16 @@ export async function exportData(server: FactorioServer) {
if (prototypesWithIcons.length === 0) {
continue;
}
const count = await exportIcons(server, modVersions, prototypesWithIcons, baseName, state);
const count = await exportIcons(server, modVersions, zipCache, prototypesWithIcons, baseName, state);
server._logger.info(`Exported ${count} ${baseName} icons`);
}
// Crop sheet to actual used height and write single spritesheet + metadata.
const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE));
state.sheet.crop({ x: 0, y: 0, w: SHEET_WIDTH, h: Math.max(usedRows * SHEET_ICON_SIZE, 1) });
zip.file("export/spritesheet.png", await state.sheet.getBuffer(JimpMime.png));
zip.file("export/metadata.json", JSON.stringify(state.metadata satisfies lib.ExportMetadata));
zip.addBuffer(await state.sheet.getBuffer(JimpMime.png), "export/spritesheet.png");
addJson("export/metadata.json", state.metadata satisfies lib.ExportMetadata);
zip.end();
server._logger.info(`Export complete: ${state.pos} icons on ${usedRows} row(s)`);
packages/host/src/patch.ts
+105 -77
View File
@@ -1,10 +1,9 @@
// Library for patching Factorio saves with scenario code.
import events from "events";
import fs from "node:fs/promises";
import JSZip from "jszip";
import path from "path";
import semver from "semver";
import { pipeline } from "node:stream/promises";
import { Type, Static } from "@sinclair/typebox";
import * as lib from "@clusterio/lib";
@@ -72,18 +71,16 @@ export class SaveModule {
};
}
static async fromSave(json: Static<typeof SaveModule.jsonSchema>, root: JSZip) {
static async fromSave(json: Static<typeof SaveModule.jsonSchema>, zip: lib.ZipArchive, root: string) {
const module = new this(lib.ModuleInfo.fromJSON(json));
module.files = new Map(await Promise.all(json.files
.map(filename => ({filename, file: root.file(filename)}))
.filter(({filename, file}) => {
if (file === null) {
lib.logger.warn(`Missing file ${filename} in save`);
}
return file !== null;
})
.map(async ({filename, file}) => [filename, await file!.async("nodebuffer")] as const)
));
for (const filename of json.files) {
const content = await zip.readFile(path.posix.join(root, filename));
if (content === null) {
lib.logger.warn(`Missing file ${filename} in save`);
continue;
}
module.files.set(filename, content);
}
return module;
}
@@ -205,7 +202,7 @@ export class PatchInfo {
};
}
static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, root: JSZip) {
static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, zip: lib.ZipArchive, root: string) {
if (json.version === undefined) {
interface ScenarioInfoV0 {
name: string;
@@ -229,7 +226,7 @@ export class PatchInfo {
return new this(
json.patch_number,
lib.ModuleInfo.fromJSON(json.scenario),
await Promise.all(json.modules.map(m => SaveModule.fromSave(m, root))),
await Promise.all(json.modules.map(m => SaveModule.fromSave(m, zip, root))),
);
}
}
@@ -396,80 +393,111 @@ const knownScenarios: Record<string, lib.ModuleInfo> = {
* @param modules - Description of the modules to patch.
*/
export async function patch(savePath: string, modules: SaveModule[]) {
let zip = await JSZip.loadAsync(await fs.readFile(savePath));
let root = zip.folder(lib.findRoot(zip))!;
const zip = await lib.ZipArchive.fromFile(savePath);
let tempSavePath: string;
try {
const root = lib.findRoot(zip);
const rootPath = (relativePath: string) => path.posix.join(root, relativePath);
let patchInfoFile = root.file("clusterio.json");
let patchInfo: PatchInfo;
if (patchInfoFile !== null) {
let content = await patchInfoFile.async("string");
patchInfo = await PatchInfo.fromSave(JSON.parse(content), root);
if (patchInfo.version > PatchInfo.currentVersion) {
throw new Error(
`Save patch version ${patchInfo.version} is newer than the patch version this ` +
`version of Clusterio can load (${PatchInfo.currentVersion})`
);
}
// Files to add or replace in the save, relative to the root
const files = new Map<string, Buffer>();
let patchInfo: PatchInfo;
const patchInfoContent = await zip.readFile(rootPath("clusterio.json"));
if (patchInfoContent !== null) {
patchInfo = await PatchInfo.fromSave(JSON.parse(patchInfoContent.toString("utf8")), zip, root);
if (patchInfo.version > PatchInfo.currentVersion) {
throw new Error(
`Save patch version ${patchInfo.version} is newer than the patch version this ` +
`version of Clusterio can load (${PatchInfo.currentVersion})`
);
}
// No info file present, try to detect if it's a known compatible scenario.
} else {
let controlFile = root.file("control.lua");
if (!controlFile) {
throw new Error("Unable to patch save, missing control.lua file.");
}
let controlStream = controlFile.nodeStream("nodebuffer");
let controlHash = await lib.hashStream(controlStream);
if (controlHash in knownScenarios) {
patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
root.file("scenario.lua", controlFile.nodeStream("nodebuffer"));
// No info file present, try to detect if it's a known compatible scenario.
} else {
throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
}
}
const controlFile = zip.file(rootPath("control.lua"));
if (!controlFile) {
throw new Error("Unable to patch save, missing control.lua file.");
}
const controlHash = await lib.hashStream(await zip.openStream(controlFile));
// Increment patch number
patchInfo.patchNumber = patchInfo.patchNumber + 1;
// Remove any existing modules from the save
if (patchInfo.version === 0) {
for (let file of root.file(/^modules\//)) {
zip.remove(file.name);
}
} else {
for (let module of patchInfo.modules) {
for (let filepath of module.files.keys()) {
const file = root.file(filepath);
if (file !== null) {
zip.remove(file.name);
}
if (controlHash in knownScenarios) {
patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
files.set("scenario.lua", await zip.readEntry(controlFile));
} else {
throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
}
}
patchInfo.modules = [];
}
reorderDependencies(modules);
// Increment patch number
patchInfo.patchNumber = patchInfo.patchNumber + 1;
// Add the modules to the save.
for (let module of modules) {
for (let [relativePath, contents] of module.files) {
root.file(relativePath, contents);
// Remove any existing modules from the save
const removed = new Set<string>();
if (patchInfo.version === 0) {
const modulesPath = rootPath("modules/");
for (const name of zip.entries.keys()) {
if (name.startsWith(modulesPath)) {
removed.add(name);
}
}
} else {
for (const module of patchInfo.modules) {
for (const filepath of module.files.keys()) {
removed.add(rootPath(filepath));
}
}
patchInfo.modules = [];
}
patchInfo.modules.push(module);
}
// Add loading code and patch info
root.file("control.lua", generateLoader(patchInfo));
root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
reorderDependencies(modules);
// Write back the save
let stream = zip.generateNodeStream({ compression: "DEFLATE" });
const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
try {
let pipe = stream.pipe(writeStream);
await events.once(pipe, "finish");
// Add the modules to the save.
for (const module of modules) {
for (const [relativePath, contents] of module.files) {
files.set(relativePath, contents);
}
patchInfo.modules.push(module);
}
// Add loading code and patch info
files.set("control.lua", Buffer.from(generateLoader(patchInfo), "utf8"));
files.set("clusterio.json", Buffer.from(JSON.stringify(patchInfo, null, "\t"), "utf8"));
// Write back the save, keeping the order of the entries in the original
const added = new Map([...files].map(([relativePath, contents]) => [rootPath(relativePath), contents]));
const output = new lib.ZipWriter();
let writeStream;
[tempSavePath, writeStream] = await lib.createTempWriteStream(savePath);
const writing = pipeline(output.outputStream, writeStream);
// Awaited below, but a write error can reject it before then.
writing.catch(() => {});
try {
for (const entry of zip.entries.values()) {
const name = entry.fileName;
if (removed.has(name)) {
continue;
}
const contents = added.get(name);
if (contents !== undefined) {
await output.addBuffer(contents, name);
added.delete(name);
} else {
await output.addEntry(zip, entry);
}
}
for (const [name, contents] of added) {
await output.addBuffer(contents, name);
}
await output.end();
await writing;
} catch (err) {
output.outputStream.destroy();
await writing.catch(() => {});
await fs.rm(tempSavePath, { force: true });
throw err;
}
} finally {
writeStream.destroy();
zip.close();
}
await fs.rename(tempSavePath, savePath);
}
packages/host/src/server.ts
+33 -22
View File
@@ -1,9 +1,11 @@
// Factorio Server interface
import fs from "node:fs/promises";
import { type FSWatcher, watch as fsWatch, writeFileSync } from "node:fs";
import { type FSWatcher, createWriteStream, watch as fsWatch, writeFileSync } from "node:fs";
import child_process from "child_process";
import path from "path";
import JSZip from "jszip";
import stream from "node:stream";
import { pipeline } from "node:stream/promises";
import type { ReadableStream } from "node:stream/web";
import events from "events";
import util from "util";
import crypto from "crypto";
@@ -112,29 +114,38 @@ async function downloadAndExtractZip(url: string, targetDir: string) {
throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
}
// Load the ZIP with JSZip
const buffer = Buffer.from(await res.arrayBuffer());
const zip = await JSZip.loadAsync(buffer);
if (!res.body) {
throw new Error(`Failed to fetch ${url}: empty response`);
}
// Extract each file/folder
await fs.mkdir(targetDir, { recursive: true });
await Promise.all(
Object.values(zip.files).map(async (entry) => {
const parts = entry.name.split("/").filter(Boolean);
const strippedPath = parts.slice(1).join("/");
if (!strippedPath) {
return; // This copies the behaviour of tar --strip-components 1
}
// Write the download to a temp file, zip files need random access to read
const tmpFilePath = `${targetDir}.tmp`;
await fs.writeFile(tmpFilePath, stream.Readable.fromWeb(res.body as ReadableStream));
try {
const zip = await lib.ZipArchive.fromFile(tmpFilePath);
try {
await fs.mkdir(targetDir, { recursive: true });
for (const entry of zip.entries.values()) {
const parts = entry.fileName.split("/").filter(Boolean);
const strippedPath = parts.slice(1).join("/");
if (!strippedPath) {
continue; // This copies the behaviour of tar --strip-components 1
}
const entryPath = path.join(targetDir, strippedPath);
if (entry.dir) {
await fs.mkdir(entryPath, { recursive: true });
} else {
await fs.mkdir(path.dirname(entryPath), { recursive: true });
await fs.writeFile(entryPath, await entry.async("nodebuffer"));
const entryPath = path.join(targetDir, strippedPath);
if (entry.fileName.endsWith("/")) {
await fs.mkdir(entryPath, { recursive: true });
} else {
await fs.mkdir(path.dirname(entryPath), { recursive: true });
await pipeline(await zip.openStream(entry), createWriteStream(entryPath));
}
}
})
);
} finally {
zip.close();
}
} finally {
await fs.rm(tmpFilePath);
}
}
/**
packages/lib/build_mod.js
+47 -39
View File
@@ -1,13 +1,40 @@
/* eslint-disable no-console */
import fs from "node:fs/promises";
import JSZip from "jszip";
import path from "node:path";
import stream from "node:stream";
import util from "node:util";
import { pipeline } from "node:stream/promises";
import yargs from "yargs";
import yazl from "yazl";
const finished = util.promisify(stream.finished);
async function addModFiles(args, info, modName, zip) {
for await (const dirent of await fs.opendir(args.sourceDir, { recursive: true })) {
if (dirent.isFile()) {
const itemPath = path.join(dirent.parentPath, dirent.name);
// On Windows the path created uses backslashes as the directory sepparator
// but the zip file needs to use forward slashes. We can't use the posix
// version of relative here as it doesn't work with Windows style paths.
let basePath = path.relative(args.sourceDir, itemPath).replace(/\\/g, "/");
if (basePath === "info.json") {
// The info.json file is overridden later.
continue;
}
zip.addFile(itemPath, path.posix.join(modName, basePath));
}
}
for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) {
let filePath = path.join(args.sourceDir, ...pathParts);
try {
await fs.access(filePath);
} catch (err) {
throw new Error(`Error reading additional file ${filePath}`, { cause: err });
}
zip.addFile(filePath, path.posix.join(modName, fileName));
}
delete info.additional_files;
zip.addBuffer(Buffer.from(JSON.stringify(info, null, "\t")), path.posix.join(modName, "info.json"));
}
async function buildMod(args, info, modName) {
if (args.build) {
@@ -15,43 +42,24 @@ async function buildMod(args, info, modName) {
modName = args.modName ?? `${info.name}_${info.version}`;
if (args.pack) {
let zip = new JSZip();
for await (const dirent of await fs.opendir(args.sourceDir, { recursive: true })) {
if (dirent.isFile()) {
const itemPath = path.join(dirent.parentPath, dirent.name);
// On Windows the path created uses backslashes as the directory sepparator
// but the zip file needs to use forward slashes. We can't use the posix
// version of relative here as it doesn't work with Windows style paths.
let basePath = path.relative(args.sourceDir, itemPath).replace(/\\/g, "/");
// eslint-disable-next-line max-depth
if (basePath === "info.json") {
// The info.json file is overridden later.
continue;
}
zip.file(path.posix.join(modName, basePath), (await fs.open(itemPath)).createReadStream());
}
}
for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) {
let filePath = path.join(args.sourceDir, ...pathParts);
let fileStream;
try {
fileStream = (await fs.open(filePath)).createReadStream();
} catch (err) {
throw new Error(`Error reading additional file ${filePath}`, { cause: err });
}
zip.file(path.posix.join(modName, fileName), fileStream);
}
delete info.additional_files;
zip.file(path.posix.join(modName, "info.json"), JSON.stringify(info, null, "\t"));
let modPath = path.join(args.outputDir, `${modName}.zip`);
console.log(`Writing ${modPath}`);
let writeStream = zip.generateNodeStream().pipe(
(await fs.open(modPath, "w")).createWriteStream()
);
await finished(writeStream);
let zip = new yazl.ZipFile();
// yazl reports errors on the zip object, not the output stream.
zip.on("error", err => zip.outputStream.destroy(err));
// addFile() can fail before the traversal is done, so start writing first.
const writing = pipeline(zip.outputStream, (await fs.open(modPath, "w")).createWriteStream());
writing.catch(() => {});
try {
await addModFiles(args, info, modName, zip);
zip.end();
await writing;
} catch (err) {
zip.outputStream.destroy();
await writing.catch(() => {});
await fs.rm(modPath, { force: true });
throw err;
}
} else {
let modDir = path.join(args.outputDir, modName);
packages/lib/package.json
+5 -2
View File
@@ -51,16 +51,19 @@
"dependencies": {
"@sinclair/typebox": "catalog:",
"@types/yauzl": "catalog:",
"@types/yazl": "catalog:",
"ajv": "^8.20.0",
"chalk": "catalog:",
"fast-deep-equal": "^3.1.3",
"jszip": "catalog:",
"set-blocking": "catalog:",
"triple-beam": "^1.4.1",
"winston": "catalog:",
"winston-transport": "catalog:",
"ws": "catalog:",
"yargs": "catalog:"
"yargs": "catalog:",
"yauzl": "catalog:",
"yazl": "catalog:"
},
"devDependencies": {
"@types/node": "catalog:",
packages/lib/src/data/ModInfo.ts
+9 -12
View File
@@ -1,10 +1,10 @@
import fs from "node:fs/promises";
import JSZip from "jszip";
import path from "path";
import { Type, Static } from "@sinclair/typebox";
import * as libHash from "../hash.js";
import * as libSchema from "../schema.js";
import { findRoot } from "../zip_ops.js";
import { ZipArchive, findRoot } from "../zip_ops.js";
import { ModRecord } from "./ModPack.js";
import {
@@ -352,18 +352,15 @@ export default class ModInfo {
static async fromModFile(modPath: string) {
let modInfo: Static<typeof ModInfo.jsonSchema>;
{
// XXX: JSZip needs the whole archive loaded in memory to work.
// This is clearly untenable and will be replaced later.
let zip = await JSZip.loadAsync(await fs.readFile(modPath));
let root = zip.folder(findRoot(zip))!;
let infoFile = root.file("info.json");
if (!infoFile) {
let zip = await ZipArchive.fromFile(modPath);
try {
let content = await zip.readFile(path.posix.join(findRoot(zip), "info.json"));
if (content === null) {
throw new Error("Mod contains no info.json file");
}
modInfo = JSON.parse(await infoFile.async("string"));
modInfo = JSON.parse(content.toString("utf8"));
} finally {
zip.close();
}
let valid = this.validateInfo(modInfo);
packages/lib/src/zip_ops.ts
+381 -3
View File
@@ -1,4 +1,122 @@
import type JSZip from "jszip";
/**
* Zip file helpers
* @module lib/zip_ops
*/
import events from "events";
import { PassThrough, type Readable } from "stream";
import util from "util";
import zlib from "zlib";
import yauzl from "yauzl";
import type yazl from "yazl";
const deflateRaw = util.promisify(zlib.deflateRaw);
/**
* Zip archive opened for random access reading
*
* Only the central directory is read up front, the content of entries is
* streamed from the archive on demand. Archives opened from a file hold
* the file open until close() is called.
*/
export class ZipArchive {
private constructor(
/** Underlying yauzl zip file. */
public zipfile: yauzl.ZipFile,
/** Entries by file name in the order they appear in the central directory. */
public entries: Map<string, yauzl.Entry>,
) { }
/**
* Open a zip archive from the file system
*
* @param zipPath - Path to the zip file to open.
*/
static async fromFile(zipPath: string) {
return this.load(await yauzl.openPromise(zipPath, { autoClose: false }));
}
/**
* Open a zip archive held in memory
*
* @param buffer - Content of the zip file.
*/
static async fromBuffer(buffer: Buffer) {
return this.load(await yauzl.fromBufferPromise(buffer));
}
private static async load(zipfile: yauzl.ZipFile) {
const entries = new Map<string, yauzl.Entry>();
try {
for await (const entry of zipfile.eachEntry()) {
entries.set(entry.fileName, entry);
}
} catch (err) {
zipfile.close();
throw err;
}
return new this(zipfile, entries);
}
/**
* Look up a file in the archive
*
* @param filePath - Path of the file in the archive.
* @returns entry for the file, or null if it does not exist.
*/
file(filePath: string) {
const entry = this.entries.get(filePath);
if (!entry || entry.fileName.endsWith("/")) {
return null;
}
return entry;
}
/**
* Open a stream of the decompressed content of an entry
*
* @param entry - Entry to read, must belong to this archive.
*/
openStream(entry: yauzl.Entry): Promise<Readable> {
return this.zipfile.openReadStreamPromise(entry);
}
/**
* Read the decompressed content of an entry
*
* @param entry - Entry to read, must belong to this archive.
*/
async readEntry(entry: yauzl.Entry) {
const chunks: Buffer[] = [];
for await (const chunk of await this.openStream(entry)) {
chunks.push(chunk);
}
return Buffer.concat(chunks);
}
/**
* Read the content of a file in the archive
*
* @param filePath - Path of the file in the archive.
* @returns content of the file, or null if it does not exist.
*/
async readFile(filePath: string) {
const entry = this.file(filePath);
if (!entry) {
return null;
}
return await this.readEntry(entry);
}
/**
* Close the archive
*
* Streams already opened are still readable until they end.
*/
close() {
this.zipfile.close();
}
}
/**
* Returns the root folder in the zip file
@@ -12,8 +130,8 @@ import type JSZip from "jszip";
* @param zip - Zip to search through.
* @returns name of the root folder.
*/
export function findRoot(zip: JSZip) {
const relativePath = Object.keys(zip.files)[0];
export function findRoot(zip: ZipArchive) {
const relativePath = zip.entries.keys().next().value;
if (relativePath === undefined) {
throw new Error("Empty zip file");
}
@@ -25,3 +143,263 @@ export function findRoot(zip: JSZip) {
return relativePath.slice(0, index);
}
/**
* Returns the output stream of a zip file being written
*
* yazl reports errors on the ZipFile object and leaves the output stream
* hanging. The stream returned here is destroyed with the error instead so
* that consumers like stream.pipeline see it.
*
* @param zipFile - zip file being written.
*/
export function zipOutputStream(zipFile: yazl.ZipFile) {
const output = zipFile.outputStream as Readable;
zipFile.on("error", err => output.destroy(err));
return output;
}
const zip64Limit = 0xffffffff;
function dosDateTime(date: Date) {
if (date.getFullYear() < 1980) {
return { date: 0x21, time: 0 };
}
const year = Math.min(date.getFullYear(), 2107);
return {
date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(),
time: (date.getHours() << 11) | (date.getMinutes() << 5) | (date.getSeconds() >> 1),
};
}
interface WriterEntry {
fileName: Buffer;
fileComment: Buffer;
versionMadeBy: number;
generalPurposeBitFlag: number;
compressionMethod: number;
lastModFileTime: number;
lastModFileDate: number;
crc32: number;
compressedSize: number;
uncompressedSize: number;
externalFileAttributes: number;
offset: number;
}
export interface ZipWriterOptions {
/** Use ZIP64 structures even when the archive does not need them. */
forceZip64?: boolean;
}
export interface ZipWriterBufferOptions {
/** Modification time to record for the file, defaults to now. */
mtime?: Date;
}
/**
* Streaming zip file writer
*
* Writes entries to outputStream as they are added. Entries from an
* existing archive are copied without decompressing them. Sizes and
* offsets past 4 GiB and more than 65535 entries use ZIP64 structures.
*/
export class ZipWriter {
/** Content of the zip file, ends when end() has been called. */
outputStream = new PassThrough();
private offset = 0;
private entries: WriterEntry[] = [];
private forceZip64: boolean;
constructor(options: ZipWriterOptions = {}) {
this.forceZip64 = options.forceZip64 ?? false;
}
/**
* Copy an entry from an archive
*
* The compressed content is copied as is along with the metadata
* Factorio writes. Extra fields are dropped.
*
* @param zip - Archive to copy from.
* @param entry - Entry to copy, must belong to zip.
*/
async addEntry(zip: ZipArchive, entry: yauzl.Entry) {
const writerEntry: WriterEntry = {
fileName: Buffer.from(entry.fileName, "utf8"),
fileComment: Buffer.from(entry.fileComment, "utf8"),
versionMadeBy: entry.versionMadeBy,
// Sizes are written in the local header so no data descriptor is
// needed, and names are re-encoded as UTF-8.
generalPurposeBitFlag: (entry.generalPurposeBitFlag & ~0x8) | 0x800,
compressionMethod: entry.compressionMethod,
lastModFileTime: entry.lastModFileTime,
lastModFileDate: entry.lastModFileDate,
crc32: entry.crc32,
compressedSize: entry.compressedSize,
uncompressedSize: entry.uncompressedSize,
externalFileAttributes: entry.externalFileAttributes,
offset: this.offset,
};
await this.write(this.localFileHeader(writerEntry));
const stream = await zip.zipfile.openReadStreamPromise(entry, { decodeFileData: false });
for await (const chunk of stream) {
await this.write(chunk);
}
this.entries.push(writerEntry);
}
/**
* Add a file with the given content
*
* @param buffer - Content of the file.
* @param fileName - Path of the file in the archive.
* @param options - Options for the entry.
*/
async addBuffer(buffer: Buffer, fileName: string, options: ZipWriterBufferOptions = {}) {
const deflated = await deflateRaw(buffer);
const { date, time } = dosDateTime(options.mtime ?? new Date());
const writerEntry: WriterEntry = {
fileName: Buffer.from(fileName, "utf8"),
fileComment: Buffer.alloc(0),
versionMadeBy: 20,
generalPurposeBitFlag: 0x800,
compressionMethod: 8,
lastModFileTime: time,
lastModFileDate: date,
crc32: zlib.crc32(buffer),
compressedSize: deflated.length,
uncompressedSize: buffer.length,
externalFileAttributes: 0,
offset: this.offset,
};
await this.write(this.localFileHeader(writerEntry));
await this.write(deflated);
this.entries.push(writerEntry);
}
/**
* Write the central directory and end the output stream
*/
async end() {
const start = this.offset;
for (const entry of this.entries) {
await this.write(this.centralDirectoryHeader(entry));
}
const size = this.offset - start;
const zip64 = this.forceZip64 || this.entries.length >= 0xffff || start >= zip64Limit || size >= zip64Limit;
if (zip64) {
const record = Buffer.alloc(56);
record.writeUInt32LE(0x06064b50, 0);
record.writeBigUInt64LE(44n, 4);
record.writeUInt16LE(45, 12); // version made by
record.writeUInt16LE(45, 14); // version needed to extract
record.writeUInt32LE(0, 16); // number of this disk
record.writeUInt32LE(0, 20); // disk with the central directory
record.writeBigUInt64LE(BigInt(this.entries.length), 24);
record.writeBigUInt64LE(BigInt(this.entries.length), 32);
record.writeBigUInt64LE(BigInt(size), 40);
record.writeBigUInt64LE(BigInt(start), 48);
const locator = Buffer.alloc(20);
locator.writeUInt32LE(0x07064b50, 0);
locator.writeUInt32LE(0, 4); // disk with the zip64 end of central directory
locator.writeBigUInt64LE(BigInt(this.offset), 8);
locator.writeUInt32LE(1, 16); // total number of disks
await this.write(record);
await this.write(locator);
}
const eocd = Buffer.alloc(22);
eocd.writeUInt32LE(0x06054b50, 0);
eocd.writeUInt16LE(0, 4); // number of this disk
eocd.writeUInt16LE(0, 6); // disk with the central directory
eocd.writeUInt16LE(zip64 ? 0xffff : this.entries.length, 8);
eocd.writeUInt16LE(zip64 ? 0xffff : this.entries.length, 10);
eocd.writeUInt32LE(zip64 ? zip64Limit : size, 12);
eocd.writeUInt32LE(zip64 ? zip64Limit : start, 16);
eocd.writeUInt16LE(0, 20); // comment length
await this.write(eocd);
this.outputStream.end();
}
private localFileHeader(entry: WriterEntry) {
const zip64 = this.forceZip64
|| entry.compressedSize >= zip64Limit
|| entry.uncompressedSize >= zip64Limit;
const extra = zip64 ? Buffer.alloc(20) : Buffer.alloc(0);
if (zip64) {
extra.writeUInt16LE(0x0001, 0);
extra.writeUInt16LE(16, 2);
extra.writeBigUInt64LE(BigInt(entry.uncompressedSize), 4);
extra.writeBigUInt64LE(BigInt(entry.compressedSize), 12);
}
const header = Buffer.alloc(30);
header.writeUInt32LE(0x04034b50, 0);
header.writeUInt16LE(zip64 ? 45 : 20, 4); // version needed to extract
header.writeUInt16LE(entry.generalPurposeBitFlag, 6);
header.writeUInt16LE(entry.compressionMethod, 8);
header.writeUInt16LE(entry.lastModFileTime, 10);
header.writeUInt16LE(entry.lastModFileDate, 12);
header.writeUInt32LE(entry.crc32, 14);
header.writeUInt32LE(zip64 ? zip64Limit : entry.compressedSize, 18);
header.writeUInt32LE(zip64 ? zip64Limit : entry.uncompressedSize, 22);
header.writeUInt16LE(entry.fileName.length, 26);
header.writeUInt16LE(extra.length, 28);
return Buffer.concat([header, entry.fileName, extra]);
}
private centralDirectoryHeader(entry: WriterEntry) {
const zip64 = this.forceZip64
|| entry.compressedSize >= zip64Limit
|| entry.uncompressedSize >= zip64Limit
|| entry.offset >= zip64Limit;
const extra = zip64 ? Buffer.alloc(28) : Buffer.alloc(0);
if (zip64) {
extra.writeUInt16LE(0x0001, 0);
extra.writeUInt16LE(24, 2);
extra.writeBigUInt64LE(BigInt(entry.uncompressedSize), 4);
extra.writeBigUInt64LE(BigInt(entry.compressedSize), 12);
extra.writeBigUInt64LE(BigInt(entry.offset), 20);
}
const header = Buffer.alloc(46);
header.writeUInt32LE(0x02014b50, 0);
header.writeUInt16LE(entry.versionMadeBy, 4);
header.writeUInt16LE(zip64 ? 45 : 20, 6); // version needed to extract
header.writeUInt16LE(entry.generalPurposeBitFlag, 8);
header.writeUInt16LE(entry.compressionMethod, 10);
header.writeUInt16LE(entry.lastModFileTime, 12);
header.writeUInt16LE(entry.lastModFileDate, 14);
header.writeUInt32LE(entry.crc32, 16);
header.writeUInt32LE(zip64 ? zip64Limit : entry.compressedSize, 20);
header.writeUInt32LE(zip64 ? zip64Limit : entry.uncompressedSize, 24);
header.writeUInt16LE(entry.fileName.length, 28);
header.writeUInt16LE(extra.length, 30);
header.writeUInt16LE(entry.fileComment.length, 32);
header.writeUInt16LE(0, 34); // disk number start
header.writeUInt16LE(0, 36); // internal file attributes
header.writeUInt32LE(entry.externalFileAttributes, 38);
header.writeUInt32LE(zip64 ? zip64Limit : entry.offset, 42);
return Buffer.concat([header, entry.fileName, extra, entry.fileComment]);
}
private async write(buffer: Buffer) {
if (this.outputStream.destroyed) {
throw this.outputStream.errored ?? new Error("Zip output stream was closed");
}
if (!this.outputStream.write(buffer)) {
// once() rejects if the stream errors, close covers a plain destroy.
const controller = new AbortController();
const { signal } = controller;
try {
await Promise.race([
events.once(this.outputStream, "drain", { signal }),
events.once(this.outputStream, "close", { signal }).then(() => {
throw this.outputStream.errored ?? new Error("Zip output stream was closed");
}),
]);
} finally {
controller.abort();
}
}
this.offset += buffer.length;
}
}
pnpm-workspace.yaml
+4 -1
View File
@@ -17,11 +17,12 @@ catalog:
'@types/set-blocking': ^2.0.2
'@types/ws': ^8.18.1
'@types/yargs': ^17.0.35
'@types/yauzl': ^3.4.0
'@types/yazl': ^3.3.1
antd: ^5.29.3
chalk: ^6.0.0
express: ^5.2.1
jsonwebtoken: ^9.0.3
jszip: ^3.10.2
react: ^18.2.0
react-dom: ^18.2.0
react-markdown: ^10.1.0
@@ -37,6 +38,8 @@ catalog:
winston-transport: ^4.9.0
ws: ^8.21.3
yargs: ^18.1.0
yauzl: ^3.4.0
yazl: ^3.3.1
catalogMode: strict
test/common.js
+36 -1
View File
@@ -1,5 +1,6 @@
import assert from "node:assert/strict";
import { compile } from "@clusterio/lib";
import yazl from "yazl";
import { compile, zipOutputStream } from "@clusterio/lib";
/**
* Generate a flat array of tests from a matrix of inputs.
@@ -55,3 +56,37 @@ export function externalTest(test) {
}
test.timeout(60000);
}
/**
* Create a zip file in memory
*
* @param {Iterable<[string, string | Buffer]>} files -
* Paths and content of the files to add, paths ending with / are directories.
* @returns {yazl.ZipFile} zip file with the output stream ready to be read.
*/
export function createZip(files) {
const zip = new yazl.ZipFile();
for (const [filePath, content] of files) {
if (filePath.endsWith("/")) {
zip.addEmptyDirectory(filePath);
} else {
zip.addBuffer(Buffer.from(content), filePath);
}
}
zip.end();
return zip;
}
/**
* Create a zip file in memory and return its content
*
* @param {Iterable<[string, string | Buffer]>} files - Paths and content of the files to add.
* @returns {Promise<Buffer>} content of the zip file.
*/
export async function createZipBuffer(files) {
const chunks = [];
for await (const chunk of zipOutputStream(createZip(files))) {
chunks.push(chunk);
}
return Buffer.concat(chunks);
}
test/host/create_scenario.js
+31 -8
View File
@@ -1,6 +1,5 @@
import assert from "node:assert/strict";
import fs from "node:fs/promises";
import JSZip from "jszip";
import path from "node:path";
import * as lib from "@clusterio/lib";
@@ -8,6 +7,7 @@ import {
createScenario, createScenarioCommand, handleCreateScenarioCommand,
} from "@clusterio/host/dist/node/src/create_scenario.js";
import { SaveModule, PatchInfo } from "@clusterio/host/dist/node/src/patch.js";
import { createZipBuffer } from "../common.js";
describe("host/create_scenario", function() {
@@ -48,14 +48,15 @@ describe("host/create_scenario", function() {
before(async function() {
await fs.rm(tempDir, { recursive: true, force: true });
await fs.mkdir(path.join(tempDir, "scenario"), { recursive: true });
const zip = new JSZip();
for (const [name, content] of scenarioFiles) {
const fsPath = path.join(tempDir, "scenario", name);
await fs.mkdir(path.dirname(fsPath), { recursive: true });
await fs.writeFile(fsPath, content);
zip.file(`scenario/${name}`, content);
}
await fs.writeFile(path.join(tempDir, "scenario.zip"), await zip.generateAsync({ type: "nodebuffer" }));
await fs.writeFile(
path.join(tempDir, "scenario.zip"),
await createZipBuffer([...scenarioFiles].map(([name, content]) => [`scenario/${name}`, content])),
);
});
it("should patch a scenario directory", async function() {
@@ -73,12 +74,16 @@ describe("host/create_scenario", function() {
it("should write a zip file when asked to", async function() {
const output = path.join(tempDir, "zipped.zip");
await createScenario(path.join(tempDir, "scenario"), output, [module], true);
const zip = await JSZip.loadAsync(await fs.readFile(output));
const zip = await lib.ZipArchive.fromFile(output);
const files = new Map();
for (const file of Object.values(zip.files)) {
if (!file.dir) {
files.set(file.name.replace(/^scenario\//, ""), await file.async("string"));
try {
for (const entry of zip.entries.values()) {
if (!entry.fileName.endsWith("/")) {
files.set(entry.fileName.replace(/^scenario\//, ""), (await zip.readEntry(entry)).toString());
}
}
} finally {
zip.close();
}
checkOutput(files);
});
@@ -105,6 +110,24 @@ describe("host/create_scenario", function() {
);
});
it("should throw if a scenario file is unreadable", async function() {
if (process.platform === "win32" || process.getuid?.() === 0) {
this.skip();
}
const scenario = path.join(tempDir, "unreadable");
await fs.cp(path.join(tempDir, "scenario"), scenario, { recursive: true });
await fs.writeFile(path.join(scenario, "unreadable.lua"), "");
await fs.chmod(path.join(scenario, "unreadable.lua"), 0o000);
for (let i = 0; i < 20; i++) {
await fs.mkdir(path.join(scenario, `dir${i}`));
await fs.writeFile(path.join(scenario, `dir${i}`, "file.lua"), "");
}
await assert.rejects(
createScenario(scenario, path.join(tempDir, "from_unreadable"), [module]),
{ code: "EACCES" },
);
});
describe("createScenarioCommand()", function() {
it("should define the output positional and options", function() {
const positionals = [];
test/host/export.js
+5 -4
View File
@@ -15,14 +15,14 @@ describe("host/src/export", function() {
describe("exportLocale()", function() {
it("returns a flattened mapping with locale definitions", async function() {
let locale = await _exportLocale(testServer, new Map(), ["base"], "en");
let locale = await _exportLocale(testServer, new Map(), new Map(), ["base"], "en");
assert.deepEqual(locale, new Map([
["test.core-a", "1"], ["test.core-b", "2"],
["test.base-a", "1"], ["test.base-b", "2"],
]));
});
it("returns a filtered locale definitions based on mod order", async function() {
let locale = await _exportLocale(testServer, new Map(), [], "en");
let locale = await _exportLocale(testServer, new Map(), new Map(), [], "en");
assert.deepEqual(locale, new Map([
["test.core-a", "1"], ["test.core-b", "2"],
]));
@@ -34,7 +34,7 @@ describe("host/src/export", function() {
describe("loadSimpleIcon()", function() {
it("defaults icon_size to 64", async function() {
const icon = await _loadSimpleIcon(
testServer, new Map(), { icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map()
testServer, new Map(), new Map(), { icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map()
);
assert.equal(icon.bitmap.width, 32);
assert.equal(icon.bitmap.height, 32);
@@ -44,7 +44,8 @@ describe("host/src/export", function() {
describe("loadLayeredIcon()", function() {
it("defaults icon_size to 64", async function() {
const icon = await _loadLayeredIcon(
testServer, new Map(), { icons: [{ icon: "__base__/graphics/icons/test-icon.png" }] }, 32, new Map()
testServer, new Map(), new Map(),
{ icons: [{ icon: "__base__/graphics/icons/test-icon.png" }] }, 32, new Map(),
);
assert.equal(icon.bitmap.width, 32);
assert.equal(icon.bitmap.height, 32);
test/host/patch.js
+2 -4
View File
@@ -1,10 +1,10 @@
import assert from "node:assert/strict";
import fs from "node:fs/promises";
import JSZip from "jszip";
import path from "node:path";
import * as lib from "@clusterio/lib";
import * as patch from "@clusterio/host/dist/node/src/patch.js";
import { createZipBuffer } from "../common.js";
describe("host/patch", function() {
@@ -262,10 +262,8 @@ describe("host/patch", function() {
describe("patch()", function() {
it("should throw on unknown scenario", async function() {
let zip = new JSZip();
zip.file("world/control.lua", "-- unknown\n");
let zipPath = path.join("temp", "test", "patch.zip");
await fs.writeFile(zipPath, await zip.generateAsync({ type: "nodebuffer" }));
await fs.writeFile(zipPath, await createZipBuffer([["world/control.lua", "-- unknown\n"]]));
await assert.rejects(
patch.patch(zipPath, []),
test/integration/export.js
+1 -1
View File
@@ -15,7 +15,7 @@ describe("Integration of lib/factorio/export", function() {
let testServer = new FactorioServer(factorioDir, writePath, {});
await testServer.init();
let locale = await _exportLocale(testServer, new Map(), ["base"], "en");
let locale = await _exportLocale(testServer, new Map(), new Map(), ["base"], "en");
assert(locale instanceof Map, "locale is not a map");
assert.equal(locale.get("entity-name.fish"), "Fish");
test/integration/patch.js
+21 -15
View File
@@ -1,6 +1,4 @@
import assert from "node:assert/strict";
import fs from "node:fs/promises";
import JSZip from "jszip";
import path from "node:path";
import { patch, SaveModule } from "@clusterio/host/dist/node/src/patch.js";
@@ -23,23 +21,31 @@ describe("Integration of lib/factorio/patch", function() {
await subdirModule.loadFiles("test/file/modules/subdir");
await patch(savePath, [testModule, subdirModule]);
let zip = await JSZip.loadAsync(await fs.readFile(savePath));
assert.equal(await zip.file("test/modules/test/test.lua").async("string"), "-- test\n");
assert.equal(await zip.file("test/modules/subdir/dir/test.lua").async("string"), "-- test\n");
assert.equal(
await zip.file("test/locale/en/test.cfg").async("string"),
"module-test=A Test\n"
);
assert.equal(
await zip.file("test/locale/en/test-locale.cfg").async("string"),
"module-test-locale=Test Locale\n"
);
let zip = await lib.ZipArchive.fromFile(savePath);
try {
assert.equal((await zip.readFile("test/modules/test/test.lua")).toString(), "-- test\n");
assert.equal((await zip.readFile("test/modules/subdir/dir/test.lua")).toString(), "-- test\n");
assert.equal(
(await zip.readFile("test/locale/en/test.cfg")).toString(),
"module-test=A Test\n"
);
assert.equal(
(await zip.readFile("test/locale/en/test-locale.cfg")).toString(),
"module-test-locale=Test Locale\n"
);
} finally {
zip.close();
}
});
it("should remove old modules in a save", async function() {
slowTest(this);
await patch(savePath, []);
let zip = await JSZip.loadAsync(await fs.readFile(savePath));
assert.equal(zip.file("test/modules/test/test.lua"), null);
let zip = await lib.ZipArchive.fromFile(savePath);
try {
assert.equal(zip.file("test/modules/test/test.lua"), null);
} finally {
zip.close();
}
});
});
});
test/lib/ModStore.test.js
+10 -13
View File
@@ -2,20 +2,18 @@ import assert from "node:assert/strict";
import path from "node:path";
import fs from "node:fs/promises";
import stream from "node:stream";
import JSZip from "jszip"; // Added for creating mock zips
// Capture native fetch before any potential mocks are applied
const nativeFetch = global.fetch;
import { ModStore, ModInfo, ModVersionEquality } from "@clusterio/lib"; // Adjust path based on compiled output
import { externalTest } from "../common.js";
import { ModStore, ModInfo, ModVersionEquality, zipOutputStream } from "@clusterio/lib";
import { externalTest, createZip, createZipBuffer } from "../common.js";
import "../setup_logging.js";
const MODS_DIR = path.join("temp", "test", "mod_store", "mods");
const CACHE_FILE = path.join(MODS_DIR, "mod-info-cache.json");
function createMockModJSZip(name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) {
const zip = new JSZip();
function mockModFiles(name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) {
const infoJson = {
name: name,
version: version,
@@ -25,16 +23,15 @@ function createMockModJSZip(name, version, factorioVersion = "1.1", dependencies
dependencies: dependencies,
description: `Description for ${name}`,
};
zip.file(`${name}/info.json`, JSON.stringify(infoJson, null, 4));
zip.file(`${name}/dummy.lua`, "-- Mock Lua file");
return zip;
return [
[`${name}/info.json`, JSON.stringify(infoJson, null, 4)],
[`${name}/dummy.lua`, "-- Mock Lua file"],
];
}
// Helper function to create a mock mod zip file
async function createMockModZip(filePath, name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) {
const zip = createMockModJSZip(name, version, factorioVersion, dependencies);
const buffer = await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" });
const buffer = await createZipBuffer(mockModFiles(name, version, factorioVersion, dependencies));
await fs.writeFile(filePath, buffer);
}
@@ -254,8 +251,8 @@ describe("lib/ModStore", function () {
if (name === "bad-download-mod") {
return { ok: false, status: 503, statusText: "Test Mock download error" };
}
const zip = createMockModJSZip(name, version);
return { ok: true, status: 200, body: stream.Readable.toWeb(zip.generateNodeStream()) };
const zip = createZip(mockModFiles(name, version));
return { ok: true, status: 200, body: stream.Readable.toWeb(zipOutputStream(zip)) };
}
// Fallback for unhandled fetches in this test
return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` };
test/lib/build_mod.js
+42
View File
@@ -0,0 +1,42 @@
import assert from "node:assert/strict";
import fs from "node:fs/promises";
import path from "node:path";
import * as libBuildMod from "@clusterio/lib/build_mod.js";
describe("lib/build_mod", function() {
describe("build()", function() {
const tempDir = path.join("temp", "test", "build_mod");
const sourceDir = path.join(tempDir, "unreadable_mod");
const outputDir = path.join(tempDir, "output");
before(async function() {
if (process.platform === "win32" || process.getuid?.() === 0) {
this.skip();
}
await fs.rm(tempDir, { recursive: true, force: true });
await fs.mkdir(sourceDir, { recursive: true });
await fs.writeFile(
path.join(sourceDir, "info.json"),
JSON.stringify({ name: "unreadable_mod", version: "1.0.0" }),
);
await fs.writeFile(path.join(sourceDir, "unreadable.lua"), "");
await fs.chmod(path.join(sourceDir, "unreadable.lua"), 0o000);
for (let i = 0; i < 20; i++) {
await fs.mkdir(path.join(sourceDir, `dir${i}`));
await fs.writeFile(path.join(sourceDir, `dir${i}`, "file.lua"), "");
}
});
after(async function() {
await fs.rm(tempDir, { recursive: true, force: true });
});
it("should reject when a source file is unreadable", async function() {
await assert.rejects(
libBuildMod.build({ build: true, pack: true, sourceDir, outputDir }),
{ code: "EACCES" },
);
assert.deepEqual(await fs.readdir(outputDir), []);
});
});
});
test/lib/zip_ops.js
+112 -21
View File
@@ -1,41 +1,132 @@
import assert from "node:assert/strict";
import JSZip from "jszip";
import zlib from "node:zlib";
import { findRoot } from "@clusterio/lib";
import { ZipArchive, ZipWriter, findRoot } from "@clusterio/lib";
import { createZipBuffer } from "../common.js";
describe("lib/factorio", function() {
async function readAll(stream) {
const chunks = [];
for await (const chunk of stream) {
chunks.push(chunk);
}
return Buffer.concat(chunks);
}
describe("lib/zip_ops", function() {
describe("class ZipArchive", function() {
it("should read files from a zip", async function() {
const zip = await ZipArchive.fromBuffer(await createZipBuffer([
["root/foo.txt", "foo"],
["root/bar.txt", "bar"],
]));
assert.deepEqual([...zip.entries.keys()], ["root/foo.txt", "root/bar.txt"]);
assert.equal((await zip.readFile("root/foo.txt")).toString(), "foo");
assert.equal(await zip.readFile("root/missing.txt"), null);
assert.equal(zip.file("root/missing.txt"), null);
assert.equal(zip.file("root/bar.txt").fileName, "root/bar.txt");
});
it("should reject invalid zips", async function() {
await assert.rejects(ZipArchive.fromBuffer(Buffer.from("not a zip file")));
});
});
describe("class ZipWriter", function() {
const sourceFiles = [
["root/", ""],
["root/foo.txt", "foo"],
["root/big.txt", "lorem ipsum ".repeat(10000)],
["root/ünïcode.txt", "unicode"],
];
async function roundTrip(forceZip64) {
const source = await ZipArchive.fromBuffer(await createZipBuffer(sourceFiles));
const writer = new ZipWriter({ forceZip64 });
const output = readAll(writer.outputStream);
for (const entry of source.entries.values()) {
await writer.addEntry(source, entry);
}
await writer.addBuffer(Buffer.from("added"), "root/added.txt", { mtime: new Date(2020, 0, 2, 3, 4, 6) });
await writer.end();
return { source, copy: await ZipArchive.fromBuffer(await output) };
}
async function assertRoundTrip(forceZip64) {
const { source, copy } = await roundTrip(forceZip64);
assert.deepEqual([...copy.entries.keys()], [...source.entries.keys(), "root/added.txt"]);
for (const [name, sourceEntry] of source.entries) {
const entry = copy.entries.get(name);
assert.equal(entry.compressionMethod, sourceEntry.compressionMethod, name);
assert.equal(entry.compressedSize, sourceEntry.compressedSize, name);
assert.equal(entry.uncompressedSize, sourceEntry.uncompressedSize, name);
assert.equal(entry.crc32, sourceEntry.crc32, name);
assert.deepEqual(entry.getLastModDate(), sourceEntry.getLastModDate({ forceDosFormat: true }), name);
if (!name.endsWith("/")) {
const content = await copy.readEntry(entry);
assert.deepEqual(content, await source.readEntry(sourceEntry), name);
assert.equal(zlib.crc32(content), entry.crc32, name);
}
}
const added = copy.entries.get("root/added.txt");
assert.equal((await copy.readEntry(added)).toString(), "added");
assert.equal(zlib.crc32("added"), added.crc32);
assert.deepEqual(added.getLastModDate(), new Date(2020, 0, 2, 3, 4, 6));
assert.equal(added.versionNeededToExtract, forceZip64 ? 45 : 20);
}
it("should copy entries and add files", async function() {
await assertRoundTrip(false);
});
it("should write ZIP64 structures when forced", async function() {
await assertRoundTrip(true);
});
it("should reject writes after the output stream is destroyed", async function() {
const writer = new ZipWriter();
writer.outputStream.on("error", () => {});
writer.outputStream.destroy(new Error("Test error"));
await assert.rejects(writer.addBuffer(Buffer.from(""), "foo.txt"), new Error("Test error"));
await assert.rejects(writer.end(), new Error("Test error"));
});
it("should fail when the output stream is destroyed while waiting for drain", async function() {
const writer = new ZipWriter();
writer.outputStream.pause();
const adding = writer.addBuffer(Buffer.alloc(1024 * 1024), "foo.txt");
writer.outputStream.destroy();
await assert.rejects(adding, new Error("Zip output stream was closed"));
});
});
describe("findRoot()", function() {
it("should find the root of a zip", function() {
let zip = new JSZip();
zip.file("root/foo.txt", "");
zip.file("root/bar.txt", "");
it("should find the root of a zip", async function() {
const zip = await ZipArchive.fromBuffer(await createZipBuffer([
["root/foo.txt", ""],
["root/bar.txt", ""],
]));
assert.equal(findRoot(zip), "root");
});
it("should throw if the first file is at the root of the zip", function() {
let zip = new JSZip();
zip.file("bar.txt", "");
zip.file("root/foo.txt", "");
it("should throw if the first file is at the root of the zip", async function() {
const zip = await ZipArchive.fromBuffer(await createZipBuffer([
["bar.txt", ""],
["root/foo.txt", ""],
]));
assert.throws(
() => findRoot(zip),
new Error("Zip contains file 'bar.txt' in root dir")
);
});
it("should return directory of first file if there are multiple root dirs", function() {
let zip = new JSZip();
zip.file("root-1/foo.txt", "");
zip.file("root-2/bar.txt", "");
it("should return directory of first file if there are multiple root dirs", async function() {
const zip = await ZipArchive.fromBuffer(await createZipBuffer([
["root-1/foo.txt", ""],
["root-2/bar.txt", ""],
]));
assert.equal(findRoot(zip), "root-1");
});
it("should throw if given an empty zip file", function() {
let zip = new JSZip();
it("should throw if given an empty zip file", async function() {
const zip = await ZipArchive.fromBuffer(await createZipBuffer([]));
assert.throws(
() => findRoot(zip),
new Error("Empty zip file")