diff --git a/package.json b/package.json index 113837db..cda98039 100644 --- a/package.json +++ b/package.json @@ -57,10 +57,11 @@ "form-data": "^4.0.6", "globals": "^17.12.0", "jsonwebtoken": "catalog:", - "jszip": "catalog:", "mocha": "^11.8.0", "typedoc": "^0.28.20", "typescript": "catalog:", - "yargs": "catalog:" + "yargs": "catalog:", + "yauzl": "catalog:", + "yazl": "catalog:" } } diff --git a/packages/controller/package.json b/packages/controller/package.json index 845fbee7..ed63dd61 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -43,7 +43,6 @@ "express": "catalog:", "finalhandler": "^2.1.1", "jsonwebtoken": "catalog:", - "jszip": "catalog:", "semver": "catalog:", "winston": "catalog:", "winston-daily-rotate-file": "catalog:", diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index da32850b..aeca0590 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -5,7 +5,6 @@ import busboy from "busboy"; import crypto from "crypto"; import events from "events"; import fs from "node:fs/promises"; -import JSZip from "jszip"; import jwt from "jsonwebtoken"; import path from "path"; import nodeStream from "stream"; @@ -266,9 +265,7 @@ async function uploadExport(req: Request, res: Response) { for await (let chunk of req) { data.push(chunk); } - let buffer: Buffer | null = Buffer.concat(data); - let zip = await JSZip.loadAsync(buffer); - buffer = null; + let zip = await lib.ZipArchive.fromBuffer(Buffer.concat(data)); // This is hardcoded to prevent path expansion attacks let exportFiles = [ @@ -288,14 +285,15 @@ async function uploadExport(req: Request, res: Response) { continue; } + let content = await zip.readEntry(file); if (filePath === "export/settings.json") { - settingPrototypes = JSON.parse(await file.async("text")); + settingPrototypes = JSON.parse(content.toString("utf8")); } let { name, ext } = path.posix.parse(filePath); - let hash = await lib.hashStream(file.nodeStream()); + let hash = await lib.hashStream(await zip.openStream(file)); assets[name] = `${name}.${hash}${ext}`; - await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), await file.async("nodebuffer")); + await lib.safeOutputFile(path.join("static", `${name}.${hash}${ext}`), content); } modPack.exportManifest = new lib.ExportManifest(assets, new Date().toISOString()); diff --git a/packages/host/package.json b/packages/host/package.json index 285e45c9..ce72524b 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -40,14 +40,14 @@ "@clusterio/lib": "workspace:^", "@sinclair/typebox": "catalog:", "jimp": "^1.6.1", - "jszip": "catalog:", "pidusage": "^4.0.1", "rcon-client": "^4.2.5", "semver": "catalog:", "set-blocking": "catalog:", "winston": "catalog:", "winston-daily-rotate-file": "catalog:", - "yargs": "catalog:" + "yargs": "catalog:", + "yazl": "catalog:" }, "devDependencies": { "@types/node": "catalog:", @@ -55,6 +55,7 @@ "@types/semver": "catalog:", "@types/set-blocking": "catalog:", "@types/yargs": "catalog:", + "@types/yazl": "catalog:", "typescript": "catalog:" } } diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index ed43019b..e2507ab9 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "path"; import pidusage from "pidusage"; +import stream from "node:stream"; import util from "util"; import type { Static } from "@sinclair/typebox"; import { exec } from "child_process"; @@ -1401,13 +1402,12 @@ end`.replace(/\r?\n/g, " "); await this.syncMods(); let zip = await exportData(this.server); - let content = await zip.generateAsync({ type: "nodebuffer" }); let url = new URL(this._host.config.get("host.controller_url")); url.pathname += "api/upload-export"; url.searchParams.set("mod_pack_id", String(this.activeModPack.id)); let response = await fetch(url, { method: "PUT", - body: content, + body: stream.Readable.toWeb(lib.zipOutputStream(zip)), duplex: "half", headers: { "Content-Type": "application/zip", diff --git a/packages/host/src/create_scenario.ts b/packages/host/src/create_scenario.ts index 6a49d168..2bc91b7e 100644 --- a/packages/host/src/create_scenario.ts +++ b/packages/host/src/create_scenario.ts @@ -1,8 +1,9 @@ // Implementation of the create-scenario command import fs from "node:fs/promises"; import os from "node:os"; -import JSZip from "jszip"; import path from "path"; +import { pipeline } from "node:stream/promises"; +import yazl from "yazl"; import * as lib from "@clusterio/lib"; import { logger } from "@clusterio/lib"; @@ -61,33 +62,50 @@ export async function createScenario( async function zipDirectory(dirPath: string, zipPath: string) { const root = path.basename(path.resolve(dirPath)); - const zip = new JSZip(); - let dirs: [string, string][] = [[dirPath, root]]; - while (dirs.length) { - let [dir, relativeDir] = dirs.pop()!; - for (let entry of await fs.readdir(dir, { withFileTypes: true })) { - let fsPath = path.join(dir, entry.name); - let relativePath = path.posix.join(relativeDir, entry.name); - if (entry.isFile()) { - zip.file(relativePath, await fs.readFile(fsPath)); - } else if (entry.isDirectory()) { - dirs.push([fsPath, relativePath]); + const zip = new yazl.ZipFile(); + const output = lib.zipOutputStream(zip); + const fd = await fs.open(zipPath, "w"); + // addFile() can fail before the traversal is done, so start writing first. + const writing = pipeline(output, fd.createWriteStream()); + writing.catch(() => {}); + try { + let dirs: [string, string][] = [[dirPath, root]]; + while (dirs.length) { + let [dir, relativeDir] = dirs.pop()!; + for (let entry of await fs.readdir(dir, { withFileTypes: true })) { + let fsPath = path.join(dir, entry.name); + let relativePath = path.posix.join(relativeDir, entry.name); + if (entry.isFile()) { + zip.addFile(fsPath, relativePath); + } else if (entry.isDirectory()) { + dirs.push([fsPath, relativePath]); + } } } + zip.end(); + } catch (err) { + output.destroy(); + await writing.catch(() => {}); + throw err; } - await fs.writeFile(zipPath, await zip.generateAsync({ type: "nodebuffer" })); + await writing; } async function extractZip(zipPath: string, dirPath: string) { - const zip = await JSZip.loadAsync(await fs.readFile(zipPath)); - const root = lib.findRoot(zip); - for (let file of Object.values(zip.files)) { - if (file.dir || !file.name.startsWith(`${root}/`)) { - continue; + const zip = await lib.ZipArchive.fromFile(zipPath); + try { + const root = lib.findRoot(zip); + for (let entry of zip.entries.values()) { + if (entry.fileName.endsWith("/") || !entry.fileName.startsWith(`${root}/`)) { + continue; + } + const fsPath = path.join(dirPath, entry.fileName.slice(root.length + 1)); + await fs.mkdir(path.dirname(fsPath), { recursive: true }); + const fd = await fs.open(fsPath, "w"); + await pipeline(await zip.openStream(entry), fd.createWriteStream()); } - const fsPath = path.join(dirPath, file.name.slice(root.length + 1)); - await fs.mkdir(path.dirname(fsPath), { recursive: true }); - await fs.writeFile(fsPath, await file.async("nodebuffer")); + } finally { + zip.close(); } } diff --git a/packages/host/src/export.ts b/packages/host/src/export.ts index eeda2594..cb7f73d5 100644 --- a/packages/host/src/export.ts +++ b/packages/host/src/export.ts @@ -2,7 +2,7 @@ import fs from "node:fs/promises"; import path from "path"; import { Jimp, JimpMime, type JimpInstance } from "jimp"; -import JSZip from "jszip"; +import yazl from "yazl"; import * as lib from "@clusterio/lib"; import * as libBuildMod from "@clusterio/lib/dist/node/build_mod.js"; @@ -64,21 +64,38 @@ async function generateExportMod(server: FactorioServer) { }); } -let zipCache = new Map(); -async function loadZip(server: FactorioServer, modVersions: Map, mod: string) { +interface ModZip { + zip: lib.ZipArchive; + root: string; +} +type ZipCache = Map; +async function loadZip(server: FactorioServer, modVersions: Map, zipCache: ZipCache, mod: string) { let modVersion = modVersions.get(mod); if (!modVersion) { throw new Error(`Got path for unknown mod ${mod}`); } let zipPath = server.writePath("mods", `${mod}_${modVersion}.zip`); - let zip = zipCache.get(zipPath); - if (!zip) { - zip = await JSZip.loadAsync(await fs.readFile(zipPath)); - zipCache.set(zipPath, zip); + let modZip = zipCache.get(zipPath); + if (!modZip) { + const zip = await lib.ZipArchive.fromFile(zipPath); + try { + modZip = { zip, root: lib.findRoot(zip) }; + } catch (err) { + zip.close(); + throw err; + } + zipCache.set(zipPath, modZip); } - return zip.folder(lib.findRoot(zip))!; + return modZip; +} + +function closeZipCache(zipCache: ZipCache) { + for (const { zip } of zipCache.values()) { + zip.close(); + } + zipCache.clear(); } /** @@ -86,11 +103,17 @@ async function loadZip(server: FactorioServer, modVersions: Map, * * @param server - The server to load the file from. * @param modVersions - Mapping of mod to version used. + * @param zipCache - Mod zips opened by this export. * @param modPath - Factorio style path to the file to load. * @returns The content of the file or null if not found. * @internal */ -async function loadFile(server: FactorioServer, modVersions: Map, modPath: string) { +async function loadFile( + server: FactorioServer, + modVersions: Map, + zipCache: ZipCache, + modPath: string, +) { let match = /^__([^\/]+)__\/(.*)$/.exec(modPath); if (!match) { server._logger.warn(`Skipping icon with bad mod path: ${JSON.stringify(modPath)}`); @@ -111,9 +134,9 @@ async function loadFile(server: FactorioServer, modVersions: Map } } - let zip; + let modZip; try { - zip = await loadZip(server, modVersions, mod); + modZip = await loadZip(server, modVersions, zipCache, mod); } catch (err: any) { if (err.code === "ENOENT") { return null; @@ -121,12 +144,7 @@ async function loadFile(server: FactorioServer, modVersions: Map throw err; } - let file = zip.file(filePath); - if (!file) { - return null; - } - - return await file.async("nodebuffer"); + return await modZip.zip.readFile(path.posix.join(modZip.root, filePath)); } type Image = JimpInstance; @@ -134,6 +152,7 @@ type IconCache = Map; async function loadIcon( server: FactorioServer, modVersions: Map, + zipCache: ZipCache, iconPath: string, iconSize: number, iconMipmaps: number, @@ -141,7 +160,7 @@ async function loadIcon( ) { let icon = iconCache.get(iconPath); if (icon === undefined) { - let fileContent = await loadFile(server, modVersions, iconPath); + let fileContent = await loadFile(server, modVersions, zipCache, iconPath); if (fileContent) { icon = await Jimp.fromBuffer(fileContent) as Image; icon.crop({ x: 0, y: 0, w: iconSize, h: iconSize }); @@ -157,11 +176,12 @@ async function loadIcon( async function loadSimpleIcon( server: FactorioServer, modVersions: Map, + zipCache: ZipCache, item: SimpleIconSpecification, size: number, iconCache: IconCache, ): Promise { - let icon = await loadIcon(server, modVersions, item.icon, item.icon_size ?? 64, 0, iconCache); + let icon = await loadIcon(server, modVersions, zipCache, item.icon, item.icon_size ?? 64, 0, iconCache); if (icon) { let iconScale = size / (item.icon_size ?? 64); if (iconScale !== 1) { @@ -175,6 +195,7 @@ async function loadSimpleIcon( async function loadLayeredIcon( server: FactorioServer, modVersions: Map, + zipCache: ZipCache, item: LayeredIconSpecification, size: number, iconCache: IconCache, @@ -190,7 +211,9 @@ async function loadLayeredIcon( for (let layer of item.icons) { let layerSize = (layer.icon_size || item.icon_size) ?? 64; - let iconLayer = await loadIcon(server, modVersions, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache); + let iconLayer = await loadIcon( + server, modVersions, zipCache, layer.icon, layerSize, layer.icon_mipmaps || 0, iconCache, + ); if (!iconLayer) { continue; @@ -308,6 +331,7 @@ async function growSheet(state: SheetState) { async function exportIcon( server: FactorioServer, modVersions: Map, + zipCache: ZipCache, state: SheetState, prototype: PrototypeWithIcon, ) { @@ -318,7 +342,7 @@ async function exportIcon( if (prototype.icons) { icon = await loadLayeredIcon( - server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache, + server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache, ); iconPos = state.pos; iconPath = prototype.icons[0].icon; @@ -328,7 +352,7 @@ async function exportIcon( iconPath = prototype.icon; if (iconPos === undefined) { icon = await loadSimpleIcon( - server, modVersions, prototype, SHEET_ICON_SIZE, state.iconCache, + server, modVersions, zipCache, prototype, SHEET_ICON_SIZE, state.iconCache, ); if (icon) { iconPos = state.pos; @@ -364,6 +388,7 @@ async function exportIcon( * * @param server - The server to load icons from. * @param modVersions - Mapping of mod name to version. + * @param zipCache - Mod zips opened by this export. * @param prototypes - Array of prototypes with icon to export for this base type. * @param baseName - name of base type (used in metadata). * @param state - Shared spritesheet state (mutated in place). @@ -373,6 +398,7 @@ async function exportIcon( async function exportIcons( server: FactorioServer, modVersions: Map, + zipCache: ZipCache, prototypes: PrototypeWithIcon[], baseName: string, state: SheetState, @@ -381,7 +407,7 @@ async function exportIcons( state.metadata[baseName] ??= metadata; let count = 0; for (let prototype of prototypes) { - const icon = await exportIcon(server, modVersions, state, prototype); + const icon = await exportIcon(server, modVersions, zipCache, state, prototype); if (icon) { count += 1; } @@ -404,6 +430,7 @@ async function exportIcons( * * @param server - The server to export the locale from. * @param modVersions - Mapping of mod name to version to export locale from. + * @param zipCache - Mod zips opened by this export. * @param modOrder - Load order of the mods. * @param languageCode - Language to export locale for. * @returns merged locale information @@ -412,6 +439,7 @@ async function exportIcons( async function exportLocale( server: FactorioServer, modVersions: Map, + zipCache: ZipCache, modOrder: string[], languageCode: string ) { @@ -445,18 +473,21 @@ async function exportLocale( continue; } - let zip; + let modZip; try { - zip = await loadZip(server, modVersions, mod); + modZip = await loadZip(server, modVersions, zipCache, mod); } catch (err: any) { if (err.code === "ENOENT") { continue; } throw err; } - for (let file of zip.file(new RegExp(`locale\\/${languageCode}\\/.*\\.cfg`))) { - let content = await file.async("nodebuffer"); - mergeLocale(lib.parse(content.toString("utf8"))); + const localePath = `${path.posix.join(modZip.root, "locale", languageCode)}/`; + for (const entry of modZip.zip.entries.values()) { + if (entry.fileName.startsWith(localePath) && entry.fileName.endsWith(".cfg")) { + let content = await modZip.zip.readEntry(entry); + mergeLocale(lib.parse(content.toString("utf8"))); + } } } @@ -470,6 +501,16 @@ async function exportLocale( * @returns zip file with exported data. */ export async function exportData(server: FactorioServer) { + const zipCache: ZipCache = new Map(); + try { + return await buildExport(server, zipCache); + } finally { + // Free up the zip files loaded during the export. + closeZipCache(zipCache); + } +} + +async function buildExport(server: FactorioServer, zipCache: ZipCache) { await generateExportMod(server); let settings: lib.ExportSettings = {}; @@ -549,16 +590,14 @@ export async function exportData(server: FactorioServer) { } } - let locale = await exportLocale(server, modVersions, modOrder, "en"); + let locale = await exportLocale(server, modVersions, zipCache, modOrder, "en"); - // Free up the memory used by zip files loaded during the export. - zipCache.clear(); - - let zip = new JSZip(); - zip.file("export/settings.json", JSON.stringify(settings satisfies lib.ExportSettings)); - zip.file("export/prototypes.json", JSON.stringify(prototypes satisfies lib.ExportPrototypes)); - zip.file("export/locale.json", JSON.stringify([...locale.entries()] satisfies lib.ExportLocale)); - zip.file("export/defines.json", JSON.stringify(defines satisfies lib.ExportDefines)); + let zip = new yazl.ZipFile(); + const addJson = (name: string, value: unknown) => zip.addBuffer(Buffer.from(JSON.stringify(value)), name); + addJson("export/settings.json", settings satisfies lib.ExportSettings); + addJson("export/prototypes.json", prototypes satisfies lib.ExportPrototypes); + addJson("export/locale.json", [...locale.entries()] satisfies lib.ExportLocale); + addJson("export/defines.json", defines satisfies lib.ExportDefines); // Build a single unified spritesheet across all prototypes with icons. const state = await createSheetState(); @@ -572,15 +611,16 @@ export async function exportData(server: FactorioServer) { if (prototypesWithIcons.length === 0) { continue; } - const count = await exportIcons(server, modVersions, prototypesWithIcons, baseName, state); + const count = await exportIcons(server, modVersions, zipCache, prototypesWithIcons, baseName, state); server._logger.info(`Exported ${count} ${baseName} icons`); } // Crop sheet to actual used height and write single spritesheet + metadata. const usedRows = Math.ceil(state.pos / (SHEET_WIDTH / SHEET_ICON_SIZE)); state.sheet.crop({ x: 0, y: 0, w: SHEET_WIDTH, h: Math.max(usedRows * SHEET_ICON_SIZE, 1) }); - zip.file("export/spritesheet.png", await state.sheet.getBuffer(JimpMime.png)); - zip.file("export/metadata.json", JSON.stringify(state.metadata satisfies lib.ExportMetadata)); + zip.addBuffer(await state.sheet.getBuffer(JimpMime.png), "export/spritesheet.png"); + addJson("export/metadata.json", state.metadata satisfies lib.ExportMetadata); + zip.end(); server._logger.info(`Export complete: ${state.pos} icons on ${usedRows} row(s)`); diff --git a/packages/host/src/patch.ts b/packages/host/src/patch.ts index f4ed658c..dbf3022d 100644 --- a/packages/host/src/patch.ts +++ b/packages/host/src/patch.ts @@ -1,10 +1,9 @@ // Library for patching Factorio saves with scenario code. -import events from "events"; import fs from "node:fs/promises"; -import JSZip from "jszip"; import path from "path"; import semver from "semver"; +import { pipeline } from "node:stream/promises"; import { Type, Static } from "@sinclair/typebox"; import * as lib from "@clusterio/lib"; @@ -72,18 +71,16 @@ export class SaveModule { }; } - static async fromSave(json: Static, root: JSZip) { + static async fromSave(json: Static, zip: lib.ZipArchive, root: string) { const module = new this(lib.ModuleInfo.fromJSON(json)); - module.files = new Map(await Promise.all(json.files - .map(filename => ({filename, file: root.file(filename)})) - .filter(({filename, file}) => { - if (file === null) { - lib.logger.warn(`Missing file ${filename} in save`); - } - return file !== null; - }) - .map(async ({filename, file}) => [filename, await file!.async("nodebuffer")] as const) - )); + for (const filename of json.files) { + const content = await zip.readFile(path.posix.join(root, filename)); + if (content === null) { + lib.logger.warn(`Missing file ${filename} in save`); + continue; + } + module.files.set(filename, content); + } return module; } @@ -205,7 +202,7 @@ export class PatchInfo { }; } - static async fromSave(json: Static, root: JSZip) { + static async fromSave(json: Static, zip: lib.ZipArchive, root: string) { if (json.version === undefined) { interface ScenarioInfoV0 { name: string; @@ -229,7 +226,7 @@ export class PatchInfo { return new this( json.patch_number, lib.ModuleInfo.fromJSON(json.scenario), - await Promise.all(json.modules.map(m => SaveModule.fromSave(m, root))), + await Promise.all(json.modules.map(m => SaveModule.fromSave(m, zip, root))), ); } } @@ -396,80 +393,111 @@ const knownScenarios: Record = { * @param modules - Description of the modules to patch. */ export async function patch(savePath: string, modules: SaveModule[]) { - let zip = await JSZip.loadAsync(await fs.readFile(savePath)); - let root = zip.folder(lib.findRoot(zip))!; + const zip = await lib.ZipArchive.fromFile(savePath); + let tempSavePath: string; + try { + const root = lib.findRoot(zip); + const rootPath = (relativePath: string) => path.posix.join(root, relativePath); - let patchInfoFile = root.file("clusterio.json"); - let patchInfo: PatchInfo; - if (patchInfoFile !== null) { - let content = await patchInfoFile.async("string"); - patchInfo = await PatchInfo.fromSave(JSON.parse(content), root); - if (patchInfo.version > PatchInfo.currentVersion) { - throw new Error( - `Save patch version ${patchInfo.version} is newer than the patch version this ` + - `version of Clusterio can load (${PatchInfo.currentVersion})` - ); - } + // Files to add or replace in the save, relative to the root + const files = new Map(); + let patchInfo: PatchInfo; + const patchInfoContent = await zip.readFile(rootPath("clusterio.json")); + if (patchInfoContent !== null) { + patchInfo = await PatchInfo.fromSave(JSON.parse(patchInfoContent.toString("utf8")), zip, root); + if (patchInfo.version > PatchInfo.currentVersion) { + throw new Error( + `Save patch version ${patchInfo.version} is newer than the patch version this ` + + `version of Clusterio can load (${PatchInfo.currentVersion})` + ); + } - // No info file present, try to detect if it's a known compatible scenario. - } else { - let controlFile = root.file("control.lua"); - if (!controlFile) { - throw new Error("Unable to patch save, missing control.lua file."); - } - let controlStream = controlFile.nodeStream("nodebuffer"); - let controlHash = await lib.hashStream(controlStream); - - if (controlHash in knownScenarios) { - patchInfo = new PatchInfo(0, knownScenarios[controlHash], []); - root.file("scenario.lua", controlFile.nodeStream("nodebuffer")); + // No info file present, try to detect if it's a known compatible scenario. } else { - throw new Error(`Unable to patch save, unknown scenario (${controlHash})`); - } - } + const controlFile = zip.file(rootPath("control.lua")); + if (!controlFile) { + throw new Error("Unable to patch save, missing control.lua file."); + } + const controlHash = await lib.hashStream(await zip.openStream(controlFile)); - // Increment patch number - patchInfo.patchNumber = patchInfo.patchNumber + 1; - - // Remove any existing modules from the save - if (patchInfo.version === 0) { - for (let file of root.file(/^modules\//)) { - zip.remove(file.name); - } - } else { - for (let module of patchInfo.modules) { - for (let filepath of module.files.keys()) { - const file = root.file(filepath); - if (file !== null) { - zip.remove(file.name); - } + if (controlHash in knownScenarios) { + patchInfo = new PatchInfo(0, knownScenarios[controlHash], []); + files.set("scenario.lua", await zip.readEntry(controlFile)); + } else { + throw new Error(`Unable to patch save, unknown scenario (${controlHash})`); } } - patchInfo.modules = []; - } - reorderDependencies(modules); + // Increment patch number + patchInfo.patchNumber = patchInfo.patchNumber + 1; - // Add the modules to the save. - for (let module of modules) { - for (let [relativePath, contents] of module.files) { - root.file(relativePath, contents); + // Remove any existing modules from the save + const removed = new Set(); + if (patchInfo.version === 0) { + const modulesPath = rootPath("modules/"); + for (const name of zip.entries.keys()) { + if (name.startsWith(modulesPath)) { + removed.add(name); + } + } + } else { + for (const module of patchInfo.modules) { + for (const filepath of module.files.keys()) { + removed.add(rootPath(filepath)); + } + } + patchInfo.modules = []; } - patchInfo.modules.push(module); - } - // Add loading code and patch info - root.file("control.lua", generateLoader(patchInfo)); - root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t")); + reorderDependencies(modules); - // Write back the save - let stream = zip.generateNodeStream({ compression: "DEFLATE" }); - const [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath); - try { - let pipe = stream.pipe(writeStream); - await events.once(pipe, "finish"); + // Add the modules to the save. + for (const module of modules) { + for (const [relativePath, contents] of module.files) { + files.set(relativePath, contents); + } + patchInfo.modules.push(module); + } + + // Add loading code and patch info + files.set("control.lua", Buffer.from(generateLoader(patchInfo), "utf8")); + files.set("clusterio.json", Buffer.from(JSON.stringify(patchInfo, null, "\t"), "utf8")); + + // Write back the save, keeping the order of the entries in the original + const added = new Map([...files].map(([relativePath, contents]) => [rootPath(relativePath), contents])); + const output = new lib.ZipWriter(); + let writeStream; + [tempSavePath, writeStream] = await lib.createTempWriteStream(savePath); + const writing = pipeline(output.outputStream, writeStream); + // Awaited below, but a write error can reject it before then. + writing.catch(() => {}); + try { + for (const entry of zip.entries.values()) { + const name = entry.fileName; + if (removed.has(name)) { + continue; + } + const contents = added.get(name); + if (contents !== undefined) { + await output.addBuffer(contents, name); + added.delete(name); + } else { + await output.addEntry(zip, entry); + } + } + for (const [name, contents] of added) { + await output.addBuffer(contents, name); + } + await output.end(); + await writing; + } catch (err) { + output.outputStream.destroy(); + await writing.catch(() => {}); + await fs.rm(tempSavePath, { force: true }); + throw err; + } } finally { - writeStream.destroy(); + zip.close(); } await fs.rename(tempSavePath, savePath); } diff --git a/packages/host/src/server.ts b/packages/host/src/server.ts index 01f0d772..dc46a0ff 100644 --- a/packages/host/src/server.ts +++ b/packages/host/src/server.ts @@ -1,9 +1,11 @@ // Factorio Server interface import fs from "node:fs/promises"; -import { type FSWatcher, watch as fsWatch, writeFileSync } from "node:fs"; +import { type FSWatcher, createWriteStream, watch as fsWatch, writeFileSync } from "node:fs"; import child_process from "child_process"; import path from "path"; -import JSZip from "jszip"; +import stream from "node:stream"; +import { pipeline } from "node:stream/promises"; +import type { ReadableStream } from "node:stream/web"; import events from "events"; import util from "util"; import crypto from "crypto"; @@ -112,29 +114,38 @@ async function downloadAndExtractZip(url: string, targetDir: string) { throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`); } - // Load the ZIP with JSZip - const buffer = Buffer.from(await res.arrayBuffer()); - const zip = await JSZip.loadAsync(buffer); + if (!res.body) { + throw new Error(`Failed to fetch ${url}: empty response`); + } - // Extract each file/folder - await fs.mkdir(targetDir, { recursive: true }); - await Promise.all( - Object.values(zip.files).map(async (entry) => { - const parts = entry.name.split("/").filter(Boolean); - const strippedPath = parts.slice(1).join("/"); - if (!strippedPath) { - return; // This copies the behaviour of tar --strip-components 1 - } + // Write the download to a temp file, zip files need random access to read + const tmpFilePath = `${targetDir}.tmp`; + await fs.writeFile(tmpFilePath, stream.Readable.fromWeb(res.body as ReadableStream)); + try { + const zip = await lib.ZipArchive.fromFile(tmpFilePath); + try { + await fs.mkdir(targetDir, { recursive: true }); + for (const entry of zip.entries.values()) { + const parts = entry.fileName.split("/").filter(Boolean); + const strippedPath = parts.slice(1).join("/"); + if (!strippedPath) { + continue; // This copies the behaviour of tar --strip-components 1 + } - const entryPath = path.join(targetDir, strippedPath); - if (entry.dir) { - await fs.mkdir(entryPath, { recursive: true }); - } else { - await fs.mkdir(path.dirname(entryPath), { recursive: true }); - await fs.writeFile(entryPath, await entry.async("nodebuffer")); + const entryPath = path.join(targetDir, strippedPath); + if (entry.fileName.endsWith("/")) { + await fs.mkdir(entryPath, { recursive: true }); + } else { + await fs.mkdir(path.dirname(entryPath), { recursive: true }); + await pipeline(await zip.openStream(entry), createWriteStream(entryPath)); + } } - }) - ); + } finally { + zip.close(); + } + } finally { + await fs.rm(tmpFilePath); + } } /** diff --git a/packages/lib/build_mod.js b/packages/lib/build_mod.js index d25ae4ba..acf5be29 100644 --- a/packages/lib/build_mod.js +++ b/packages/lib/build_mod.js @@ -1,13 +1,40 @@ /* eslint-disable no-console */ import fs from "node:fs/promises"; -import JSZip from "jszip"; import path from "node:path"; -import stream from "node:stream"; -import util from "node:util"; +import { pipeline } from "node:stream/promises"; import yargs from "yargs"; +import yazl from "yazl"; -const finished = util.promisify(stream.finished); +async function addModFiles(args, info, modName, zip) { + for await (const dirent of await fs.opendir(args.sourceDir, { recursive: true })) { + if (dirent.isFile()) { + const itemPath = path.join(dirent.parentPath, dirent.name); + // On Windows the path created uses backslashes as the directory sepparator + // but the zip file needs to use forward slashes. We can't use the posix + // version of relative here as it doesn't work with Windows style paths. + let basePath = path.relative(args.sourceDir, itemPath).replace(/\\/g, "/"); + if (basePath === "info.json") { + // The info.json file is overridden later. + continue; + } + zip.addFile(itemPath, path.posix.join(modName, basePath)); + } + } + + for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) { + let filePath = path.join(args.sourceDir, ...pathParts); + try { + await fs.access(filePath); + } catch (err) { + throw new Error(`Error reading additional file ${filePath}`, { cause: err }); + } + zip.addFile(filePath, path.posix.join(modName, fileName)); + } + delete info.additional_files; + + zip.addBuffer(Buffer.from(JSON.stringify(info, null, "\t")), path.posix.join(modName, "info.json")); +} async function buildMod(args, info, modName) { if (args.build) { @@ -15,43 +42,24 @@ async function buildMod(args, info, modName) { modName = args.modName ?? `${info.name}_${info.version}`; if (args.pack) { - let zip = new JSZip(); - for await (const dirent of await fs.opendir(args.sourceDir, { recursive: true })) { - if (dirent.isFile()) { - const itemPath = path.join(dirent.parentPath, dirent.name); - // On Windows the path created uses backslashes as the directory sepparator - // but the zip file needs to use forward slashes. We can't use the posix - // version of relative here as it doesn't work with Windows style paths. - let basePath = path.relative(args.sourceDir, itemPath).replace(/\\/g, "/"); - // eslint-disable-next-line max-depth - if (basePath === "info.json") { - // The info.json file is overridden later. - continue; - } - zip.file(path.posix.join(modName, basePath), (await fs.open(itemPath)).createReadStream()); - } - } - - for (let [fileName, pathParts] of Object.entries(info.additional_files || {})) { - let filePath = path.join(args.sourceDir, ...pathParts); - let fileStream; - try { - fileStream = (await fs.open(filePath)).createReadStream(); - } catch (err) { - throw new Error(`Error reading additional file ${filePath}`, { cause: err }); - } - zip.file(path.posix.join(modName, fileName), fileStream); - } - delete info.additional_files; - - zip.file(path.posix.join(modName, "info.json"), JSON.stringify(info, null, "\t")); - let modPath = path.join(args.outputDir, `${modName}.zip`); console.log(`Writing ${modPath}`); - let writeStream = zip.generateNodeStream().pipe( - (await fs.open(modPath, "w")).createWriteStream() - ); - await finished(writeStream); + let zip = new yazl.ZipFile(); + // yazl reports errors on the zip object, not the output stream. + zip.on("error", err => zip.outputStream.destroy(err)); + // addFile() can fail before the traversal is done, so start writing first. + const writing = pipeline(zip.outputStream, (await fs.open(modPath, "w")).createWriteStream()); + writing.catch(() => {}); + try { + await addModFiles(args, info, modName, zip); + zip.end(); + await writing; + } catch (err) { + zip.outputStream.destroy(); + await writing.catch(() => {}); + await fs.rm(modPath, { force: true }); + throw err; + } } else { let modDir = path.join(args.outputDir, modName); diff --git a/packages/lib/package.json b/packages/lib/package.json index e112f20f..9254901c 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -51,16 +51,19 @@ "dependencies": { "@sinclair/typebox": "catalog:", + "@types/yauzl": "catalog:", + "@types/yazl": "catalog:", "ajv": "^8.20.0", "chalk": "catalog:", "fast-deep-equal": "^3.1.3", - "jszip": "catalog:", "set-blocking": "catalog:", "triple-beam": "^1.4.1", "winston": "catalog:", "winston-transport": "catalog:", "ws": "catalog:", - "yargs": "catalog:" + "yargs": "catalog:", + "yauzl": "catalog:", + "yazl": "catalog:" }, "devDependencies": { "@types/node": "catalog:", diff --git a/packages/lib/src/data/ModInfo.ts b/packages/lib/src/data/ModInfo.ts index dc702d91..c44c8a6e 100644 --- a/packages/lib/src/data/ModInfo.ts +++ b/packages/lib/src/data/ModInfo.ts @@ -1,10 +1,10 @@ import fs from "node:fs/promises"; -import JSZip from "jszip"; +import path from "path"; import { Type, Static } from "@sinclair/typebox"; import * as libHash from "../hash.js"; import * as libSchema from "../schema.js"; -import { findRoot } from "../zip_ops.js"; +import { ZipArchive, findRoot } from "../zip_ops.js"; import { ModRecord } from "./ModPack.js"; import { @@ -352,18 +352,15 @@ export default class ModInfo { static async fromModFile(modPath: string) { let modInfo: Static; - { - // XXX: JSZip needs the whole archive loaded in memory to work. - // This is clearly untenable and will be replaced later. - let zip = await JSZip.loadAsync(await fs.readFile(modPath)); - let root = zip.folder(findRoot(zip))!; - - let infoFile = root.file("info.json"); - if (!infoFile) { + let zip = await ZipArchive.fromFile(modPath); + try { + let content = await zip.readFile(path.posix.join(findRoot(zip), "info.json")); + if (content === null) { throw new Error("Mod contains no info.json file"); } - - modInfo = JSON.parse(await infoFile.async("string")); + modInfo = JSON.parse(content.toString("utf8")); + } finally { + zip.close(); } let valid = this.validateInfo(modInfo); diff --git a/packages/lib/src/zip_ops.ts b/packages/lib/src/zip_ops.ts index 1c96a6b8..b7087caf 100644 --- a/packages/lib/src/zip_ops.ts +++ b/packages/lib/src/zip_ops.ts @@ -1,4 +1,122 @@ -import type JSZip from "jszip"; +/** + * Zip file helpers + * @module lib/zip_ops + */ +import events from "events"; +import { PassThrough, type Readable } from "stream"; +import util from "util"; +import zlib from "zlib"; +import yauzl from "yauzl"; +import type yazl from "yazl"; + +const deflateRaw = util.promisify(zlib.deflateRaw); + + +/** + * Zip archive opened for random access reading + * + * Only the central directory is read up front, the content of entries is + * streamed from the archive on demand. Archives opened from a file hold + * the file open until close() is called. + */ +export class ZipArchive { + private constructor( + /** Underlying yauzl zip file. */ + public zipfile: yauzl.ZipFile, + /** Entries by file name in the order they appear in the central directory. */ + public entries: Map, + ) { } + + /** + * Open a zip archive from the file system + * + * @param zipPath - Path to the zip file to open. + */ + static async fromFile(zipPath: string) { + return this.load(await yauzl.openPromise(zipPath, { autoClose: false })); + } + + /** + * Open a zip archive held in memory + * + * @param buffer - Content of the zip file. + */ + static async fromBuffer(buffer: Buffer) { + return this.load(await yauzl.fromBufferPromise(buffer)); + } + + private static async load(zipfile: yauzl.ZipFile) { + const entries = new Map(); + try { + for await (const entry of zipfile.eachEntry()) { + entries.set(entry.fileName, entry); + } + } catch (err) { + zipfile.close(); + throw err; + } + return new this(zipfile, entries); + } + + /** + * Look up a file in the archive + * + * @param filePath - Path of the file in the archive. + * @returns entry for the file, or null if it does not exist. + */ + file(filePath: string) { + const entry = this.entries.get(filePath); + if (!entry || entry.fileName.endsWith("/")) { + return null; + } + return entry; + } + + /** + * Open a stream of the decompressed content of an entry + * + * @param entry - Entry to read, must belong to this archive. + */ + openStream(entry: yauzl.Entry): Promise { + return this.zipfile.openReadStreamPromise(entry); + } + + /** + * Read the decompressed content of an entry + * + * @param entry - Entry to read, must belong to this archive. + */ + async readEntry(entry: yauzl.Entry) { + const chunks: Buffer[] = []; + for await (const chunk of await this.openStream(entry)) { + chunks.push(chunk); + } + return Buffer.concat(chunks); + } + + /** + * Read the content of a file in the archive + * + * @param filePath - Path of the file in the archive. + * @returns content of the file, or null if it does not exist. + */ + async readFile(filePath: string) { + const entry = this.file(filePath); + if (!entry) { + return null; + } + return await this.readEntry(entry); + } + + /** + * Close the archive + * + * Streams already opened are still readable until they end. + */ + close() { + this.zipfile.close(); + } +} /** * Returns the root folder in the zip file @@ -12,8 +130,8 @@ import type JSZip from "jszip"; * @param zip - Zip to search through. * @returns name of the root folder. */ -export function findRoot(zip: JSZip) { - const relativePath = Object.keys(zip.files)[0]; +export function findRoot(zip: ZipArchive) { + const relativePath = zip.entries.keys().next().value; if (relativePath === undefined) { throw new Error("Empty zip file"); } @@ -25,3 +143,263 @@ export function findRoot(zip: JSZip) { return relativePath.slice(0, index); } + +/** + * Returns the output stream of a zip file being written + * + * yazl reports errors on the ZipFile object and leaves the output stream + * hanging. The stream returned here is destroyed with the error instead so + * that consumers like stream.pipeline see it. + * + * @param zipFile - zip file being written. + */ +export function zipOutputStream(zipFile: yazl.ZipFile) { + const output = zipFile.outputStream as Readable; + zipFile.on("error", err => output.destroy(err)); + return output; +} + +const zip64Limit = 0xffffffff; + +function dosDateTime(date: Date) { + if (date.getFullYear() < 1980) { + return { date: 0x21, time: 0 }; + } + const year = Math.min(date.getFullYear(), 2107); + return { + date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(), + time: (date.getHours() << 11) | (date.getMinutes() << 5) | (date.getSeconds() >> 1), + }; +} + +interface WriterEntry { + fileName: Buffer; + fileComment: Buffer; + versionMadeBy: number; + generalPurposeBitFlag: number; + compressionMethod: number; + lastModFileTime: number; + lastModFileDate: number; + crc32: number; + compressedSize: number; + uncompressedSize: number; + externalFileAttributes: number; + offset: number; +} + +export interface ZipWriterOptions { + /** Use ZIP64 structures even when the archive does not need them. */ + forceZip64?: boolean; +} + +export interface ZipWriterBufferOptions { + /** Modification time to record for the file, defaults to now. */ + mtime?: Date; +} + +/** + * Streaming zip file writer + * + * Writes entries to outputStream as they are added. Entries from an + * existing archive are copied without decompressing them. Sizes and + * offsets past 4 GiB and more than 65535 entries use ZIP64 structures. + */ +export class ZipWriter { + /** Content of the zip file, ends when end() has been called. */ + outputStream = new PassThrough(); + private offset = 0; + private entries: WriterEntry[] = []; + private forceZip64: boolean; + + constructor(options: ZipWriterOptions = {}) { + this.forceZip64 = options.forceZip64 ?? false; + } + + /** + * Copy an entry from an archive + * + * The compressed content is copied as is along with the metadata + * Factorio writes. Extra fields are dropped. + * + * @param zip - Archive to copy from. + * @param entry - Entry to copy, must belong to zip. + */ + async addEntry(zip: ZipArchive, entry: yauzl.Entry) { + const writerEntry: WriterEntry = { + fileName: Buffer.from(entry.fileName, "utf8"), + fileComment: Buffer.from(entry.fileComment, "utf8"), + versionMadeBy: entry.versionMadeBy, + // Sizes are written in the local header so no data descriptor is + // needed, and names are re-encoded as UTF-8. + generalPurposeBitFlag: (entry.generalPurposeBitFlag & ~0x8) | 0x800, + compressionMethod: entry.compressionMethod, + lastModFileTime: entry.lastModFileTime, + lastModFileDate: entry.lastModFileDate, + crc32: entry.crc32, + compressedSize: entry.compressedSize, + uncompressedSize: entry.uncompressedSize, + externalFileAttributes: entry.externalFileAttributes, + offset: this.offset, + }; + await this.write(this.localFileHeader(writerEntry)); + const stream = await zip.zipfile.openReadStreamPromise(entry, { decodeFileData: false }); + for await (const chunk of stream) { + await this.write(chunk); + } + this.entries.push(writerEntry); + } + + /** + * Add a file with the given content + * + * @param buffer - Content of the file. + * @param fileName - Path of the file in the archive. + * @param options - Options for the entry. + */ + async addBuffer(buffer: Buffer, fileName: string, options: ZipWriterBufferOptions = {}) { + const deflated = await deflateRaw(buffer); + const { date, time } = dosDateTime(options.mtime ?? new Date()); + const writerEntry: WriterEntry = { + fileName: Buffer.from(fileName, "utf8"), + fileComment: Buffer.alloc(0), + versionMadeBy: 20, + generalPurposeBitFlag: 0x800, + compressionMethod: 8, + lastModFileTime: time, + lastModFileDate: date, + crc32: zlib.crc32(buffer), + compressedSize: deflated.length, + uncompressedSize: buffer.length, + externalFileAttributes: 0, + offset: this.offset, + }; + await this.write(this.localFileHeader(writerEntry)); + await this.write(deflated); + this.entries.push(writerEntry); + } + + /** + * Write the central directory and end the output stream + */ + async end() { + const start = this.offset; + for (const entry of this.entries) { + await this.write(this.centralDirectoryHeader(entry)); + } + const size = this.offset - start; + const zip64 = this.forceZip64 || this.entries.length >= 0xffff || start >= zip64Limit || size >= zip64Limit; + if (zip64) { + const record = Buffer.alloc(56); + record.writeUInt32LE(0x06064b50, 0); + record.writeBigUInt64LE(44n, 4); + record.writeUInt16LE(45, 12); // version made by + record.writeUInt16LE(45, 14); // version needed to extract + record.writeUInt32LE(0, 16); // number of this disk + record.writeUInt32LE(0, 20); // disk with the central directory + record.writeBigUInt64LE(BigInt(this.entries.length), 24); + record.writeBigUInt64LE(BigInt(this.entries.length), 32); + record.writeBigUInt64LE(BigInt(size), 40); + record.writeBigUInt64LE(BigInt(start), 48); + const locator = Buffer.alloc(20); + locator.writeUInt32LE(0x07064b50, 0); + locator.writeUInt32LE(0, 4); // disk with the zip64 end of central directory + locator.writeBigUInt64LE(BigInt(this.offset), 8); + locator.writeUInt32LE(1, 16); // total number of disks + await this.write(record); + await this.write(locator); + } + const eocd = Buffer.alloc(22); + eocd.writeUInt32LE(0x06054b50, 0); + eocd.writeUInt16LE(0, 4); // number of this disk + eocd.writeUInt16LE(0, 6); // disk with the central directory + eocd.writeUInt16LE(zip64 ? 0xffff : this.entries.length, 8); + eocd.writeUInt16LE(zip64 ? 0xffff : this.entries.length, 10); + eocd.writeUInt32LE(zip64 ? zip64Limit : size, 12); + eocd.writeUInt32LE(zip64 ? zip64Limit : start, 16); + eocd.writeUInt16LE(0, 20); // comment length + await this.write(eocd); + this.outputStream.end(); + } + + private localFileHeader(entry: WriterEntry) { + const zip64 = this.forceZip64 + || entry.compressedSize >= zip64Limit + || entry.uncompressedSize >= zip64Limit; + const extra = zip64 ? Buffer.alloc(20) : Buffer.alloc(0); + if (zip64) { + extra.writeUInt16LE(0x0001, 0); + extra.writeUInt16LE(16, 2); + extra.writeBigUInt64LE(BigInt(entry.uncompressedSize), 4); + extra.writeBigUInt64LE(BigInt(entry.compressedSize), 12); + } + const header = Buffer.alloc(30); + header.writeUInt32LE(0x04034b50, 0); + header.writeUInt16LE(zip64 ? 45 : 20, 4); // version needed to extract + header.writeUInt16LE(entry.generalPurposeBitFlag, 6); + header.writeUInt16LE(entry.compressionMethod, 8); + header.writeUInt16LE(entry.lastModFileTime, 10); + header.writeUInt16LE(entry.lastModFileDate, 12); + header.writeUInt32LE(entry.crc32, 14); + header.writeUInt32LE(zip64 ? zip64Limit : entry.compressedSize, 18); + header.writeUInt32LE(zip64 ? zip64Limit : entry.uncompressedSize, 22); + header.writeUInt16LE(entry.fileName.length, 26); + header.writeUInt16LE(extra.length, 28); + return Buffer.concat([header, entry.fileName, extra]); + } + + private centralDirectoryHeader(entry: WriterEntry) { + const zip64 = this.forceZip64 + || entry.compressedSize >= zip64Limit + || entry.uncompressedSize >= zip64Limit + || entry.offset >= zip64Limit; + const extra = zip64 ? Buffer.alloc(28) : Buffer.alloc(0); + if (zip64) { + extra.writeUInt16LE(0x0001, 0); + extra.writeUInt16LE(24, 2); + extra.writeBigUInt64LE(BigInt(entry.uncompressedSize), 4); + extra.writeBigUInt64LE(BigInt(entry.compressedSize), 12); + extra.writeBigUInt64LE(BigInt(entry.offset), 20); + } + const header = Buffer.alloc(46); + header.writeUInt32LE(0x02014b50, 0); + header.writeUInt16LE(entry.versionMadeBy, 4); + header.writeUInt16LE(zip64 ? 45 : 20, 6); // version needed to extract + header.writeUInt16LE(entry.generalPurposeBitFlag, 8); + header.writeUInt16LE(entry.compressionMethod, 10); + header.writeUInt16LE(entry.lastModFileTime, 12); + header.writeUInt16LE(entry.lastModFileDate, 14); + header.writeUInt32LE(entry.crc32, 16); + header.writeUInt32LE(zip64 ? zip64Limit : entry.compressedSize, 20); + header.writeUInt32LE(zip64 ? zip64Limit : entry.uncompressedSize, 24); + header.writeUInt16LE(entry.fileName.length, 28); + header.writeUInt16LE(extra.length, 30); + header.writeUInt16LE(entry.fileComment.length, 32); + header.writeUInt16LE(0, 34); // disk number start + header.writeUInt16LE(0, 36); // internal file attributes + header.writeUInt32LE(entry.externalFileAttributes, 38); + header.writeUInt32LE(zip64 ? zip64Limit : entry.offset, 42); + return Buffer.concat([header, entry.fileName, extra, entry.fileComment]); + } + + private async write(buffer: Buffer) { + if (this.outputStream.destroyed) { + throw this.outputStream.errored ?? new Error("Zip output stream was closed"); + } + if (!this.outputStream.write(buffer)) { + // once() rejects if the stream errors, close covers a plain destroy. + const controller = new AbortController(); + const { signal } = controller; + try { + await Promise.race([ + events.once(this.outputStream, "drain", { signal }), + events.once(this.outputStream, "close", { signal }).then(() => { + throw this.outputStream.errored ?? new Error("Zip output stream was closed"); + }), + ]); + } finally { + controller.abort(); + } + } + this.offset += buffer.length; + } +} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 301cf5d0..9bc1b526 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -17,11 +17,12 @@ catalog: '@types/set-blocking': ^2.0.2 '@types/ws': ^8.18.1 '@types/yargs': ^17.0.35 + '@types/yauzl': ^3.4.0 + '@types/yazl': ^3.3.1 antd: ^5.29.3 chalk: ^6.0.0 express: ^5.2.1 jsonwebtoken: ^9.0.3 - jszip: ^3.10.2 react: ^18.2.0 react-dom: ^18.2.0 react-markdown: ^10.1.0 @@ -37,6 +38,8 @@ catalog: winston-transport: ^4.9.0 ws: ^8.21.3 yargs: ^18.1.0 + yauzl: ^3.4.0 + yazl: ^3.3.1 catalogMode: strict diff --git a/test/common.js b/test/common.js index 12bf426e..c3244f72 100644 --- a/test/common.js +++ b/test/common.js @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; -import { compile } from "@clusterio/lib"; +import yazl from "yazl"; +import { compile, zipOutputStream } from "@clusterio/lib"; /** * Generate a flat array of tests from a matrix of inputs. @@ -55,3 +56,37 @@ export function externalTest(test) { } test.timeout(60000); } + +/** + * Create a zip file in memory + * + * @param {Iterable<[string, string | Buffer]>} files - + * Paths and content of the files to add, paths ending with / are directories. + * @returns {yazl.ZipFile} zip file with the output stream ready to be read. + */ +export function createZip(files) { + const zip = new yazl.ZipFile(); + for (const [filePath, content] of files) { + if (filePath.endsWith("/")) { + zip.addEmptyDirectory(filePath); + } else { + zip.addBuffer(Buffer.from(content), filePath); + } + } + zip.end(); + return zip; +} + +/** + * Create a zip file in memory and return its content + * + * @param {Iterable<[string, string | Buffer]>} files - Paths and content of the files to add. + * @returns {Promise} content of the zip file. + */ +export async function createZipBuffer(files) { + const chunks = []; + for await (const chunk of zipOutputStream(createZip(files))) { + chunks.push(chunk); + } + return Buffer.concat(chunks); +} diff --git a/test/host/create_scenario.js b/test/host/create_scenario.js index 2324c198..de032da7 100644 --- a/test/host/create_scenario.js +++ b/test/host/create_scenario.js @@ -1,6 +1,5 @@ import assert from "node:assert/strict"; import fs from "node:fs/promises"; -import JSZip from "jszip"; import path from "node:path"; import * as lib from "@clusterio/lib"; @@ -8,6 +7,7 @@ import { createScenario, createScenarioCommand, handleCreateScenarioCommand, } from "@clusterio/host/dist/node/src/create_scenario.js"; import { SaveModule, PatchInfo } from "@clusterio/host/dist/node/src/patch.js"; +import { createZipBuffer } from "../common.js"; describe("host/create_scenario", function() { @@ -48,14 +48,15 @@ describe("host/create_scenario", function() { before(async function() { await fs.rm(tempDir, { recursive: true, force: true }); await fs.mkdir(path.join(tempDir, "scenario"), { recursive: true }); - const zip = new JSZip(); for (const [name, content] of scenarioFiles) { const fsPath = path.join(tempDir, "scenario", name); await fs.mkdir(path.dirname(fsPath), { recursive: true }); await fs.writeFile(fsPath, content); - zip.file(`scenario/${name}`, content); } - await fs.writeFile(path.join(tempDir, "scenario.zip"), await zip.generateAsync({ type: "nodebuffer" })); + await fs.writeFile( + path.join(tempDir, "scenario.zip"), + await createZipBuffer([...scenarioFiles].map(([name, content]) => [`scenario/${name}`, content])), + ); }); it("should patch a scenario directory", async function() { @@ -73,12 +74,16 @@ describe("host/create_scenario", function() { it("should write a zip file when asked to", async function() { const output = path.join(tempDir, "zipped.zip"); await createScenario(path.join(tempDir, "scenario"), output, [module], true); - const zip = await JSZip.loadAsync(await fs.readFile(output)); + const zip = await lib.ZipArchive.fromFile(output); const files = new Map(); - for (const file of Object.values(zip.files)) { - if (!file.dir) { - files.set(file.name.replace(/^scenario\//, ""), await file.async("string")); + try { + for (const entry of zip.entries.values()) { + if (!entry.fileName.endsWith("/")) { + files.set(entry.fileName.replace(/^scenario\//, ""), (await zip.readEntry(entry)).toString()); + } } + } finally { + zip.close(); } checkOutput(files); }); @@ -105,6 +110,24 @@ describe("host/create_scenario", function() { ); }); + it("should throw if a scenario file is unreadable", async function() { + if (process.platform === "win32" || process.getuid?.() === 0) { + this.skip(); + } + const scenario = path.join(tempDir, "unreadable"); + await fs.cp(path.join(tempDir, "scenario"), scenario, { recursive: true }); + await fs.writeFile(path.join(scenario, "unreadable.lua"), ""); + await fs.chmod(path.join(scenario, "unreadable.lua"), 0o000); + for (let i = 0; i < 20; i++) { + await fs.mkdir(path.join(scenario, `dir${i}`)); + await fs.writeFile(path.join(scenario, `dir${i}`, "file.lua"), ""); + } + await assert.rejects( + createScenario(scenario, path.join(tempDir, "from_unreadable"), [module]), + { code: "EACCES" }, + ); + }); + describe("createScenarioCommand()", function() { it("should define the output positional and options", function() { const positionals = []; diff --git a/test/host/export.js b/test/host/export.js index 6c551b7c..48c9eaea 100644 --- a/test/host/export.js +++ b/test/host/export.js @@ -15,14 +15,14 @@ describe("host/src/export", function() { describe("exportLocale()", function() { it("returns a flattened mapping with locale definitions", async function() { - let locale = await _exportLocale(testServer, new Map(), ["base"], "en"); + let locale = await _exportLocale(testServer, new Map(), new Map(), ["base"], "en"); assert.deepEqual(locale, new Map([ ["test.core-a", "1"], ["test.core-b", "2"], ["test.base-a", "1"], ["test.base-b", "2"], ])); }); it("returns a filtered locale definitions based on mod order", async function() { - let locale = await _exportLocale(testServer, new Map(), [], "en"); + let locale = await _exportLocale(testServer, new Map(), new Map(), [], "en"); assert.deepEqual(locale, new Map([ ["test.core-a", "1"], ["test.core-b", "2"], ])); @@ -34,7 +34,7 @@ describe("host/src/export", function() { describe("loadSimpleIcon()", function() { it("defaults icon_size to 64", async function() { const icon = await _loadSimpleIcon( - testServer, new Map(), { icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map() + testServer, new Map(), new Map(), { icon: "__base__/graphics/icons/test-icon.png" }, 32, new Map() ); assert.equal(icon.bitmap.width, 32); assert.equal(icon.bitmap.height, 32); @@ -44,7 +44,8 @@ describe("host/src/export", function() { describe("loadLayeredIcon()", function() { it("defaults icon_size to 64", async function() { const icon = await _loadLayeredIcon( - testServer, new Map(), { icons: [{ icon: "__base__/graphics/icons/test-icon.png" }] }, 32, new Map() + testServer, new Map(), new Map(), + { icons: [{ icon: "__base__/graphics/icons/test-icon.png" }] }, 32, new Map(), ); assert.equal(icon.bitmap.width, 32); assert.equal(icon.bitmap.height, 32); diff --git a/test/host/patch.js b/test/host/patch.js index 8262e5b6..1bebefca 100644 --- a/test/host/patch.js +++ b/test/host/patch.js @@ -1,10 +1,10 @@ import assert from "node:assert/strict"; import fs from "node:fs/promises"; -import JSZip from "jszip"; import path from "node:path"; import * as lib from "@clusterio/lib"; import * as patch from "@clusterio/host/dist/node/src/patch.js"; +import { createZipBuffer } from "../common.js"; describe("host/patch", function() { @@ -262,10 +262,8 @@ describe("host/patch", function() { describe("patch()", function() { it("should throw on unknown scenario", async function() { - let zip = new JSZip(); - zip.file("world/control.lua", "-- unknown\n"); let zipPath = path.join("temp", "test", "patch.zip"); - await fs.writeFile(zipPath, await zip.generateAsync({ type: "nodebuffer" })); + await fs.writeFile(zipPath, await createZipBuffer([["world/control.lua", "-- unknown\n"]])); await assert.rejects( patch.patch(zipPath, []), diff --git a/test/integration/export.js b/test/integration/export.js index 6555801a..7e6b0d45 100644 --- a/test/integration/export.js +++ b/test/integration/export.js @@ -15,7 +15,7 @@ describe("Integration of lib/factorio/export", function() { let testServer = new FactorioServer(factorioDir, writePath, {}); await testServer.init(); - let locale = await _exportLocale(testServer, new Map(), ["base"], "en"); + let locale = await _exportLocale(testServer, new Map(), new Map(), ["base"], "en"); assert(locale instanceof Map, "locale is not a map"); assert.equal(locale.get("entity-name.fish"), "Fish"); diff --git a/test/integration/patch.js b/test/integration/patch.js index 8f04302a..8b818777 100644 --- a/test/integration/patch.js +++ b/test/integration/patch.js @@ -1,6 +1,4 @@ import assert from "node:assert/strict"; -import fs from "node:fs/promises"; -import JSZip from "jszip"; import path from "node:path"; import { patch, SaveModule } from "@clusterio/host/dist/node/src/patch.js"; @@ -23,23 +21,31 @@ describe("Integration of lib/factorio/patch", function() { await subdirModule.loadFiles("test/file/modules/subdir"); await patch(savePath, [testModule, subdirModule]); - let zip = await JSZip.loadAsync(await fs.readFile(savePath)); - assert.equal(await zip.file("test/modules/test/test.lua").async("string"), "-- test\n"); - assert.equal(await zip.file("test/modules/subdir/dir/test.lua").async("string"), "-- test\n"); - assert.equal( - await zip.file("test/locale/en/test.cfg").async("string"), - "module-test=A Test\n" - ); - assert.equal( - await zip.file("test/locale/en/test-locale.cfg").async("string"), - "module-test-locale=Test Locale\n" - ); + let zip = await lib.ZipArchive.fromFile(savePath); + try { + assert.equal((await zip.readFile("test/modules/test/test.lua")).toString(), "-- test\n"); + assert.equal((await zip.readFile("test/modules/subdir/dir/test.lua")).toString(), "-- test\n"); + assert.equal( + (await zip.readFile("test/locale/en/test.cfg")).toString(), + "module-test=A Test\n" + ); + assert.equal( + (await zip.readFile("test/locale/en/test-locale.cfg")).toString(), + "module-test-locale=Test Locale\n" + ); + } finally { + zip.close(); + } }); it("should remove old modules in a save", async function() { slowTest(this); await patch(savePath, []); - let zip = await JSZip.loadAsync(await fs.readFile(savePath)); - assert.equal(zip.file("test/modules/test/test.lua"), null); + let zip = await lib.ZipArchive.fromFile(savePath); + try { + assert.equal(zip.file("test/modules/test/test.lua"), null); + } finally { + zip.close(); + } }); }); }); diff --git a/test/lib/ModStore.test.js b/test/lib/ModStore.test.js index 4c7d1b0b..309f3117 100644 --- a/test/lib/ModStore.test.js +++ b/test/lib/ModStore.test.js @@ -2,20 +2,18 @@ import assert from "node:assert/strict"; import path from "node:path"; import fs from "node:fs/promises"; import stream from "node:stream"; -import JSZip from "jszip"; // Added for creating mock zips // Capture native fetch before any potential mocks are applied const nativeFetch = global.fetch; -import { ModStore, ModInfo, ModVersionEquality } from "@clusterio/lib"; // Adjust path based on compiled output -import { externalTest } from "../common.js"; +import { ModStore, ModInfo, ModVersionEquality, zipOutputStream } from "@clusterio/lib"; +import { externalTest, createZip, createZipBuffer } from "../common.js"; import "../setup_logging.js"; const MODS_DIR = path.join("temp", "test", "mod_store", "mods"); const CACHE_FILE = path.join(MODS_DIR, "mod-info-cache.json"); -function createMockModJSZip(name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) { - const zip = new JSZip(); +function mockModFiles(name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) { const infoJson = { name: name, version: version, @@ -25,16 +23,15 @@ function createMockModJSZip(name, version, factorioVersion = "1.1", dependencies dependencies: dependencies, description: `Description for ${name}`, }; - zip.file(`${name}/info.json`, JSON.stringify(infoJson, null, 4)); - zip.file(`${name}/dummy.lua`, "-- Mock Lua file"); - - return zip; + return [ + [`${name}/info.json`, JSON.stringify(infoJson, null, 4)], + [`${name}/dummy.lua`, "-- Mock Lua file"], + ]; } // Helper function to create a mock mod zip file async function createMockModZip(filePath, name, version, factorioVersion = "1.1", dependencies = ["base >= 1.1"]) { - const zip = createMockModJSZip(name, version, factorioVersion, dependencies); - const buffer = await zip.generateAsync({ type: "nodebuffer", compression: "DEFLATE" }); + const buffer = await createZipBuffer(mockModFiles(name, version, factorioVersion, dependencies)); await fs.writeFile(filePath, buffer); } @@ -254,8 +251,8 @@ describe("lib/ModStore", function () { if (name === "bad-download-mod") { return { ok: false, status: 503, statusText: "Test Mock download error" }; } - const zip = createMockModJSZip(name, version); - return { ok: true, status: 200, body: stream.Readable.toWeb(zip.generateNodeStream()) }; + const zip = createZip(mockModFiles(name, version)); + return { ok: true, status: 200, body: stream.Readable.toWeb(zipOutputStream(zip)) }; } // Fallback for unhandled fetches in this test return { ok: false, status: 404, statusText: `Test Mock 404: ${urlString}` }; diff --git a/test/lib/build_mod.js b/test/lib/build_mod.js new file mode 100644 index 00000000..709d579b --- /dev/null +++ b/test/lib/build_mod.js @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; + +import * as libBuildMod from "@clusterio/lib/build_mod.js"; + + +describe("lib/build_mod", function() { + describe("build()", function() { + const tempDir = path.join("temp", "test", "build_mod"); + const sourceDir = path.join(tempDir, "unreadable_mod"); + const outputDir = path.join(tempDir, "output"); + before(async function() { + if (process.platform === "win32" || process.getuid?.() === 0) { + this.skip(); + } + await fs.rm(tempDir, { recursive: true, force: true }); + await fs.mkdir(sourceDir, { recursive: true }); + await fs.writeFile( + path.join(sourceDir, "info.json"), + JSON.stringify({ name: "unreadable_mod", version: "1.0.0" }), + ); + await fs.writeFile(path.join(sourceDir, "unreadable.lua"), ""); + await fs.chmod(path.join(sourceDir, "unreadable.lua"), 0o000); + for (let i = 0; i < 20; i++) { + await fs.mkdir(path.join(sourceDir, `dir${i}`)); + await fs.writeFile(path.join(sourceDir, `dir${i}`, "file.lua"), ""); + } + }); + after(async function() { + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it("should reject when a source file is unreadable", async function() { + await assert.rejects( + libBuildMod.build({ build: true, pack: true, sourceDir, outputDir }), + { code: "EACCES" }, + ); + assert.deepEqual(await fs.readdir(outputDir), []); + }); + }); +}); diff --git a/test/lib/zip_ops.js b/test/lib/zip_ops.js index 99739d82..6e94df44 100644 --- a/test/lib/zip_ops.js +++ b/test/lib/zip_ops.js @@ -1,41 +1,132 @@ import assert from "node:assert/strict"; -import JSZip from "jszip"; +import zlib from "node:zlib"; -import { findRoot } from "@clusterio/lib"; +import { ZipArchive, ZipWriter, findRoot } from "@clusterio/lib"; +import { createZipBuffer } from "../common.js"; -describe("lib/factorio", function() { +async function readAll(stream) { + const chunks = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + return Buffer.concat(chunks); +} + + +describe("lib/zip_ops", function() { + describe("class ZipArchive", function() { + it("should read files from a zip", async function() { + const zip = await ZipArchive.fromBuffer(await createZipBuffer([ + ["root/foo.txt", "foo"], + ["root/bar.txt", "bar"], + ])); + assert.deepEqual([...zip.entries.keys()], ["root/foo.txt", "root/bar.txt"]); + assert.equal((await zip.readFile("root/foo.txt")).toString(), "foo"); + assert.equal(await zip.readFile("root/missing.txt"), null); + assert.equal(zip.file("root/missing.txt"), null); + assert.equal(zip.file("root/bar.txt").fileName, "root/bar.txt"); + }); + it("should reject invalid zips", async function() { + await assert.rejects(ZipArchive.fromBuffer(Buffer.from("not a zip file"))); + }); + }); + + describe("class ZipWriter", function() { + const sourceFiles = [ + ["root/", ""], + ["root/foo.txt", "foo"], + ["root/big.txt", "lorem ipsum ".repeat(10000)], + ["root/ünïcode.txt", "unicode"], + ]; + + async function roundTrip(forceZip64) { + const source = await ZipArchive.fromBuffer(await createZipBuffer(sourceFiles)); + const writer = new ZipWriter({ forceZip64 }); + const output = readAll(writer.outputStream); + for (const entry of source.entries.values()) { + await writer.addEntry(source, entry); + } + await writer.addBuffer(Buffer.from("added"), "root/added.txt", { mtime: new Date(2020, 0, 2, 3, 4, 6) }); + await writer.end(); + return { source, copy: await ZipArchive.fromBuffer(await output) }; + } + + async function assertRoundTrip(forceZip64) { + const { source, copy } = await roundTrip(forceZip64); + assert.deepEqual([...copy.entries.keys()], [...source.entries.keys(), "root/added.txt"]); + for (const [name, sourceEntry] of source.entries) { + const entry = copy.entries.get(name); + assert.equal(entry.compressionMethod, sourceEntry.compressionMethod, name); + assert.equal(entry.compressedSize, sourceEntry.compressedSize, name); + assert.equal(entry.uncompressedSize, sourceEntry.uncompressedSize, name); + assert.equal(entry.crc32, sourceEntry.crc32, name); + assert.deepEqual(entry.getLastModDate(), sourceEntry.getLastModDate({ forceDosFormat: true }), name); + if (!name.endsWith("/")) { + const content = await copy.readEntry(entry); + assert.deepEqual(content, await source.readEntry(sourceEntry), name); + assert.equal(zlib.crc32(content), entry.crc32, name); + } + } + const added = copy.entries.get("root/added.txt"); + assert.equal((await copy.readEntry(added)).toString(), "added"); + assert.equal(zlib.crc32("added"), added.crc32); + assert.deepEqual(added.getLastModDate(), new Date(2020, 0, 2, 3, 4, 6)); + assert.equal(added.versionNeededToExtract, forceZip64 ? 45 : 20); + } + + it("should copy entries and add files", async function() { + await assertRoundTrip(false); + }); + it("should write ZIP64 structures when forced", async function() { + await assertRoundTrip(true); + }); + it("should reject writes after the output stream is destroyed", async function() { + const writer = new ZipWriter(); + writer.outputStream.on("error", () => {}); + writer.outputStream.destroy(new Error("Test error")); + await assert.rejects(writer.addBuffer(Buffer.from(""), "foo.txt"), new Error("Test error")); + await assert.rejects(writer.end(), new Error("Test error")); + }); + it("should fail when the output stream is destroyed while waiting for drain", async function() { + const writer = new ZipWriter(); + writer.outputStream.pause(); + const adding = writer.addBuffer(Buffer.alloc(1024 * 1024), "foo.txt"); + writer.outputStream.destroy(); + await assert.rejects(adding, new Error("Zip output stream was closed")); + }); + }); + describe("findRoot()", function() { - it("should find the root of a zip", function() { - let zip = new JSZip(); - zip.file("root/foo.txt", ""); - zip.file("root/bar.txt", ""); - + it("should find the root of a zip", async function() { + const zip = await ZipArchive.fromBuffer(await createZipBuffer([ + ["root/foo.txt", ""], + ["root/bar.txt", ""], + ])); assert.equal(findRoot(zip), "root"); }); - it("should throw if the first file is at the root of the zip", function() { - let zip = new JSZip(); - zip.file("bar.txt", ""); - zip.file("root/foo.txt", ""); - + it("should throw if the first file is at the root of the zip", async function() { + const zip = await ZipArchive.fromBuffer(await createZipBuffer([ + ["bar.txt", ""], + ["root/foo.txt", ""], + ])); assert.throws( () => findRoot(zip), new Error("Zip contains file 'bar.txt' in root dir") ); }); - it("should return directory of first file if there are multiple root dirs", function() { - let zip = new JSZip(); - zip.file("root-1/foo.txt", ""); - zip.file("root-2/bar.txt", ""); - + it("should return directory of first file if there are multiple root dirs", async function() { + const zip = await ZipArchive.fromBuffer(await createZipBuffer([ + ["root-1/foo.txt", ""], + ["root-2/bar.txt", ""], + ])); assert.equal(findRoot(zip), "root-1"); }); - it("should throw if given an empty zip file", function() { - let zip = new JSZip(); - + it("should throw if given an empty zip file", async function() { + const zip = await ZipArchive.fromBuffer(await createZipBuffer([])); assert.throws( () => findRoot(zip), new Error("Empty zip file")