Files
factorio-clusterio/packages/lib/src/zip_ops.ts
T
2026-11-20 16:11:36 +00:00

438 lines
14 KiB
TypeScript

/**
* Zip file helpers
* @module lib/zip_ops
*/
import events from "events";
import { PassThrough, type Readable } from "stream";
import util from "util";
import zlib from "zlib";
import yauzl from "yauzl";
const deflateRaw = util.promisify(zlib.deflateRaw);
/**
* Entry in a ZipArchive
*
* Local subset of yauzl.Entry so the published declarations do not import
* yauzl, which would make @types/yauzl a runtime dependency of lib.
*/
export interface ZipEntry {
fileName: string;
fileComment: string;
versionMadeBy: number;
versionNeededToExtract: number;
generalPurposeBitFlag: number;
compressionMethod: number;
lastModFileTime: number;
lastModFileDate: number;
crc32: number;
compressedSize: number;
uncompressedSize: number;
externalFileAttributes: number;
getLastModDate(options?: { timezone?: "local" | "UTC" | null, forceDosFormat?: boolean }): Date;
}
/**
* Zip archive opened for random access reading
*
* Only the central directory is read up front, the content of entries is
* streamed from the archive on demand. Archives opened from a file hold
* the file open until close() is called.
*/
export class ZipArchive {
private constructor(
private zipfile: yauzl.ZipFile,
/** Entries by file name in the order they appear in the central directory. */
public entries: Map<string, ZipEntry>,
) { }
/**
* Open a zip archive from the file system
*
* @param zipPath - Path to the zip file to open.
*/
static async fromFile(zipPath: string) {
return this.load(await yauzl.openPromise(zipPath, { autoClose: false }));
}
/**
* Open a zip archive held in memory
*
* @param buffer - Content of the zip file.
*/
static async fromBuffer(buffer: Buffer) {
return this.load(await yauzl.fromBufferPromise(buffer));
}
private static async load(zipfile: yauzl.ZipFile) {
const entries = new Map<string, yauzl.Entry>();
try {
for await (const entry of zipfile.eachEntry()) {
entries.set(entry.fileName, entry);
}
} catch (err) {
zipfile.close();
throw err;
}
return new this(zipfile, entries);
}
/**
* Look up a file in the archive
*
* @param filePath - Path of the file in the archive.
* @returns entry for the file, or null if it does not exist.
*/
file(filePath: string) {
const entry = this.entries.get(filePath);
if (!entry || entry.fileName.endsWith("/")) {
return null;
}
return entry;
}
/**
* Open a stream of the decompressed content of an entry
*
* @param entry - Entry to read, must belong to this archive.
*/
openStream(entry: ZipEntry): Promise<Readable> {
return this.zipfile.openReadStreamPromise(entry as yauzl.Entry);
}
/**
* Open a stream of the content of an entry as stored in the archive
*
* @param entry - Entry to read, must belong to this archive.
*/
openRawStream(entry: ZipEntry): Promise<Readable> {
return this.zipfile.openReadStreamPromise(entry as yauzl.Entry, { decodeFileData: false });
}
/**
* Read the decompressed content of an entry
*
* @param entry - Entry to read, must belong to this archive.
*/
async readEntry(entry: ZipEntry) {
const chunks: Buffer[] = [];
for await (const chunk of await this.openStream(entry)) {
chunks.push(chunk);
}
return Buffer.concat(chunks);
}
/**
* Read the content of a file in the archive
*
* @param filePath - Path of the file in the archive.
* @returns content of the file, or null if it does not exist.
*/
async readFile(filePath: string) {
const entry = this.file(filePath);
if (!entry) {
return null;
}
return await this.readEntry(entry);
}
/**
* Close the archive
*
* Streams already opened are still readable until they end.
*/
close() {
this.zipfile.close();
}
}
/**
* Returns the root folder in the zip file
*
* Returns the name of the folder that the first entry in the zip file is
* contained in. Throws an error if this is not in a folder.
*
* This matches the logic Factorio uses when determining the folder to look
* for content in a zip file.
*
* @param zip - Zip to search through.
* @returns name of the root folder.
*/
export function findRoot(zip: ZipArchive) {
const relativePath = zip.entries.keys().next().value;
if (relativePath === undefined) {
throw new Error("Empty zip file");
}
let index = relativePath.indexOf("/");
if (index === -1) {
throw new Error(`Zip contains file '${relativePath}' in root dir`);
}
return relativePath.slice(0, index);
}
/**
* Returns the output stream of a zip file being written
*
* yazl reports errors on the ZipFile object and leaves the output stream
* hanging. The stream returned here is destroyed with the error instead so
* that consumers like stream.pipeline see it.
*
* @param zipFile - zip file being written.
*/
export function zipOutputStream(zipFile: {
outputStream: NodeJS.ReadableStream,
on(event: "error", listener: (err: Error) => void): unknown,
}) {
const output = zipFile.outputStream as Readable;
zipFile.on("error", err => output.destroy(err));
return output;
}
const zip64Limit = 0xffffffff;
function dosDateTime(date: Date) {
if (date.getFullYear() < 1980) {
return { date: 0x21, time: 0 };
}
const year = Math.min(date.getFullYear(), 2107);
return {
date: ((year - 1980) << 9) | ((date.getMonth() + 1) << 5) | date.getDate(),
time: (date.getHours() << 11) | (date.getMinutes() << 5) | (date.getSeconds() >> 1),
};
}
interface WriterEntry {
fileName: Buffer;
fileComment: Buffer;
versionMadeBy: number;
generalPurposeBitFlag: number;
compressionMethod: number;
lastModFileTime: number;
lastModFileDate: number;
crc32: number;
compressedSize: number;
uncompressedSize: number;
externalFileAttributes: number;
offset: number;
}
export interface ZipWriterOptions {
/** Use ZIP64 structures even when the archive does not need them. */
forceZip64?: boolean;
}
export interface ZipWriterBufferOptions {
/** Modification time to record for the file, defaults to now. */
mtime?: Date;
}
/**
* Streaming zip file writer
*
* Writes entries to outputStream as they are added. Entries from an
* existing archive are copied without decompressing them. Sizes and
* offsets past 4 GiB and more than 65535 entries use ZIP64 structures.
*/
export class ZipWriter {
/** Content of the zip file, ends when end() has been called. */
outputStream = new PassThrough();
private offset = 0;
private entries: WriterEntry[] = [];
private forceZip64: boolean;
constructor(options: ZipWriterOptions = {}) {
this.forceZip64 = options.forceZip64 ?? false;
}
/**
* Copy an entry from an archive
*
* The compressed content is copied as is along with the metadata
* Factorio writes. Extra fields are dropped.
*
* @param zip - Archive to copy from.
* @param entry - Entry to copy, must belong to zip.
*/
async addEntry(zip: ZipArchive, entry: ZipEntry) {
const writerEntry: WriterEntry = {
fileName: Buffer.from(entry.fileName, "utf8"),
fileComment: Buffer.from(entry.fileComment, "utf8"),
versionMadeBy: entry.versionMadeBy,
// Sizes are written in the local header so no data descriptor is
// needed, and names are re-encoded as UTF-8.
generalPurposeBitFlag: (entry.generalPurposeBitFlag & ~0x8) | 0x800,
compressionMethod: entry.compressionMethod,
lastModFileTime: entry.lastModFileTime,
lastModFileDate: entry.lastModFileDate,
crc32: entry.crc32,
compressedSize: entry.compressedSize,
uncompressedSize: entry.uncompressedSize,
externalFileAttributes: entry.externalFileAttributes,
offset: this.offset,
};
await this.write(this.localFileHeader(writerEntry));
const stream = await zip.openRawStream(entry);
for await (const chunk of stream) {
await this.write(chunk);
}
this.entries.push(writerEntry);
}
/**
* Add a file with the given content
*
* @param buffer - Content of the file.
* @param fileName - Path of the file in the archive.
* @param options - Options for the entry.
*/
async addBuffer(buffer: Buffer, fileName: string, options: ZipWriterBufferOptions = {}) {
const deflated = await deflateRaw(buffer);
const { date, time } = dosDateTime(options.mtime ?? new Date());
const writerEntry: WriterEntry = {
fileName: Buffer.from(fileName, "utf8"),
fileComment: Buffer.alloc(0),
versionMadeBy: 20,
generalPurposeBitFlag: 0x800,
compressionMethod: 8,
lastModFileTime: time,
lastModFileDate: date,
crc32: zlib.crc32(buffer),
compressedSize: deflated.length,
uncompressedSize: buffer.length,
externalFileAttributes: 0,
offset: this.offset,
};
await this.write(this.localFileHeader(writerEntry));
await this.write(deflated);
this.entries.push(writerEntry);
}
/**
* Write the central directory and end the output stream
*/
async end() {
const start = this.offset;
for (const entry of this.entries) {
await this.write(this.centralDirectoryHeader(entry));
}
const size = this.offset - start;
const zip64 = this.forceZip64 || this.entries.length >= 0xffff || start >= zip64Limit || size >= zip64Limit;
if (zip64) {
const record = Buffer.alloc(56);
record.writeUInt32LE(0x06064b50, 0);
record.writeBigUInt64LE(44n, 4);
record.writeUInt16LE(45, 12); // version made by
record.writeUInt16LE(45, 14); // version needed to extract
record.writeUInt32LE(0, 16); // number of this disk
record.writeUInt32LE(0, 20); // disk with the central directory
record.writeBigUInt64LE(BigInt(this.entries.length), 24);
record.writeBigUInt64LE(BigInt(this.entries.length), 32);
record.writeBigUInt64LE(BigInt(size), 40);
record.writeBigUInt64LE(BigInt(start), 48);
const locator = Buffer.alloc(20);
locator.writeUInt32LE(0x07064b50, 0);
locator.writeUInt32LE(0, 4); // disk with the zip64 end of central directory
locator.writeBigUInt64LE(BigInt(this.offset), 8);
locator.writeUInt32LE(1, 16); // total number of disks
await this.write(record);
await this.write(locator);
}
const eocd = Buffer.alloc(22);
eocd.writeUInt32LE(0x06054b50, 0);
eocd.writeUInt16LE(0, 4); // number of this disk
eocd.writeUInt16LE(0, 6); // disk with the central directory
eocd.writeUInt16LE(zip64 ? 0xffff : this.entries.length, 8);
eocd.writeUInt16LE(zip64 ? 0xffff : this.entries.length, 10);
eocd.writeUInt32LE(zip64 ? zip64Limit : size, 12);
eocd.writeUInt32LE(zip64 ? zip64Limit : start, 16);
eocd.writeUInt16LE(0, 20); // comment length
await this.write(eocd);
this.outputStream.end();
}
private localFileHeader(entry: WriterEntry) {
const zip64 = this.forceZip64
|| entry.compressedSize >= zip64Limit
|| entry.uncompressedSize >= zip64Limit;
const extra = zip64 ? Buffer.alloc(20) : Buffer.alloc(0);
if (zip64) {
extra.writeUInt16LE(0x0001, 0);
extra.writeUInt16LE(16, 2);
extra.writeBigUInt64LE(BigInt(entry.uncompressedSize), 4);
extra.writeBigUInt64LE(BigInt(entry.compressedSize), 12);
}
const header = Buffer.alloc(30);
header.writeUInt32LE(0x04034b50, 0);
header.writeUInt16LE(zip64 ? 45 : 20, 4); // version needed to extract
header.writeUInt16LE(entry.generalPurposeBitFlag, 6);
header.writeUInt16LE(entry.compressionMethod, 8);
header.writeUInt16LE(entry.lastModFileTime, 10);
header.writeUInt16LE(entry.lastModFileDate, 12);
header.writeUInt32LE(entry.crc32, 14);
header.writeUInt32LE(zip64 ? zip64Limit : entry.compressedSize, 18);
header.writeUInt32LE(zip64 ? zip64Limit : entry.uncompressedSize, 22);
header.writeUInt16LE(entry.fileName.length, 26);
header.writeUInt16LE(extra.length, 28);
return Buffer.concat([header, entry.fileName, extra]);
}
private centralDirectoryHeader(entry: WriterEntry) {
const zip64 = this.forceZip64
|| entry.compressedSize >= zip64Limit
|| entry.uncompressedSize >= zip64Limit
|| entry.offset >= zip64Limit;
const extra = zip64 ? Buffer.alloc(28) : Buffer.alloc(0);
if (zip64) {
extra.writeUInt16LE(0x0001, 0);
extra.writeUInt16LE(24, 2);
extra.writeBigUInt64LE(BigInt(entry.uncompressedSize), 4);
extra.writeBigUInt64LE(BigInt(entry.compressedSize), 12);
extra.writeBigUInt64LE(BigInt(entry.offset), 20);
}
const header = Buffer.alloc(46);
header.writeUInt32LE(0x02014b50, 0);
header.writeUInt16LE(entry.versionMadeBy, 4);
header.writeUInt16LE(zip64 ? 45 : 20, 6); // version needed to extract
header.writeUInt16LE(entry.generalPurposeBitFlag, 8);
header.writeUInt16LE(entry.compressionMethod, 10);
header.writeUInt16LE(entry.lastModFileTime, 12);
header.writeUInt16LE(entry.lastModFileDate, 14);
header.writeUInt32LE(entry.crc32, 16);
header.writeUInt32LE(zip64 ? zip64Limit : entry.compressedSize, 20);
header.writeUInt32LE(zip64 ? zip64Limit : entry.uncompressedSize, 24);
header.writeUInt16LE(entry.fileName.length, 28);
header.writeUInt16LE(extra.length, 30);
header.writeUInt16LE(entry.fileComment.length, 32);
header.writeUInt16LE(0, 34); // disk number start
header.writeUInt16LE(0, 36); // internal file attributes
header.writeUInt32LE(entry.externalFileAttributes, 38);
header.writeUInt32LE(zip64 ? zip64Limit : entry.offset, 42);
return Buffer.concat([header, entry.fileName, extra, entry.fileComment]);
}
private async write(buffer: Buffer) {
if (this.outputStream.destroyed) {
throw this.outputStream.errored ?? new Error("Zip output stream was closed");
}
if (!this.outputStream.write(buffer)) {
// once() rejects if the stream errors, close covers a plain destroy.
const controller = new AbortController();
const { signal } = controller;
try {
await Promise.race([
events.once(this.outputStream, "drain", { signal }),
events.once(this.outputStream, "close", { signal }).then(() => {
throw this.outputStream.errored ?? new Error("Zip output stream was closed");
}),
]);
} finally {
controller.abort();
}
}
this.offset += buffer.length;
}
}