mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-14 15:24:52 +09:00
* Replace jszip with yauzl and yazl jszip loads a whole archive into memory to read it and buffers the output when writing. Saves and mods can be several hundred megabytes, so read zips with yauzl, which only reads the central directory and streams entries on demand, and write them with yazl, which streams the output. lib gains a ZipArchive wrapper around yauzl with the lookup helpers the callers need, findRoot() now takes it instead of a JSZip. Saves are patched by streaming every kept entry into a new zip, mod info.json is read straight from the file, exports are streamed to the controller and Factorio zip downloads go through a temp file instead of a buffer. Patching now inflates and deflates each entry again where jszip reused the compressed data, on a 336 MB synthetic save that is 7 s instead of 0.7 s, with peak memory going from 416 MB to 151 MB. Closes #1002 Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Copy save entries without recompressing them yazl inflates and deflates every entry it is given, so patching a save took ten times longer than with jszip, which reused the compressed data. Add a ZipWriter to lib that streams entries out as they are added and copies entries from a ZipArchive byte for byte, with ZIP64 structures when sizes or offsets need them, and use it in patch(). On a 336 MB synthetic save patching now takes 0.4 s at 143 MB peak RSS against 0.75 s at 417 MB with jszip. Factorio loads the result. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Handle save write errors before the patch loop awaits them A write error on the temporary save, such as a full disk, rejected the pipeline promise while addEntry() or addBuffer() was still pending. Nothing was listening yet, so the host's unhandledRejection handler exited the process instead of failing the instance start. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Keep the export zip cache local to each export The cache was module level, so when two instances on one host exported at once, the first to finish closed archives the other was still reading. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Start writing zips before adding files from disk yazl reads files passed to addFile() as soon as they are added. An unreadable file emitted an error while the directory traversal was still running, before the output pipeline existed, and the process exited on the uncaught error. build_mod and zipDirectory now start the pipeline first and reject instead. build_mod also removes the partial zip on failure. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
190 lines
5.6 KiB
TypeScript
190 lines
5.6 KiB
TypeScript
// Implementation of the create-scenario command
|
|
import fs from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "path";
|
|
import { pipeline } from "node:stream/promises";
|
|
import yazl from "yazl";
|
|
|
|
import * as lib from "@clusterio/lib";
|
|
import { logger } from "@clusterio/lib";
|
|
|
|
import { loadModules, patch, SaveModule } from "./patch.js";
|
|
import { findVersion } from "./server.js";
|
|
|
|
|
|
/**
|
|
* Create a scenario patched with the given modules
|
|
*
|
|
* Runs the save patching on the scenario at scenarioPath and writes the
|
|
* result to outputPath. The scenario can be a directory, a zipped
|
|
* scenario or a save.
|
|
*
|
|
* @param scenarioPath - Path to the scenario to patch.
|
|
* @param outputPath - Path to write the patched scenario to.
|
|
* @param modules - Description of the modules to patch.
|
|
* @param zip - Write a zip file instead of a directory.
|
|
*/
|
|
export async function createScenario(
|
|
scenarioPath: string,
|
|
outputPath: string,
|
|
modules: SaveModule[],
|
|
zip = false,
|
|
) {
|
|
try {
|
|
await fs.stat(outputPath);
|
|
throw new Error(`${outputPath} already exists`);
|
|
} catch (err: any) {
|
|
if (err.code !== "ENOENT") {
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-scenario-"));
|
|
try {
|
|
const tempPath = path.join(tempDir, "scenario.zip");
|
|
if ((await fs.stat(scenarioPath)).isDirectory()) {
|
|
await zipDirectory(scenarioPath, tempPath);
|
|
} else {
|
|
await fs.copyFile(scenarioPath, tempPath);
|
|
}
|
|
|
|
await patch(tempPath, modules);
|
|
if (zip) {
|
|
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
await fs.copyFile(tempPath, outputPath);
|
|
} else {
|
|
await extractZip(tempPath, outputPath);
|
|
}
|
|
} finally {
|
|
await fs.rm(tempDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function zipDirectory(dirPath: string, zipPath: string) {
|
|
const root = path.basename(path.resolve(dirPath));
|
|
const zip = new yazl.ZipFile();
|
|
const output = lib.zipOutputStream(zip);
|
|
const fd = await fs.open(zipPath, "w");
|
|
// addFile() can fail before the traversal is done, so start writing first.
|
|
const writing = pipeline(output, fd.createWriteStream());
|
|
writing.catch(() => {});
|
|
try {
|
|
let dirs: [string, string][] = [[dirPath, root]];
|
|
while (dirs.length) {
|
|
let [dir, relativeDir] = dirs.pop()!;
|
|
for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
|
|
let fsPath = path.join(dir, entry.name);
|
|
let relativePath = path.posix.join(relativeDir, entry.name);
|
|
if (entry.isFile()) {
|
|
zip.addFile(fsPath, relativePath);
|
|
} else if (entry.isDirectory()) {
|
|
dirs.push([fsPath, relativePath]);
|
|
}
|
|
}
|
|
}
|
|
zip.end();
|
|
} catch (err) {
|
|
output.destroy();
|
|
await writing.catch(() => {});
|
|
throw err;
|
|
}
|
|
await writing;
|
|
}
|
|
|
|
async function extractZip(zipPath: string, dirPath: string) {
|
|
const zip = await lib.ZipArchive.fromFile(zipPath);
|
|
try {
|
|
const root = lib.findRoot(zip);
|
|
for (let entry of zip.entries.values()) {
|
|
if (entry.fileName.endsWith("/") || !entry.fileName.startsWith(`${root}/`)) {
|
|
continue;
|
|
}
|
|
const fsPath = path.join(dirPath, entry.fileName.slice(root.length + 1));
|
|
await fs.mkdir(path.dirname(fsPath), { recursive: true });
|
|
const fd = await fs.open(fsPath, "w");
|
|
await pipeline(await zip.openStream(entry), fd.createWriteStream());
|
|
}
|
|
} finally {
|
|
zip.close();
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Yargs create-scenario command
|
|
*
|
|
* @param yargs - yargs command builder.
|
|
*/
|
|
export function createScenarioCommand(yargs: any) {
|
|
yargs
|
|
.positional("output", {
|
|
describe: "Path to write the scenario to",
|
|
type: "string",
|
|
})
|
|
.option("zip", {
|
|
type: "boolean", nargs: 0, default: false,
|
|
describe: "Write the scenario as a zip file instead of a directory",
|
|
})
|
|
.option("scenario", {
|
|
nargs: 1,
|
|
describe:
|
|
"Path to a scenario directory, zipped scenario or save to patch. " +
|
|
"Defaults to the freeplay scenario of the Factorio install in host.factorio_directory",
|
|
type: "string",
|
|
})
|
|
.option("factorio-version", {
|
|
nargs: 1,
|
|
describe: "Factorio version to take the freeplay scenario from",
|
|
default: "latest",
|
|
type: "string",
|
|
})
|
|
.option("plugins", {
|
|
describe: "Plugins to patch modules from, defaults to all plugins enabled in the host config",
|
|
type: "array",
|
|
})
|
|
;
|
|
}
|
|
|
|
/**
|
|
* Handle yargs create-scenario command
|
|
*
|
|
* @param args - yargs args object.
|
|
* @param hostConfig - Host config.
|
|
* @param pluginInfos - Plugins available to the host.
|
|
*/
|
|
export async function handleCreateScenarioCommand(
|
|
args: Record<string, unknown>,
|
|
hostConfig: lib.HostConfig,
|
|
pluginInfos: lib.PluginNodeEnvInfo[],
|
|
) {
|
|
let selected: lib.PluginNodeEnvInfo[];
|
|
if (args.plugins) {
|
|
selected = (args.plugins as string[]).map(name => {
|
|
const pluginInfo = pluginInfos.find(info => info.name === name);
|
|
if (!pluginInfo) {
|
|
throw new Error(`Plugin ${name} is not in the plugin list`);
|
|
}
|
|
return pluginInfo;
|
|
});
|
|
} else {
|
|
selected = pluginInfos.filter(
|
|
info => info.instanceEntrypoint && hostConfig.get(`${info.name}.load_plugin`)
|
|
);
|
|
}
|
|
|
|
let scenarioPath = args.scenario as string | undefined;
|
|
if (scenarioPath === undefined) {
|
|
const factorioDir = hostConfig.get("host.factorio_directory");
|
|
const version = args.factorioVersion as lib.TargetVersion;
|
|
if (!lib.isTargetVersion(version)) {
|
|
throw new Error(`Invalid Factorio version ${version}`);
|
|
}
|
|
const [dataDir] = await findVersion(factorioDir, version);
|
|
scenarioPath = path.join(dataDir, "base", "scenarios", "freeplay");
|
|
}
|
|
|
|
const modules = await loadModules(selected);
|
|
logger.info(`Patching ${scenarioPath} with modules ${[...modules.keys()].join(", ")}`);
|
|
await createScenario(scenarioPath, args.output as string, [...modules.values()], Boolean(args.zip));
|
|
logger.info(`Created scenario ${args.output}`);
|
|
}
|