Refactor module patching logic

Break out the module info and patch info into data classes and pack all
the files for a module into one more easily modified structure.
This commit is contained in:
Hornwitser committed 2023-08-25 16:33:02 +02:00
1 parent b1a6628bfa
commit a1148c7ec9
6 files changed
+339 -206

No files matched your search

+223 -98
View File
@@ -1,44 +1,181 @@
// Library for patching Factorio saves with scenario code.
"use strict";
import events from "events";
import fs from "fs-extra";
import JSZip from "jszip";
import path from "path";
import semver from "semver";
import { Type, Static } from "@sinclair/typebox";
import * as lib from "@clusterio/lib";
interface ScenarioInfo {
name: string;
modules: string[];
/**
* Describes a module that can be patched into a save
*/
export class SaveModule {
constructor(
/** Module */
public info: lib.ModuleInfo,
/** Files and their content that will be patched into the save */
public files = new Map<string, Buffer>,
) { }
async loadFiles(moduleDirectory: string) {
let dirs: [string, string][] = [[moduleDirectory, ""]];
while (dirs.length) {
let [dir, relativeDir] = dirs.pop()!;
for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
let fsPath = path.join(dir, entry.name);
let relativePath = path.posix.join(relativeDir, entry.name);
let savePath = path.posix.join("modules", this.info.name, relativePath);
if (entry.isFile()) {
this.files.set(savePath, await fs.readFile(fsPath));
} else if (entry.isDirectory()) {
dirs.push([fsPath, relativePath]);
}
}
}
}
static jsonSchema = Type.Object({
...lib.ModuleInfo.jsonSchema.properties,
"files": Type.Array(Type.String()),
});
toJSON() {
return {
...this.info.toJSON(),
files: [...this.files.keys()],
};
}
static async fromSave(json: Static<typeof SaveModule.jsonSchema>, root: JSZip) {
const module = new this(lib.ModuleInfo.fromJSON(json));
module.files = new Map(
await Promise.all(json.files.map(async f => [f, await root.file(f)!.async("nodebuffer")] as const))
);
return module;
}
static async fromPlugin(plugin: lib.BasePlugin) {
let pluginPackagePath = require.resolve(path.posix.join(plugin.info.requirePath, "package.json"));
let moduleDirectory = path.join(path.dirname(pluginPackagePath), "module");
if (!await fs.pathExists(moduleDirectory)) {
return null;
}
let moduleJsonPath = path.join(moduleDirectory, "module.json");
let moduleJson;
try {
moduleJson = {
name: plugin.info.name,
version: plugin.info.version,
dependencies: { "clusterio": "*" },
...JSON.parse(await fs.readFile(moduleJsonPath, "utf8")),
}
} catch (err: any) {
throw new Error(`Loading module/module.json in plugin ${plugin.info.name} failed: ${err.message}`);
}
if (!lib.ModuleInfo.validate(moduleJson)) {
throw new Error(
`module/module.json in plugin ${plugin.info.name} failed validation:\n` +
`${JSON.stringify(lib.ModuleInfo.validate.errors, null, 4)}`
);
}
let module = new SaveModule(lib.ModuleInfo.fromJSON(moduleJson));
await module.loadFiles(moduleDirectory);
return module;
}
static async fromDirectory(moduleDirectory: string) {
let name = path.basename(moduleDirectory);
let moduleJsonPath = path.join(moduleDirectory, "module.json");
let moduleJson;
try {
moduleJson = JSON.parse(await fs.readFile(moduleJsonPath, "utf8"));
} catch (err: any) {
throw new Error(`Loading ${name}/module.json failed: ${err.message}`);
}
if (!lib.ModuleInfo.validate(moduleJson)) {
throw new Error(
`${name}/module.json failed validation:\n` +
`${JSON.stringify(lib.ModuleInfo.validate.errors, null, 4)}`
);
}
if (moduleJson.name !== name) {
throw new Error(`Expected name of module ${moduleJson.name} to match the directory name ${name}`);
}
let module = new SaveModule(lib.ModuleInfo.fromJSON(moduleJson));
await module.loadFiles(moduleDirectory);
return module;
}
}
export interface ModuleInfo {
name: string;
version: string;
path: string;
dependencies: Record<string, string>;
load: string[];
require: string[];
export class PatchInfo {
static currentVersion = 1;
constructor(
public patchNumber: number,
public scenario: lib.ModuleInfo,
public modules: SaveModule[],
public version = PatchInfo.currentVersion,
) { }
static jsonSchema = Type.Object({
"version": Type.Number(),
"patch_number": Type.Number(),
"scenario": lib.ModuleInfo.jsonSchema,
"modules": Type.Array(SaveModule.jsonSchema),
});
toJSON() {
return {
version: PatchInfo.currentVersion,
patch_number: this.patchNumber,
scenario: this.scenario.toJSON(),
modules: this.modules.map(m => m.toJSON()),
};
}
static async fromSave(json: Static<typeof PatchInfo.jsonSchema>, root: JSZip) {
if (json.version === undefined) {
interface ScenarioInfoV0 {
name: string;
modules: string[];
}
interface PatchInfoV0 {
patch_number: number;
scenario: ScenarioInfoV0;
modules: { name: string, files: { path: string, load: boolean, require: boolean }[], }[];
}
const info = json as unknown as PatchInfoV0;
return new this(
info.patch_number,
new lib.ModuleInfo(info.scenario.name, "0.0.0", info.scenario.modules),
[],
0,
);
}
return new this(
json.patch_number,
lib.ModuleInfo.fromJSON(json.scenario),
await Promise.all(json.modules.map(m => SaveModule.fromSave(m, root))),
);
}
}
interface PatchInfo {
patch_number: number;
scenario: ScenarioInfo;
modules: { name: string, files: { path: string, load: boolean, require: boolean }[], }[];
}
const knownScenarios: Record<string, ScenarioInfo> = {
const knownScenarios: Record<string, lib.ModuleInfo> = {
// First seen in 0.17.63
"4e866186ebe297f1038fd325b09df1a1f5e2fdd1": {
"name": "freeplay",
"modules": [
"freeplay",
"silo-script",
],
},
"4e866186ebe297f1038fd325b09df1a1f5e2fdd1": new lib.ModuleInfo("freeplay", "0.17.63", ["freeplay", "silo-script"]),
};
/**
@@ -52,15 +189,18 @@ function generateLoader(patchInfo: PatchInfo) {
let lines = [
"-- Auto generated scenario module loader created by Clusterio",
"-- Modifications to this file will be lost when loaded in Clusterio",
`clusterio_patch_number = ${patchInfo.patch_number}`,
`clusterio_patch_number = ${patchInfo.patchNumber}`,
"",
'local event_handler = require("event_handler")',
"",
"-- Scenario modules",
];
for (let moduleName of patchInfo["scenario"]["modules"]) {
lines.push(`event_handler.add_lib(require("${moduleName}"))`);
for (let requirePath of patchInfo.scenario.load) {
lines.push(`event_handler.add_lib(require("${requirePath}"))`);
}
for (let requirePath of patchInfo.scenario.require) {
lines.push(`require("${requirePath}")`);
}
lines.push(...[
@@ -68,15 +208,14 @@ function generateLoader(patchInfo: PatchInfo) {
"-- Clusterio modules",
]);
for (let module of patchInfo["modules"]) {
for (let file of module["files"]) {
let requirePath = `modules/${module.name}/${file["path"].slice(0, -4)}`;
if (file["load"]) {
lines.push(`event_handler.add_lib(require("${requirePath}"))`);
}
if (file["require"]) {
lines.push(`require("${requirePath}")`);
}
for (let module of patchInfo.modules) {
for (let file of module.info.load) {
let requirePath = `modules/${module.info.name}/${file.replace(/\.lua$/i, "")}`;
lines.push(`event_handler.add_lib(require("${requirePath}"))`);
}
for (let file of module.info.require) {
let requirePath = `modules/${module.info.name}/${file.replace(/\.lua$/i, "")}`;
lines.push(`require("${requirePath}")`);
}
}
@@ -96,32 +235,32 @@ function generateLoader(patchInfo: PatchInfo) {
* @param modules - Array of modules to reorder
* @internal
*/
function reorderDependencies(modules: ModuleInfo[]) {
function reorderDependencies(modules: SaveModule[]) {
let index = 0;
let present = new Map();
let hold = new Map();
let present = new Map<string, string>();
let hold = new Map<string, [SaveModule]>();
reorder: while (index < modules.length) {
let module = modules[index];
if (semver.valid(module.version) === null) {
throw new Error(`Invalid version '${module.version}' for module ${module.name}`);
if (semver.valid(module.info.version) === null) {
throw new Error(`Invalid version '${module.info.version}' for module ${module.info.name}`);
}
for (let [dependency, requirement] of Object.entries(module.dependencies)) {
for (let [dependency, requirement] of module.info.dependencies) {
if (semver.validRange(requirement) === null) {
throw new Error(
`Invalid version range '${requirement}' for dependency ${dependency} on module ${module.name}`
`Invalid version range '${requirement}' for dependency ${dependency} on module ${module.info.name}`
);
}
if (present.has(dependency)) {
if (!semver.satisfies(present.get(dependency), requirement)) {
throw new Error(`Module ${module.name} requires ${dependency} ${requirement}`);
if (!semver.satisfies(present.get(dependency)!, requirement)) {
throw new Error(`Module ${module.info.name} requires ${dependency} ${requirement}`);
}
// We have an unmet dependency, take it out and continue
} else {
if (hold.has(dependency)) {
hold.get(dependency).push(module);
hold.get(dependency)!.push(module);
} else {
hold.set(dependency, [module]);
}
@@ -131,12 +270,12 @@ function reorderDependencies(modules: ModuleInfo[]) {
}
// No unmet dependencies, record and continue
present.set(module.name, module.version);
present.set(module.info.name, module.info.version);
index += 1;
if (hold.has(module.name)) {
modules.splice(index, 0, ...hold.get(module.name));
hold.delete(module.name);
if (hold.has(module.info.name)) {
modules.splice(index, 0, ...hold.get(module.info.name)!);
hold.delete(module.info.name);
}
}
@@ -148,10 +287,10 @@ function reorderDependencies(modules: ModuleInfo[]) {
// on a module that is missing, the module is part of a dependency loop, or the
// the module depends on a module that satisfy any of these conditions.
let remaining = new Map();
let remaining = new Map<string, SaveModule>();
for (let heldModules of hold.values()) {
for (let module of heldModules) {
remaining.set(module.name, module);
remaining.set(module.info.name, module);
}
}
@@ -160,22 +299,22 @@ function reorderDependencies(modules: ModuleInfo[]) {
let cycle: string[] = [];
while (true) {
// Find an unmet dependency
let dependency = Object.keys(module.dependencies).find(name => !present.has(name));
let dependency = [...module.info.dependencies.keys()].find(name => !present.has(name))!;
if (!remaining.has(dependency)) {
// There's no module being held up by this dependency, the
// dependency is missing.
throw new Error(`Missing dependency ${dependency} for module ${module.name}`);
throw new Error(`Missing dependency ${dependency} for module ${module.info.name}`);
}
if (cycle.includes(module.name)) {
cycle.push(module.name);
cycle.splice(0, cycle.indexOf(module.name));
if (cycle.includes(module.info.name)) {
cycle.push(module.info.name);
cycle.splice(0, cycle.indexOf(module.info.name));
throw new Error(`Module dependency loop detected: ${cycle.join(" -> ")}`);
}
cycle.push(module.name);
module = remaining.get(dependency);
cycle.push(module.info.name);
module = remaining.get(dependency)!;
}
}
}
@@ -191,7 +330,7 @@ function reorderDependencies(modules: ModuleInfo[]) {
* @param savePath - Path to the Factorio save to patch.
* @param modules - Description of the modules to patch.
*/
export async function patch(savePath: string, modules: ModuleInfo[]) {
export async function patch(savePath: string, modules: SaveModule[]) {
let zip = await JSZip.loadAsync(await fs.readFile(savePath));
let root = zip.folder(lib.findRoot(zip))!;
@@ -199,7 +338,13 @@ export async function patch(savePath: string, modules: ModuleInfo[]) {
let patchInfo: PatchInfo;
if (patchInfoFile !== null) {
let content = await patchInfoFile.async("string");
patchInfo = JSON.parse(content);
patchInfo = await PatchInfo.fromSave(JSON.parse(content), root);
if (patchInfo.version > PatchInfo.currentVersion) {
throw new Error(
`Save patch version ${patchInfo.version} is newer than the patch version this ` +
`version of Clusterio can load (${PatchInfo.currentVersion})`
);
}
// No info file present, try to detect if it's a known compatible scenario.
} else {
@@ -211,62 +356,42 @@ export async function patch(savePath: string, modules: ModuleInfo[]) {
let controlHash = await lib.hashStream(controlStream);
if (controlHash in knownScenarios) {
patchInfo = {
"patch_number": 0,
"scenario": knownScenarios[controlHash],
"modules": [],
};
patchInfo = new PatchInfo(0, knownScenarios[controlHash], []);
} else {
throw new Error(`Unable to patch save, unknown scenario (${controlHash})`);
}
}
// Increment patch number
patchInfo["patch_number"] = (patchInfo["patch_number"] || 0) + 1;
patchInfo.patchNumber = patchInfo.patchNumber + 1;
// Remove any existing modules from the save
patchInfo["modules"] = [];
for (let file of root.file(/^modules\//)) {
zip.remove(file.name);
if (patchInfo.version === 0) {
for (let file of root.file(/^modules\//)) {
zip.remove(file.name);
}
} else {
for (let module of patchInfo.modules) {
for (let file of module.files.keys()) {
zip.remove(root.file(file)!.name)
}
}
patchInfo.modules = [];
}
reorderDependencies(modules);
// Add the modules to the save.
for (let module of modules) {
let moduleDir = root.folder(`modules/${module.name}`)!;
let moduleInfo = {
"name": module.name,
"files": [] as { path: string, load: boolean, require: boolean }[],
};
let dirs: [string, string][] = [[module.path, ""]];
while (dirs.length) {
let [dir, relativeDir] = dirs.pop()!;
for (let entry of await fs.readdir(dir, { withFileTypes: true })) {
let fsPath = path.join(dir, entry.name);
let relativePath = path.posix.join(relativeDir, entry.name);
if (entry.isFile()) {
moduleDir.file(relativePath, await fs.readFile(fsPath));
moduleInfo["files"].push({
"path": relativePath,
"load": module.load.includes(relativePath),
"require": module.require.includes(relativePath),
});
} else if (entry.isDirectory()) {
dirs.push([fsPath, relativePath]);
}
}
for (let [relativePath, contents] of module.files) {
root.file(relativePath, contents);
}
patchInfo["modules"].push(moduleInfo);
patchInfo.modules.push(module);
}
// Add loading code and patch info
root.file("control.lua", generateLoader(patchInfo));
root.file("clusterio.json", JSON.stringify(patchInfo, null, 4));
root.file("clusterio.json", JSON.stringify(patchInfo, null, "\t"));
// Write back the save
let tempSavePath = `${savePath}.tmp`;