mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-22 16:13:12 +09:00
Reading a mod zip needs zip_ops, which pulled yauzl into the web interface bundle through the data classes. ModStore is node only, so the zip reading now lives there. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
384 lines
12 KiB
TypeScript
384 lines
12 KiB
TypeScript
import fs from "node:fs/promises";
|
|
import path from "path";
|
|
import events from "events";
|
|
import { Static } from "@sinclair/typebox";
|
|
import { logger } from "./logging.js";
|
|
import { downloadFile, atomicOverwriteFile } from "./file_ops.js";
|
|
import { hashFile } from "./hash.js";
|
|
import { ZipArchive, findRoot } from "./zip_ops.js";
|
|
import { ModPortalReleaseSchema, ModPortalDetailsSchema, ModNameVersionPair } from "./data/messages_mod.js";
|
|
import { ModInfo, ModPack, MajorMinorVersion, ModVersionEquality, SourceVersion } from "./data/index.js";
|
|
|
|
export interface ModStoreEvents {
|
|
/** A stored mod was created, updated or deleted */
|
|
"change": [ mod: ModInfo ];
|
|
}
|
|
|
|
interface ModsInfoResponse {
|
|
pagination: any,
|
|
results: Array<Static<typeof ModPortalDetailsSchema>>,
|
|
}
|
|
|
|
/**
|
|
* Interface describing the structure of the pagination info from the portal API
|
|
*/
|
|
interface ModPortalPagination {
|
|
count: number;
|
|
links: { next?: string; prev?: string; };
|
|
page: number;
|
|
page_count: number;
|
|
page_size: number;
|
|
}
|
|
|
|
/**
|
|
* Interface describing the structure of the portal API response for a single page
|
|
*/
|
|
interface ModPortalResponse {
|
|
pagination: ModPortalPagination;
|
|
results: Static<typeof ModPortalDetailsSchema>[];
|
|
}
|
|
|
|
export default class ModStore extends events.EventEmitter<ModStoreEvents> {
|
|
constructor(
|
|
public modsDirectory: string,
|
|
public files: Map<string, ModInfo>,
|
|
) {
|
|
super();
|
|
}
|
|
|
|
getMod(name: string, version: SourceVersion, sha1?: string) {
|
|
const mod = this.files.get(ModInfo.filename(name, version));
|
|
if (!mod || sha1 && sha1 !== mod.sha1) {
|
|
return undefined;
|
|
}
|
|
return mod;
|
|
}
|
|
|
|
mods() { return this.files.values(); }
|
|
|
|
/**
|
|
* Read the info.json of a mod zip file along with its size, mtime and hash
|
|
*
|
|
* @param modPath - Path to the mod zip file.
|
|
* @returns Mod info for the file.
|
|
*/
|
|
static async readModInfo(modPath: string) {
|
|
let modInfo: Static<typeof ModInfo.jsonSchema>;
|
|
let zip = await ZipArchive.fromFile(modPath);
|
|
try {
|
|
let content = await zip.readFile(path.posix.join(findRoot(zip), "info.json"));
|
|
if (content === null) {
|
|
throw new Error("Mod contains no info.json file");
|
|
}
|
|
modInfo = JSON.parse(content.toString("utf8"));
|
|
} finally {
|
|
zip.close();
|
|
}
|
|
|
|
let valid = ModInfo.validateInfo(modInfo);
|
|
if (!valid) {
|
|
throw new Error("Mod's info.json is not valid");
|
|
}
|
|
|
|
const stat = await fs.stat(modPath);
|
|
return ModInfo.fromJSON({
|
|
...modInfo,
|
|
|
|
size: stat.size,
|
|
sha1: await hashFile(modPath),
|
|
mtime_ms: stat.mtimeMs,
|
|
updated_at_ms: stat.mtimeMs,
|
|
is_deleted: false,
|
|
});
|
|
}
|
|
|
|
static async loadFile(modsDirectory: string, file: string) {
|
|
let modInfo = await ModStore.readModInfo(path.join(modsDirectory, file));
|
|
if (modInfo.filename !== file) {
|
|
throw new Error(`filename does not match the expected name ${modInfo.filename}.`);
|
|
}
|
|
return modInfo;
|
|
}
|
|
|
|
async loadFile(file: string) {
|
|
let modInfo = await ModStore.loadFile(this.modsDirectory, file);
|
|
this.addMod(modInfo);
|
|
return modInfo;
|
|
}
|
|
|
|
async deleteFile(file: string) {
|
|
let modInfo = this.files.get(file);
|
|
if (!modInfo) {
|
|
throw new Error(`Mod ${file} does not exist`);
|
|
}
|
|
|
|
await fs.unlink(path.join(this.modsDirectory, file));
|
|
this.files.delete(file);
|
|
modInfo.isDeleted = true;
|
|
modInfo.updatedAtMs = Date.now();
|
|
this.emit("change", modInfo);
|
|
}
|
|
|
|
addMod(modInfo: ModInfo) {
|
|
this.files.set(modInfo.filename, modInfo);
|
|
this.emit("change", modInfo);
|
|
}
|
|
|
|
async loadMod(name: string, version: SourceVersion) {
|
|
return await this.loadFile(ModInfo.filename(name, version));
|
|
}
|
|
|
|
async deleteMod(name: string, version: SourceVersion) {
|
|
return await this.deleteFile(ModInfo.filename(name, version));
|
|
}
|
|
|
|
static async fromDirectory(modsDirectory: string) {
|
|
type ModInfoCache = Static<typeof ModInfo.jsonSchema>[];
|
|
const cacheFilename = path.join(modsDirectory, "mod-info-cache.json");
|
|
let cache = new Map<string, ModInfo>();
|
|
try {
|
|
const content = await fs.readFile(cacheFilename, "utf8");
|
|
cache = new Map((JSON.parse(content) as ModInfoCache).map(item => {
|
|
const mod = ModInfo.fromJSON(item);
|
|
return [mod.filename, mod];
|
|
}));
|
|
} catch (err: any) {
|
|
if (err.code !== "ENOENT") {
|
|
logger.warn(`Error loading ${cacheFilename}:\n${err.stack}`);
|
|
}
|
|
}
|
|
|
|
const files = new Map<string, ModInfo>();
|
|
for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) {
|
|
if (entry.isDirectory()) {
|
|
logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`);
|
|
continue;
|
|
}
|
|
|
|
if (!entry.name.toLowerCase().endsWith(".zip")) {
|
|
continue;
|
|
}
|
|
|
|
if (entry.name.toLowerCase().endsWith(".tmp.zip")) {
|
|
continue;
|
|
}
|
|
|
|
const cachedModInfo = cache.get(entry.name);
|
|
if (cachedModInfo) {
|
|
const stat = await fs.stat(path.join(modsDirectory, entry.name));
|
|
if (cachedModInfo.mtimeMs === stat.mtimeMs) {
|
|
files.set(cachedModInfo.filename, cachedModInfo);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
logger.info(`Loading info for Mod ${entry.name}`);
|
|
let modInfo: ModInfo;
|
|
try {
|
|
modInfo = await this.loadFile(modsDirectory, entry.name);
|
|
} catch (err: any) {
|
|
logger.error(`Error loading mod ${entry.name}: ${err.message}`);
|
|
continue;
|
|
}
|
|
files.set(modInfo.filename, modInfo);
|
|
}
|
|
|
|
if (files.size) {
|
|
logger.info(`Loaded info for ${files.size} mods`);
|
|
const content: ModInfoCache = [...files.values()].map(mod => mod.toJSON());
|
|
await atomicOverwriteFile(cacheFilename, JSON.stringify(content));
|
|
}
|
|
|
|
return new this(modsDirectory, files);
|
|
}
|
|
|
|
/**
|
|
* Download a matching version from a selection of available versions.
|
|
*
|
|
* @param releases - Array of mod releases taken from the mod portal.
|
|
* @param version - The required version to install.
|
|
* @param username - Factorio username to download using.
|
|
* @param token - Factorio token to download using.
|
|
*/
|
|
private async downloadModFromReleases(
|
|
releases: Array<Static<typeof ModPortalReleaseSchema>>, version: ModVersionEquality,
|
|
username: string, token: string
|
|
) {
|
|
const selectedRelease = releases.find(release => version.testVersion(release.version));
|
|
|
|
if (selectedRelease === undefined) {
|
|
logger.warn(`Mod release matching version ${version} not found.`);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
const url = new URL(selectedRelease.download_url, "https://mods.factorio.com/");
|
|
url.searchParams.set("username", username);
|
|
url.searchParams.set("token", token);
|
|
const filePath = path.join(this.modsDirectory, selectedRelease.file_name);
|
|
|
|
await downloadFile(url, filePath, "overwrite");
|
|
await this.loadFile(selectedRelease.file_name);
|
|
|
|
} catch (err: any) {
|
|
logger.error(`Failed to download or load mod ${selectedRelease.file_name} (${version}): ${err.message}`);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Downloads all provided mods, does not perform chunking or filtering of builtins
|
|
*
|
|
* @param mods - An array of mods to download
|
|
* @param username - Factorio username used for auth to download mods
|
|
* @param token - Factorio token used for auth to download mods
|
|
* @param factorioVersion - Factorio version to filter for (mods are not guaranteed to work between Middle versions)
|
|
*/
|
|
private async downloadModsChunk(
|
|
mods: ModNameVersionPair[],
|
|
username: string, token: string,
|
|
factorioVersion: MajorMinorVersion,
|
|
) {
|
|
const url = new URL("https://mods.factorio.com/api/mods");
|
|
url.searchParams.set("page_size", "max");
|
|
url.searchParams.set("version", factorioVersion);
|
|
const response = await fetch(url, {
|
|
method: "POST",
|
|
body: new URLSearchParams({ namelist: mods.map(m => m.name).join(",") }),
|
|
});
|
|
|
|
if (response.status !== 200) {
|
|
// Surface the portal's own error (e.g. an unsupported version) rather
|
|
// than a generic status line.
|
|
let detail = `${response.status} ${response.statusText}`;
|
|
try {
|
|
const body = await response.json() as { message?: string };
|
|
if (body && typeof body.message === "string") {
|
|
detail = body.message;
|
|
}
|
|
} catch {
|
|
// Response body was not JSON; keep the status line.
|
|
}
|
|
throw new Error(`Mod portal request to ${url} failed: ${detail}`);
|
|
}
|
|
|
|
const modReleases = (await response.json() as ModsInfoResponse).results;
|
|
await Promise.all(modReleases.map((mod) => {
|
|
const modVersion = mods.find(m => m.name === mod.name)!.version;
|
|
if (mod.releases) {
|
|
return this.downloadModFromReleases(mod.releases, modVersion, username, token);
|
|
} else if (mod.latest_release) {
|
|
return this.downloadModFromReleases([mod.latest_release], modVersion, username, token);
|
|
}
|
|
throw new Error(`Mod ${mod.name} has no releases or latest_release`);
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* Downloads all provided mods to the mod dir.
|
|
*
|
|
* Handles chunking to prevent errors due to URL length.
|
|
* Will re-download if mod is already present, if this is undesired use filterInstalled first.
|
|
*
|
|
* @param mods - An array of mods to download.
|
|
* @param username - Factorio username used for auth to download mods.
|
|
* @param token - Factorio token used for auth to download mods.
|
|
* @param factorioVersion - Factorio version to filter for (mods are not guaranteed to work between Middle versions)
|
|
*/
|
|
async downloadMods(
|
|
mods: ModNameVersionPair[], username: string, token: string, factorioVersion: MajorMinorVersion = "1.1"
|
|
) {
|
|
const chunkSize = 500;
|
|
const futures: Promise<void>[] = [];
|
|
const builtInMods = new Set(ModPack.getBuiltinModNames(factorioVersion));
|
|
mods = mods.filter(mod => !(builtInMods.has(mod.name) || mod.name === "core"));
|
|
|
|
for (let i = 0; i < mods.length; i += chunkSize) {
|
|
futures.push(
|
|
this.downloadModsChunk(mods.slice(i, i + chunkSize), username, token, factorioVersion)
|
|
);
|
|
}
|
|
|
|
await Promise.all(futures);
|
|
}
|
|
|
|
/**
|
|
* Filter an array of mods based on if they are installed.
|
|
*
|
|
* @param mods - Array of mods to filer.
|
|
* @param installed - `true` to keep installed mods, `false` to keep uninstalled mods.
|
|
* @returns Filtered array of mods based on install state.
|
|
*/
|
|
filterInstalled(mods: ModNameVersionPair[], installed: boolean) {
|
|
const installedMods = Array.from(this.mods());
|
|
return mods.filter(mod => (
|
|
installed === installedMods.some(m => mod.name === m.name && mod.version.testVersion(m.version))
|
|
));
|
|
}
|
|
|
|
/**
|
|
* Fetch all mods from the Factorio Mod Portal for a specific Factorio version.
|
|
*
|
|
* Handles pagination automatically to retrieve the complete list.
|
|
*
|
|
* @param factorioVersion - The Factorio version to fetch mods for (e.g., "1.1").
|
|
* @param pageSize - The page size for fetching mods.
|
|
* @param hide_deprecated - Whether to exclude deprecated mods.
|
|
* @returns An array of mod details.
|
|
*/
|
|
static async fetchAllModsFromPortal(
|
|
factorioVersion: string,
|
|
pageSize: number,
|
|
hide_deprecated: boolean = false
|
|
): Promise<Static<typeof ModPortalDetailsSchema>[]> {
|
|
logger.info(`Fetching all mods for Factorio version ${factorioVersion}...`);
|
|
let currentPage = 1;
|
|
let allMods: Static<typeof ModPortalDetailsSchema>[] = [];
|
|
let hasMorePages = true;
|
|
|
|
while (hasMorePages) {
|
|
const url = new URL("https://mods.factorio.com/api/mods");
|
|
url.searchParams.set("page_size", String(pageSize));
|
|
url.searchParams.set("version", factorioVersion);
|
|
url.searchParams.set("page", String(currentPage));
|
|
url.searchParams.set("hide_deprecated", String(hide_deprecated));
|
|
|
|
logger.verbose(`Fetching mod portal page ${currentPage}: ${url.toString()}`);
|
|
const response = await fetch(url.toString());
|
|
|
|
if (!response.ok) {
|
|
let errorMessage = `Mod portal fetch page ${currentPage} failed: `;
|
|
errorMessage += `${response.status} ${response.statusText}`;
|
|
try {
|
|
errorMessage += ` - ${await response.text()}`;
|
|
} catch (bodyError) {
|
|
logger.warn("Failed to read response body for failed portal request");
|
|
}
|
|
throw new Error(errorMessage);
|
|
}
|
|
|
|
const data = await response.json() as ModPortalResponse;
|
|
allMods = allMods.concat(data.results);
|
|
|
|
hasMorePages = currentPage < data.pagination.page_count;
|
|
currentPage += 1;
|
|
}
|
|
logger.info(`Successfully fetched ${allMods.length} mods for Factorio version ${factorioVersion}.`);
|
|
return allMods;
|
|
}
|
|
|
|
/**
|
|
* Fetch the full release list for a mod including dependencies.
|
|
*
|
|
* @param modName - Name of the mod to fetch the releases of
|
|
* @returns Mod portal response
|
|
*/
|
|
static async fetchModReleases(modName: string) {
|
|
const url = `https://mods.factorio.com/api/mods/${modName}/full`;
|
|
const response = await fetch(url);
|
|
if (!response.ok) {
|
|
throw Error(`Fetch: ${url} returned ${response.status} ${response.statusText}`);
|
|
}
|
|
return await response.json() as ModsInfoResponse["results"][number];
|
|
}
|
|
}
|