Merge pull request #712 from O2theC/auto-mod-update - basic mod portal integration

Add mod portal integration and mod downloads to the web interface
This commit is contained in:
Daniel authored and GitHub committed 2025-04-13 21:52:07 +02:00
commit f21a3f45bf
10 files changed
+1834 -15

No files matched your search

+208 -8
View File
@@ -2,15 +2,41 @@ import fs from "fs-extra";
import path from "path";
import { Static } from "@sinclair/typebox";
import { logger } from "./logging";
import { ModInfo } from "./data";
import { integerModVersion, ModInfo, ModPack, ModRecord } from "./data";
import TypedEventEmitter from "./TypedEventEmitter";
import { safeOutputFile } from "./file_ops";
import { Writable } from "stream";
import { ModPortalReleaseSchema, ModPortalDetailsSchema } from "./data/messages_mod";
export interface ModStoreEvents {
/** A stored mod was created, updated or deleted */
change: (mod: ModInfo) => void;
}
interface ModsInfoResponse {
pagination: any,
results: Array<Static<typeof ModPortalDetailsSchema>>,
}
/**
* Interface describing the structure of the pagination info from the portal API
*/
interface ModPortalPagination {
count: number;
links: { next?: string; prev?: string; };
page: number;
page_count: number;
page_size: number;
}
/**
* Interface describing the structure of the portal API response for a single page
*/
interface ModPortalResponse {
pagination: ModPortalPagination;
results: Static<typeof ModPortalDetailsSchema>[];
}
export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, ModStoreEvents> {
constructor(
public modsDirectory: string,
@@ -27,11 +53,6 @@ export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, Mo
return mod;
}
hasMod(name: string, version: string, sha1?: string): boolean {
const mod = this.files.get(ModInfo.filename(name, version));
return Boolean(mod && (!sha1 || sha1 !== mod.sha1));
}
mods() { return this.files.values(); }
static async loadFile(modsDirectory: string, file: string) {
@@ -44,8 +65,7 @@ export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, Mo
async loadFile(file: string) {
let modInfo = await ModStore.loadFile(this.modsDirectory, file);
this.files.set(modInfo.filename, modInfo);
this.emit("change", modInfo);
this.addMod(modInfo);
return modInfo;
}
@@ -134,4 +154,184 @@ export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, Mo
return new this(modsDirectory, files);
}
private async downloadModFromReleases(releases: Array<Static<typeof ModPortalReleaseSchema>>, version: string,
username: string, token: string
) {
const selectedRelease = releases.find(release => release.version === version);
if (selectedRelease === undefined) {
logger.warn(`Mod release matching version ${version} not found.`);
return;
}
try {
const url = new URL(`https://mods.factorio.com/${selectedRelease.download_url}`);
url.searchParams.set("username", username);
url.searchParams.set("token", token);
const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
const finalFileName = `${this.modsDirectory}/${selectedRelease.file_name}`;
await ModStore.downloadFile(url, tempFileName);
await fs.rename(tempFileName, finalFileName);
await this.loadFile(selectedRelease.file_name);
} catch (err: any) {
logger.error(`Failed to download or load mod ${selectedRelease.file_name} (${version}): ${err.message}`);
// Attempt to clean up temp file if it exists
try {
const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
await fs.unlink(tempFileName);
} catch (unlinkErr: any) {
if (unlinkErr.code !== "ENOENT") { // Ignore if file doesn't exist
logger.warn(`Failed to clean up temp file ${selectedRelease.file_name}.tmp: ${unlinkErr.message}`);
}
}
}
}
private async downloadModsChunk(
modNames: string[], modVersions: string[], username: string, token: string, factorioVersion: string
) {
const url = new URL("https://mods.factorio.com/api/mods");
url.searchParams.set("page_size", "max");
url.searchParams.set("version", factorioVersion);
const response = await fetch(url, {
method: "POST",
body: new URLSearchParams({ namelist: modNames.join(",") }),
});
if (response.status !== 200) {
throw Error(`Fetch: ${url} returned ${response.status} ${response.statusText}`);
}
const mods = (await response.json() as ModsInfoResponse).results;
await Promise.all(mods.map((mod) => {
if (mod.releases) {
return this.downloadModFromReleases(mod.releases,
modVersions[modNames.indexOf(mod.name)], username, token);
} else if (mod.latest_release) {
return this.downloadModFromReleases([mod.latest_release],
modVersions[modNames.indexOf(mod.name)], username, token);
}
throw new Error(`Mod ${mod.name} has no releases or latest_release`);
}));
}
/**
*
* downloads mods to the mod dir, will not download mods for which are already in the mod directory
*
* @param modMap a map of what mods to download and what version to download
* @param username factorio username used for auth to download mods
* @param token factorio token used for auth to download mods
* @param factorioVersion factorio version to filter for (mods are not guaranteed to work between Middle versions)
*
*/
async downloadMods(modMap: Map<string, string>, username: string, token: string, factorioVersion: string = "1.1") {
const chunkSize = 500;
let futures: Promise<void>[] = [];
modMap = new Map(modMap);
// Truncate factorioVersion to major.minor (e.g., "1.1.100" -> "1.1") for API/builtin checks
const versionParts = factorioVersion.split(".");
if (versionParts.length > 2) {
factorioVersion = versionParts.slice(0, 2).join(".");
}
const builtInMods = new Set(ModPack.getBuiltinModNames(factorioVersion));
// get rid of mods already downloaded, built-in, or core
modMap.forEach((modVersion, modName, map) => {
const filename = `${modName}_${modVersion}.zip`;
// Filter existing, built-in from getBuiltinModNames, and explicitly filter "core"
if (this.files.has(filename) || builtInMods.has(modName) || modName === "core") {
map.delete(modName);
}
});
const modNames = Array.from(modMap.keys());
const modVersions = Array.from(modMap.values());
for (let i = 0; i < modMap.size; i += chunkSize) {
futures.push(
this.downloadModsChunk(modNames.slice(i, i + chunkSize), modVersions.slice(i, i + chunkSize),
username, token, factorioVersion)
);
}
await Promise.all(futures);
}
static async downloadFile(url: string | URL, filePath: string) {
try {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
}
const fileStream = fs.createWriteStream(filePath);
// Check if the response body exists and pipe it to the file
if (response.body) {
const writer = Writable.toWeb(fileStream);
await response.body.pipeTo(writer);
} else {
throw new Error("Response body is missing.");
}
} catch (error) {
logger.error("Error downloading file:", error);
throw error;
}
}
/**
* Fetch all mods from the Factorio Mod Portal for a specific Factorio version.
*
* Handles pagination automatically to retrieve the complete list.
*
* @param factorioVersion - The Factorio version to fetch mods for (e.g., "1.1").
* @param pageSize - The page size for fetching mods.
* @param hide_deprecated - Whether to exclude deprecated mods.
* @returns An array of mod details.
*/
static async fetchAllModsFromPortal(
factorioVersion: string,
pageSize: number,
hide_deprecated: boolean = false
): Promise<Static<typeof ModPortalDetailsSchema>[]> {
logger.info(`Fetching all mods for Factorio version ${factorioVersion}...`);
let currentPage = 1;
let allMods: Static<typeof ModPortalDetailsSchema>[] = [];
let hasMorePages = true;
while (hasMorePages) {
const url = new URL("https://mods.factorio.com/api/mods");
url.searchParams.set("page_size", String(pageSize));
url.searchParams.set("version", factorioVersion);
url.searchParams.set("page", String(currentPage));
url.searchParams.set("hide_deprecated", String(hide_deprecated));
logger.verbose(`Fetching mod portal page ${currentPage}: ${url.toString()}`);
const response = await fetch(url.toString());
if (!response.ok) {
let errorMessage = `Mod portal fetch page ${currentPage} failed: `;
errorMessage += `${response.status} ${response.statusText}`;
try {
errorMessage += ` - ${await response.text()}`;
} catch (bodyError) {
logger.warn("Failed to read response body for failed portal request");
}
throw new Error(errorMessage);
}
const data = await response.json() as ModPortalResponse;
allMods = allMods.concat(data.results);
hasMorePages = currentPage < data.pagination.page_count;
currentPage += 1;
}
logger.info(`Successfully fetched ${allMods.length} mods for Factorio version ${factorioVersion}.`);
return allMods;
}
}
+14
View File
@@ -26,6 +26,8 @@ export interface ControllerConfigFields {
"controller.default_mod_pack_id": number | null;
"controller.default_role_id": number | null;
"controller.autosave_interval": number;
"controller.mod_portal_cache_duration_minutes": number;
"controller.mod_portal_page_size": number;
}
/**
@@ -197,6 +199,18 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
optional: true,
initialValue: 1,
},
"controller.mod_portal_cache_duration_minutes": {
title: "Mod Portal Cache Duration",
description: "Duration in minutes to cache mod portal API responses.",
type: "number",
initialValue: 30,
},
"controller.mod_portal_page_size": {
title: "Mod Portal Page Size",
description: "Maximum number of results per page when querying the Factorio mod portal API.",
type: "number",
initialValue: 1000,
},
};
}
+107
View File
@@ -197,6 +197,77 @@ export class ModSearchRequest {
};
}
// Define the structure for the latest release info from the portal
export const ModPortalReleaseSchema = Type.Object({
version: Type.String(),
// Match the structure from ModStore's ModRelease/ModDetails
info_json: Type.Object({ factorio_version: Type.String() }),
released_at: Type.String(), // ISO 8601 date string
download_url: Type.String(),
file_name: Type.String(),
sha1: Type.String(),
});
// Define the structure for mod details returned by the portal API
// This should align with the ModDetails interface in ModStore.ts
export const ModPortalDetailsSchema = Type.Object({
name: Type.String(),
title: Type.String(),
summary: Type.String(),
owner: Type.String(),
downloads_count: Type.Integer(),
category: Type.Optional(Type.String()),
score: Type.Optional(Type.Number()),
latest_release: Type.Optional(ModPortalReleaseSchema),
releases: Type.Optional(Type.Array(ModPortalReleaseSchema)),
});
export class ModPortalGetAllRequest {
declare ["constructor"]: typeof ModPortalGetAllRequest;
static type = "request" as const;
static src = "control" as const;
static dst = "controller" as const;
static permission = "core.mod.search_portal" as const;
// Define allowed Factorio versions
static allowedVersions = [
Type.Literal("0.13"), Type.Literal("0.14"), Type.Literal("0.15"),
Type.Literal("0.16"), Type.Literal("0.17"), Type.Literal("0.18"),
Type.Literal("1.0"), Type.Literal("1.1"), Type.Literal("2.0"),
] as const;
constructor(
public factorioVersion: Static<typeof ModPortalGetAllRequest.allowedVersions[number]>,
public hide_deprecated?: boolean,
) { }
static jsonSchema = Type.Object({
"factorioVersion": Type.Union([...this.allowedVersions]), // Use Union for validation
"hide_deprecated": Type.Optional(Type.Boolean()),
});
static fromJSON(json: Static<typeof this.jsonSchema>) {
return new this(json.factorioVersion, json.hide_deprecated);
}
// Define the Response class inline
static Response = class ModPortalGetAllResponse {
declare ["constructor"]: typeof ModPortalGetAllResponse;
constructor(
public mods: Static<typeof ModPortalDetailsSchema>[],
) { }
static jsonSchema = Type.Object({
mods: Type.Array(ModPortalDetailsSchema),
});
static fromJSON(json: Static<typeof this.jsonSchema>) {
// No transformation needed if schema matches API structure
return new this(json.mods);
}
};
}
export class ModDownloadRequest {
declare ["constructor"]: typeof ModDownloadRequest;
static type = "request" as const;
@@ -284,3 +355,39 @@ export class ModUpdatesEvent {
return new this(json.updates.map(update => ModInfo.fromJSON(update)));
}
}
/**
* Request mods to be downloaded from the Factorio mod portal to the controller.
*
* The controller handles the actual download process asynchronously.
* Requires Factorio credentials to be configured on the controller if the portal
* requires authentication for downloads.
*
* @param name - Name of the mod to download.
* @param version - Version of the mod to download.
* @param factorioVersion - Factorio version context for the download.
*/
export class ModPortalDownloadRequest {
declare ["constructor"]: typeof ModPortalDownloadRequest;
static type = "request" as const;
static src = "control" as const;
static dst = "controller" as const;
static permission = "core.mod.download_from_portal" as const;
static Response = ModInfo;
constructor(
public name: string,
public version: string,
public factorioVersion: string,
) { }
static jsonSchema = Type.Object({
"name": Type.String(),
"version": Type.String(),
"factorioVersion": Type.String(),
});
static fromJSON(json: Static<typeof this.jsonSchema>) {
return new this(json.name, json.version, json.factorioVersion);
}
}
+2
View File
@@ -97,10 +97,12 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
mod.ModGetRequest,
mod.ModListRequest,
mod.ModSearchRequest,
mod.ModPortalGetAllRequest,
mod.ModDownloadRequest,
mod.ModDeleteRequest,
mod.ModPackUpdatesEvent,
mod.ModUpdatesEvent,
mod.ModPortalDownloadRequest,
user.PermissionListRequest,
user.RoleListRequest,
+13 -1
View File
@@ -373,6 +373,12 @@ definePermission({
description: "Search through the list of mods stored on the controller.",
grantByDefault: true,
});
definePermission({
name: "core.mod.search_portal",
title: "Search mod portal",
description: "Search for mods on the Factorio mod portal.",
grantByDefault: true,
});
definePermission({
name: "core.mod.subscribe",
title: "Subscribe to mod updates",
@@ -381,7 +387,7 @@ definePermission({
});
definePermission({
name: "core.mod.upload",
title: "Upload mod",
title: "Upload mods",
description: "Upload mods to the controller.",
});
definePermission({
@@ -395,6 +401,12 @@ definePermission({
title: "Delete mod",
description: "Delete mods stored on the controller.",
});
definePermission({
name: "core.mod.download_from_portal",
title: "Download mods from portal",
description: "Allow downloading mods directly from the Factorio mod portal to the controller.",
grantByDefault: true,
});
definePermission({
name: "core.permission.list",