mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-19 23:33:13 +09:00
Merge pull request #712 from O2theC/auto-mod-update - basic mod portal integration
Add mod portal integration and mod downloads to the web interface
This commit is contained in:
10 files changed
+1834
-15
No files matched your search
@@ -2,15 +2,41 @@ import fs from "fs-extra";
|
||||
import path from "path";
|
||||
import { Static } from "@sinclair/typebox";
|
||||
import { logger } from "./logging";
|
||||
import { ModInfo } from "./data";
|
||||
import { integerModVersion, ModInfo, ModPack, ModRecord } from "./data";
|
||||
import TypedEventEmitter from "./TypedEventEmitter";
|
||||
import { safeOutputFile } from "./file_ops";
|
||||
import { Writable } from "stream";
|
||||
import { ModPortalReleaseSchema, ModPortalDetailsSchema } from "./data/messages_mod";
|
||||
|
||||
export interface ModStoreEvents {
|
||||
/** A stored mod was created, updated or deleted */
|
||||
change: (mod: ModInfo) => void;
|
||||
}
|
||||
|
||||
interface ModsInfoResponse {
|
||||
pagination: any,
|
||||
results: Array<Static<typeof ModPortalDetailsSchema>>,
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface describing the structure of the pagination info from the portal API
|
||||
*/
|
||||
interface ModPortalPagination {
|
||||
count: number;
|
||||
links: { next?: string; prev?: string; };
|
||||
page: number;
|
||||
page_count: number;
|
||||
page_size: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface describing the structure of the portal API response for a single page
|
||||
*/
|
||||
interface ModPortalResponse {
|
||||
pagination: ModPortalPagination;
|
||||
results: Static<typeof ModPortalDetailsSchema>[];
|
||||
}
|
||||
|
||||
export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, ModStoreEvents> {
|
||||
constructor(
|
||||
public modsDirectory: string,
|
||||
@@ -27,11 +53,6 @@ export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, Mo
|
||||
return mod;
|
||||
}
|
||||
|
||||
hasMod(name: string, version: string, sha1?: string): boolean {
|
||||
const mod = this.files.get(ModInfo.filename(name, version));
|
||||
return Boolean(mod && (!sha1 || sha1 !== mod.sha1));
|
||||
}
|
||||
|
||||
mods() { return this.files.values(); }
|
||||
|
||||
static async loadFile(modsDirectory: string, file: string) {
|
||||
@@ -44,8 +65,7 @@ export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, Mo
|
||||
|
||||
async loadFile(file: string) {
|
||||
let modInfo = await ModStore.loadFile(this.modsDirectory, file);
|
||||
this.files.set(modInfo.filename, modInfo);
|
||||
this.emit("change", modInfo);
|
||||
this.addMod(modInfo);
|
||||
return modInfo;
|
||||
}
|
||||
|
||||
@@ -134,4 +154,184 @@ export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, Mo
|
||||
|
||||
return new this(modsDirectory, files);
|
||||
}
|
||||
|
||||
private async downloadModFromReleases(releases: Array<Static<typeof ModPortalReleaseSchema>>, version: string,
|
||||
username: string, token: string
|
||||
) {
|
||||
const selectedRelease = releases.find(release => release.version === version);
|
||||
|
||||
if (selectedRelease === undefined) {
|
||||
logger.warn(`Mod release matching version ${version} not found.`);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const url = new URL(`https://mods.factorio.com/${selectedRelease.download_url}`);
|
||||
url.searchParams.set("username", username);
|
||||
url.searchParams.set("token", token);
|
||||
const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
|
||||
const finalFileName = `${this.modsDirectory}/${selectedRelease.file_name}`;
|
||||
|
||||
await ModStore.downloadFile(url, tempFileName);
|
||||
await fs.rename(tempFileName, finalFileName);
|
||||
await this.loadFile(selectedRelease.file_name);
|
||||
|
||||
} catch (err: any) {
|
||||
logger.error(`Failed to download or load mod ${selectedRelease.file_name} (${version}): ${err.message}`);
|
||||
// Attempt to clean up temp file if it exists
|
||||
try {
|
||||
const tempFileName = `${this.modsDirectory}/${selectedRelease.file_name}.tmp`;
|
||||
await fs.unlink(tempFileName);
|
||||
} catch (unlinkErr: any) {
|
||||
if (unlinkErr.code !== "ENOENT") { // Ignore if file doesn't exist
|
||||
logger.warn(`Failed to clean up temp file ${selectedRelease.file_name}.tmp: ${unlinkErr.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadModsChunk(
|
||||
modNames: string[], modVersions: string[], username: string, token: string, factorioVersion: string
|
||||
) {
|
||||
const url = new URL("https://mods.factorio.com/api/mods");
|
||||
url.searchParams.set("page_size", "max");
|
||||
url.searchParams.set("version", factorioVersion);
|
||||
const response = await fetch(url, {
|
||||
method: "POST",
|
||||
body: new URLSearchParams({ namelist: modNames.join(",") }),
|
||||
});
|
||||
if (response.status !== 200) {
|
||||
throw Error(`Fetch: ${url} returned ${response.status} ${response.statusText}`);
|
||||
}
|
||||
const mods = (await response.json() as ModsInfoResponse).results;
|
||||
|
||||
await Promise.all(mods.map((mod) => {
|
||||
if (mod.releases) {
|
||||
return this.downloadModFromReleases(mod.releases,
|
||||
modVersions[modNames.indexOf(mod.name)], username, token);
|
||||
} else if (mod.latest_release) {
|
||||
return this.downloadModFromReleases([mod.latest_release],
|
||||
modVersions[modNames.indexOf(mod.name)], username, token);
|
||||
}
|
||||
throw new Error(`Mod ${mod.name} has no releases or latest_release`);
|
||||
}));
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
*
|
||||
* downloads mods to the mod dir, will not download mods for which are already in the mod directory
|
||||
*
|
||||
* @param modMap a map of what mods to download and what version to download
|
||||
* @param username factorio username used for auth to download mods
|
||||
* @param token factorio token used for auth to download mods
|
||||
* @param factorioVersion factorio version to filter for (mods are not guaranteed to work between Middle versions)
|
||||
*
|
||||
*/
|
||||
async downloadMods(modMap: Map<string, string>, username: string, token: string, factorioVersion: string = "1.1") {
|
||||
const chunkSize = 500;
|
||||
let futures: Promise<void>[] = [];
|
||||
modMap = new Map(modMap);
|
||||
|
||||
// Truncate factorioVersion to major.minor (e.g., "1.1.100" -> "1.1") for API/builtin checks
|
||||
const versionParts = factorioVersion.split(".");
|
||||
if (versionParts.length > 2) {
|
||||
factorioVersion = versionParts.slice(0, 2).join(".");
|
||||
}
|
||||
|
||||
const builtInMods = new Set(ModPack.getBuiltinModNames(factorioVersion));
|
||||
|
||||
// get rid of mods already downloaded, built-in, or core
|
||||
modMap.forEach((modVersion, modName, map) => {
|
||||
const filename = `${modName}_${modVersion}.zip`;
|
||||
// Filter existing, built-in from getBuiltinModNames, and explicitly filter "core"
|
||||
if (this.files.has(filename) || builtInMods.has(modName) || modName === "core") {
|
||||
map.delete(modName);
|
||||
}
|
||||
});
|
||||
|
||||
const modNames = Array.from(modMap.keys());
|
||||
const modVersions = Array.from(modMap.values());
|
||||
for (let i = 0; i < modMap.size; i += chunkSize) {
|
||||
futures.push(
|
||||
this.downloadModsChunk(modNames.slice(i, i + chunkSize), modVersions.slice(i, i + chunkSize),
|
||||
username, token, factorioVersion)
|
||||
);
|
||||
}
|
||||
await Promise.all(futures);
|
||||
}
|
||||
|
||||
static async downloadFile(url: string | URL, filePath: string) {
|
||||
try {
|
||||
const response = await fetch(url);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const fileStream = fs.createWriteStream(filePath);
|
||||
|
||||
// Check if the response body exists and pipe it to the file
|
||||
if (response.body) {
|
||||
const writer = Writable.toWeb(fileStream);
|
||||
await response.body.pipeTo(writer);
|
||||
} else {
|
||||
throw new Error("Response body is missing.");
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error("Error downloading file:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch all mods from the Factorio Mod Portal for a specific Factorio version.
|
||||
*
|
||||
* Handles pagination automatically to retrieve the complete list.
|
||||
*
|
||||
* @param factorioVersion - The Factorio version to fetch mods for (e.g., "1.1").
|
||||
* @param pageSize - The page size for fetching mods.
|
||||
* @param hide_deprecated - Whether to exclude deprecated mods.
|
||||
* @returns An array of mod details.
|
||||
*/
|
||||
static async fetchAllModsFromPortal(
|
||||
factorioVersion: string,
|
||||
pageSize: number,
|
||||
hide_deprecated: boolean = false
|
||||
): Promise<Static<typeof ModPortalDetailsSchema>[]> {
|
||||
logger.info(`Fetching all mods for Factorio version ${factorioVersion}...`);
|
||||
let currentPage = 1;
|
||||
let allMods: Static<typeof ModPortalDetailsSchema>[] = [];
|
||||
let hasMorePages = true;
|
||||
|
||||
while (hasMorePages) {
|
||||
const url = new URL("https://mods.factorio.com/api/mods");
|
||||
url.searchParams.set("page_size", String(pageSize));
|
||||
url.searchParams.set("version", factorioVersion);
|
||||
url.searchParams.set("page", String(currentPage));
|
||||
url.searchParams.set("hide_deprecated", String(hide_deprecated));
|
||||
|
||||
logger.verbose(`Fetching mod portal page ${currentPage}: ${url.toString()}`);
|
||||
const response = await fetch(url.toString());
|
||||
|
||||
if (!response.ok) {
|
||||
let errorMessage = `Mod portal fetch page ${currentPage} failed: `;
|
||||
errorMessage += `${response.status} ${response.statusText}`;
|
||||
try {
|
||||
errorMessage += ` - ${await response.text()}`;
|
||||
} catch (bodyError) {
|
||||
logger.warn("Failed to read response body for failed portal request");
|
||||
}
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
|
||||
const data = await response.json() as ModPortalResponse;
|
||||
allMods = allMods.concat(data.results);
|
||||
|
||||
hasMorePages = currentPage < data.pagination.page_count;
|
||||
currentPage += 1;
|
||||
}
|
||||
logger.info(`Successfully fetched ${allMods.length} mods for Factorio version ${factorioVersion}.`);
|
||||
return allMods;
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,8 @@ export interface ControllerConfigFields {
|
||||
"controller.default_mod_pack_id": number | null;
|
||||
"controller.default_role_id": number | null;
|
||||
"controller.autosave_interval": number;
|
||||
"controller.mod_portal_cache_duration_minutes": number;
|
||||
"controller.mod_portal_page_size": number;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -197,6 +199,18 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
|
||||
optional: true,
|
||||
initialValue: 1,
|
||||
},
|
||||
"controller.mod_portal_cache_duration_minutes": {
|
||||
title: "Mod Portal Cache Duration",
|
||||
description: "Duration in minutes to cache mod portal API responses.",
|
||||
type: "number",
|
||||
initialValue: 30,
|
||||
},
|
||||
"controller.mod_portal_page_size": {
|
||||
title: "Mod Portal Page Size",
|
||||
description: "Maximum number of results per page when querying the Factorio mod portal API.",
|
||||
type: "number",
|
||||
initialValue: 1000,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -197,6 +197,77 @@ export class ModSearchRequest {
|
||||
};
|
||||
}
|
||||
|
||||
// Define the structure for the latest release info from the portal
|
||||
export const ModPortalReleaseSchema = Type.Object({
|
||||
version: Type.String(),
|
||||
// Match the structure from ModStore's ModRelease/ModDetails
|
||||
info_json: Type.Object({ factorio_version: Type.String() }),
|
||||
released_at: Type.String(), // ISO 8601 date string
|
||||
download_url: Type.String(),
|
||||
file_name: Type.String(),
|
||||
sha1: Type.String(),
|
||||
});
|
||||
|
||||
// Define the structure for mod details returned by the portal API
|
||||
// This should align with the ModDetails interface in ModStore.ts
|
||||
export const ModPortalDetailsSchema = Type.Object({
|
||||
name: Type.String(),
|
||||
title: Type.String(),
|
||||
summary: Type.String(),
|
||||
owner: Type.String(),
|
||||
downloads_count: Type.Integer(),
|
||||
category: Type.Optional(Type.String()),
|
||||
score: Type.Optional(Type.Number()),
|
||||
latest_release: Type.Optional(ModPortalReleaseSchema),
|
||||
releases: Type.Optional(Type.Array(ModPortalReleaseSchema)),
|
||||
});
|
||||
|
||||
export class ModPortalGetAllRequest {
|
||||
declare ["constructor"]: typeof ModPortalGetAllRequest;
|
||||
static type = "request" as const;
|
||||
static src = "control" as const;
|
||||
static dst = "controller" as const;
|
||||
static permission = "core.mod.search_portal" as const;
|
||||
|
||||
// Define allowed Factorio versions
|
||||
static allowedVersions = [
|
||||
Type.Literal("0.13"), Type.Literal("0.14"), Type.Literal("0.15"),
|
||||
Type.Literal("0.16"), Type.Literal("0.17"), Type.Literal("0.18"),
|
||||
Type.Literal("1.0"), Type.Literal("1.1"), Type.Literal("2.0"),
|
||||
] as const;
|
||||
|
||||
constructor(
|
||||
public factorioVersion: Static<typeof ModPortalGetAllRequest.allowedVersions[number]>,
|
||||
public hide_deprecated?: boolean,
|
||||
) { }
|
||||
|
||||
static jsonSchema = Type.Object({
|
||||
"factorioVersion": Type.Union([...this.allowedVersions]), // Use Union for validation
|
||||
"hide_deprecated": Type.Optional(Type.Boolean()),
|
||||
});
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
return new this(json.factorioVersion, json.hide_deprecated);
|
||||
}
|
||||
|
||||
// Define the Response class inline
|
||||
static Response = class ModPortalGetAllResponse {
|
||||
declare ["constructor"]: typeof ModPortalGetAllResponse;
|
||||
constructor(
|
||||
public mods: Static<typeof ModPortalDetailsSchema>[],
|
||||
) { }
|
||||
|
||||
static jsonSchema = Type.Object({
|
||||
mods: Type.Array(ModPortalDetailsSchema),
|
||||
});
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
// No transformation needed if schema matches API structure
|
||||
return new this(json.mods);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export class ModDownloadRequest {
|
||||
declare ["constructor"]: typeof ModDownloadRequest;
|
||||
static type = "request" as const;
|
||||
@@ -284,3 +355,39 @@ export class ModUpdatesEvent {
|
||||
return new this(json.updates.map(update => ModInfo.fromJSON(update)));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Request mods to be downloaded from the Factorio mod portal to the controller.
|
||||
*
|
||||
* The controller handles the actual download process asynchronously.
|
||||
* Requires Factorio credentials to be configured on the controller if the portal
|
||||
* requires authentication for downloads.
|
||||
*
|
||||
* @param name - Name of the mod to download.
|
||||
* @param version - Version of the mod to download.
|
||||
* @param factorioVersion - Factorio version context for the download.
|
||||
*/
|
||||
export class ModPortalDownloadRequest {
|
||||
declare ["constructor"]: typeof ModPortalDownloadRequest;
|
||||
static type = "request" as const;
|
||||
static src = "control" as const;
|
||||
static dst = "controller" as const;
|
||||
static permission = "core.mod.download_from_portal" as const;
|
||||
static Response = ModInfo;
|
||||
|
||||
constructor(
|
||||
public name: string,
|
||||
public version: string,
|
||||
public factorioVersion: string,
|
||||
) { }
|
||||
|
||||
static jsonSchema = Type.Object({
|
||||
"name": Type.String(),
|
||||
"version": Type.String(),
|
||||
"factorioVersion": Type.String(),
|
||||
});
|
||||
|
||||
static fromJSON(json: Static<typeof this.jsonSchema>) {
|
||||
return new this(json.name, json.version, json.factorioVersion);
|
||||
}
|
||||
}
|
||||
@@ -97,10 +97,12 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
|
||||
mod.ModGetRequest,
|
||||
mod.ModListRequest,
|
||||
mod.ModSearchRequest,
|
||||
mod.ModPortalGetAllRequest,
|
||||
mod.ModDownloadRequest,
|
||||
mod.ModDeleteRequest,
|
||||
mod.ModPackUpdatesEvent,
|
||||
mod.ModUpdatesEvent,
|
||||
mod.ModPortalDownloadRequest,
|
||||
|
||||
user.PermissionListRequest,
|
||||
user.RoleListRequest,
|
||||
|
||||
@@ -373,6 +373,12 @@ definePermission({
|
||||
description: "Search through the list of mods stored on the controller.",
|
||||
grantByDefault: true,
|
||||
});
|
||||
definePermission({
|
||||
name: "core.mod.search_portal",
|
||||
title: "Search mod portal",
|
||||
description: "Search for mods on the Factorio mod portal.",
|
||||
grantByDefault: true,
|
||||
});
|
||||
definePermission({
|
||||
name: "core.mod.subscribe",
|
||||
title: "Subscribe to mod updates",
|
||||
@@ -381,7 +387,7 @@ definePermission({
|
||||
});
|
||||
definePermission({
|
||||
name: "core.mod.upload",
|
||||
title: "Upload mod",
|
||||
title: "Upload mods",
|
||||
description: "Upload mods to the controller.",
|
||||
});
|
||||
definePermission({
|
||||
@@ -395,6 +401,12 @@ definePermission({
|
||||
title: "Delete mod",
|
||||
description: "Delete mods stored on the controller.",
|
||||
});
|
||||
definePermission({
|
||||
name: "core.mod.download_from_portal",
|
||||
title: "Download mods from portal",
|
||||
description: "Allow downloading mods directly from the Factorio mod portal to the controller.",
|
||||
grantByDefault: true,
|
||||
});
|
||||
|
||||
definePermission({
|
||||
name: "core.permission.list",
|
||||
|
||||
Reference in new issue
Block a user