From a769c3179a2a00be393750cf69d4da611188fa9f Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Wed, 11 Dec 2024 20:28:33 -0500 Subject: [PATCH 01/28] made modRelease and modDetails interface and made a rough function to get the latest version of a mod from it's name --- .../lib/src/factorio/mod_update_thingy.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 packages/lib/src/factorio/mod_update_thingy.ts diff --git a/packages/lib/src/factorio/mod_update_thingy.ts b/packages/lib/src/factorio/mod_update_thingy.ts new file mode 100644 index 00000000..d55cf335 --- /dev/null +++ b/packages/lib/src/factorio/mod_update_thingy.ts @@ -0,0 +1,31 @@ +import { ModInfo } from "../data"; + + +// the example comments are from the mod maraxsis + +interface modRelease { + download_url: string, // /download/maraxsis/670de301df21001f6147fd1c -- append to mods.factorio.com to download file, auth required + file_name: string, // maraxsis_1.8.0.zip -- the file name of the downloaded file ^^ + info_json: { factorio_version: string } // 2.0 -- seems to just hold what factorio version it supports, due to it being nested, it could other info, not sure + released_at: string, // 2024-10-15T03:35:29.892000Z -- some kind of time stamp, i'm sure that can be imported/converted to a useable format if it useful to have + sha1: string, // e227ee560d93485c7f215c9451eaa9cc38d81c98 -- it's a hash, not much to say + version: string // 1.8.0 -- just a normal version string +} + +interface modDetails { + category: string, // content -- what mod catogory is this apart of, i think a mod can only be apart of 1 category + download_count: number, // 3653 -- how many downloads does the mod have as a number + name: string, // maraxsis -- the name of the mod + owner: string, // notnotmelon -- the name of the author of the mod + releases: Array, // an array of all releases and version of the mod, the type is declared above + score: number, // 1054.8333333333333 -- not sure, seems some kind of float or double, based on name, it's prob used for sorting purposes + summary: string, // the mod description + thumbnail: string, // append path to assets-mods.factorio.com to get full url to thumbnail file + title: string // the full title of the mod, shown on the mod page +} + + +async function getLatestVersion(mod: ModInfo) { + const modStuff: modDetails = await (await fetch("https://mods.factorio.com/api/mods/" + mod.name)).json() as modDetails + const latestVersion = modStuff.releases[modStuff.releases.length].version +} From 71c1cffcacac1ec715df7824f68bdc98bf4fe726 Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Thu, 12 Dec 2024 11:59:37 -0500 Subject: [PATCH 02/28] moved interfaces to ModStore and made fetchLatestVersionMod functions --- packages/lib/src/ModStore.ts | 106 +++++++++++++++++- .../lib/src/factorio/mod_update_thingy.ts | 31 ----- 2 files changed, 105 insertions(+), 32 deletions(-) delete mode 100644 packages/lib/src/factorio/mod_update_thingy.ts diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index e53fdb39..979df95d 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -1,8 +1,9 @@ import fs from "fs-extra"; import path from "path"; -import { Static } from "@sinclair/typebox"; +import { Static, StringContentEncodingOption, Type } from "@sinclair/typebox"; import { logger } from "./logging"; import { ModInfo } from "./data"; +import * as libSchema from "./schema"; import TypedEventEmitter from "./TypedEventEmitter"; import { safeOutputFile } from "./file_ops"; @@ -11,6 +12,58 @@ export interface ModStoreEvents { change: (mod: ModInfo) => void; } +interface ModRelease { + download_url: string, + // /download/maraxsis/670de301df21001f6147fd1c -- append to mods.factorio.com to download file, auth required + file_name: string, + // maraxsis_1.8.0.zip -- the file name of the downloaded file ^^ + info_json: { + dependencies?: Array, + factorio_version: string; + }; + released_at: string, + // 2024-10-15T03:35:29.892000Z -- (ISO 8601) + sha1: string, + // e227ee560d93485c7f215c9451eaa9cc38d81c98 + version: string; // 1.8.0 -- just a normal version string +} + +interface ModImage { + id: string, + thumbnail: string, + url: string; +} + +interface ModLicense { + description: string, + id: string, + name: string, + title: string, + url: string; +} + +interface ModDetails { + category: string, // content -- what mod catogory is this apart of, a mod can only be apart of 1 category + download_count: number, // 3653 -- how many downloads does the mod have as a number + name: string, // maraxsis -- the name of the mod + owner: string, // notnotmelon -- the name of the author of the mod + releases: Array, // an array of all releases and version of the mod + score: number, // 1054.8333333333333 -- it's prob used for sorting purposes + summary: string, // the mod description + thumbnail: string, // append path to assets-mods.factorio.com to get full url to thumbnail file + title: string, // the full title of the mod, shown on the mod page + changelog?: string, + created_at?: string, + description?: string, + homepage?: string, + images?: Array, + license?: ModLicense, + source_url?: string, + tags?: Array, + updated_at?: string, + deprecated?: boolean; +} + export default class ModStore extends TypedEventEmitter { constructor( public modsDirectory: string, @@ -134,4 +187,55 @@ export default class ModStore extends TypedEventEmitter { + const modDetails: ModDetails = await ( + await fetch(`https://mods.factorio.com/api/mods/${modName}`) + ).json() as ModDetails; + return modDetails.releases[modDetails.releases.length].version; + } + + // gets the latest version of the mod provided as a string + static async fetchLatestVersionModInfo(modName: string): Promise { + const modDetails: ModDetails = await ( + await fetch(`https://mods.factorio.com/api/mods/${modName}/full`) + ).json() as ModDetails; + const latestRelease: ModRelease = modDetails.releases[modDetails.releases.length]; + let modInfo = new ModInfo(); + const info_json = latestRelease.info_json; + + // info.json fields + if (modDetails.name) { modInfo.name = modDetails.name; } + if (latestRelease.version) { modInfo.version = latestRelease.version; } + if (modDetails.title) { modInfo.title = modDetails.title; } + if (modDetails.owner) { modInfo.author = modDetails.owner; } + if (modDetails.homepage) { modInfo.homepage = modDetails.homepage; } + if (modDetails.description) { modInfo.description = modDetails.description; } + if (info_json.factorio_version) { modInfo.factorioVersion = info_json.factorio_version; } + if (info_json.dependencies) { modInfo.dependencies = info_json.dependencies; } + + const infoJsonSchema = Type.Object({ + "name": Type.String(), + "version": Type.String(), + "title": Type.String(), + "author": Type.String(), + "contact": Type.Optional(Type.String()), + "homepage": Type.Optional(Type.String()), + "description": Type.Optional(Type.String()), + "factorio_version": Type.Optional(Type.String()), + "dependencies": Type.Optional(Type.Array(Type.String())), + }); + + const validateInfo = libSchema.compile>(infoJsonSchema as any); + + const valid = validateInfo(modInfo); + + if (valid) { + return modInfo; + } + + return modInfo; + + } } diff --git a/packages/lib/src/factorio/mod_update_thingy.ts b/packages/lib/src/factorio/mod_update_thingy.ts deleted file mode 100644 index d55cf335..00000000 --- a/packages/lib/src/factorio/mod_update_thingy.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { ModInfo } from "../data"; - - -// the example comments are from the mod maraxsis - -interface modRelease { - download_url: string, // /download/maraxsis/670de301df21001f6147fd1c -- append to mods.factorio.com to download file, auth required - file_name: string, // maraxsis_1.8.0.zip -- the file name of the downloaded file ^^ - info_json: { factorio_version: string } // 2.0 -- seems to just hold what factorio version it supports, due to it being nested, it could other info, not sure - released_at: string, // 2024-10-15T03:35:29.892000Z -- some kind of time stamp, i'm sure that can be imported/converted to a useable format if it useful to have - sha1: string, // e227ee560d93485c7f215c9451eaa9cc38d81c98 -- it's a hash, not much to say - version: string // 1.8.0 -- just a normal version string -} - -interface modDetails { - category: string, // content -- what mod catogory is this apart of, i think a mod can only be apart of 1 category - download_count: number, // 3653 -- how many downloads does the mod have as a number - name: string, // maraxsis -- the name of the mod - owner: string, // notnotmelon -- the name of the author of the mod - releases: Array, // an array of all releases and version of the mod, the type is declared above - score: number, // 1054.8333333333333 -- not sure, seems some kind of float or double, based on name, it's prob used for sorting purposes - summary: string, // the mod description - thumbnail: string, // append path to assets-mods.factorio.com to get full url to thumbnail file - title: string // the full title of the mod, shown on the mod page -} - - -async function getLatestVersion(mod: ModInfo) { - const modStuff: modDetails = await (await fetch("https://mods.factorio.com/api/mods/" + mod.name)).json() as modDetails - const latestVersion = modStuff.releases[modStuff.releases.length].version -} From a2a3bcd4f874ebe4d529524392b8e6d09ea0b0c5 Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Thu, 12 Dec 2024 17:56:40 -0500 Subject: [PATCH 03/28] adjusted function input for getting latest mod version and adding chunking to not hit request limits --- packages/lib/src/ModStore.ts | 153 +++++++++++++++-------------------- 1 file changed, 64 insertions(+), 89 deletions(-) diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 979df95d..c552690f 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -2,7 +2,7 @@ import fs from "fs-extra"; import path from "path"; import { Static, StringContentEncodingOption, Type } from "@sinclair/typebox"; import { logger } from "./logging"; -import { ModInfo } from "./data"; +import { integerModVersion, ModInfo } from "./data"; import * as libSchema from "./schema"; import TypedEventEmitter from "./TypedEventEmitter"; import { safeOutputFile } from "./file_ops"; @@ -12,56 +12,36 @@ export interface ModStoreEvents { change: (mod: ModInfo) => void; } +interface ModDownloadRequest { + name: string; + version?: string; + sha1?: string; +} + interface ModRelease { download_url: string, - // /download/maraxsis/670de301df21001f6147fd1c -- append to mods.factorio.com to download file, auth required file_name: string, - // maraxsis_1.8.0.zip -- the file name of the downloaded file ^^ - info_json: { - dependencies?: Array, - factorio_version: string; - }; - released_at: string, - // 2024-10-15T03:35:29.892000Z -- (ISO 8601) - sha1: string, - // e227ee560d93485c7f215c9451eaa9cc38d81c98 - version: string; // 1.8.0 -- just a normal version string + info_json: { factorio_version: string; }, + releasted_at: string, // ISO 8601 + version: string, + sha1: string; } -interface ModImage { - id: string, - thumbnail: string, - url: string; -} - -interface ModLicense { - description: string, - id: string, - name: string, - title: string, - url: string; -} interface ModDetails { - category: string, // content -- what mod catogory is this apart of, a mod can only be apart of 1 category - download_count: number, // 3653 -- how many downloads does the mod have as a number - name: string, // maraxsis -- the name of the mod - owner: string, // notnotmelon -- the name of the author of the mod - releases: Array, // an array of all releases and version of the mod - score: number, // 1054.8333333333333 -- it's prob used for sorting purposes - summary: string, // the mod description - thumbnail: string, // append path to assets-mods.factorio.com to get full url to thumbnail file - title: string, // the full title of the mod, shown on the mod page - changelog?: string, - created_at?: string, - description?: string, - homepage?: string, - images?: Array, - license?: ModLicense, - source_url?: string, - tags?: Array, - updated_at?: string, - deprecated?: boolean; + name: string, + title: string, + owner: string, + summary: string, + downloads_count: number, + category: string, + score: number, + releases: Array, +} + +interface ModsInfoResponse { + pagination: any, + results: Array, } export default class ModStore extends TypedEventEmitter { @@ -188,54 +168,49 @@ export default class ModStore extends TypedEventEmitter { - const modDetails: ModDetails = await ( - await fetch(`https://mods.factorio.com/api/mods/${modName}`) - ).json() as ModDetails; - return modDetails.releases[modDetails.releases.length].version; + static getLatestVersionFromReleases(releases: Array): string | undefined { + let latestVersion = "0.0.0"; + releases.forEach((release) => { + if (integerModVersion(latestVersion) < integerModVersion(release.version)) { + latestVersion = release.version; + } + }); + if (latestVersion === "0.0.0") { + return undefined; + } + return latestVersion; } - // gets the latest version of the mod provided as a string - static async fetchLatestVersionModInfo(modName: string): Promise { - const modDetails: ModDetails = await ( - await fetch(`https://mods.factorio.com/api/mods/${modName}/full`) - ).json() as ModDetails; - const latestRelease: ModRelease = modDetails.releases[modDetails.releases.length]; - let modInfo = new ModInfo(); - const info_json = latestRelease.info_json; - - // info.json fields - if (modDetails.name) { modInfo.name = modDetails.name; } - if (latestRelease.version) { modInfo.version = latestRelease.version; } - if (modDetails.title) { modInfo.title = modDetails.title; } - if (modDetails.owner) { modInfo.author = modDetails.owner; } - if (modDetails.homepage) { modInfo.homepage = modDetails.homepage; } - if (modDetails.description) { modInfo.description = modDetails.description; } - if (info_json.factorio_version) { modInfo.factorioVersion = info_json.factorio_version; } - if (info_json.dependencies) { modInfo.dependencies = info_json.dependencies; } - - const infoJsonSchema = Type.Object({ - "name": Type.String(), - "version": Type.String(), - "title": Type.String(), - "author": Type.String(), - "contact": Type.Optional(Type.String()), - "homepage": Type.Optional(Type.String()), - "description": Type.Optional(Type.String()), - "factorio_version": Type.Optional(Type.String()), - "dependencies": Type.Optional(Type.Array(Type.String())), + static async getLatestVersionsChunk(modNames: Array): Promise<{ [key: string]: string | undefined; }> { + const url = new URL("https://mods.factorio.com/api/mods"); + url.searchParams.set("page_size", "max"); + const response = await fetch(url, { + method: "POST", + body: new URLSearchParams({ namelist: modNames }), + }); + if (response.status !== 200) { + throw Error(`Fetch: ${url} returned ${response.status} ${response.statusText}`); + } + const mods = (await response.json() as ModsInfoResponse).results; + const versions: { [key: string]: string | undefined; } = {}; + mods.forEach((mod) => { + const modName = mod.name; + let latestVersion = ModStore.getLatestVersionFromReleases(mod.releases); + versions[modName] = latestVersion; }); - const validateInfo = libSchema.compile>(infoJsonSchema as any); - - const valid = validateInfo(modInfo); - - if (valid) { - return modInfo; - } - - return modInfo; - + return versions; } + + static async getLatestVersions(modNames: Array): Promise<{ [key: string]: string | undefined; }> { + const chunkSize = 500; + const chunks: Array<{ [key: string]: string | undefined; }> = []; + for (let i = 0; i < modNames.length; i += chunkSize) { + chunks.push(await ModStore.getLatestVersionsChunk(modNames.slice(i, i + chunkSize))); + } + const versions = chunks.reduce((acc, curr) => ({ ...acc, ...curr }), {}); + return versions; + } + + } From 61249e5e7e4cd90d63477ccddcd22887542a8340 Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Thu, 12 Dec 2024 19:57:33 -0500 Subject: [PATCH 04/28] no more undefines in that return dict --- packages/lib/src/ModStore.ts | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index c552690f..f6e6ff2a 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -6,6 +6,7 @@ import { integerModVersion, ModInfo } from "./data"; import * as libSchema from "./schema"; import TypedEventEmitter from "./TypedEventEmitter"; import { safeOutputFile } from "./file_ops"; +import { BinaryLike } from "crypto"; export interface ModStoreEvents { /** A stored mod was created, updated or deleted */ @@ -35,7 +36,7 @@ interface ModDetails { summary: string, downloads_count: number, category: string, - score: number, + score?: number, releases: Array, } @@ -168,20 +169,20 @@ export default class ModStore extends TypedEventEmitter): string | undefined { - let latestVersion = "0.0.0"; + private static getLatestVersionFromReleases(releases: Array): string | undefined { + if (releases.length === 0) { + return undefined; + } + let latestVersion = releases[0].version; releases.forEach((release) => { if (integerModVersion(latestVersion) < integerModVersion(release.version)) { latestVersion = release.version; } }); - if (latestVersion === "0.0.0") { - return undefined; - } return latestVersion; } - static async getLatestVersionsChunk(modNames: Array): Promise<{ [key: string]: string | undefined; }> { + private static async getLatestVersionsChunk(modNames: Array): Promise<{ [key: string]: string; }> { const url = new URL("https://mods.factorio.com/api/mods"); url.searchParams.set("page_size", "max"); const response = await fetch(url, { @@ -192,25 +193,32 @@ export default class ModStore extends TypedEventEmitter { const modName = mod.name; let latestVersion = ModStore.getLatestVersionFromReleases(mod.releases); - versions[modName] = latestVersion; + if (latestVersion !== undefined) { versions[modName] = latestVersion; } + }); return versions; } - static async getLatestVersions(modNames: Array): Promise<{ [key: string]: string | undefined; }> { + /** + * + * @param modNames an array of mod names to check for their latest version + * @returns a dict/object with the keys being mod names and values being their latest version, + * it is not guarteed that all mods submited will be returned + */ + static async getLatestVersions(modNames: Array): Promise<{ [key: string]: string; }> { const chunkSize = 500; - const chunks: Array<{ [key: string]: string | undefined; }> = []; + const chunks: Array<{ [key: string]: string; }> = []; for (let i = 0; i < modNames.length; i += chunkSize) { chunks.push(await ModStore.getLatestVersionsChunk(modNames.slice(i, i + chunkSize))); } const versions = chunks.reduce((acc, curr) => ({ ...acc, ...curr }), {}); + return versions; } - } From 24c07c0ed12b6a1980a4c89c786e04879cfd9cc6 Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Thu, 12 Dec 2024 20:03:05 -0500 Subject: [PATCH 05/28] it use maps now, much safer --- packages/lib/src/ModStore.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index f6e6ff2a..43b1b55a 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -182,7 +182,7 @@ export default class ModStore extends TypedEventEmitter): Promise<{ [key: string]: string; }> { + private static async getLatestVersionsChunk(modNames: Array): Promise> { const url = new URL("https://mods.factorio.com/api/mods"); url.searchParams.set("page_size", "max"); const response = await fetch(url, { @@ -193,11 +193,11 @@ export default class ModStore extends TypedEventEmitter = new Map(); mods.forEach((mod) => { const modName = mod.name; let latestVersion = ModStore.getLatestVersionFromReleases(mod.releases); - if (latestVersion !== undefined) { versions[modName] = latestVersion; } + if (latestVersion !== undefined) { versions.set(modName, latestVersion); } }); @@ -210,14 +210,15 @@ export default class ModStore extends TypedEventEmitter): Promise<{ [key: string]: string; }> { + static async getLatestVersions(modNames: Array): Promise> { const chunkSize = 500; - const chunks: Array<{ [key: string]: string; }> = []; + const chunks: Array> = []; for (let i = 0; i < modNames.length; i += chunkSize) { chunks.push(await ModStore.getLatestVersionsChunk(modNames.slice(i, i + chunkSize))); } - const versions = chunks.reduce((acc, curr) => ({ ...acc, ...curr }), {}); - + const versions = chunks.reduce((acc, curr) => ( + new Map([...acc, ...curr]) + ), new Map()); return versions; } From 64e429c5f62e9851da9e504040cdf5e18197579d Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Thu, 12 Dec 2024 20:30:51 -0500 Subject: [PATCH 06/28] got rid of pesky type definitions for function returns and better map merge --- packages/lib/src/ModStore.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 43b1b55a..eecaeabf 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -169,7 +169,7 @@ export default class ModStore extends TypedEventEmitter): string | undefined { + private static getLatestVersionFromReleases(releases: Array) { if (releases.length === 0) { return undefined; } @@ -182,7 +182,7 @@ export default class ModStore extends TypedEventEmitter): Promise> { + private static async getLatestVersionsChunk(modNames: string[]) { const url = new URL("https://mods.factorio.com/api/mods"); url.searchParams.set("page_size", "max"); const response = await fetch(url, { @@ -207,18 +207,19 @@ export default class ModStore extends TypedEventEmitter): Promise> { + static async getLatestVersions(modNames: string[]) { const chunkSize = 500; - const chunks: Array> = []; + let versions = new Map(); for (let i = 0; i < modNames.length; i += chunkSize) { - chunks.push(await ModStore.getLatestVersionsChunk(modNames.slice(i, i + chunkSize))); + const chunk = (await ModStore.getLatestVersionsChunk(modNames.slice(i, i + chunkSize))); + + for (const [modName, version] of chunk) { + versions.set(modName, version); + } } - const versions = chunks.reduce((acc, curr) => ( - new Map([...acc, ...curr]) - ), new Map()); return versions; } From 4080c1025dc1cd5e99e774783fea084385c71cfe Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Fri, 13 Dec 2024 17:05:09 -0500 Subject: [PATCH 07/28] added download file function --- packages/lib/src/ModStore.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index eecaeabf..155a1a72 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -7,6 +7,7 @@ import * as libSchema from "./schema"; import TypedEventEmitter from "./TypedEventEmitter"; import { safeOutputFile } from "./file_ops"; import { BinaryLike } from "crypto"; +import { Writable } from "stream"; export interface ModStoreEvents { /** A stored mod was created, updated or deleted */ @@ -223,4 +224,27 @@ export default class ModStore extends TypedEventEmitter Date: Fri, 13 Dec 2024 17:05:28 -0500 Subject: [PATCH 08/28] fixed mod export command --- docs/migrating-from-2.0.0-alpha.13.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/migrating-from-2.0.0-alpha.13.md b/docs/migrating-from-2.0.0-alpha.13.md index adddebdd..272e947a 100644 --- a/docs/migrating-from-2.0.0-alpha.13.md +++ b/docs/migrating-from-2.0.0-alpha.13.md @@ -8,7 +8,7 @@ Please study the entire migration guide carefully before attempting to update an If you're using mods you need to create mod pack string for the mod configurations you're currently running. You can do this by running the follow RCON command on an instance that is currently running the mod configuration you want to use (you can also run this command in single player and find the string printed in Factorio's [Log file](https://wiki.factorio.com/Log_file)): - /sc local m={name="pack",description="",factorio_version=game.active_mods.base,mods={},settings={startup={},["runtime-global"]={},["runtime-per-user"]={}}} for n,v in pairs(game.active_mods) do table.insert(m.mods,{name=n,enabled=true,version=v}) end local function set(s,c) for n,v in pairs(s) do m.settings[c][n]=v end end set(settings.startup,"startup") set(settings.global,"runtime-global") if game.player then set(game.player.mod_settings,"runtime-per-user") else set(settings.player,"runtime-per-user") end local e=game.encode_string(game.table_to_json(m)) log(e) rcon.print(e) + /sc local a={name="pack",description="",factorio_version=script.active_mods.base,mods={},settings={startup={},["runtime-global"]={},["runtime-per-user"]={}}}for b,c in pairs(script.active_mods)do table.insert(a.mods,{name=b,enabled=true,version=c})end;local function d(e,f)for b,c in pairs(e)do a.settings[f][b]=c end end;d(settings.startup,"startup")d(settings.global,"runtime-global")if game.player then d(game.player.mod_settings,"runtime-per-user")else d(settings.player,"runtime-per-user")end;local g=helpers.encode_string(helpers.table_to_json(a))log(g)rcon.print(g) This should produce a [Mod Pack String](https://forums.factorio.com/viewtopic.php?f=96&t=103578) which you need to copy and paste somewhere to keep it for later. Mod Pack strings looks something like this: From e1c29bcde1dc3ed9b826ad8f83e40812af8e365a Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Fri, 13 Dec 2024 22:49:20 -0500 Subject: [PATCH 09/28] added functions to download mods from a Map of names and versions --- packages/lib/src/ModStore.ts | 57 ++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 155a1a72..24cd6bcb 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -8,6 +8,7 @@ import TypedEventEmitter from "./TypedEventEmitter"; import { safeOutputFile } from "./file_ops"; import { BinaryLike } from "crypto"; import { Writable } from "stream"; +import { read } from "fs"; export interface ModStoreEvents { /** A stored mod was created, updated or deleted */ @@ -211,7 +212,10 @@ export default class ModStore extends TypedEventEmitter(); for (let i = 0; i < modNames.length; i += chunkSize) { @@ -224,7 +228,56 @@ export default class ModStore extends TypedEventEmitter, version: string, + username: string, token: string + ) { + if (releases.length === 0) { } + releases.forEach(async (release) => { + if (release.version === version) { + const url = new URL(`https://mods.factorio.com/${release.download_url}`); + url.searchParams.set("username", username); + url.searchParams.set("token", token); + const fileName = `${this.modsDirectory}/${release.file_name}.tmp`; + await ModStore.downloadFile(url, fileName); + await fs.rename(fileName, `${this.modsDirectory}/${release.file_name}`); + } + }); + } + + private async downloadModsChunk(modNames: string[], modVersions: string[], username: string, token: string) { + const url = new URL("https://mods.factorio.com/api/mods"); + url.searchParams.set("page_size", "max"); + const response = await fetch(url, { + method: "POST", + body: new URLSearchParams({ namelist: modNames }), + }); + if (response.status !== 200) { + throw Error(`Fetch: ${url} returned ${response.status} ${response.statusText}`); + } + const mods = (await response.json() as ModsInfoResponse).results; + let futures: Promise[] = []; + mods.forEach((mod) => { + futures.push(this.downloadModFromReleases(mod.releases, modVersions[modNames.indexOf(mod.name)], + username, token)); + }); + futures.forEach(async (future) => await future); + } + + async downloadMods(modMap: Map, username: string, token: string) { + const chunkSize = 500; + let futures: Promise[] = []; + const modNames = Array.from(modMap.keys()); + const modVersions = Array.from(modMap.values()); + for (let i = 0; i < modMap.size; i += chunkSize) { + futures.push( + this.downloadModsChunk(modNames.slice(i, i + chunkSize), modVersions.slice(i, i + chunkSize), + username, token) + ); + } + futures.forEach(async (future) => await future); + } + + static async downloadFile(url: string | URL, filePath: string) { try { const response = await fetch(url); From 61a241b02d4f63b6cbf7a537e9443b37e436a371 Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Sat, 14 Dec 2024 17:57:56 -0500 Subject: [PATCH 10/28] added function to update modpacks --- packages/lib/src/ModStore.ts | 93 ++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 8 deletions(-) diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 24cd6bcb..5013b4ef 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -2,7 +2,7 @@ import fs from "fs-extra"; import path from "path"; import { Static, StringContentEncodingOption, Type } from "@sinclair/typebox"; import { logger } from "./logging"; -import { integerModVersion, ModInfo } from "./data"; +import { integerModVersion, ModInfo, ModPack, ModRecord } from "./data"; import * as libSchema from "./schema"; import TypedEventEmitter from "./TypedEventEmitter"; import { safeOutputFile } from "./file_ops"; @@ -184,9 +184,10 @@ export default class ModStore extends TypedEventEmitter = new Map(); + const versions = new Map(); mods.forEach((mod) => { const modName = mod.name; let latestVersion = ModStore.getLatestVersionFromReleases(mod.releases); @@ -209,17 +210,21 @@ export default class ModStore extends TypedEventEmitter= 3) { + factorioVersion = factorioVersion.slice(0, factorioVersion.lastIndexOf(".")); + } const chunkSize = 500; let versions = new Map(); for (let i = 0; i < modNames.length; i += chunkSize) { - const chunk = (await ModStore.getLatestVersionsChunk(modNames.slice(i, i + chunkSize))); + const chunk = (await ModStore.getLatestVersionsChunk(modNames.slice(i, i + chunkSize), factorioVersion)); for (const [modName, version] of chunk) { versions.set(modName, version); @@ -232,6 +237,7 @@ export default class ModStore extends TypedEventEmitter { if (release.version === version) { const url = new URL(`https://mods.factorio.com/${release.download_url}`); @@ -240,13 +246,17 @@ export default class ModStore extends TypedEventEmitter await future); } - async downloadMods(modMap: Map, username: string, token: string) { + + /** + * + * downloads mods to the mod dir, will not download mods for which are already in the mod directory + * + * @param modMap a map of what mods to download and what version to download + * @param username factorio username used for auth to download mods + * @param token factorio token used for auth to download mods + * @param factorioVersion factorio version to filter for (mods are not guaranteed to work between Middle versions) + * + */ + async downloadMods(modMap: Map, username: string, token: string, factorioVersion: string = "1.1") { const chunkSize = 500; let futures: Promise[] = []; const modNames = Array.from(modMap.keys()); const modVersions = Array.from(modMap.values()); + if (factorioVersion.split(".").length >= 3) { + factorioVersion = factorioVersion.slice(0, factorioVersion.lastIndexOf(".")); + } + + // get rid of mods already downloaded, why download what is already there + modMap.forEach((modVersion, modName, map) => { + const filename = `${modName}_${modVersion}.zip`; + if (this.files.has(filename)) { + map.delete(modName); + } + }); + + for (let i = 0; i < modMap.size; i += chunkSize) { futures.push( this.downloadModsChunk(modNames.slice(i, i + chunkSize), modVersions.slice(i, i + chunkSize), - username, token) + username, token, factorioVersion) ); } futures.forEach(async (future) => await future); @@ -300,4 +334,47 @@ export default class ModStore extends TypedEventEmitter { + const currentVerScore = integerModVersion(modPack.mods.get(modName)?.version as string); + const latestVerScore = integerModVersion(modVersion); + const haveLatestVer = this.files.has(`${modName}_${modVersion}.zip`); + if (currentVerScore >= latestVerScore) { + map.delete(modName); + return; + } + if (haveLatestVer) { + let mod = modPack.mods.get(modName) as ModRecord; + mod.version = modVersion; + map.delete(modName); + } + }); + + // download latest version of mods + await this.downloadMods(modLatestVersion, username, token, factorioVersion); + + // switch mods to use said latest version + modLatestVersion.forEach((modVersion, modName, map) => { + const haveLatestVer = this.files.has(`${modName}_${modVersion}.zip`); + if (haveLatestVer) { + let mod = modPack.mods.get(modName) as ModRecord; + mod.version = modVersion; + } + }); + } + } From a60a3bcbd7a84140cf1a22eb2d86f41769cdc6ad Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Sun, 15 Dec 2024 11:48:12 -0500 Subject: [PATCH 11/28] fixed type, split download modpack function into downloadmods and updatemods , returns a map of mods and their latest versions --- packages/lib/src/ModStore.ts | 100 +++++++++++++++++++++++++---------- 1 file changed, 73 insertions(+), 27 deletions(-) diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 5013b4ef..7d77179c 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -9,6 +9,8 @@ import { safeOutputFile } from "./file_ops"; import { BinaryLike } from "crypto"; import { Writable } from "stream"; import { read } from "fs"; +import { Value } from "@sinclair/typebox/value"; +import { keyword$DataError } from "ajv/dist/compile/errors"; export interface ModStoreEvents { /** A stored mod was created, updated or deleted */ @@ -25,7 +27,7 @@ interface ModRelease { download_url: string, file_name: string, info_json: { factorio_version: string; }, - releasted_at: string, // ISO 8601 + released_at: string, // ISO 8601 version: string, sha1: string; } @@ -203,7 +205,6 @@ export default class ModStore extends TypedEventEmitter= 3) { factorioVersion = factorioVersion.slice(0, factorioVersion.lastIndexOf(".")); } + modNames = modNames.filter((modName, index, array) => !(modName in ModPack.getBuiltinModNames(factorioVersion)) + ); + const chunkSize = 500; let versions = new Map(); for (let i = 0; i < modNames.length; i += chunkSize) { @@ -238,17 +242,21 @@ export default class ModStore extends TypedEventEmitter { - if (release.version === version) { - const url = new URL(`https://mods.factorio.com/${release.download_url}`); - url.searchParams.set("username", username); - url.searchParams.set("token", token); - const fileName = `${this.modsDirectory}/${release.file_name}.tmp`; - await ModStore.downloadFile(url, fileName); - await fs.rename(fileName, `${this.modsDirectory}/${release.file_name}`); - this.addMod(await this.loadFile(`${this.modsDirectory}/${release.file_name}`)); - } - }); + // releases.find() // work on this + + const latestRelease = releases.find(release => release.version === version); + + if (latestRelease === undefined) { + return; + } + + const url = new URL(`https://mods.factorio.com/${latestRelease.download_url}`); + url.searchParams.set("username", username); + url.searchParams.set("token", token); + const fileName = `${this.modsDirectory}/${latestRelease.file_name}.tmp`; + await ModStore.downloadFile(url, fileName); + await fs.rename(fileName, `${this.modsDirectory}/${latestRelease.file_name}`); + this.addMod(await this.loadFile(`${this.modsDirectory}/${latestRelease.file_name}`)); } private async downloadModsChunk( @@ -265,12 +273,9 @@ export default class ModStore extends TypedEventEmitter[] = []; - mods.forEach((mod) => { - futures.push(this.downloadModFromReleases(mod.releases, modVersions[modNames.indexOf(mod.name)], - username, token)); - }); - futures.forEach(async (future) => await future); + + await Promise.all(mods.map((mod) => this.downloadModFromReleases(mod.releases, + modVersions[modNames.indexOf(mod.name)], username, token))); } @@ -287,8 +292,7 @@ export default class ModStore extends TypedEventEmitter, username: string, token: string, factorioVersion: string = "1.1") { const chunkSize = 500; let futures: Promise[] = []; - const modNames = Array.from(modMap.keys()); - const modVersions = Array.from(modMap.values()); + modMap = new Map(modMap); if (factorioVersion.split(".").length >= 3) { factorioVersion = factorioVersion.slice(0, factorioVersion.lastIndexOf(".")); } @@ -296,19 +300,20 @@ export default class ModStore extends TypedEventEmitter { const filename = `${modName}_${modVersion}.zip`; - if (this.files.has(filename)) { + if (modName in ModPack.getBuiltinModNames(factorioVersion) || this.files.has(filename)) { map.delete(modName); } }); - + const modNames = Array.from(modMap.keys()); + const modVersions = Array.from(modMap.values()); for (let i = 0; i < modMap.size; i += chunkSize) { futures.push( this.downloadModsChunk(modNames.slice(i, i + chunkSize), modVersions.slice(i, i + chunkSize), username, token, factorioVersion) ); } - futures.forEach(async (future) => await future); + await Promise.all(futures); } static async downloadFile(url: string | URL, filePath: string) { @@ -334,6 +339,43 @@ export default class ModStore extends TypedEventEmitter(); + + modPack.mods.forEach((value, key, map) => { + currentVersions.set(key, value.version); + }); + const factorioVersion = modPack.factorioVersion; + + const latestVersions = await ModStore.getLatestVersions(modNames, factorioVersion); + + let versionChanges = new Map(); + for (const modName in latestVersions.keys()) { + if (!latestVersions.has(modName)) { continue; } + + const latestVersion = latestVersions.get(modName) as string; + + const currentVerScore = integerModVersion(currentVersions.get(modName) as string); + const latestVerScore = integerModVersion(latestVersion); + const haveLatestVer = this.files.has(`${modName}_${latestVersion}.zip`); + + if (currentVerScore >= latestVerScore) { + continue; + } + if (modName in ModPack.getBuiltinModNames(modPack.factorioVersion)) { + continue; + } + if (haveLatestVer) { + versionChanges.set(modName, latestVersion); + } + } + + return versionChanges; + + + } + /** * goes through a modpack and updates all it's mods, whether that is switching to a newer version or @@ -343,11 +385,11 @@ export default class ModStore extends TypedEventEmitter { const currentVerScore = integerModVersion(modPack.mods.get(modName)?.version as string); @@ -357,18 +399,22 @@ export default class ModStore extends TypedEventEmitter { + modLatestVersion.forEach((modVersion, modName, _map) => { const haveLatestVer = this.files.has(`${modName}_${modVersion}.zip`); if (haveLatestVer) { let mod = modPack.mods.get(modName) as ModRecord; From 05790966db4807242fc0b4ae7e9d6bdc664637f7 Mon Sep 17 00:00:00 2001 From: O2theC <25ocarter@gmail.com> Date: Sun, 15 Dec 2024 11:48:50 -0500 Subject: [PATCH 12/28] removed unused imports --- packages/lib/src/ModStore.ts | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 7d77179c..0f85cfe3 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -1,16 +1,11 @@ import fs from "fs-extra"; import path from "path"; -import { Static, StringContentEncodingOption, Type } from "@sinclair/typebox"; +import { Static} from "@sinclair/typebox"; import { logger } from "./logging"; import { integerModVersion, ModInfo, ModPack, ModRecord } from "./data"; -import * as libSchema from "./schema"; import TypedEventEmitter from "./TypedEventEmitter"; import { safeOutputFile } from "./file_ops"; -import { BinaryLike } from "crypto"; import { Writable } from "stream"; -import { read } from "fs"; -import { Value } from "@sinclair/typebox/value"; -import { keyword$DataError } from "ajv/dist/compile/errors"; export interface ModStoreEvents { /** A stored mod was created, updated or deleted */ From 2ae3f9a600688fbd841a03aaef325c743a2bf981 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sat, 15 Mar 2025 23:13:33 +0100 Subject: [PATCH 13/28] Add mod portal search --- packages/controller/src/ControlConnection.ts | 17 ++ packages/lib/src/ModStore.ts | 125 ++++++++++++- packages/lib/src/data/messages_mod.ts | 85 +++++++++ packages/lib/src/link/messages.ts | 1 + packages/lib/src/permissions.ts | 6 + packages/web_ui/src/components/ModsPage.tsx | 186 ++++++++++++++++++- 6 files changed, 409 insertions(+), 11 deletions(-) diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 8ef23516..315e6b92 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -98,6 +98,7 @@ export default class ControlConnection extends BaseConnection { this.handle(lib.ModGetRequest, this.handleModGetRequest.bind(this)); this.handle(lib.ModListRequest, this.handleModListRequest.bind(this)); this.handle(lib.ModSearchRequest, this.handleModSearchRequest.bind(this)); + this.handle(lib.ModPortalSearchRequest, this.handleModPortalSearchRequest.bind(this)); this.handle(lib.ModDeleteRequest, this.handleModDeleteRequest.bind(this)); this.handle(lib.LogSetSubscriptionsRequest, this.handleLogSetSubscriptionsRequest.bind(this)); this.handle(lib.LogQueryRequest, this.handleLogQueryRequest.bind(this)); @@ -514,6 +515,22 @@ export default class ControlConnection extends BaseConnection { }; } + async handleModPortalSearchRequest(request: lib.ModPortalSearchRequest) { + try { + return lib.ModStore.searchModPortal( + request.query, + request.factorioVersion, + request.page, + request.pageSize || 10, + request.sort, + request.sortOrder || "asc" + ); + } catch (error: any) { + logger.error(`Error searching mod portal: ${error}`); + throw new lib.RequestError(`Failed to search mod portal: ${error.message}`); + } + } + async handleModDeleteRequest(request: lib.ModDeleteRequest) { await this._controller.modStore.deleteMod(request.name, request.version); } diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 0f85cfe3..0c3b8fb6 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -1,6 +1,6 @@ import fs from "fs-extra"; import path from "path"; -import { Static} from "@sinclair/typebox"; +import { Static } from "@sinclair/typebox"; import { logger } from "./logging"; import { integerModVersion, ModInfo, ModPack, ModRecord } from "./data"; import TypedEventEmitter from "./TypedEventEmitter"; @@ -36,7 +36,8 @@ interface ModDetails { downloads_count: number, category: string, score?: number, - releases: Array, + releases?: Array, + latest_release?: ModRelease, } interface ModsInfoResponse { @@ -196,9 +197,14 @@ export default class ModStore extends TypedEventEmitter(); mods.forEach((mod) => { const modName = mod.name; - let latestVersion = ModStore.getLatestVersionFromReleases(mod.releases); - if (latestVersion !== undefined) { versions.set(modName, latestVersion); } - + if (mod.releases) { + let latestVersion = ModStore.getLatestVersionFromReleases(mod.releases); + if (latestVersion !== undefined) { versions.set(modName, latestVersion); } + } else if (mod.latest_release) { + versions.set(modName, mod.latest_release.version); + } else { + logger.warn(`Mod ${modName} has no releases or latest_release`); + } }); return versions; } @@ -217,8 +223,9 @@ export default class ModStore extends TypedEventEmitter= 3) { factorioVersion = factorioVersion.slice(0, factorioVersion.lastIndexOf(".")); } - modNames = modNames.filter((modName, index, array) => !(modName in ModPack.getBuiltinModNames(factorioVersion)) - ); + + const builtinMods = ModPack.getBuiltinModNames(factorioVersion); + modNames = modNames.filter(modName => !(modName in builtinMods)); const chunkSize = 500; let versions = new Map(); @@ -269,8 +276,16 @@ export default class ModStore extends TypedEventEmitter this.downloadModFromReleases(mod.releases, - modVersions[modNames.indexOf(mod.name)], username, token))); + await Promise.all(mods.map((mod) => { + if (mod.releases) { + return this.downloadModFromReleases(mod.releases, + modVersions[modNames.indexOf(mod.name)], username, token); + } else if (mod.latest_release) { + return this.downloadModFromReleases([mod.latest_release], + modVersions[modNames.indexOf(mod.name)], username, token); + } + throw new Error(`Mod ${mod.name} has no releases or latest_release`); + })); } @@ -418,4 +433,96 @@ export default class ModStore extends TypedEventEmitter { + // Find the latest release for the specified Factorio version + const latestRelease = mod.latest_release; + + const transformedMod = { + name: mod.name, + title: mod.title, + summary: mod.summary, + owner: mod.owner, + downloads_count: mod.downloads_count, + latest_release: latestRelease ? { + version: latestRelease.version, + factorio_version: latestRelease.info_json.factorio_version, + released_at: latestRelease.released_at, + download_url: latestRelease.download_url, + file_name: latestRelease.file_name, + sha1: latestRelease.sha1, + } : undefined, + }; + return transformedMod; + }).filter(mod => mod.latest_release); // Only include mods with a release for the specified version + + return { + queryIssues: [], + pageCount: Math.ceil(data.pagination.count / pageSize), + resultCount: data.pagination.count, + results, + }; + } } diff --git a/packages/lib/src/data/messages_mod.ts b/packages/lib/src/data/messages_mod.ts index 60fc6563..3dd61955 100644 --- a/packages/lib/src/data/messages_mod.ts +++ b/packages/lib/src/data/messages_mod.ts @@ -197,6 +197,91 @@ export class ModSearchRequest { }; } +export class ModPortalSearchRequest { + declare ["constructor"]: typeof ModPortalSearchRequest; + static type = "request" as const; + static src = "control" as const; + static dst = "controller" as const; + static permission = "core.mod.search_portal" as const; + + constructor( + public query: string, + public factorioVersion: string, + public page: number, + public pageSize?: number, + public sort?: string, + public sortOrder?: string, + ) { } + + static jsonSchema = Type.Object({ + "query": Type.String(), + "factorioVersion": Type.String(), + "page": Type.Integer(), + "pageSize": Type.Optional(Type.Integer()), + "sort": Type.Optional(Type.String()), + "sortOrder": Type.Optional(Type.String()), + }); + + static fromJSON(json: Static) { + return new this(json.query, json.factorioVersion, json.page, json.pageSize, json.sort, json.sortOrder); + } + + static Response = class Response { + constructor( + public queryIssues: string[], + public pageCount: number, + public resultCount: number, + public results: { + name: string, + title: string, + summary: string, + owner: string, + downloads_count: number, + latest_release: { + version: string, + factorio_version: string, + released_at: string, + download_url: string, + file_name: string, + sha1: string, + }, + }[], + ) { } + + static jsonSchema = Type.Object({ + "queryIssues": Type.Array(Type.String()), + "pageCount": Type.Integer(), + "resultCount": Type.Integer(), + "results": Type.Array( + Type.Object({ + "name": Type.String(), + "title": Type.String(), + "summary": Type.String(), + "owner": Type.String(), + "downloads_count": Type.Number(), + "latest_release": Type.Object({ + "version": Type.String(), + "factorio_version": Type.String(), + "released_at": Type.String(), + "download_url": Type.String(), + "file_name": Type.String(), + "sha1": Type.String(), + }), + }), + ), + }); + + static fromJSON(json: Static) { + return new this( + json.queryIssues, + json.pageCount, + json.resultCount, + json.results + ); + } + }; +} + export class ModDownloadRequest { declare ["constructor"]: typeof ModDownloadRequest; static type = "request" as const; diff --git a/packages/lib/src/link/messages.ts b/packages/lib/src/link/messages.ts index 4182db3a..b1c17826 100644 --- a/packages/lib/src/link/messages.ts +++ b/packages/lib/src/link/messages.ts @@ -97,6 +97,7 @@ export const dataClasses: (RequestClass | EventClass) mod.ModGetRequest, mod.ModListRequest, mod.ModSearchRequest, + mod.ModPortalSearchRequest, mod.ModDownloadRequest, mod.ModDeleteRequest, mod.ModPackUpdatesEvent, diff --git a/packages/lib/src/permissions.ts b/packages/lib/src/permissions.ts index 306047de..4e64684d 100644 --- a/packages/lib/src/permissions.ts +++ b/packages/lib/src/permissions.ts @@ -373,6 +373,12 @@ definePermission({ description: "Search through the list of mods stored on the controller.", grantByDefault: true, }); +definePermission({ + name: "core.mod.search_portal", + title: "Search mod portal", + description: "Search for mods on the Factorio mod portal.", + grantByDefault: true, +}); definePermission({ name: "core.mod.subscribe", title: "Subscribe to mod updates", diff --git a/packages/web_ui/src/components/ModsPage.tsx b/packages/web_ui/src/components/ModsPage.tsx index eeec0224..767b32bf 100644 --- a/packages/web_ui/src/components/ModsPage.tsx +++ b/packages/web_ui/src/components/ModsPage.tsx @@ -1,8 +1,9 @@ -import React, { Fragment, useState, useContext } from "react"; +import React, { Fragment, useState, useContext, useEffect } from "react"; import { useNavigate } from "react-router-dom"; import { Button, Form, Input, Modal, Popconfirm, Space, Table, Typography, Upload } from "antd"; import ImportOutlined from "@ant-design/icons/ImportOutlined"; import PlusOutlined from "@ant-design/icons/PlusOutlined"; +import SearchOutlined from "@ant-design/icons/SearchOutlined"; import * as lib from "@clusterio/lib"; @@ -129,6 +130,184 @@ function CreateModPackButton() { ; } +function SearchModsButton() { + const control = useContext(ControlContext); + const account = useAccount(); + const [open, setOpen] = useState(false); + const [form] = Form.useForm(); + const [searchText, setSearchText] = useState(""); + const [factorioVersion, setFactorioVersion] = useState("1.1"); + const [modResults, setModResults] = useState([]); + const [loading, setLoading] = useState(false); + const [page, setPage] = useState(1); + const [pageSize, setPageSize] = useState(10); + const [totalResults, setTotalResults] = useState(0); + const [totalPages, setTotalPages] = useState(0); + const [sort, setSort] = useState("name"); + const [sortOrder, setSortOrder] = useState<"asc" | "desc">("asc"); + + // Search mods when search parameters change + useEffect(() => { + if (!open) { return; } + + setLoading(true); + let canceled = false; + + control.send(new lib.ModPortalSearchRequest( + searchText, + factorioVersion, + page, + pageSize, + sort, + sortOrder + )).then(response => { + if (canceled) { return; } + + setModResults(response.results); + setTotalResults(response.resultCount); + setTotalPages(response.pageCount); + setLoading(false); + }).catch(error => { + if (canceled) { return; } + setLoading(false); + }); + + return () => { + canceled = true; + }; + }, [open, searchText, factorioVersion, page, pageSize, sort, sortOrder, control]); + + const handleSearch = () => { + const values = form.getFieldsValue(); + setSearchText(values.name || ""); + setFactorioVersion(values.factorioVersion || "1.1"); + setPage(1); // Reset to first page on new search + }; + + const handleTableChange = ( + pagination: any, + filters: any, + sorter: any + ) => { + if (pagination.current !== page) { + setPage(pagination.current); + } + if (pagination.pageSize !== pageSize) { + setPageSize(pagination.pageSize); + } + + if (sorter.field && sorter.order) { + setSort(sorter.field); + setSortOrder(sorter.order === "ascend" ? "asc" : "desc"); + } + }; + + return <> + + { setOpen(false); }} + width={800} + footer={[ + , + , + ]} + > +
+ + + + + + +
+ + record.name} + loading={loading} + onChange={handleTableChange} + pagination={{ + current: page, + pageSize: pageSize, + total: totalResults, + showSizeChanger: true, + pageSizeOptions: ["10", "20", "50"], + }} + expandable={{ + expandedRowRender: record => ( +
+

Summary: {record.summary}

+

Downloads: {record.downloads_count}

+

Latest Release:

+
    +
  • Version: {record.latest_release.version}
    • +
  • Factorio Version: {record.latest_release.info_json.factorio_version}
    • +
  • Released: {new Date(record.latest_release.released_at).toLocaleString()}
    • +
+ {account.hasPermission("core.mod.download") && ( + + )} +
+ ), + }} + columns={[ + { + title: "Name", + dataIndex: "name", + key: "name", + sorter: true, + sortOrder: sort === "name" ? (sortOrder === "asc" ? "ascend" : "descend") : undefined, + }, + { + title: "Title", + dataIndex: "title", + key: "title", + sorter: true, + sortOrder: sort === "title" ? (sortOrder === "asc" ? "ascend" : "descend") : undefined, + }, + { + title: "Author", + dataIndex: "owner", + key: "owner", + sorter: true, + sortOrder: sort === "owner" ? (sortOrder === "asc" ? "ascend" : "descend") : undefined, + }, + { + title: "Latest Version", + key: "version", + render: (_, record) => record.latest_release.version, + }, + ]} + /> + + ; +} + export default function ModsPage() { let account = useAccount(); let control = useContext(ControlContext); @@ -223,7 +402,10 @@ export default function ModsPage() { }, })} /> - + + + {uploadButton} + } /> Date: Tue, 8 Apr 2025 20:52:56 +0200 Subject: [PATCH 14/28] Fix mod description rendering --- packages/web_ui/src/components/ModsPage.tsx | 32 +++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/packages/web_ui/src/components/ModsPage.tsx b/packages/web_ui/src/components/ModsPage.tsx index 767b32bf..ef8d83d4 100644 --- a/packages/web_ui/src/components/ModsPage.tsx +++ b/packages/web_ui/src/components/ModsPage.tsx @@ -247,15 +247,27 @@ function SearchModsButton() { expandable={{ expandedRowRender: record => (
-

Summary: {record.summary}

-

Downloads: {record.downloads_count}

+

Summary: {record.summary ?? "N/A"}

+

Downloads: {record.downloads_count ?? "N/A"}

Latest Release:

-
    -
  • Version: {record.latest_release.version}
    • -
  • Factorio Version: {record.latest_release.info_json.factorio_version}
    • -
  • Released: {new Date(record.latest_release.released_at).toLocaleString()}
    • -
- {account.hasPermission("core.mod.download") && ( + {record.latest_release ? ( +
    +
  • Version: {record.latest_release.version ?? "N/A"}
    • +
  • + Factorio Version: { + record.latest_release.factorio_version ?? "N/A" + } +
    • +
  • + Released: { + record.latest_release.released_at + ? new Date(record.latest_release.released_at).toLocaleString() + : "N/A" + } +
    • +
+ ) :

No release information available.

} + {account.hasPermission("core.mod.download") && record.latest_release && ( @@ -300,7 +314,7 @@ function SearchModsButton() { { title: "Latest Version", key: "version", - render: (_, record) => record.latest_release.version, + render: (_, record) => record.latest_release?.version ?? "N/A", }, ]} /> From 3bf0d48641936df9f28bd55d737c1690f5621c14 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Tue, 8 Apr 2025 20:53:06 +0200 Subject: [PATCH 15/28] Fix linter warnings --- packages/web_ui/src/components/ModsPage.tsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/packages/web_ui/src/components/ModsPage.tsx b/packages/web_ui/src/components/ModsPage.tsx index ef8d83d4..3b0654c4 100644 --- a/packages/web_ui/src/components/ModsPage.tsx +++ b/packages/web_ui/src/components/ModsPage.tsx @@ -172,6 +172,7 @@ function SearchModsButton() { setLoading(false); }); + // eslint-disable-next-line consistent-return return () => { canceled = true; }; @@ -202,6 +203,13 @@ function SearchModsButton() { } }; + const getColumnSortOrder = (columnKey: string): "ascend" | "descend" | undefined => { + if (sort === columnKey) { + return sortOrder === "asc" ? "ascend" : "descend"; + } + return undefined; + }; + return <> Date: Tue, 8 Apr 2025 22:31:41 +0200 Subject: [PATCH 16/28] Search mods clientside --- packages/controller/src/ControlConnection.ts | 24 +- packages/lib/src/ModStore.ts | 149 +++++------ packages/lib/src/data/messages_mod.ts | 108 ++++---- packages/lib/src/link/messages.ts | 2 +- packages/web_ui/src/components/ModsPage.tsx | 250 +++++++++++++++---- 5 files changed, 326 insertions(+), 207 deletions(-) diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 315e6b92..0ce1d46c 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -98,7 +98,7 @@ export default class ControlConnection extends BaseConnection { this.handle(lib.ModGetRequest, this.handleModGetRequest.bind(this)); this.handle(lib.ModListRequest, this.handleModListRequest.bind(this)); this.handle(lib.ModSearchRequest, this.handleModSearchRequest.bind(this)); - this.handle(lib.ModPortalSearchRequest, this.handleModPortalSearchRequest.bind(this)); + this.handle(lib.ModPortalGetAllRequest, this.handleModPortalGetAllRequest.bind(this)); this.handle(lib.ModDeleteRequest, this.handleModDeleteRequest.bind(this)); this.handle(lib.LogSetSubscriptionsRequest, this.handleLogSetSubscriptionsRequest.bind(this)); this.handle(lib.LogQueryRequest, this.handleLogQueryRequest.bind(this)); @@ -515,19 +515,23 @@ export default class ControlConnection extends BaseConnection { }; } - async handleModPortalSearchRequest(request: lib.ModPortalSearchRequest) { + + /** + * Handle request to fetch all mods from the Factorio Mod Portal for a given version. + * @param request - The request object containing factorioVersion. + */ + async handleModPortalGetAllRequest(request: lib.ModPortalGetAllRequest) { try { - return lib.ModStore.searchModPortal( - request.query, + const mods = await lib.ModStore.fetchAllModsFromPortal( request.factorioVersion, - request.page, - request.pageSize || 10, - request.sort, - request.sortOrder || "asc" + request.hide_deprecated ); + // The Response class is defined inline within the Request class + return new lib.ModPortalGetAllRequest.Response(mods); } catch (error: any) { - logger.error(`Error searching mod portal: ${error}`); - throw new lib.RequestError(`Failed to search mod portal: ${error.message}`); + logger.error(`Error fetching all mods from portal (${request.factorioVersion}): ${error.message}`); + // Propagate a user-friendly error back to the client + throw new lib.RequestError(`Portal mod fetch failed: ${error.message}`); } } diff --git a/packages/lib/src/ModStore.ts b/packages/lib/src/ModStore.ts index 0c3b8fb6..6698a431 100644 --- a/packages/lib/src/ModStore.ts +++ b/packages/lib/src/ModStore.ts @@ -45,6 +45,28 @@ interface ModsInfoResponse { results: Array, } +/** + * Interface describing the structure of the pagination info from the portal API + */ +interface ModPortalPagination { + count: number; + links: { next?: string; prev?: string; }; + page: number; + page_count: number; + page_size: number; +} + +/** + * Interface describing the structure of the portal API response for a single page + */ +interface ModPortalResponse { + pagination: ModPortalPagination; + results: ModDetails[]; +} + +const MOD_PORTAL_API_URL = "https://mods.factorio.com/api/mods"; +const MOD_PORTAL_MAX_PAGE_SIZE = 10000; // Or largest allowed by API + export default class ModStore extends TypedEventEmitter { constructor( public modsDirectory: string, @@ -434,95 +456,60 @@ export default class ModStore extends TypedEventEmitter { + logger.info(`Fetching all mods for Factorio version ${factorioVersion}...`); + let currentPage = 1; + let allMods: ModDetails[] = []; + let hasMorePages = true; - // Construct the URL for the Factorio mod portal API - const url = new URL("https://mods.factorio.com/api/mods"); + try { + while (hasMorePages) { + const url = new URL(MOD_PORTAL_API_URL); + url.searchParams.set("page_size", String(MOD_PORTAL_MAX_PAGE_SIZE)); + url.searchParams.set("version", factorioVersion); + url.searchParams.set("page", String(currentPage)); + url.searchParams.set("hide_deprecated", String(hide_deprecated)); - // Add query parameters - url.searchParams.set("page", page.toString()); - url.searchParams.set("page_size", pageSize.toString()); - url.searchParams.set("version", factorioVersion); + logger.verbose(`Fetching mod portal page ${currentPage}: ${url.toString()}`); + const response = await fetch(url.toString()); - if (query) { - url.searchParams.set("q", query); - } + if (!response.ok) { + let errorDetail = ""; + try { + errorDetail = await response.text(); + } catch (bodyError) { + logger.warn("Failed to read response body for failed portal request"); + } + let errorMessage = `Mod portal fetch page ${currentPage} failed: `; + errorMessage += `${response.status} ${response.statusText}`; + if (errorDetail) { + errorMessage += ` - ${errorDetail}`; + } + throw new Error(errorMessage); + } - // Handle sorting - if (sort) { - let sortField = sort; - // Map frontend sort fields to API sort fields - if (sort === "name") { - sortField = "name"; - } else if (sort === "title") { - sortField = "title"; - } else if (sort === "downloads") { - sortField = "downloads_count"; - } else if (sort === "updated") { - sortField = "updated_at"; + const data = await response.json() as ModPortalResponse; + allMods = allMods.concat(data.results); + + hasMorePages = currentPage < data.pagination.page_count; + currentPage += 1; } - - url.searchParams.set("sort", sortField); - url.searchParams.set("sort_order", sortOrder); + logger.info(`Successfully fetched ${allMods.length} mods for Factorio version ${factorioVersion}.`); + return allMods; + } catch (err: any) { + logger.error(err); + throw new Error(err); } - - // Make the request to the Factorio mod portal - const response = await fetch(url); - - if (!response.ok) { - throw new Error(`Failed to fetch from mod portal: ${response.status} ${response.statusText}`); - } - - const data = await response.json() as ModsInfoResponse; - logger.info(`Mod portal search results: ${JSON.stringify(data)}`); - - // Transform the response to match our expected format - const results = data.results.map(mod => { - // Find the latest release for the specified Factorio version - const latestRelease = mod.latest_release; - - const transformedMod = { - name: mod.name, - title: mod.title, - summary: mod.summary, - owner: mod.owner, - downloads_count: mod.downloads_count, - latest_release: latestRelease ? { - version: latestRelease.version, - factorio_version: latestRelease.info_json.factorio_version, - released_at: latestRelease.released_at, - download_url: latestRelease.download_url, - file_name: latestRelease.file_name, - sha1: latestRelease.sha1, - } : undefined, - }; - return transformedMod; - }).filter(mod => mod.latest_release); // Only include mods with a release for the specified version - - return { - queryIssues: [], - pageCount: Math.ceil(data.pagination.count / pageSize), - resultCount: data.pagination.count, - results, - }; } } diff --git a/packages/lib/src/data/messages_mod.ts b/packages/lib/src/data/messages_mod.ts index 3dd61955..e495bef1 100644 --- a/packages/lib/src/data/messages_mod.ts +++ b/packages/lib/src/data/messages_mod.ts @@ -197,87 +197,73 @@ export class ModSearchRequest { }; } -export class ModPortalSearchRequest { - declare ["constructor"]: typeof ModPortalSearchRequest; +// Define the structure for the latest release info from the portal +const ModPortalReleaseSchema = Type.Object({ + version: Type.String(), + // Match the structure from ModStore's ModRelease/ModDetails + info_json: Type.Object({ factorio_version: Type.String() }), + released_at: Type.String(), // ISO 8601 date string + download_url: Type.String(), + file_name: Type.String(), + sha1: Type.String(), +}); + +// Define the structure for mod details returned by the portal API +// This should align with the ModDetails interface in ModStore.ts +const ModPortalDetailsSchema = Type.Object({ + name: Type.String(), + title: Type.String(), + summary: Type.String(), + owner: Type.String(), + downloads_count: Type.Integer(), + category: Type.Optional(Type.String()), // Add optional category + score: Type.Optional(Type.Number()), // Add optional score + latest_release: Type.Optional(ModPortalReleaseSchema), + // Add releases array if needed: releases: Type.Optional(Type.Array(ModPortalReleaseSchema)), +}); + +export class ModPortalGetAllRequest { + declare ["constructor"]: typeof ModPortalGetAllRequest; static type = "request" as const; static src = "control" as const; static dst = "controller" as const; - static permission = "core.mod.search_portal" as const; + static permission = "core.mod.search_portal" as const; // Reuse search permission for now + + // Define allowed Factorio versions + static allowedVersions = [ + Type.Literal("0.13"), Type.Literal("0.14"), Type.Literal("0.15"), + Type.Literal("0.16"), Type.Literal("0.17"), Type.Literal("0.18"), + Type.Literal("1.0"), Type.Literal("1.1"), Type.Literal("2.0"), + ] as const; constructor( - public query: string, - public factorioVersion: string, - public page: number, - public pageSize?: number, - public sort?: string, - public sortOrder?: string, + public factorioVersion: Static, + public hide_deprecated?: boolean, ) { } static jsonSchema = Type.Object({ - "query": Type.String(), - "factorioVersion": Type.String(), - "page": Type.Integer(), - "pageSize": Type.Optional(Type.Integer()), - "sort": Type.Optional(Type.String()), - "sortOrder": Type.Optional(Type.String()), + "factorioVersion": Type.Union([...this.allowedVersions]), // Use Union for validation + "hide_deprecated": Type.Optional(Type.Boolean()), }); static fromJSON(json: Static) { - return new this(json.query, json.factorioVersion, json.page, json.pageSize, json.sort, json.sortOrder); + return new this(json.factorioVersion, json.hide_deprecated); } - static Response = class Response { + // Define the Response class inline + static Response = class ModPortalGetAllResponse { + declare ["constructor"]: typeof ModPortalGetAllResponse; constructor( - public queryIssues: string[], - public pageCount: number, - public resultCount: number, - public results: { - name: string, - title: string, - summary: string, - owner: string, - downloads_count: number, - latest_release: { - version: string, - factorio_version: string, - released_at: string, - download_url: string, - file_name: string, - sha1: string, - }, - }[], + public mods: Static[], ) { } static jsonSchema = Type.Object({ - "queryIssues": Type.Array(Type.String()), - "pageCount": Type.Integer(), - "resultCount": Type.Integer(), - "results": Type.Array( - Type.Object({ - "name": Type.String(), - "title": Type.String(), - "summary": Type.String(), - "owner": Type.String(), - "downloads_count": Type.Number(), - "latest_release": Type.Object({ - "version": Type.String(), - "factorio_version": Type.String(), - "released_at": Type.String(), - "download_url": Type.String(), - "file_name": Type.String(), - "sha1": Type.String(), - }), - }), - ), + mods: Type.Array(ModPortalDetailsSchema), }); static fromJSON(json: Static) { - return new this( - json.queryIssues, - json.pageCount, - json.resultCount, - json.results - ); + // No transformation needed if schema matches API structure + return new this(json.mods); } }; } diff --git a/packages/lib/src/link/messages.ts b/packages/lib/src/link/messages.ts index b1c17826..2a275125 100644 --- a/packages/lib/src/link/messages.ts +++ b/packages/lib/src/link/messages.ts @@ -97,7 +97,7 @@ export const dataClasses: (RequestClass | EventClass) mod.ModGetRequest, mod.ModListRequest, mod.ModSearchRequest, - mod.ModPortalSearchRequest, + mod.ModPortalGetAllRequest, mod.ModDownloadRequest, mod.ModDeleteRequest, mod.ModPackUpdatesEvent, diff --git a/packages/web_ui/src/components/ModsPage.tsx b/packages/web_ui/src/components/ModsPage.tsx index 3b0654c4..644d6352 100644 --- a/packages/web_ui/src/components/ModsPage.tsx +++ b/packages/web_ui/src/components/ModsPage.tsx @@ -1,9 +1,10 @@ -import React, { Fragment, useState, useContext, useEffect } from "react"; +import React, { Fragment, useState, useContext, useEffect, useMemo } from "react"; import { useNavigate } from "react-router-dom"; -import { Button, Form, Input, Modal, Popconfirm, Space, Table, Typography, Upload } from "antd"; +import { Button, Form, Input, Modal, Popconfirm, Space, Table, Typography, Upload, Alert, Select } from "antd"; import ImportOutlined from "@ant-design/icons/ImportOutlined"; import PlusOutlined from "@ant-design/icons/PlusOutlined"; import SearchOutlined from "@ant-design/icons/SearchOutlined"; +import { Static } from "@sinclair/typebox"; import * as lib from "@clusterio/lib"; @@ -22,6 +23,9 @@ import UploadButton from "./UploadButton"; const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare; +// Define the allowed Factorio versions based on the lib schema +type FactorioVersion = Static<(typeof lib.ModPortalGetAllRequest)["allowedVersions"][number]>; + function ImportModPackButton() { let control = useContext(ControlContext); let navigate = useNavigate(); @@ -130,45 +134,71 @@ function CreateModPackButton() { ; } +// Define a basic type for the mod data we expect back from the new backend request +// This should ideally match the structure returned by the backend. +// We reuse the ModPortalMod structure name for convenience, but it's fetched via backend. +interface ModPortalMod { + name: string; + title: string; + owner: string; + summary: string; + downloads_count: number; + category: string; + score?: number; + latest_release?: { + download_url: string; + file_name: string; + info_json: { factorio_version: string; }; + released_at: string; + version: string; + sha1: string; + }; +} + function SearchModsButton() { const control = useContext(ControlContext); const account = useAccount(); const [open, setOpen] = useState(false); const [form] = Form.useForm(); const [searchText, setSearchText] = useState(""); - const [factorioVersion, setFactorioVersion] = useState("1.1"); - const [modResults, setModResults] = useState([]); - const [loading, setLoading] = useState(false); + const [factorioVersion, setFactorioVersion] = useState("2.0"); + + // State for all mods fetched from backend + const [allMods, setAllMods] = useState([]); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + // State for client-side pagination and sorting const [page, setPage] = useState(1); const [pageSize, setPageSize] = useState(10); - const [totalResults, setTotalResults] = useState(0); - const [totalPages, setTotalPages] = useState(0); const [sort, setSort] = useState("name"); const [sortOrder, setSortOrder] = useState<"asc" | "desc">("asc"); - // Search mods when search parameters change + // Fetch all mods from backend when modal opens or version changes useEffect(() => { - if (!open) { return; } + if (!open || !factorioVersion) { + setAllMods([]); // Clear mods if modal closed or no version + return; + } setLoading(true); + setError(null); let canceled = false; - control.send(new lib.ModPortalSearchRequest( - searchText, - factorioVersion, - page, - pageSize, - sort, - sortOrder - )).then(response => { + // Use the hypothetical new backend request + control.send( + // Factorio version state now matches the required type + new lib.ModPortalGetAllRequest(factorioVersion) + ).then((response: any) => { if (canceled) { return; } - - setModResults(response.results); - setTotalResults(response.resultCount); - setTotalPages(response.pageCount); + // *** Backend Change Needed: Ensure the response has a 'mods' array *** + setAllMods(response?.mods || []); // Added optional chaining setLoading(false); - }).catch(error => { + }).catch(err => { if (canceled) { return; } + notifyErrorHandler("Error fetching mods from portal")(err); + setError(err); + setAllMods([]); setLoading(false); }); @@ -176,15 +206,80 @@ function SearchModsButton() { return () => { canceled = true; }; - }, [open, searchText, factorioVersion, page, pageSize, sort, sortOrder, control]); + // Re-fetch when modal opens or factorio version changes + }, [open, factorioVersion, control]); - const handleSearch = () => { - const values = form.getFieldsValue(); - setSearchText(values.name || ""); - setFactorioVersion(values.factorioVersion || "1.1"); - setPage(1); // Reset to first page on new search + + // Memoize the filtered, sorted, and paginated mods (logic remains the same) + const displayedMods = useMemo(() => { + let filtered = allMods; + + // Apply search filter (case-insensitive) + if (searchText) { + const lowerSearchText = searchText.toLowerCase(); + filtered = filtered.filter(mod => mod.name.toLowerCase().includes(lowerSearchText) + || mod.title.toLowerCase().includes(lowerSearchText) + ); + } + + // Apply sorting + if (sort && sortOrder) { + filtered = [...filtered].sort((a, b) => { + let aValue: any = a[sort as keyof ModPortalMod] ?? ""; + let bValue: any = b[sort as keyof ModPortalMod] ?? ""; + + // Handle specific types if necessary (e.g., numbers, dates) + if (sort === "downloads_count") { + aValue = a.downloads_count ?? 0; + bValue = b.downloads_count ?? 0; + return sortOrder === "asc" ? aValue - bValue : bValue - aValue; + } + + // Default string comparison + const comparison = strcmp(String(aValue), String(bValue)); + return sortOrder === "asc" ? comparison : -comparison; + }); + } + + // Apply pagination + const startIndex = (page - 1) * pageSize; + return filtered.slice(startIndex, startIndex + pageSize); + + }, [allMods, searchText, sort, sortOrder, page, pageSize]); + + // Calculate total results based on the filtered list *before* pagination (logic remains the same) + const totalResults = useMemo(() => { + let filtered = allMods; + if (searchText) { + const lowerSearchText = searchText.toLowerCase(); + filtered = filtered.filter(mod => mod.name.toLowerCase().includes(lowerSearchText) + || mod.title.toLowerCase().includes(lowerSearchText) + ); + } + return filtered.length; + }, [allMods, searchText]); + + // Update search text and factorio version from form + const handleSearch = (changedValues: any, allValues: any) => { + const nameValue = allValues.name; + const versionValue = allValues.factorioVersion as FactorioVersion | undefined; + + // Only trigger state updates if values actually changed + if (nameValue !== searchText) { + setSearchText(nameValue || ""); + } + if (versionValue && versionValue !== factorioVersion) { + setFactorioVersion(versionValue); + setAllMods([]); // Clear mods when version changes, useEffect will fetch new ones + setLoading(true); // Show loading indicator immediately + } + // Reset page only if search text or version changed + if (nameValue !== searchText || (versionValue && versionValue !== factorioVersion)) { + setPage(1); + } }; + // Update pagination and sorting state when table changes (logic remains the same) const handleTableChange = ( pagination: any, filters: any, @@ -195,14 +290,23 @@ function SearchModsButton() { } if (pagination.pageSize !== pageSize) { setPageSize(pagination.pageSize); + setPage(1); // Reset to first page if page size changes } - if (sorter.field && sorter.order) { + // Update sort state if sorter exists and has changed + if (sorter.field && (sorter.field !== sort || sorter.order !== (sortOrder === "asc" ? "ascend" : "descend"))) { setSort(sorter.field); setSortOrder(sorter.order === "ascend" ? "asc" : "desc"); + setPage(1); // Reset to first page on sort change + } else if (!sorter.field && sort !== "name") { // Reset only if not already default + // Clear sort if column header clicked without order - default to name ascending + setSort("name"); + setSortOrder("asc"); + setPage(1); } }; + // Helper to get antd sort order format (logic remains the same) const getColumnSortOrder = (columnKey: string): "ascend" | "descend" | undefined => { if (sort === columnKey) { return sortOrder === "asc" ? "ascend" : "descend"; @@ -213,7 +317,7 @@ function SearchModsButton() { return <> { setOpen(false); }} width={800} @@ -221,27 +325,48 @@ function SearchModsButton() { , - , + // Remove explicit search button as form updates trigger search/filter ]} >
- - + + - - + + + {/* Display error message if fetching failed */} + {error && } +
record.name} loading={loading} onChange={handleTableChange} @@ -250,20 +375,20 @@ function SearchModsButton() { pageSize: pageSize, total: totalResults, showSizeChanger: true, - pageSizeOptions: ["10", "20", "50"], + pageSizeOptions: ["10", "20", "50", "100"], }} expandable={{ expandedRowRender: record => (

Summary: {record.summary ?? "N/A"}

-

Downloads: {record.downloads_count ?? "N/A"}

+

Downloads: {record.downloads_count?.toLocaleString() ?? "N/A"}

Latest Release:

{record.latest_release ? (
  • Version: {record.latest_release.version ?? "N/A"}
  • Factorio Version: { - record.latest_release.factorio_version ?? "N/A" + record.latest_release.info_json?.factorio_version ?? "N/A" }
  • @@ -275,21 +400,28 @@ function SearchModsButton() {
) :

No release information available.

} + {/* Download button logic remains the same, assuming backend handles downloads */} {account.hasPermission("core.mod.download") && record.latest_release && ( @@ -298,6 +430,7 @@ function SearchModsButton() { ), }} columns={[ + // Columns remain largely the same { title: "Name", dataIndex: "name", @@ -319,6 +452,15 @@ function SearchModsButton() { sorter: true, sortOrder: getColumnSortOrder("owner"), }, + { + title: "Downloads", + dataIndex: "downloads_count", + key: "downloads_count", + sorter: true, + sortOrder: getColumnSortOrder("downloads_count"), + align: "right", + render: (count) => count?.toLocaleString() ?? "N/A", + }, { title: "Latest Version", key: "version", From d50230becc204bbfac33a8ded9c28900add597d0 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Tue, 8 Apr 2025 22:42:03 +0200 Subject: [PATCH 17/28] Cache mods on controller --- packages/controller/src/ControlConnection.ts | 10 +++++++++- packages/controller/src/Controller.ts | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 0ce1d46c..a1c661cc 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -515,17 +515,25 @@ export default class ControlConnection extends BaseConnection { }; } - /** * Handle request to fetch all mods from the Factorio Mod Portal for a given version. * @param request - The request object containing factorioVersion. */ async handleModPortalGetAllRequest(request: lib.ModPortalGetAllRequest) { + const cacheKey = `${request.factorioVersion}-${request.hide_deprecated}`; + const cachedData = this._controller.modPortalCache.get(cacheKey); + + if (cachedData && Date.now() - cachedData.timestamp < Controller.MOD_PORTAL_CACHE_DURATION) { + return new lib.ModPortalGetAllRequest.Response(cachedData.data); + } + try { + logger.info(`Fetching mod portal data for ${cacheKey}`); const mods = await lib.ModStore.fetchAllModsFromPortal( request.factorioVersion, request.hide_deprecated ); + this._controller.modPortalCache.set(cacheKey, { timestamp: Date.now(), data: mods }); // The Response class is defined inline within the Request class return new lib.ModPortalGetAllRequest.Response(mods); } catch (error: any) { diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 1517677a..08d2e6d8 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -96,6 +96,13 @@ export default class Controller { clusterLogIndex: lib.LogIndex | null = null; clusterLogBuildInterval: ReturnType | null = null; + // Cache for mod portal requests + static MOD_PORTAL_CACHE_DURATION = 30 * 60 * 1000; // 30 minutes + modPortalCache = new Map(); + + /** @type {?module:lib.ProcessConnector} */ + process = null; + static async bootstrap(config: lib.ControllerConfig) { let databaseDirectory = config.get("controller.database_directory"); await fs.ensureDir(databaseDirectory); @@ -288,7 +295,7 @@ export default class Controller { let tls_cert = this.config.get("controller.tls_certificate"); let tls_key = this.config.get("controller.tls_private_key"); - if (httpsPort && (!tls_cert || !tls_key)) { + if (httpsPort && (!tls_cert || !tls_key)) { throw new lib.StartupError( "tls_certificate and tls_private_key must be configure in order to use https_port" ); From 2936973a75a49444e47976411a84b25cd5655426 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Tue, 8 Apr 2025 22:54:16 +0200 Subject: [PATCH 18/28] Link to mod portal --- packages/web_ui/src/components/ModsPage.tsx | 34 ++++++++++++++++----- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/packages/web_ui/src/components/ModsPage.tsx b/packages/web_ui/src/components/ModsPage.tsx index 644d6352..722853da 100644 --- a/packages/web_ui/src/components/ModsPage.tsx +++ b/packages/web_ui/src/components/ModsPage.tsx @@ -1,9 +1,7 @@ import React, { Fragment, useState, useContext, useEffect, useMemo } from "react"; import { useNavigate } from "react-router-dom"; -import { Button, Form, Input, Modal, Popconfirm, Space, Table, Typography, Upload, Alert, Select } from "antd"; -import ImportOutlined from "@ant-design/icons/ImportOutlined"; -import PlusOutlined from "@ant-design/icons/PlusOutlined"; -import SearchOutlined from "@ant-design/icons/SearchOutlined"; +import { Button, Form, Input, Modal, Popconfirm, Space, Table, Typography, Upload, Alert, Select, Tooltip } from "antd"; +import { ImportOutlined, PlusOutlined, SearchOutlined } from "@ant-design/icons"; import { Static } from "@sinclair/typebox"; import * as lib from "@clusterio/lib"; @@ -320,7 +318,7 @@ function SearchModsButton() { title="Search Mod Portal" open={open} onCancel={() => { setOpen(false); }} - width={800} + width={1000} footer={[ + {account.hasPermission("core.mod.download") && record.latest_release && ( From c6f10e21955b18160b4a1fc14201545c67287ea9 Mon Sep 17 00:00:00 2001 From: Danielv123 Date: Sun, 13 Apr 2025 14:22:57 +0200 Subject: [PATCH 24/28] Refactor to avoid duplicated code --- packages/web_ui/src/components/ModsPage.tsx | 65 +++++++-------------- 1 file changed, 20 insertions(+), 45 deletions(-) diff --git a/packages/web_ui/src/components/ModsPage.tsx b/packages/web_ui/src/components/ModsPage.tsx index 13d98490..38924d07 100644 --- a/packages/web_ui/src/components/ModsPage.tsx +++ b/packages/web_ui/src/components/ModsPage.tsx @@ -28,6 +28,9 @@ const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" // Define the allowed Factorio versions based on the lib schema type FactorioVersion = Static<(typeof lib.ModPortalGetAllRequest)["allowedVersions"][number]>; +// Type alias for the mod structure derived from the library response +type ModPortalModType = InstanceType["mods"][number]; + function ImportModPackButton() { let control = useContext(ControlContext); let navigate = useNavigate(); @@ -136,27 +139,6 @@ function CreateModPackButton() { ; } -// Define a basic type for the mod data we expect back from the new backend request -// This should ideally match the structure returned by the backend. -// We reuse the ModPortalMod structure name for convenience, but it's fetched via backend. -interface ModPortalMod { - name: string; - title: string; - owner: string; - summary: string; - downloads_count: number; - category: string; - score?: number; - latest_release?: { - download_url: string; - file_name: string; - info_json: { factorio_version: string; }; - released_at: string; - version: string; - sha1: string; - }; -} - function SearchModsButton() { const control = useContext(ControlContext); const account = useAccount(); @@ -166,7 +148,7 @@ function SearchModsButton() { const [factorioVersion, setFactorioVersion] = useState("2.0"); // State for all mods fetched from backend - const [allMods, setAllMods] = useState([]); + const [allMods, setAllMods] = useState([]); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); @@ -214,14 +196,11 @@ function SearchModsButton() { setError(null); let canceled = false; - // Use the hypothetical new backend request control.send( - // Factorio version state now matches the required type new lib.ModPortalGetAllRequest(factorioVersion) ).then((response: any) => { if (canceled) { return; } - // *** Backend Change Needed: Ensure the response has a 'mods' array *** - setAllMods(response?.mods || []); // Added optional chaining + setAllMods(response.mods || []); setLoading(false); }).catch(err => { if (canceled) { return; } @@ -254,8 +233,8 @@ function SearchModsButton() { // Apply sorting if (sort && sortOrder) { filtered = [...filtered].sort((a, b) => { - let aValue: any = a[sort as keyof ModPortalMod] ?? ""; - let bValue: any = b[sort as keyof ModPortalMod] ?? ""; + let aValue: any = a[sort as keyof ModPortalModType] ?? ""; + let bValue: any = b[sort as keyof ModPortalModType] ?? ""; // Handle specific types if necessary (e.g., numbers, dates) if (sort === "downloads_count") { @@ -354,7 +333,6 @@ function SearchModsButton() { , - // Remove explicit search button as form updates trigger search/filter ]} > @@ -431,14 +407,13 @@ function SearchModsButton() { ) :

No release information available.

} - {/* Download button logic remains the same, assuming backend handles downloads */} - {account.hasPermission("core.mod.download") && record.latest_release && ( + {account.hasPermission("core.mod.download_from_portal") && record.latest_release && ( - {account.hasPermission("core.mod.download") && record.latest_release && ( + {account.hasPermission("core.mod.download_from_portal") && record.latest_release && (