Merge pull request #821 from Cooldude2606/feature/version-selector

Add drop down version selector based on Wube version archive

packages/controller/src/ControlConnection.ts

@@ -126,6 +126,7 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this));
 		this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this));
 		this.handle(lib.DebugDumpWsRequest, this.handleDebugDumpWsRequest.bind(this));
+		this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this));
 	}
 
 	validateIngress(message: lib.MessageRequest | lib.MessageEvent) {
@@ -1222,6 +1223,10 @@ export default class ControlConnection extends BaseConnection {
 		this.connector._socket!.clusterio_ignore_dump = true;
 		this._controller.debugEvents.on("message", this.ws_dumper);
 	}
+
+	async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) {
+		return await this._controller.factorioVersions.get(request.maxAgeMs);
+	}
 }
 
 module.exports = ControlConnection;

packages/controller/src/Controller.ts

@@ -105,6 +105,8 @@ export default class Controller {
 
 	// Cache for mod portal requests
 	modPortalCache = new Map<string, { timestamp: number, data: any[] }>();
+	// Cache for factorio versions
+	factorioVersions = new lib.ValueCache(lib.fetchFactorioVersions);
 
 	static async bootstrap(config: lib.ControllerConfig) {
 		let databaseDirectory = config.get("controller.database_directory");

packages/host/src/server.ts

@@ -79,7 +79,7 @@ function versionOrder(a: string, b: string) {
  * @returns Array with path to data dir and version found.
  * @internal
  */
-async function findVersion(factorioDir: string, targetVersion: lib.TargetVersion) {
+async function findVersion(factorioDir: string, targetVersion: lib.TargetVersion): Promise<[string, lib.FullVersion]> {
 
 	// There are two supported setups: having the factorio dir be the actual
 	// install directory, and having the factorio dir be a folder containing
@@ -87,8 +87,10 @@ async function findVersion(factorioDir: string, targetVersion: lib.TargetVersion
 
 	let simpleVersion = await getVersion(path.join(factorioDir, "data", "changelog.txt"));
 	if (simpleVersion !== null) {
-		if (simpleVersion === targetVersion || targetVersion === "latest") {
-			return [path.join(factorioDir, "data"), simpleVersion];
+		if (simpleVersion === targetVersion || simpleVersion.startsWith(targetVersion) || targetVersion === "latest") {
+			if (lib.isFullVersion(simpleVersion)) {
+				return [path.join(factorioDir, "data"), simpleVersion];
+			}
 		}
 
 		throw new Error(
@@ -96,14 +98,14 @@ async function findVersion(factorioDir: string, targetVersion: lib.TargetVersion
 		);
 	}
 
-	let versions = new Map();
+	let versions = new Map<lib.FullVersion, string>();
 	for (let entry of await fs.readdir(factorioDir, { withFileTypes: true })) {
 		if (!entry.isDirectory()) {
 			continue;
 		}
 
 		let version = await getVersion(path.join(factorioDir, entry.name, "data", "changelog.txt"));
-		if (version === null) {
+		if (version === null || !lib.isFullVersion(version)) {
 			continue;
 		}
 
@@ -118,9 +120,17 @@ async function findVersion(factorioDir: string, targetVersion: lib.TargetVersion
 		throw new Error(`Unable to find any Factorio install in ${factorioDir}`);
 	}
 
+	if (lib.isPartialVersion(targetVersion)) {
+		const sorted = [...versions.keys()].sort(versionOrder);
+		const latest = sorted.find(version => version.startsWith(targetVersion));
+		if (latest) {
+			return [path.join(factorioDir, versions.get(latest)!, "data"), latest];
+		}
+	}
+
 	if (targetVersion === "latest") {
-		let latest = [...versions.keys()].sort(versionOrder)[0];
-		return [path.join(factorioDir, versions.get(latest), "data"), latest];
+		const latest = [...versions.keys()].sort(versionOrder)[0];
+		return [path.join(factorioDir, versions.get(latest)!, "data"), latest];
 	}
 
 	throw new Error(`Unable to find Factorio version ${targetVersion}`);

packages/lib/browser.ts

@@ -15,3 +15,4 @@ export * from "./src/subscriptions";
 
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as RateLimiter } from "./src/RateLimiter";
+export { default as ValueCache } from "./src/ValueCache";

packages/lib/index.ts

@@ -9,6 +9,7 @@ export * from "./src/config";
 export * from "./src/data";
 export * from "./src/database";
 export * from "./src/errors";
+export * from "./src/external";
 export * from "./src/factorio";
 export * from "./src/file_ops";
 export * from "./src/hash";
@@ -35,3 +36,4 @@ export * from "./src/rce_ops";
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as ModStore } from "./src/ModStore";
 export { default as RateLimiter } from "./src/RateLimiter";
+export { default as ValueCache } from "./src/ValueCache";

packages/lib/src/ValueCache.ts

@@ -0,0 +1,64 @@
+
+/**
+ * Represents an async function which can fetch a value for a ValueCache
+ */
+export interface ValueCacheFetch<T> {
+	(): Promise<T>;
+}
+
+/**
+ * In-memory single value cache
+ *
+ * Stores the last time a value was fetched and measures staleness
+ * Concurrent calls will share the same in-flight fetch
+ */
+export default class ValueCache<T> {
+	private ongoingFetch: Promise<T> | null = null;
+	private timestamp: number | null = null;
+	private value: T | null = null; // Only valid when timestamp is not null
+
+	/**
+	 * Create a new in memory single value cache
+	 *
+	 * @param fetch A function that fetches and returns a fresh value
+	 */
+	constructor(
+		private readonly fetch: ValueCacheFetch<T>,
+	) {}
+
+	/**
+	 * Get the cached value, will fetch a fresh value if stale or missing
+	 *
+	 * @param maxAgeMs Maximum allowed cache age in milliseconds.
+	 *                 Defaults to 0, meaning it will always fetch a new value.
+	 * @throws TypeError if maxAgeMs is invalid
+	 * @throws Propagates errors from underlying fetch method
+	 */
+	get(maxAgeMs: number = 0): Promise<T> {
+		if (maxAgeMs < 0) {
+			throw new TypeError("maxAgeMs must be a non-negative number");
+		}
+
+		const now = Date.now();
+
+		if (this.timestamp && now - this.timestamp < maxAgeMs) {
+			return Promise.resolve(this.value as T);
+		}
+
+		if (this.ongoingFetch) {
+			return this.ongoingFetch;
+		}
+
+		this.ongoingFetch = this.fetch()
+			.then(value => {
+				this.timestamp = Date.now();
+				this.value = value;
+				return value;
+			})
+			.finally(() => {
+				this.ongoingFetch = null;
+			});
+
+		return this.ongoingFetch;
+	}
+}

packages/lib/src/config/definitions.ts

@@ -479,6 +479,7 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
 			restartRequired: true,
 			type: "string",
 			initialValue: "latest",
+			inputComponent: "target_version",
 		},
 		"factorio.executable_path": {
 			description:

packages/lib/src/data/messages_controller.ts

@@ -2,6 +2,7 @@ import { Type, Static } from "@sinclair/typebox";
 import { JsonString, StringEnum, StringKey, plainJson } from "./composites";
 import { levels } from "../logging";
 import { ControllerConfig, HostConfig } from "../config";
+import { ExternalFactorioVersionSchema } from "../external";
 
 export class ControllerStopRequest {
 	declare ["constructor"]: typeof ControllerStopRequest;
@@ -391,3 +392,25 @@ export class DebugWsMessageEvent {
 		return new this(json.direction, json.content);
 	}
 }
+
+export class FactorioVersionsRequest {
+	declare ["constructor"]: typeof FactorioVersionsRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.external.get_factorio_versions" as const;
+
+	constructor(
+		public maxAgeMs: number = 5 * 60 * 1000, // Default 5 minutes
+	) {}
+
+	static jsonSchema = Type.Object({
+		"maxAgeMs": Type.Number(),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.maxAgeMs);
+	}
+
+	static Response = plainJson(Type.Array(ExternalFactorioVersionSchema));
+}

packages/lib/src/external/FactorioVersions.ts

@@ -0,0 +1,69 @@
+import { Static, Type } from "@sinclair/typebox";
+import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data";
+
+const ARCHIVE_URL = "https://factorio.com/download/archive";
+
+/**
+ * Represents the parsed result of a factorio version
+ */
+export const ExternalFactorioVersionSchema = Type.Object({
+	stable: Type.Boolean(),
+	version: FullVersionSchema,
+	headlessUrl: Type.String(),
+});
+
+export type ExternalFactorioVersion = Static<typeof ExternalFactorioVersionSchema>
+
+/**
+ * Fetch all factorio versions that support headless
+ *
+ * @returns A list of all factorio versions and their download URL
+ * @throws Network errors and non-ok status responses
+ */
+export async function fetchFactorioVersions(): Promise<ExternalFactorioVersion[]> {
+	let res;
+
+	try {
+		res = await fetch(ARCHIVE_URL);
+	} catch (err) {
+		throw new Error(`Failed to fetch Factorio versions from ${ARCHIVE_URL}: ${err}`);
+	}
+
+	if (!res.ok) {
+		throw new Error(`Failed to fetch Factorio versions from ${ARCHIVE_URL}: HTTP ${res.status} ${res.statusText}`);
+	}
+
+	const html = await res.text();
+	return parseFactorioVersions(html);
+}
+
+/**
+ * Parse all versions from the factorio archive webpage
+ *
+ * @param html The HTML content of the archive page
+ * @returns A list of all factorio versions and their download URL
+ */
+function parseFactorioVersions(html: string): ExternalFactorioVersion[] {
+	const versions = new Map<string, ExternalFactorioVersion>();
+
+	// Match <a class=" slot-button-inline version-button-stable " href="/download/archive/2.0.72">2.0.72</a>
+	// eslint-disable-next-line max-len
+	const anchorRegex = /<a[^>]*class="[^"]*version-button-(?<kind>stable|experimental)[^"]*"[^>]*href="[^"]*\/download\/archive\/(?<version>\d+\.\d+.\d+)"[^>]*>/g;
+
+	let match: RegExpExecArray | null;
+	const minimumMultiplayerVersion = integerFullVersion("0.12.35");
+	while ((match = anchorRegex.exec(html)) !== null) {
+		const kind = match.groups?.kind;
+		const version = match.groups?.version;
+
+		if (kind && version && isFullVersion(version) && integerFullVersion(version) >= minimumMultiplayerVersion) {
+			versions.set(version, {
+				version: version,
+				stable: kind === "stable",
+				headlessUrl: `www.factorio.com/get-download/${version}/headless/linux64`,
+			});
+		}
+	}
+
+	return [...versions.values()];
+}

packages/lib/src/external/index.ts

@@ -0,0 +1,5 @@
+/**
+ * Library for interfacing with external apis
+ * @module lib/external
+ */
+export * from "./FactorioVersions";

packages/lib/src/link/messages.ts

@@ -35,6 +35,7 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	controller.SystemInfoUpdateEvent,
 	controller.DebugDumpWsRequest,
 	controller.DebugWsMessageEvent,
+	controller.FactorioVersionsRequest,
 
 	subscriptions.SubscriptionRequest,
 

packages/lib/src/permissions.ts

@@ -568,3 +568,9 @@ definePermission({
 	title: "Dump WebSocket",
 	description: "Dump all WebSocket communicatation from the controller.",
 });
+
+definePermission({
+	name: "core.external.get_factorio_versions",
+	title: "Get factorio versions",
+	description: "Get list of all factorio versions from Wube",
+});

packages/lib/tsconfig.browser.json

@@ -6,6 +6,7 @@
 		"./src/api.ts",
 		"./src/config/*.ts",
 		"./src/data/*.ts",
+		"./src/external/*.ts",
 		"./src/datastore.ts",
 		"./src/errors.ts",
 		"./src/factorio/exchange_string.ts",
@@ -25,6 +26,7 @@
 		"./src/ExponentialBackoff.ts",
 		"./src/RateLimiter.ts",
 		"./src/TypedEventEmitter.ts",
+		"./src/ValueCache.ts",
 	],
 	"compilerOptions": {
 		"rootDir": ".",

packages/web_ui/src/bootstrap.tsx

@@ -9,6 +9,8 @@ import App from "./components/App";
 import BaseWebPlugin, { InputComponent } from "./BaseWebPlugin";
 import InputRole from "./components/InputRole";
 import InputModPack from "./components/InputModPack";
+import InputTargetVersion from "./components/InputTargetVersion";
+import InputPartialVersion from "./components/InputPartialVersion";
 import { Control, ControlConnector } from "./util/websocket";
 
 const { ConsoleTransport, WebConsoleFormat, logger } = lib;
@@ -109,6 +111,8 @@ async function loadPlugins(pluginInfos: lib.PluginWebpackEnvInfo[], control: Con
 
 function inputComponentsFromPlugins(plugins: Map<string, BaseWebPlugin>) {
 	const inputComponents: Record<string, InputComponent> = {
+		"partial_version": InputPartialVersion,
+		"target_version": InputTargetVersion,
 		"mod_pack": InputModPack,
 		"role": InputRole,
 	};

packages/web_ui/src/components/InputPartialVersion.tsx

@@ -0,0 +1,68 @@
+import React, { useContext, useEffect, useState } from "react";
+import { TreeSelect } from "antd";
+
+import * as lib from "@clusterio/lib";
+import { InputComponentProps } from "../BaseWebPlugin";
+import { useAccount } from "../model/account";
+import ControlContext from "./ControlContext";
+
+const defaultFieldDefinition: lib.FieldDefinition = { type: "string", optional: false };
+
+export default function InputPartialVersion (
+	props: Omit<InputComponentProps, "fieldDefinition"> & {
+		fieldDefinition?: lib.FieldDefinition;
+		className?: string;
+	}
+) {
+	const fieldDefinition = props.fieldDefinition || defaultFieldDefinition;
+	const [versions, setVersions] = useState<readonly lib.PartialVersion[]>(lib.ApiVersions);
+	const control = useContext(ControlContext);
+	const account = useAccount();
+
+	useEffect(() => {
+		(async () => {
+			if (account.hasPermission("core.external.get_factorio_versions")) {
+				const res = await control.factorioVersions.get(5 * 60 * 1000);
+				setVersions(res.map(v => v.version));
+			}
+		})();
+	}, [control]);
+
+	// Split the versions into groups based on major minor
+	const groups = new Map<lib.PartialVersion, lib.PartialVersion[]>();
+
+	for (const version of versions) {
+		const [major, minor] = version.split(".");
+		const key = `${major}.${minor}` as lib.PartialVersion;
+
+		if (!groups.has(key)) {
+			groups.set(key, []);
+		}
+		groups.get(key)!.push(version);
+	}
+
+	// Construct the tree to be disabled
+	const tree = [...groups.entries()].map(([majorMinor, patchVersions]) => (
+		{
+			title: majorMinor,
+			value: majorMinor,
+			key: majorMinor,
+			children: patchVersions.map((v) => ({
+				title: v,
+				value: v,
+				key: v,
+			})),
+		}
+	));
+
+	return <TreeSelect
+		showSearch
+		style={{ minWidth: 175 }}
+		onChange={(value) => props.onChange(value ?? null)}
+		value={props.value}
+		treeData={tree}
+		allowClear={fieldDefinition.optional}
+		className={props.className}
+		disabled={props.disabled}
+	/>;
+}

packages/web_ui/src/components/InputTargetVersion.tsx

@@ -0,0 +1,71 @@
+import React, { useContext, useEffect, useState } from "react";
+import { TreeSelect } from "antd";
+
+import * as lib from "@clusterio/lib";
+import { InputComponentProps } from "../BaseWebPlugin";
+import { useAccount } from "../model/account";
+import ControlContext from "./ControlContext";
+
+const defaultFieldDefinition: lib.FieldDefinition = { type: "string", optional: false };
+
+export default function InputTargetVersion (
+	props: Omit<InputComponentProps, "fieldDefinition"> & {
+		fieldDefinition?: lib.FieldDefinition;
+		className?: string;
+	}
+) {
+	const fieldDefinition = props.fieldDefinition || defaultFieldDefinition;
+	const [versions, setVersions] = useState<readonly lib.TargetVersion[]>(lib.ApiVersions);
+	const control = useContext(ControlContext);
+	const account = useAccount();
+
+	useEffect(() => {
+		(async () => {
+			if (account.hasPermission("core.external.get_factorio_versions")) {
+				const res = await control.factorioVersions.get(5 * 60 * 1000);
+				setVersions(res.map(v => v.version));
+			}
+		})();
+	}, [control]);
+
+	// Split the versions into groups based on major minor
+	const groups = new Map<lib.TargetVersion, lib.TargetVersion[]>();
+
+	for (const version of versions) {
+		const [major, minor] = version.split(".");
+		const key = `${major}.${minor}` as lib.PartialVersion;
+
+		if (!groups.has(key)) {
+			groups.set(key, []);
+		}
+		groups.get(key)!.push(version);
+	}
+
+	// Construct the tree to be disabled
+	const tree = [...groups.entries()].map(([majorMinor, patchVersions]) => (
+		{
+			title: majorMinor,
+			value: majorMinor,
+			key: majorMinor,
+			children: patchVersions.map((v) => ({
+				title: v,
+				value: v,
+				key: v,
+			})),
+		}
+	));
+
+	// Add "latest" option for TargetVersion
+	tree.unshift({ title: "latest", value: "latest", key: "latest", children: [] });
+
+	return <TreeSelect
+		showSearch
+		style={{ minWidth: 175 }}
+		onChange={(value) => props.onChange(value ?? null)}
+		value={props.value}
+		treeData={tree}
+		allowClear={fieldDefinition.optional}
+		className={props.className}
+		disabled={props.disabled}
+	/>;
+}

packages/web_ui/src/components/ModPackViewPage.tsx

@@ -26,6 +26,7 @@ import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import SectionHeader from "./SectionHeader";
 import ModDetails from "./ModDetails";
+import InputPartialVersion from "./InputPartialVersion";
 
 const { logger } = lib;
 const { Text } = Typography;
@@ -1288,7 +1289,9 @@ export default function ModPackViewPage() {
 							message: "Must be an a.b or a.b.c version number.",
 						}]}
 					>
-						<Input
+						<InputPartialVersion
+							value={form.getFieldValue("factorioVersion")}
+							onChange={(value: any) => form.setFieldValue("factorioVersion", value)}
 							className={hasChange(changes, { type: "factorioVersion" }) ? "changed" : ""}
 						/>
 					</Form.Item>

packages/web_ui/src/components/ModsPage.tsx

@@ -21,6 +21,7 @@ import SectionHeader from "./SectionHeader";
 import ModDetails from "./ModDetails";
 import { Dropzone } from "./Dropzone";
 import UploadButton from "./UploadButton";
+import InputPartialVersion from "./InputPartialVersion";
 
 const strcmp = new Intl.Collator(undefined, { numeric: true, sensitivity: "base" }).compare;
 
@@ -128,7 +129,10 @@ function CreateModPackButton() {
 						message: "Must be an a.b or a.b.c version number.",
 					}]}
 				>
-					<Input />
+					<InputPartialVersion
+						value={form.getFieldValue("factorioVersion")}
+						onChange={(value) => form.setFieldValue("factorioVersion", value)}
+					/>
 				</Form.Item>
 			</Form>
 		</Modal>

packages/web_ui/src/util/websocket.ts

@@ -69,6 +69,9 @@ export class Control extends lib.Link {
 	users = new lib.EventSubscriber(lib.UserUpdatesEvent, this);
 	roles = new lib.EventSubscriber(lib.RoleUpdatesEvent, this);
 
+	/** Cache for factorio versions to avoid repeat calls to the controller */
+	factorioVersions = new lib.ValueCache(this.requestFactorioVersions.bind(this));
+
 	declare connector: ControlConnector;
 
 	constructor(
@@ -105,6 +108,10 @@ export class Control extends lib.Link {
 		this.handle(lib.DebugWsMessageEvent, this.handleDebugWsMessageEvent.bind(this));
 	}
 
+	requestFactorioVersions() {
+		return this.send(new lib.FactorioVersionsRequest());
+	}
+
 	async handleAccountUpdateEvent(event: lib.AccountUpdateEvent) {
 		this.accountRoles = event.roles??null;
 		this.emitAccountUpdate();