Add ValueCache and fetchFactorioVersions

.gitignore

@@ -30,6 +30,9 @@ dist/
 node_modules
 
 Factorio*
+!*Factorio*.ts
+!*Factorio*.tsx
+!*Factorio*.js
 /factorio
 !/test/file/factorio
 !/test/lib/factorio

packages/lib/index.ts

@@ -9,6 +9,7 @@ export * from "./src/config";
 export * from "./src/data";
 export * from "./src/database";
 export * from "./src/errors";
+export * from "./src/external";
 export * from "./src/factorio";
 export * from "./src/file_ops";
 export * from "./src/hash";
@@ -35,3 +36,4 @@ export * from "./src/rce_ops";
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as ModStore } from "./src/ModStore";
 export { default as RateLimiter } from "./src/RateLimiter";
+export { default as ValueCache } from "./src/ValueCache";

packages/lib/src/ValueCache.ts

@@ -0,0 +1,64 @@
+
+/**
+ * Represents an async function which can fetch a value for a ValueCache
+ */
+export interface ValueCacheFetch<T> {
+	(): Promise<T>;
+}
+
+/**
+ * In-memory single value cache
+ *
+ * Stores the last time a value was fetched and measures staleness
+ * Concurrent calls will share the same in-flight fetch
+ */
+export default class ValueCache<T> {
+	private ongoingFetch: Promise<T> | null = null;
+	private timestamp: number | null = null;
+	private value: T | null = null; // Only valid when timestamp is not null
+
+	/**
+	 * Create a new in memory single value cache
+	 *
+	 * @param fetch A function that fetches and returns a fresh value
+	 */
+	constructor(
+		private readonly fetch: ValueCacheFetch<T>,
+	) {}
+
+	/**
+	 * Get the cached value, will fetch a fresh value if stale or missing
+	 *
+	 * @param maxAgeMs Maximum allowed cache age in milliseconds.
+	 *                 Defaults to 0, meaning it will always fetch a new value.
+	 * @throws TypeError if maxAgeMs is invalid
+	 * @throws Propagates errors from underlying fetch method
+	 */
+	get(maxAgeMs: number = 0): Promise<T> {
+		if (maxAgeMs < 0) {
+			throw new TypeError("maxAgeMs must be a non-negative number");
+		}
+
+		const now = Date.now();
+
+		if (this.timestamp && now - this.timestamp < maxAgeMs) {
+			return Promise.resolve(this.value as T);
+		}
+
+		if (this.ongoingFetch) {
+			return this.ongoingFetch;
+		}
+
+		this.ongoingFetch = this.fetch()
+			.then(value => {
+				this.timestamp = Date.now();
+				this.value = value;
+				return value;
+			})
+			.finally(() => {
+				this.ongoingFetch = null;
+			});
+
+		return this.ongoingFetch;
+	}
+}

packages/lib/src/external/FactorioVersions.ts

@@ -0,0 +1,65 @@
+import { FullVersion, isFullVersion } from "../data";
+
+const ARCHIVE_URL = "https://factorio.com/download/archive";
+
+/**
+ * Represents the parsed result of a factorio version
+ */
+export type ExternalFactorioVersion = {
+	stable: boolean;
+	version: FullVersion;
+	downloadUrl: string;
+};
+
+/**
+ * Fetch all factorio versions that support headless
+ *
+ * @returns A list of all factorio versions and their download URL
+ * @throws Network errors and non-ok status responses
+ */
+export async function fetchFactorioVersions(): Promise<ExternalFactorioVersion[]> {
+	let res;
+
+	try {
+		res = await fetch(ARCHIVE_URL);
+	} catch (err) {
+		throw new Error(`Failed to fetch Factorio versions from ${ARCHIVE_URL}: ${err}`);
+	}
+
+	if (!res.ok) {
+		throw new Error(`Failed to fetch Factorio versions from ${ARCHIVE_URL}: HTTP ${res.status} ${res.statusText}`);
+	}
+
+	const html = await res.text();
+	return parseFactorioVersions(html);
+}
+
+/**
+ * Parse all versions from the factorio archive webpage
+ *
+ * @param html The HTML content of the archive page
+ * @returns A list of all factorio versions and their download URL
+ */
+function parseFactorioVersions(html: string): ExternalFactorioVersion[] {
+	const versions = new Map<string, ExternalFactorioVersion>();
+
+	// Match <a class=" slot-button-inline version-button-stable " href="/download/archive/2.0.72">2.0.72</a>
+	// eslint-disable-next-line max-len
+	const anchorRegex = /<a[^>]*class="[^"]*version-button-(?<kind>stable|experimental)[^"]*"[^>]*href="[^"]*\/download\/archive\/(?<version>\d+\.\d+.\d+)"[^>]*>/g;
+
+	let match: RegExpExecArray | null;
+	while ((match = anchorRegex.exec(html)) !== null) {
+		const kind = match.groups?.kind;
+		const version = match.groups?.version;
+
+		if (kind && version && isFullVersion(version)) {
+			versions.set(version, {
+				version: version,
+				stable: kind === "stable",
+				downloadUrl: `${ARCHIVE_URL}/${version}`,
+			});
+		}
+	}
+
+	return [...versions.values()];
+}

packages/lib/src/external/index.ts

@@ -0,0 +1,5 @@
+/**
+ * Library for interfacing with external apis
+ * @module lib/external
+ */
+export * from "./FactorioVersions";

test/lib/ValueCache.test.js

@@ -0,0 +1,127 @@
+const assert = require("assert").strict;
+const { ValueCache, wait } = require("@clusterio/lib");
+
+describe("ValueCache", function () {
+	describe("constructor", function () {
+		it("constructs with a fetch function", function() {
+			const fetcher = async () => 42;
+			const cache = new ValueCache(fetcher);
+			assert.ok(cache);
+		});
+	});
+
+	describe("get", function() {
+		it("throws TypeError for negative maxAgeMs", async function() {
+			const fetcher = async () => 42;
+			const cache = new ValueCache(fetcher);
+
+			assert.throws(() => cache.get(-1), err => {
+				assert.equal(err instanceof TypeError, true);
+				assert.match(String(err.message), /maxAgeMs/);
+				return true;
+			});
+		});
+		it("uses a default of 0 for maxAgeMs", async function() {
+			let callCount = 0;
+			const fetcher = async () => {
+				callCount += 1;
+				return callCount;
+			};
+
+			const cache = new ValueCache(fetcher);
+
+			const v1 = await cache.get();
+			const v2 = await cache.get();
+
+			assert.equal(v1, 1);
+			assert.equal(v2, 2);
+			assert.equal(callCount, 2);
+		});
+		it("returns cached value when within maxAgeMs", async function() {
+			let callCount = 0;
+			const fetcher = async () => {
+				callCount += 1;
+				return callCount;
+			};
+
+			const cache = new ValueCache(fetcher);
+
+			const v1 = await cache.get(1000);
+			const v2 = await cache.get(1000);
+
+			assert.equal(v1, 1);
+			assert.equal(v2, 1);
+			assert.equal(callCount, 1);
+		});
+		it("refreshes the cache when older than maxAgeMs", async function() {
+			let callCount = 0;
+			const fetcher = async () => {
+				callCount += 1;
+				return callCount;
+			};
+
+			const cache = new ValueCache(fetcher);
+
+			const v1 = await cache.get(50);
+			const v2 = await cache.get(1000); // Use cached value
+			await wait(100);
+			const v3 = await cache.get(50); // Stale after 100ms wait
+			const v4 = await cache.get(1000); // Use cached value
+
+			assert.equal(v1, 1);
+			assert.equal(v2, 1);
+			assert.equal(v3, 2);
+			assert.equal(v4, 2);
+			assert.equal(callCount, 2);
+		});
+		it("shares in-flight fetch between concurrent callers", async function() {
+			let callCount = 0;
+			let resolveFetcher;
+			const fetcher = () => (
+				new Promise(resolve => {
+					callCount += 1;
+					resolveFetcher = resolve;
+				})
+			);
+
+			const cache = new ValueCache(fetcher);
+
+			const p1 = cache.get(1000);
+			const p2 = cache.get(1000);
+
+			assert.equal(callCount, 1);
+
+			resolveFetcher(42);
+			const [v1, v2] = await Promise.all([p1, p2]);
+
+			assert.equal(v1, 42);
+			assert.equal(v2, 42);
+			assert.equal(callCount, 1);
+		});
+		it("propagates errors from the fetcher and clears ongoing fetch", async function() {
+			let callCount = 0;
+			const error = new Error("fetch failed");
+			const fetcher = async () => {
+				callCount += 1;
+				throw error;
+			};
+
+			const cache = new ValueCache(fetcher);
+
+			await assert.rejects(cache.get(), err => {
+				assert.equal(err, error);
+				return true;
+			});
+
+			assert.equal(callCount, 1);
+
+			// After a failure, a new attempt to fetch should be made
+			await assert.rejects(cache.get(), err => {
+				assert.equal(err, error);
+				return true;
+			});
+
+			assert.equal(callCount, 2);
+		});
+	});
+});

test/lib/external/FactorioVersions.test.js

@@ -0,0 +1,122 @@
+const assert = require("assert").strict;
+const { readFile } = require("fs-extra");
+
+const { fetchFactorioVersions } = require("@clusterio/lib");
+const { slowTest } = require("../../integration");
+
+describe("FactorioVersions", function() {
+	describe("fetchFactorioVersions", function() {
+		const originalFetch = global.fetch;
+
+		beforeEach(() => {
+			global.fetch = undefined;
+		});
+
+		afterEach(() => {
+			global.fetch = originalFetch;
+		});
+
+		function mockFetch(html) {
+			const mockResponse = {
+				ok: true,
+				status: 200,
+				statusText: "OK",
+				text: async () => html,
+			};
+
+			global.fetch = async () => mockResponse;
+		}
+
+		it("fetches and parsed versions with stable / experimental flags", async function() {
+			mockFetch(`
+				<html>
+					<body>
+						<a class="
+							slot-button-inline
+							version-button-experimental
+						" href="/download/archive/2.0.73">
+						2.0.73
+						</a>
+					</body>
+					<body>
+						<a class="
+							slot-button-inline
+							version-button-stable
+						" href="/download/archive/2.0.72">
+						2.0.72
+						</a>
+					</body>
+				</html>
+			`);
+
+			const versions = await fetchFactorioVersions();
+			assert.deepEqual(versions, [
+				{ stable: false, version: "2.0.73", downloadUrl: "https://factorio.com/download/archive/2.0.73" },
+				{ stable: true, version: "2.0.72", downloadUrl: "https://factorio.com/download/archive/2.0.72" },
+			]);
+		});
+		it("deduplicates versions by last occurrence", async function() {
+			mockFetch(`
+				<html>
+					<body>
+						<a class="
+							slot-button-inline
+							version-button-experimental
+						" href="/download/archive/2.0.73">
+						2.0.73
+						</a>
+					</body>
+					<body>
+						<a class="
+							slot-button-inline
+							version-button-stable
+						" href="/download/archive/2.0.73">
+						2.0.73
+						</a>
+					</body>
+				</html>
+			`);
+
+			const versions = await fetchFactorioVersions();
+			assert.deepEqual(versions, [
+				{ stable: true, version: "2.0.73", downloadUrl: "https://factorio.com/download/archive/2.0.73" },
+			]);
+		});
+		it("thorws on HTTP error status", async function() {
+			const mockResponse = {
+				ok: false,
+				status: 500,
+				statusText: "Server Error",
+				text: async () => "",
+			};
+
+			global.fetch = async () => mockResponse;
+
+			await assert.rejects(fetchFactorioVersions(), /HTTP 500 Server Error/);
+		});
+		it("thorws on network error status", async function() {
+			global.fetch = async () => {
+				throw new Error("Network Error");
+			};
+
+			await assert.rejects(fetchFactorioVersions(), /Network Error/);
+		});
+		it("parses versions from saved html file", async function() {
+			mockFetch(await readFile("test/file/html/factorio-download-archive.html"));
+
+			const versions = await fetchFactorioVersions();
+			assert.equal(versions.length, 111);
+			assert.equal(versions.filter(v => v.stable).length, 53);
+			assert.equal(versions.filter(v => !v.stable).length, 58);
+		});
+		it("parses versions from live api", async function() {
+			slowTest(this);
+			global.fetch = originalFetch;
+
+			const versions = await fetchFactorioVersions();
+			assert.ok(versions.length >= 111);
+			assert.ok(versions.filter(v => v.stable).length >= 53);
+			assert.ok(versions.filter(v => !v.stable).length >= 58);
+		});
+	});
+});