Add automatic headless download and update

packages/controller/src/HostConnection.ts

@@ -81,6 +81,7 @@ export default class HostConnection extends BaseConnection {
 		this.handle(lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsUpdatesEvent.bind(this));
 		this.handle(lib.LogMessageEvent, this.handleLogMessageEvent.bind(this));
 		this.handle(lib.InstancePlayerUpdateEvent, this.handleInstancePlayerUpdateEvent.bind(this));
+		this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this));
 		this.snoopEvent(lib.InstanceAdminlistUpdateEvent, this.snoopInstanceAdminlistUpdateEvent.bind(this));
 		this.snoopEvent(lib.InstanceBanlistUpdateEvent, this.snoopInstanceBanlistUpdateEvent.bind(this));
 		this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.snoopInstanceWhitelistUpdateEvent.bind(this));
@@ -363,6 +364,10 @@ export default class HostConnection extends BaseConnection {
 		});
 	}
 
+	async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) {
+		return await this._controller.factorioVersions.get(request.maxAgeMs);
+	}
+
 	async snoopInstanceAdminlistUpdateEvent(event: lib.InstanceAdminlistUpdateEvent, src: lib.Address) {
 		if (src.type !== lib.Address.instance) {
 			return;

packages/host/src/Instance.ts

@@ -639,8 +639,10 @@ end`.replace(/\r?\n/g, " ");
 
 	async init(pluginInfos: lib.PluginNodeEnvInfo[]) {
 		this.notifyStatus("starting");
+		const factorioVersions = await this.sendTo("controller", new lib.FactorioVersionsRequest());
 		try {
 			await this._loadStats();
+			await this.server.checkForUpdates(factorioVersions);
 			await this.server.init();
 		} catch (err) {
 			this.notifyExit();

packages/host/src/server.ts

@@ -2,6 +2,7 @@
 import fs from "fs-extra";
 import child_process from "child_process";
 import path from "path";
+import JSZip from "jszip";
 import events from "events";
 import util from "util";
 import crypto from "crypto";
@@ -9,6 +10,7 @@ import { Rcon } from "rcon-client";
 
 import * as lib from "@clusterio/lib";
 
+const execFileAsync = util.promisify(child_process.execFile);
 
 /**
  * Determines the version of Factorio the datadir is pointing to by
@@ -66,13 +68,109 @@ function versionOrder(a: string, b: string) {
 	return 0;
 }
 
+/**
+ * Find all installed factorio versions and return them as a set
+ *
+ * @param factorioDir - Path to Factorio installation dirs.
+ * @returns Set of installed factorio versions
+ */
+async function listFactorioVersions(factorioDir: string) {
+	const versions = new Set<lib.FullVersion>();
+
+	for (const name of await fs.readdir(factorioDir)) {
+		// We do not use withFileTypes or lstat because we want to follow symlinks
+		const fullPath = path.join(factorioDir, name);
+		try {
+			const stats = await fs.stat(fullPath);
+			if (!stats.isDirectory()) {
+				continue;
+			}
+		} catch {
+			continue;
+		}
+
+		const version = await getFactorioVersion(fullPath);
+		if (version === null || !lib.isFullVersion(version)) {
+			continue;
+		}
+
+		versions.add(version);
+	}
+
+	return versions;
+}
+
+/**
+ * Downloads a ZIP file from a URL using native fetch and extracts it
+ *
+ * @param url - The URL of the ZIP file
+ * @param targetDir - Directory to extract the ZIP into
+ * @throws Network error if status of request is not ok
+ */
+async function downloadAndExtractZip(url: string, targetDir: string) {
+	const res = await fetch(url);
+	if (!res.ok) {
+		throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
+	}
+
+  	// Load the ZIP with JSZip
+	const buffer = Buffer.from(await res.arrayBuffer());
+	const zip = await JSZip.loadAsync(buffer);
+
+  	// Extract each file/folder
+	await fs.ensureDir(targetDir);
+	await Promise.all(
+		Object.values(zip.files).map(async (entry) => {
+			const entryPath = path.join(targetDir, entry.name);
+			if (entry.dir) {
+				await fs.ensureDir(entryPath);
+			} else {
+				await fs.ensureDir(path.dirname(entryPath));
+				await fs.writeFile(entryPath, await entry.async("nodebuffer"));
+			}
+		})
+	);
+}
+
+/**
+ * Downloads a tar.gz file from a URL using native fetch and extracts it
+ *
+ * @param url - The URL of the ZIP file
+ * @param targetDir - Directory to extract the ZIP into
+ * @throws Network error if status of request is not ok
+ */
+async function downloadAndExtractTarGz(url: string, targetDir: string) {
+	const res = await fetch(url);
+	if (!res.ok) {
+		throw new Error(`Failed to fetch ${url}: ${res.status} ${res.statusText}`);
+	}
+
+	// Write the buffer to a temp file
+	const tmpFilePath = `${targetDir}.tmp`;
+	const buffer = Buffer.from(await res.arrayBuffer());
+	await fs.writeFile(tmpFilePath, buffer);
+
+	// Execute tar CLI to extract
+	try {
+		await execFileAsync("tar", ["-xzf", tmpFilePath, "-C", targetDir, "--strip-components", "1"]);
+	} catch (err: any) {
+		if (err.code === "ENOENT") {
+			throw new Error("error executing tar command, do you have 'xz-utils' installed?");
+		}
+		throw err;
+	} finally {
+		// Clean up temp file
+		await fs.remove(tmpFilePath);
+	}
+}
+
 /**
  * Find factorio data directory of the given version
  *
  * Searches the given factorio dir for an installation of Factorio of
  * the target version.
  *
- * @param factorioDir - Path to Factorio installation dir(s).
+ * @param factorioDir - Path to Factorio installation dirs.
  * @param targetVersion -
  *     Version to look for, supports the special value "latest" for the
  *     latest version available.
@@ -860,6 +958,34 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 		this._state = "init";
 	}
 
+	/**
+	 * Check of updated versions to download
+	 *
+	 * Should be called before init for the new version to be detected during init
+	 */
+	async checkForUpdates(availableVersions: lib.ExternalFactorioVersion[]) {
+		const targetVersion = this._targetVersion;
+		const latestVersion = availableVersions
+			.filter(v => (
+				targetVersion === "latest" || v.version === targetVersion || v.version.startsWith(targetVersion)
+			))
+			.sort((v1, v2) => versionOrder(v1.version, v2.version))
+			[0];
+
+		const installedVersions = await listFactorioVersions(this._factorioDir);
+		if (!installedVersions.has(latestVersion.version)) {
+			// Download the newer version
+			const v = latestVersion.version;
+			if (process.platform === "linux") {
+				this._logger.info(`A newer version of factorio is available (${v}), starting download...`);
+				const dst = path.join(this._factorioDir, latestVersion.version);
+				await downloadAndExtractTarGz(latestVersion.headlessUrl, dst);
+			} else {
+				this._logger.info(`A newer version of factorio is available (${v}) but must be manually downloaded`);
+			}
+		}
+	}
+
 	/**
 	 * The version of Factorio in use. This will be the actual version if
 	 * "latest" (the default) was specified as `factorioVersion` to the constructor.
@@ -1412,6 +1538,7 @@ export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 // For testing only
 export const _versionOrder = versionOrder;
 export const _findVersion = findVersion;
+export const _listFactorioVersions = listFactorioVersions;
 export const _randomDynamicPort = randomDynamicPort;
 export const _generatePassword = generatePassword;
 export const _parseOutput = parseOutput;

packages/lib/src/data/messages_controller.ts

@@ -402,7 +402,7 @@ export class DebugWsMessageEvent {
 export class FactorioVersionsRequest {
 	declare ["constructor"]: typeof FactorioVersionsRequest;
 	static type = "request" as const;
-	static src = "control" as const;
+	static src = ["control", "instance"] as const;
 	static dst = "controller" as const;
 	static permission = "core.external.get_factorio_versions" as const;
 

packages/lib/src/external/FactorioVersions.ts

@@ -60,7 +60,7 @@ function parseFactorioVersions(html: string): ExternalFactorioVersion[] {
 			versions.set(version, {
 				version: version,
 				stable: kind === "stable",
-				headlessUrl: `www.factorio.com/get-download/${version}/headless/linux64`,
+				headlessUrl: `https://www.factorio.com/get-download/${version}/headless/linux64`,
 			});
 		}
 	}

test/file/factorio/0.1.2/data/base/locale/en/base.cfg

@@ -0,0 +1,4 @@
+; Test data for exportLocale
+[test]
+base-a=1
+base-b=2

test/file/factorio/0.1.2/data/changelog.txt

@@ -0,0 +1,3 @@
+---------------------------------------------------------------------------------------------------
+Version: 0.1.2
+About: Dummy changelog.txt for FactorioServer tests

test/file/factorio/0.1.2/data/core/locale/en/core.cfg

@@ -0,0 +1,4 @@
+; Test data for exportLocale
+[test]
+core-a=1
+core-b=2

test/file/factorio/not-full-version/data/base/locale/en/base.cfg

@@ -0,0 +1,4 @@
+; Test data for exportLocale
+[test]
+base-a=1
+base-b=2

test/file/factorio/not-full-version/data/changelog.txt

@@ -0,0 +1,3 @@
+---------------------------------------------------------------------------------------------------
+Version: 0.1
+About: Dummy changelog.txt for FactorioServer tests

test/file/factorio/not-full-version/data/core/locale/en/core.cfg

@@ -0,0 +1,4 @@
+; Test data for exportLocale
+[test]
+core-a=1
+core-b=2

test/host/server.js

@@ -41,8 +41,8 @@ describe("host/server", function() {
 		it("should search given directory for latest Factorio install", async function() {
 			const installDir = path.join("test", "file", "factorio");
 			const [dir, version] = await hostServer._findVersion(installDir, "latest");
-			assert.equal(dir, path.join(installDir, "0.1.1", "data"));
-			assert.equal(version, "0.1.1");
+			assert.equal(dir, path.join(installDir, "0.1.2", "data"));
+			assert.equal(version, "0.1.2");
 		});
 		it("should search given directory for given Factorio install", async function() {
 			const installDir = path.join("test", "file", "factorio");
@@ -53,14 +53,14 @@ describe("host/server", function() {
 		it("should search given directory for partly given Factorio install", async function() {
 			const installDir = path.join("test", "file", "factorio");
 			const [dir, version] = await hostServer._findVersion(installDir, "0.1");
-			assert.equal(dir, path.join(installDir, "0.1.1", "data"));
-			assert.equal(version, "0.1.1");
+			assert.equal(dir, path.join(installDir, "0.1.2", "data"));
+			assert.equal(version, "0.1.2");
 		});
 		it("should reject if no factorio install with the given version was found", async function() {
 			let installDir = path.join("test", "file", "factorio");
 			await assert.rejects(
-				hostServer._findVersion(installDir, "0.1.2"),
-				new Error("Unable to find Factorio version 0.1.2")
+				hostServer._findVersion(installDir, "0.1.3"),
+				new Error("Unable to find Factorio version 0.1.3")
 			);
 		});
 		it("should reject if no factorio install was found", async function() {
@@ -72,6 +72,14 @@ describe("host/server", function() {
 		});
 	});
 
+	describe("_listFactorioVersions()", function() {
+		it("should list all versions in a directory", async function() {
+			const installDir = path.join("test", "file", "factorio");
+			const installedVersions = await hostServer._listFactorioVersions(installDir);
+			assert.deepEqual(installedVersions, new Set(["0.1.1", "0.1.2"]));
+		});
+	});
+
 	describe("randomDynamicPort()", function() {
 		it("should return a port number", function() {
 			let port = hostServer._randomDynamicPort();
@@ -144,7 +152,7 @@ describe("host/server", function() {
 
 		describe(".version", function() {
 			it("should return the version detected", function() {
-				assert.equal(server.version, "0.1.1");
+				assert.equal(server.version, "0.1.2");
 			});
 		});
 

test/lib/external/FactorioVersions.test.js

@@ -54,12 +54,12 @@ describe("FactorioVersions", function() {
 				{
 					stable: false,
 					version: "2.0.73",
-					headlessUrl: "www.factorio.com/get-download/2.0.73/headless/linux64",
+					headlessUrl: "https://www.factorio.com/get-download/2.0.73/headless/linux64",
 				},
 				{
 					stable: true,
 					version: "2.0.72",
-					headlessUrl: "www.factorio.com/get-download/2.0.72/headless/linux64",
+					headlessUrl: "https://www.factorio.com/get-download/2.0.72/headless/linux64",
 				},
 			]);
 		});
@@ -90,7 +90,7 @@ describe("FactorioVersions", function() {
 				{
 					stable: true,
 					version: "2.0.73",
-					headlessUrl: "www.factorio.com/get-download/2.0.73/headless/linux64",
+					headlessUrl: "https://www.factorio.com/get-download/2.0.73/headless/linux64",
 				},
 			]);
 		});