Refactor controller mod storage into ModStore

Structure the storage of mods on the controller into into a reusable
component in preparation for the host using the same kind of mod storage
to retrieve and cache the mods it needs to run mod packs on instances.

packages/controller/src/BaseConnection.ts

@@ -1,9 +1,13 @@
+import path from "path";
+import fs from "fs-extra";
+
 import type Controller from "./Controller";
 import type WsServerConnector from "./WsServerConnector";
 
 import * as lib from "@clusterio/lib";
 const { logger } = lib;
 import ControllerRouter from "./ControllerRouter";
+import * as routes from "./routes";
 
 
 /**
@@ -28,6 +32,7 @@ export default class BaseConnection extends lib.Link {
 
 		this.handle(lib.ModPackGetRequest, this.handleModPackGetRequest.bind(this));
 		this.handle(lib.ModPackGetDefaultRequest, this.handleModPackGetDefaultRequest.bind(this));
+		this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this));
 	}
 
 	async disconnect(code: number, reason: string) {
@@ -68,4 +73,29 @@ export default class BaseConnection extends lib.Link {
 		}
 		return modPack;
 	}
+
+	getMod(mod: { name: string, version: string, sha1?: string }) {
+		let filename = lib.ModInfo.filename(mod.name, mod.version);
+		let modInfo = this._controller.modStore.files.get(filename);
+		if (!modInfo) {
+			throw new lib.RequestError(`Mod ${filename} does not exist on controller`);
+		}
+		if (mod.sha1 && mod.sha1 !== modInfo.sha1) {
+			throw new lib.RequestError(`Mod ${filename} checksum does not match controller's checksum`);
+		}
+		return modInfo;
+	}
+
+	async handleModDownloadRequest(request: lib.ModDownloadRequest) {
+		let mod = this.getMod(request);
+		let modPath = path.join(this._controller.config.get("controller.mods_directory"), mod.filename);
+
+		let stream = await routes.createProxyStream(this._controller.app);
+		stream.filename = mod.filename;
+		stream.source = fs.createReadStream(modPath);
+		stream.mime = "application/zip";
+		stream.size = String(mod.size);
+
+		return stream.id;
+	}
 }

packages/controller/src/ControlConnection.ts

@@ -2,8 +2,6 @@ import type WsServerConnector from "./WsServerConnector";
 import type { HostInfo } from "./HostConnection";
 
 import events from "events";
-import fs from "fs-extra";
-import path from "path";
 
 import * as lib from "@clusterio/lib";
 const { logFilter, logger } = lib;
@@ -100,7 +98,6 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.ModGetRequest, this.handleModGetRequest.bind(this));
 		this.handle(lib.ModListRequest, this.handleModListRequest.bind(this));
 		this.handle(lib.ModSearchRequest, this.handleModSearchRequest.bind(this));
-		this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this));
 		this.handle(lib.ModDeleteRequest, this.handleModDeleteRequest.bind(this));
 		this.handle(lib.LogSetSubscriptionsRequest, this.handleLogSetSubscriptionsRequest.bind(this));
 		this.handle(lib.LogQueryRequest, this.handleLogQueryRequest.bind(this));
@@ -461,17 +458,11 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleModGetRequest(request: lib.ModGetRequest) {
-		let { name, version } = request;
-		let filename = `${name}_${version}.zip`;
-		let mod = this._controller.mods.get(filename);
-		if (!mod) {
-			throw new lib.RequestError(`Mod ${filename} does not exist`);
-		}
-		return mod;
+		return this.getMod(request);
 	}
 
 	async handleModListRequest() {
-		return [...this._controller.mods.values()];
+		return [...this._controller.modStore.mods()];
 	}
 
 	static termsMatchesMod(terms: lib.ParsedTerm[], mod: lib.ModInfo) {
@@ -511,7 +502,7 @@ export default class ControlConnection extends BaseConnection {
 
 		type ModVersions = { name: string, versions: lib.ModInfo[] };
 		let results: Map<string, ModVersions> = new Map();
-		for (let mod of this._controller.mods.values()) {
+		for (let mod of this._controller.modStore.mods()) {
 			if (
 				mod.factorioVersion !== factorioVersion
 				|| !ControlConnection.termsMatchesMod(query.terms, mod)
@@ -563,26 +554,8 @@ export default class ControlConnection extends BaseConnection {
 		};
 	}
 
-	async handleModDownloadRequest(request: lib.ModDownloadRequest) {
-		let { name, version } = request;
-		let filename = `${name}_${version}.zip`;
-		let mod = this._controller.mods.get(filename);
-		if (!mod) {
-			throw new lib.RequestError(`Mod ${filename} does not exist`);
-		}
-		let modPath = path.join(this._controller.config.get("controller.mods_directory"), mod.filename);
-
-		let stream = await routes.createProxyStream(this._controller.app);
-		stream.filename = mod.filename;
-		stream.source = fs.createReadStream(modPath);
-		stream.mime = "application/zip";
-		stream.size = String(mod.size);
-
-		return stream.id;
-	}
-
 	async handleModDeleteRequest(request: lib.ModDeleteRequest) {
-		await this._controller.deleteMod(request.name, request.version);
+		await this._controller.modStore.deleteMod(request.name, request.version);
 	}
 
 	async handleLogSetSubscriptionsRequest(request: lib.LogSetSubscriptionsRequest) {

packages/controller/src/Controller.ts

@@ -102,13 +102,13 @@ export default class Controller {
 
 		let modsDirectory = config.get("controller.mods_directory");
 		await fs.ensureDir(modsDirectory);
-		const mods = await Controller.loadModInfos(modsDirectory);
+		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
 
 		return [
 			hosts,
 			instances,
 			modPacks,
-			mods,
+			modStore,
 			userManager,
 		] as const;
 	}
@@ -125,8 +125,8 @@ export default class Controller {
 		public instances = new Map<number, InstanceInfo>(),
 		/** Mapping of mod pack id to mod pack */
 		public modPacks = new Map<number, lib.ModPack>(),
-		/** Mapping of mod names to mod infos */
-		public mods = new Map<string, lib.ModInfo>(),
+		/** Mods stored on the controller */
+		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
 		/** User and roles manager for the cluster */
 		public userManager = new UserManager(config),
 	) {
@@ -142,6 +142,10 @@ export default class Controller {
 		this.wsServer = new WsServer(this);
 		this.subscriptions = new lib.SubscriptionController(this);
 
+		this.modStore.on("change", mod => {
+			this.modUpdated(mod);
+		});
+
 		// Handle subscriptions for all internal properties
 		this.subscriptions.handle(lib.HostUpdateEvent);
 		this.subscriptions.handle(lib.InstanceDetailsUpdateEvent);
@@ -511,40 +515,6 @@ export default class Controller {
 		await lib.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, "\t"));
 	}
 
-	static async loadModInfos(modsDirectory: string): Promise<Map<string, lib.ModInfo>> {
-		let mods = new Map();
-		for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) {
-			if (entry.isDirectory()) {
-				logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`);
-				continue;
-			}
-
-			if (entry.name.toLowerCase().endsWith(".tmp.zip")) {
-				continue;
-			}
-
-			logger.info(`Loading info for Mod ${entry.name}`);
-			let modInfo;
-			try {
-				modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, entry.name));
-			} catch (err: any) {
-				logger.error(`Error loading mod ${entry.name}: ${err.message}`);
-				continue;
-			}
-
-			if (modInfo.filename !== entry.name) {
-				logger.error(
-					`Mod ${entry.name}'s filename does not match the expected ` +
-					`name ${modInfo.filename}, it will be ignored.`
-				);
-				continue;
-			}
-
-			mods.set(modInfo.filename, modInfo);
-		}
-		return mods;
-	}
-
 	static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
 		let manifest = {};
 		try {
@@ -557,20 +527,6 @@ export default class Controller {
 		return manifest;
 	}
 
-	async deleteMod(name: string, version: string): Promise<void> {
-		let filename = `${name}_${version}.zip`;
-		let mod = this.mods.get(filename);
-		if (!mod) {
-			throw new Error(`Mod ${filename} does not exist`);
-		}
-
-		let modsDirectory = this.config.get("controller.mods_directory");
-		await fs.unlink(path.join(modsDirectory, filename));
-		this.mods.delete(filename);
-		mod.isDeleted = true;
-		this.modUpdated(mod);
-	}
-
 	/**
 	 * Query controller log
 	 *

packages/controller/src/routes.ts

@@ -561,8 +561,7 @@ async function uploadMod(req: Request, res: Response) {
 			const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename));
 			checkModName(modInfo.filename);
 			await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename));
-			req.app.locals.controller.mods.set(modInfo.filename, modInfo);
-			req.app.locals.controller.modUpdated(modInfo);
+			req.app.locals.controller.modStore.addMod(modInfo);
 			mods.push(modInfo.toJSON());
 
 		} catch (err: any) {