Refactor controller mod storage into ModStore

Structure the storage of mods on the controller into into a reusable
component in preparation for the host using the same kind of mod storage
to retrieve and cache the mods it needs to run mod packs on instances.

packages/controller/src/BaseConnection.ts

@@ -1,9 +1,13 @@
+import path from "path";
+import fs from "fs-extra";
+
 import type Controller from "./Controller";
 import type WsServerConnector from "./WsServerConnector";
 
 import * as lib from "@clusterio/lib";
 const { logger } = lib;
 import ControllerRouter from "./ControllerRouter";
+import * as routes from "./routes";
 
 
 /**
@@ -28,6 +32,7 @@ export default class BaseConnection extends lib.Link {
 
 		this.handle(lib.ModPackGetRequest, this.handleModPackGetRequest.bind(this));
 		this.handle(lib.ModPackGetDefaultRequest, this.handleModPackGetDefaultRequest.bind(this));
+		this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this));
 	}
 
 	async disconnect(code: number, reason: string) {
@@ -68,4 +73,29 @@ export default class BaseConnection extends lib.Link {
 		}
 		return modPack;
 	}
+
+	getMod(mod: { name: string, version: string, sha1?: string }) {
+		let filename = lib.ModInfo.filename(mod.name, mod.version);
+		let modInfo = this._controller.modStore.files.get(filename);
+		if (!modInfo) {
+			throw new lib.RequestError(`Mod ${filename} does not exist on controller`);
+		}
+		if (mod.sha1 && mod.sha1 !== modInfo.sha1) {
+			throw new lib.RequestError(`Mod ${filename} checksum does not match controller's checksum`);
+		}
+		return modInfo;
+	}
+
+	async handleModDownloadRequest(request: lib.ModDownloadRequest) {
+		let mod = this.getMod(request);
+		let modPath = path.join(this._controller.config.get("controller.mods_directory"), mod.filename);
+
+		let stream = await routes.createProxyStream(this._controller.app);
+		stream.filename = mod.filename;
+		stream.source = fs.createReadStream(modPath);
+		stream.mime = "application/zip";
+		stream.size = String(mod.size);
+
+		return stream.id;
+	}
 }

packages/controller/src/ControlConnection.ts

@@ -2,8 +2,6 @@ import type WsServerConnector from "./WsServerConnector";
 import type { HostInfo } from "./HostConnection";
 
 import events from "events";
-import fs from "fs-extra";
-import path from "path";
 
 import * as lib from "@clusterio/lib";
 const { logFilter, logger } = lib;
@@ -100,7 +98,6 @@ export default class ControlConnection extends BaseConnection {
 		this.handle(lib.ModGetRequest, this.handleModGetRequest.bind(this));
 		this.handle(lib.ModListRequest, this.handleModListRequest.bind(this));
 		this.handle(lib.ModSearchRequest, this.handleModSearchRequest.bind(this));
-		this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this));
 		this.handle(lib.ModDeleteRequest, this.handleModDeleteRequest.bind(this));
 		this.handle(lib.LogSetSubscriptionsRequest, this.handleLogSetSubscriptionsRequest.bind(this));
 		this.handle(lib.LogQueryRequest, this.handleLogQueryRequest.bind(this));
@@ -461,17 +458,11 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleModGetRequest(request: lib.ModGetRequest) {
-		let { name, version } = request;
-		let filename = `${name}_${version}.zip`;
-		let mod = this._controller.mods.get(filename);
-		if (!mod) {
-			throw new lib.RequestError(`Mod ${filename} does not exist`);
-		}
-		return mod;
+		return this.getMod(request);
 	}
 
 	async handleModListRequest() {
-		return [...this._controller.mods.values()];
+		return [...this._controller.modStore.mods()];
 	}
 
 	static termsMatchesMod(terms: lib.ParsedTerm[], mod: lib.ModInfo) {
@@ -511,7 +502,7 @@ export default class ControlConnection extends BaseConnection {
 
 		type ModVersions = { name: string, versions: lib.ModInfo[] };
 		let results: Map<string, ModVersions> = new Map();
-		for (let mod of this._controller.mods.values()) {
+		for (let mod of this._controller.modStore.mods()) {
 			if (
 				mod.factorioVersion !== factorioVersion
 				|| !ControlConnection.termsMatchesMod(query.terms, mod)
@@ -563,26 +554,8 @@ export default class ControlConnection extends BaseConnection {
 		};
 	}
 
-	async handleModDownloadRequest(request: lib.ModDownloadRequest) {
-		let { name, version } = request;
-		let filename = `${name}_${version}.zip`;
-		let mod = this._controller.mods.get(filename);
-		if (!mod) {
-			throw new lib.RequestError(`Mod ${filename} does not exist`);
-		}
-		let modPath = path.join(this._controller.config.get("controller.mods_directory"), mod.filename);
-
-		let stream = await routes.createProxyStream(this._controller.app);
-		stream.filename = mod.filename;
-		stream.source = fs.createReadStream(modPath);
-		stream.mime = "application/zip";
-		stream.size = String(mod.size);
-
-		return stream.id;
-	}
-
 	async handleModDeleteRequest(request: lib.ModDeleteRequest) {
-		await this._controller.deleteMod(request.name, request.version);
+		await this._controller.modStore.deleteMod(request.name, request.version);
 	}
 
 	async handleLogSetSubscriptionsRequest(request: lib.LogSetSubscriptionsRequest) {

packages/controller/src/Controller.ts

@@ -102,13 +102,13 @@ export default class Controller {
 
 		let modsDirectory = config.get("controller.mods_directory");
 		await fs.ensureDir(modsDirectory);
-		const mods = await Controller.loadModInfos(modsDirectory);
+		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
 
 		return [
 			hosts,
 			instances,
 			modPacks,
-			mods,
+			modStore,
 			userManager,
 		] as const;
 	}
@@ -125,8 +125,8 @@ export default class Controller {
 		public instances = new Map<number, InstanceInfo>(),
 		/** Mapping of mod pack id to mod pack */
 		public modPacks = new Map<number, lib.ModPack>(),
-		/** Mapping of mod names to mod infos */
-		public mods = new Map<string, lib.ModInfo>(),
+		/** Mods stored on the controller */
+		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
 		/** User and roles manager for the cluster */
 		public userManager = new UserManager(config),
 	) {
@@ -142,6 +142,10 @@ export default class Controller {
 		this.wsServer = new WsServer(this);
 		this.subscriptions = new lib.SubscriptionController(this);
 
+		this.modStore.on("change", mod => {
+			this.modUpdated(mod);
+		});
+
 		// Handle subscriptions for all internal properties
 		this.subscriptions.handle(lib.HostUpdateEvent);
 		this.subscriptions.handle(lib.InstanceDetailsUpdateEvent);
@@ -511,40 +515,6 @@ export default class Controller {
 		await lib.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, "\t"));
 	}
 
-	static async loadModInfos(modsDirectory: string): Promise<Map<string, lib.ModInfo>> {
-		let mods = new Map();
-		for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) {
-			if (entry.isDirectory()) {
-				logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`);
-				continue;
-			}
-
-			if (entry.name.toLowerCase().endsWith(".tmp.zip")) {
-				continue;
-			}
-
-			logger.info(`Loading info for Mod ${entry.name}`);
-			let modInfo;
-			try {
-				modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, entry.name));
-			} catch (err: any) {
-				logger.error(`Error loading mod ${entry.name}: ${err.message}`);
-				continue;
-			}
-
-			if (modInfo.filename !== entry.name) {
-				logger.error(
-					`Mod ${entry.name}'s filename does not match the expected ` +
-					`name ${modInfo.filename}, it will be ignored.`
-				);
-				continue;
-			}
-
-			mods.set(modInfo.filename, modInfo);
-		}
-		return mods;
-	}
-
 	static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
 		let manifest = {};
 		try {
@@ -557,20 +527,6 @@ export default class Controller {
 		return manifest;
 	}
 
-	async deleteMod(name: string, version: string): Promise<void> {
-		let filename = `${name}_${version}.zip`;
-		let mod = this.mods.get(filename);
-		if (!mod) {
-			throw new Error(`Mod ${filename} does not exist`);
-		}
-
-		let modsDirectory = this.config.get("controller.mods_directory");
-		await fs.unlink(path.join(modsDirectory, filename));
-		this.mods.delete(filename);
-		mod.isDeleted = true;
-		this.modUpdated(mod);
-	}
-
 	/**
 	 * Query controller log
 	 *

packages/controller/src/routes.ts

@@ -561,8 +561,7 @@ async function uploadMod(req: Request, res: Response) {
 			const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename));
 			checkModName(modInfo.filename);
 			await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename));
-			req.app.locals.controller.mods.set(modInfo.filename, modInfo);
-			req.app.locals.controller.modUpdated(modInfo);
+			req.app.locals.controller.modStore.addMod(modInfo);
 			mods.push(modInfo.toJSON());
 
 		} catch (err: any) {

packages/lib/index.ts

@@ -29,4 +29,5 @@ export * from "./src/zip_ops";
 export * from "./src/subscriptions";
 
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
+export { default as ModStore } from "./src/ModStore";
 export { default as RateLimiter } from "./src/RateLimiter";

packages/lib/src/ModStore.ts

@@ -0,0 +1,110 @@
+import fs from "fs-extra";
+import path from "path";
+import { logger } from "./logging";
+import { ModInfo } from "./data";
+import EventEmitter from "events";
+
+abstract class TypedEventEmitter<
+	Keys extends string,
+	Events extends Record<Keys, (...args: [...any]) => void>,
+> extends EventEmitter {
+	declare ["on"]: <E extends Keys>(event: E, listener: Events[E]) => this;
+	declare ["off"]: <E extends Keys>(event: E, listener: Events[E]) => this;
+	declare ["once"]: <E extends Keys>(event: E, listener: Events[E]) => this;
+	declare ["emit"]: <E extends Keys>(event: E, ...args: Parameters<Events[E]>) => boolean;
+}
+
+export interface ModStoreEvents {
+	/** A stored mod was created, updated or deleted */
+	change: (mod: ModInfo) => void;
+}
+
+export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, ModStoreEvents> {
+	constructor(
+		public modsDirectory: string,
+		public files: Map<string, ModInfo>,
+	) {
+		super();
+	}
+
+	getMod(name: string, version: string, sha1?: string) {
+		const mod = this.files.get(ModInfo.filename(name, version));
+		if (!mod || sha1 && sha1 !== mod.sha1) {
+			return undefined;
+		}
+		return mod;
+	}
+
+	hasMod(name: string, version: string, sha1?: string): boolean {
+		const mod = this.files.get(ModInfo.filename(name, version));
+		return Boolean(mod && (!sha1 || sha1 !== mod.sha1));
+	}
+
+	mods() { return this.files.values(); }
+
+	static async loadFile(modsDirectory: string, file: string) {
+		let modInfo = await ModInfo.fromModFile(path.join(modsDirectory, file));
+		if (modInfo.filename !== file) {
+			throw new Error(`filename does not match the expected name ${modInfo.filename}.`);
+		}
+		return modInfo;
+	}
+
+	async loadFile(file: string) {
+		let modInfo = await ModStore.loadFile(this.modsDirectory, file);
+		this.files.set(modInfo.filename, modInfo);
+		this.emit("change", modInfo);
+		return modInfo;
+	}
+
+	async deleteFile(file: string) {
+		let modInfo = this.files.get(file);
+		if (!modInfo) {
+			throw new Error(`Mod ${file} does not exist`);
+		}
+
+		await fs.unlink(path.join(this.modsDirectory, file));
+		this.files.delete(file);
+		modInfo.isDeleted = true;
+		this.emit("change", modInfo);
+	}
+
+	addMod(modInfo: ModInfo) {
+		this.files.set(modInfo.filename, modInfo);
+		this.emit("change", modInfo);
+	}
+
+	async loadMod(name: string, version: string) {
+		return await this.loadFile(ModInfo.filename(name, version));
+	}
+
+	async deleteMod(name: string, version: string) {
+		return await this.deleteFile(ModInfo.filename(name, version));
+	}
+
+	static async fromDirectory(modsDirectory: string) {
+		const files = new Map<string, ModInfo>();
+		for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) {
+			if (entry.isDirectory()) {
+				logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`);
+				continue;
+			}
+
+			if (entry.name.toLowerCase().endsWith(".tmp.zip")) {
+				continue;
+			}
+
+			logger.info(`Loading info for Mod ${entry.name}`);
+			let modInfo: ModInfo;
+			try {
+				modInfo = await this.loadFile(modsDirectory, entry.name);
+			} catch (err: any) {
+				logger.error(`Error loading mod ${entry.name}: ${err.message}`);
+				continue;
+			}
+			files.set(modInfo.filename, modInfo);
+		}
+
+		return new this(modsDirectory, files);
+	}
+}

packages/lib/src/data/ModInfo.ts

@@ -90,7 +90,17 @@ export default class ModInfo {
 	 * Expected name of zip file containing this mod.
 	 */
 	get filename() {
-		return `${this.name}_${this.version}.zip`;
+		return ModInfo.filename(this.name, this.version);
+	}
+
+	/**
+	 * Expected name of zip file containing mod with given name and version.
+	 * @param name - Mod's name
+	 * @param version - Mod's version
+	 * @returns string containing {name}_{version}.zip
+	 */
+	static filename(name: string, version: string) {
+		return `${name}_${version}.zip`;
 	}
 
 	/**

packages/lib/src/data/messages_mod.ts

@@ -114,15 +114,17 @@ export class ModGetRequest {
 	constructor(
 		public name: string,
 		public version: string,
+		public sha1?: string,
 	) { }
 
 	static jsonSchema = Type.Object({
 		"name": Type.String(),
 		"version": Type.String(),
+		"sha1": Type.Optional(Type.String()),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.name, json.version);
+		return new this(json.name, json.version, json.sha1);
 	}
 
 	static Response = ModInfo;
@@ -198,22 +200,24 @@ export class ModSearchRequest {
 export class ModDownloadRequest {
 	declare ["constructor"]: typeof ModDownloadRequest;
 	static type = "request" as const;
-	static src = "control" as const;
+	static src = ["host", "control"] as const;
 	static dst = "controller" as const;
 	static permission = "core.mod.download" as const;
 
 	constructor(
 		public name: string,
 		public version: string,
+		public sha1?: string,
 	) { }
 
 	static jsonSchema = Type.Object({
 		"name": Type.String(),
 		"version": Type.String(),
+		"sha1": Type.Optional(Type.String()),
 	});
 
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.name, json.version);
+		return new this(json.name, json.version, json.sha1);
 	}
 
 	static Response = JsonString;

packages/web_ui/src/components/ModsPage.tsx

@@ -132,7 +132,7 @@ export default function ModsPage() {
 	let [modList] = useModList();
 	let [modPackList] = useModPackList();
 
-	function actions(mod: lib.ModInfo|lib.ModRecord) {
+	function actions(mod: lib.ModInfo) {
 		return <Space>
 			{account.hasPermission("core.mod.download")
 				&& <Typography.Link
@@ -151,7 +151,7 @@ export default function ModsPage() {
 			}
 			{account.hasPermission("core.mod.delete")
 				&& <Popconfirm
-					title={`Delete ${mod.name}_${mod.version}.zip?`}
+					title={`Delete ${mod.filename}?`}
 					onConfirm={event => {
 						control.send(
 							new lib.ModDeleteRequest(mod.name, mod.version)