Refactor controller mod storage into ModStore

Structure the storage of mods on the controller into into a reusable
component in preparation for the host using the same kind of mod storage
to retrieve and cache the mods it needs to run mod packs on instances.
This commit is contained in:
Hornwitser committed 2023-12-03 19:59:14 +01:00
1 parent 448cfbbb51
commit 853866492b
9 files changed
+174 -91

No files matched your search

+30
View File
@@ -1,9 +1,13 @@
import path from "path";
import fs from "fs-extra";
import type Controller from "./Controller"; import type Controller from "./Controller";
import type WsServerConnector from "./WsServerConnector"; import type WsServerConnector from "./WsServerConnector";
import * as lib from "@clusterio/lib"; import * as lib from "@clusterio/lib";
const { logger } = lib; const { logger } = lib;
import ControllerRouter from "./ControllerRouter"; import ControllerRouter from "./ControllerRouter";
import * as routes from "./routes";
/** /**
@@ -28,6 +32,7 @@ export default class BaseConnection extends lib.Link {
this.handle(lib.ModPackGetRequest, this.handleModPackGetRequest.bind(this)); this.handle(lib.ModPackGetRequest, this.handleModPackGetRequest.bind(this));
this.handle(lib.ModPackGetDefaultRequest, this.handleModPackGetDefaultRequest.bind(this)); this.handle(lib.ModPackGetDefaultRequest, this.handleModPackGetDefaultRequest.bind(this));
this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this));
} }
async disconnect(code: number, reason: string) { async disconnect(code: number, reason: string) {
@@ -68,4 +73,29 @@ export default class BaseConnection extends lib.Link {
} }
return modPack; return modPack;
} }
getMod(mod: { name: string, version: string, sha1?: string }) {
let filename = lib.ModInfo.filename(mod.name, mod.version);
let modInfo = this._controller.modStore.files.get(filename);
if (!modInfo) {
throw new lib.RequestError(`Mod ${filename} does not exist on controller`);
}
if (mod.sha1 && mod.sha1 !== modInfo.sha1) {
throw new lib.RequestError(`Mod ${filename} checksum does not match controller's checksum`);
}
return modInfo;
}
async handleModDownloadRequest(request: lib.ModDownloadRequest) {
let mod = this.getMod(request);
let modPath = path.join(this._controller.config.get("controller.mods_directory"), mod.filename);
let stream = await routes.createProxyStream(this._controller.app);
stream.filename = mod.filename;
stream.source = fs.createReadStream(modPath);
stream.mime = "application/zip";
stream.size = String(mod.size);
return stream.id;
}
} }
+4 -31
View File
@@ -2,8 +2,6 @@ import type WsServerConnector from "./WsServerConnector";
import type { HostInfo } from "./HostConnection"; import type { HostInfo } from "./HostConnection";
import events from "events"; import events from "events";
import fs from "fs-extra";
import path from "path";
import * as lib from "@clusterio/lib"; import * as lib from "@clusterio/lib";
const { logFilter, logger } = lib; const { logFilter, logger } = lib;
@@ -100,7 +98,6 @@ export default class ControlConnection extends BaseConnection {
this.handle(lib.ModGetRequest, this.handleModGetRequest.bind(this)); this.handle(lib.ModGetRequest, this.handleModGetRequest.bind(this));
this.handle(lib.ModListRequest, this.handleModListRequest.bind(this)); this.handle(lib.ModListRequest, this.handleModListRequest.bind(this));
this.handle(lib.ModSearchRequest, this.handleModSearchRequest.bind(this)); this.handle(lib.ModSearchRequest, this.handleModSearchRequest.bind(this));
this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this));
this.handle(lib.ModDeleteRequest, this.handleModDeleteRequest.bind(this)); this.handle(lib.ModDeleteRequest, this.handleModDeleteRequest.bind(this));
this.handle(lib.LogSetSubscriptionsRequest, this.handleLogSetSubscriptionsRequest.bind(this)); this.handle(lib.LogSetSubscriptionsRequest, this.handleLogSetSubscriptionsRequest.bind(this));
this.handle(lib.LogQueryRequest, this.handleLogQueryRequest.bind(this)); this.handle(lib.LogQueryRequest, this.handleLogQueryRequest.bind(this));
@@ -461,17 +458,11 @@ export default class ControlConnection extends BaseConnection {
} }
async handleModGetRequest(request: lib.ModGetRequest) { async handleModGetRequest(request: lib.ModGetRequest) {
let { name, version } = request; return this.getMod(request);
let filename = `${name}_${version}.zip`;
let mod = this._controller.mods.get(filename);
if (!mod) {
throw new lib.RequestError(`Mod ${filename} does not exist`);
}
return mod;
} }
async handleModListRequest() { async handleModListRequest() {
return [...this._controller.mods.values()]; return [...this._controller.modStore.mods()];
} }
static termsMatchesMod(terms: lib.ParsedTerm[], mod: lib.ModInfo) { static termsMatchesMod(terms: lib.ParsedTerm[], mod: lib.ModInfo) {
@@ -511,7 +502,7 @@ export default class ControlConnection extends BaseConnection {
type ModVersions = { name: string, versions: lib.ModInfo[] }; type ModVersions = { name: string, versions: lib.ModInfo[] };
let results: Map<string, ModVersions> = new Map(); let results: Map<string, ModVersions> = new Map();
for (let mod of this._controller.mods.values()) { for (let mod of this._controller.modStore.mods()) {
if ( if (
mod.factorioVersion !== factorioVersion mod.factorioVersion !== factorioVersion
|| !ControlConnection.termsMatchesMod(query.terms, mod) || !ControlConnection.termsMatchesMod(query.terms, mod)
@@ -563,26 +554,8 @@ export default class ControlConnection extends BaseConnection {
}; };
} }
async handleModDownloadRequest(request: lib.ModDownloadRequest) {
let { name, version } = request;
let filename = `${name}_${version}.zip`;
let mod = this._controller.mods.get(filename);
if (!mod) {
throw new lib.RequestError(`Mod ${filename} does not exist`);
}
let modPath = path.join(this._controller.config.get("controller.mods_directory"), mod.filename);
let stream = await routes.createProxyStream(this._controller.app);
stream.filename = mod.filename;
stream.source = fs.createReadStream(modPath);
stream.mime = "application/zip";
stream.size = String(mod.size);
return stream.id;
}
async handleModDeleteRequest(request: lib.ModDeleteRequest) { async handleModDeleteRequest(request: lib.ModDeleteRequest) {
await this._controller.deleteMod(request.name, request.version); await this._controller.modStore.deleteMod(request.name, request.version);
} }
async handleLogSetSubscriptionsRequest(request: lib.LogSetSubscriptionsRequest) { async handleLogSetSubscriptionsRequest(request: lib.LogSetSubscriptionsRequest) {
+8 -52
View File
@@ -102,13 +102,13 @@ export default class Controller {
let modsDirectory = config.get("controller.mods_directory"); let modsDirectory = config.get("controller.mods_directory");
await fs.ensureDir(modsDirectory); await fs.ensureDir(modsDirectory);
const mods = await Controller.loadModInfos(modsDirectory); const modStore = await lib.ModStore.fromDirectory(modsDirectory);
return [ return [
hosts, hosts,
instances, instances,
modPacks, modPacks,
mods, modStore,
userManager, userManager,
] as const; ] as const;
} }
@@ -125,8 +125,8 @@ export default class Controller {
public instances = new Map<number, InstanceInfo>(), public instances = new Map<number, InstanceInfo>(),
/** Mapping of mod pack id to mod pack */ /** Mapping of mod pack id to mod pack */
public modPacks = new Map<number, lib.ModPack>(), public modPacks = new Map<number, lib.ModPack>(),
/** Mapping of mod names to mod infos */ /** Mods stored on the controller */
public mods = new Map<string, lib.ModInfo>(), public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
/** User and roles manager for the cluster */ /** User and roles manager for the cluster */
public userManager = new UserManager(config), public userManager = new UserManager(config),
) { ) {
@@ -142,6 +142,10 @@ export default class Controller {
this.wsServer = new WsServer(this); this.wsServer = new WsServer(this);
this.subscriptions = new lib.SubscriptionController(this); this.subscriptions = new lib.SubscriptionController(this);
this.modStore.on("change", mod => {
this.modUpdated(mod);
});
// Handle subscriptions for all internal properties // Handle subscriptions for all internal properties
this.subscriptions.handle(lib.HostUpdateEvent); this.subscriptions.handle(lib.HostUpdateEvent);
this.subscriptions.handle(lib.InstanceDetailsUpdateEvent); this.subscriptions.handle(lib.InstanceDetailsUpdateEvent);
@@ -511,40 +515,6 @@ export default class Controller {
await lib.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, "\t")); await lib.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, "\t"));
} }
static async loadModInfos(modsDirectory: string): Promise<Map<string, lib.ModInfo>> {
let mods = new Map();
for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) {
if (entry.isDirectory()) {
logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`);
continue;
}
if (entry.name.toLowerCase().endsWith(".tmp.zip")) {
continue;
}
logger.info(`Loading info for Mod ${entry.name}`);
let modInfo;
try {
modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, entry.name));
} catch (err: any) {
logger.error(`Error loading mod ${entry.name}: ${err.message}`);
continue;
}
if (modInfo.filename !== entry.name) {
logger.error(
`Mod ${entry.name}'s filename does not match the expected ` +
`name ${modInfo.filename}, it will be ignored.`
);
continue;
}
mods.set(modInfo.filename, modInfo);
}
return mods;
}
static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> { static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
let manifest = {}; let manifest = {};
try { try {
@@ -557,20 +527,6 @@ export default class Controller {
return manifest; return manifest;
} }
async deleteMod(name: string, version: string): Promise<void> {
let filename = `${name}_${version}.zip`;
let mod = this.mods.get(filename);
if (!mod) {
throw new Error(`Mod ${filename} does not exist`);
}
let modsDirectory = this.config.get("controller.mods_directory");
await fs.unlink(path.join(modsDirectory, filename));
this.mods.delete(filename);
mod.isDeleted = true;
this.modUpdated(mod);
}
/** /**
* Query controller log * Query controller log
* *
+1 -2
View File
@@ -561,8 +561,7 @@ async function uploadMod(req: Request, res: Response) {
const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename)); const modInfo = await lib.ModInfo.fromModFile(path.join(modsDirectory, tempFilename));
checkModName(modInfo.filename); checkModName(modInfo.filename);
await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename)); await fs.rename(path.join(modsDirectory, tempFilename), path.join(modsDirectory, modInfo.filename));
req.app.locals.controller.mods.set(modInfo.filename, modInfo); req.app.locals.controller.modStore.addMod(modInfo);
req.app.locals.controller.modUpdated(modInfo);
mods.push(modInfo.toJSON()); mods.push(modInfo.toJSON());
} catch (err: any) { } catch (err: any) {
+1
View File
@@ -29,4 +29,5 @@ export * from "./src/zip_ops";
export * from "./src/subscriptions"; export * from "./src/subscriptions";
export { default as ExponentialBackoff } from "./src/ExponentialBackoff"; export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
export { default as ModStore } from "./src/ModStore";
export { default as RateLimiter } from "./src/RateLimiter"; export { default as RateLimiter } from "./src/RateLimiter";
+110
View File
@@ -0,0 +1,110 @@
import fs from "fs-extra";
import path from "path";
import { logger } from "./logging";
import { ModInfo } from "./data";
import EventEmitter from "events";
abstract class TypedEventEmitter<
Keys extends string,
Events extends Record<Keys, (...args: [...any]) => void>,
> extends EventEmitter {
declare ["on"]: <E extends Keys>(event: E, listener: Events[E]) => this;
declare ["off"]: <E extends Keys>(event: E, listener: Events[E]) => this;
declare ["once"]: <E extends Keys>(event: E, listener: Events[E]) => this;
declare ["emit"]: <E extends Keys>(event: E, ...args: Parameters<Events[E]>) => boolean;
}
export interface ModStoreEvents {
/** A stored mod was created, updated or deleted */
change: (mod: ModInfo) => void;
}
export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, ModStoreEvents> {
constructor(
public modsDirectory: string,
public files: Map<string, ModInfo>,
) {
super();
}
getMod(name: string, version: string, sha1?: string) {
const mod = this.files.get(ModInfo.filename(name, version));
if (!mod || sha1 && sha1 !== mod.sha1) {
return undefined;
}
return mod;
}
hasMod(name: string, version: string, sha1?: string): boolean {
const mod = this.files.get(ModInfo.filename(name, version));
return Boolean(mod && (!sha1 || sha1 !== mod.sha1));
}
mods() { return this.files.values(); }
static async loadFile(modsDirectory: string, file: string) {
let modInfo = await ModInfo.fromModFile(path.join(modsDirectory, file));
if (modInfo.filename !== file) {
throw new Error(`filename does not match the expected name ${modInfo.filename}.`);
}
return modInfo;
}
async loadFile(file: string) {
let modInfo = await ModStore.loadFile(this.modsDirectory, file);
this.files.set(modInfo.filename, modInfo);
this.emit("change", modInfo);
return modInfo;
}
async deleteFile(file: string) {
let modInfo = this.files.get(file);
if (!modInfo) {
throw new Error(`Mod ${file} does not exist`);
}
await fs.unlink(path.join(this.modsDirectory, file));
this.files.delete(file);
modInfo.isDeleted = true;
this.emit("change", modInfo);
}
addMod(modInfo: ModInfo) {
this.files.set(modInfo.filename, modInfo);
this.emit("change", modInfo);
}
async loadMod(name: string, version: string) {
return await this.loadFile(ModInfo.filename(name, version));
}
async deleteMod(name: string, version: string) {
return await this.deleteFile(ModInfo.filename(name, version));
}
static async fromDirectory(modsDirectory: string) {
const files = new Map<string, ModInfo>();
for (let entry of await fs.readdir(modsDirectory, { withFileTypes: true })) {
if (entry.isDirectory()) {
logger.warn(`Ignoring ${entry.name}${path.sep} in mods directory.`);
continue;
}
if (entry.name.toLowerCase().endsWith(".tmp.zip")) {
continue;
}
logger.info(`Loading info for Mod ${entry.name}`);
let modInfo: ModInfo;
try {
modInfo = await this.loadFile(modsDirectory, entry.name);
} catch (err: any) {
logger.error(`Error loading mod ${entry.name}: ${err.message}`);
continue;
}
files.set(modInfo.filename, modInfo);
}
return new this(modsDirectory, files);
}
}
+11 -1
View File
@@ -90,7 +90,17 @@ export default class ModInfo {
* Expected name of zip file containing this mod. * Expected name of zip file containing this mod.
*/ */
get filename() { get filename() {
return `${this.name}_${this.version}.zip`; return ModInfo.filename(this.name, this.version);
}
/**
* Expected name of zip file containing mod with given name and version.
* @param name - Mod's name
* @param version - Mod's version
* @returns string containing {name}_{version}.zip
*/
static filename(name: string, version: string) {
return `${name}_${version}.zip`;
} }
/** /**
+7 -3
View File
@@ -114,15 +114,17 @@ export class ModGetRequest {
constructor( constructor(
public name: string, public name: string,
public version: string, public version: string,
public sha1?: string,
) { } ) { }
static jsonSchema = Type.Object({ static jsonSchema = Type.Object({
"name": Type.String(), "name": Type.String(),
"version": Type.String(), "version": Type.String(),
"sha1": Type.Optional(Type.String()),
}); });
static fromJSON(json: Static<typeof this.jsonSchema>) { static fromJSON(json: Static<typeof this.jsonSchema>) {
return new this(json.name, json.version); return new this(json.name, json.version, json.sha1);
} }
static Response = ModInfo; static Response = ModInfo;
@@ -198,22 +200,24 @@ export class ModSearchRequest {
export class ModDownloadRequest { export class ModDownloadRequest {
declare ["constructor"]: typeof ModDownloadRequest; declare ["constructor"]: typeof ModDownloadRequest;
static type = "request" as const; static type = "request" as const;
static src = "control" as const; static src = ["host", "control"] as const;
static dst = "controller" as const; static dst = "controller" as const;
static permission = "core.mod.download" as const; static permission = "core.mod.download" as const;
constructor( constructor(
public name: string, public name: string,
public version: string, public version: string,
public sha1?: string,
) { } ) { }
static jsonSchema = Type.Object({ static jsonSchema = Type.Object({
"name": Type.String(), "name": Type.String(),
"version": Type.String(), "version": Type.String(),
"sha1": Type.Optional(Type.String()),
}); });
static fromJSON(json: Static<typeof this.jsonSchema>) { static fromJSON(json: Static<typeof this.jsonSchema>) {
return new this(json.name, json.version); return new this(json.name, json.version, json.sha1);
} }
static Response = JsonString; static Response = JsonString;
+2 -2
View File
@@ -132,7 +132,7 @@ export default function ModsPage() {
let [modList] = useModList(); let [modList] = useModList();
let [modPackList] = useModPackList(); let [modPackList] = useModPackList();
function actions(mod: lib.ModInfo|lib.ModRecord) { function actions(mod: lib.ModInfo) {
return <Space> return <Space>
{account.hasPermission("core.mod.download") {account.hasPermission("core.mod.download")
&& <Typography.Link && <Typography.Link
@@ -151,7 +151,7 @@ export default function ModsPage() {
} }
{account.hasPermission("core.mod.delete") {account.hasPermission("core.mod.delete")
&& <Popconfirm && <Popconfirm
title={`Delete ${mod.name}_${mod.version}.zip?`} title={`Delete ${mod.filename}?`}
onConfirm={event => { onConfirm={event => {
control.send( control.send(
new lib.ModDeleteRequest(mod.name, mod.version) new lib.ModDeleteRequest(mod.name, mod.version)