diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 5db6fcba..20651843 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -397,8 +397,7 @@ export default class ControlConnection extends BaseConnection { if (this._controller.modPacks.has(modPack.id)) { throw new lib.RequestError(`Mod pack with ID ${modPack.id} already exist`); } - this._controller.modPacks.set(modPack.id, modPack); - this._controller.modPacksUpdated([modPack]); + this._controller.modPacks.set(modPack); } async handleModPackUpdateRequest(request: lib.ModPackUpdateRequest) { @@ -406,8 +405,7 @@ export default class ControlConnection extends BaseConnection { if (modPack.id === undefined || !this._controller.modPacks.has(modPack.id)) { throw new lib.RequestError(`Mod pack with ID ${modPack.id} does not exist`); } - this._controller.modPacks.set(modPack.id, modPack); - this._controller.modPacksUpdated([modPack]); + this._controller.modPacks.set(modPack); } async handleModPackDeleteRequest(request: lib.ModPackDeleteRequest) { @@ -416,9 +414,7 @@ export default class ControlConnection extends BaseConnection { if (!modPack) { throw new lib.RequestError(`Mod pack with ID ${id} does not exist`); } - modPack.isDeleted = true; - this._controller.modPacks.delete(id); - this._controller.modPacksUpdated([modPack]); + this._controller.modPacks.delete(modPack); } async handleModGetRequest(request: lib.ModGetRequest) { diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index d4950775..2badf027 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -61,10 +61,6 @@ export default class Controller { hostsDirty = false; /** True if {@link Controller.instances} has changed since last time it was saved */ instancesDirty = false; - /** True if {@link Controller.saves} has changed since last time it was saved */ - savesDirty = false; - /** True if {@link Controller.modPacks} has changed since last time it was saved */ - modPacksDirty = false; httpServer: http.Server | null = null; httpServerCloser: HttpCloser | null = null; @@ -114,9 +110,12 @@ export default class Controller { const systems = await Controller.loadSystems(path.join(databaseDirectory, "systems.json")); const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json")); const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json")); - const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json")); - const modPacks = new lib.Datastore(...await new lib.JsonIdDatastoreProvider( + const saves = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "saves.json"), lib.SaveDetails + ).bootstrap()); + + const modPacks = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( path.join(databaseDirectory, "mod-packs.json"), lib.ModPack ).bootstrap()); @@ -155,9 +154,9 @@ export default class Controller { /** Mapping of instance id to instance info */ public instances = new Map(), /** Mapping of save id to save details */ - public saves = new Map(), + public saves = new lib.SubscribableDatastore(), /** Mapping of mod pack id to mod pack */ - public modPacks = new lib.Datastore(), + public modPacks = new lib.SubscribableDatastore(), /** Mods stored on the controller */ public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()), /** User and roles manager for the cluster */ @@ -183,10 +182,15 @@ export default class Controller { this.subscriptions.handle(lib.SystemInfoUpdateEvent, this.handleSystemInfoSubscription.bind(this)); this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this)); this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this)); + this.subscriptions.handle( lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsSubscription.bind(this) ); + this.saves.on("update", this.savesUpdated.bind(this)); + this.subscriptions.handle(lib.ModPackUpdatesEvent, this.handleModPackSubscription.bind(this)); + this.modPacks.on("update", this.modPacksUpdated.bind(this)); + this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this)); this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this)); } @@ -565,11 +569,7 @@ export default class Controller { await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances); } - if (this.savesDirty) { - this.savesDirty = false; - await Controller.saveSaves(path.join(databaseDirectory, "saves.json"), this.saves); - } - + await this.saves.save(); await this.modPacks.save(); if (this.userManager.dirty) { @@ -667,23 +667,6 @@ export default class Controller { await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t")); } - static async loadSaves(filePath: string): Promise> { - let json: Static[]; - try { - json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); - } - return new Map(json.map(s => lib.SaveDetails.fromJSON(s)).map(s => [s.id, s])); - } - - static async saveSaves(filePath: string, saves: Map) { - await lib.safeOutputFile(filePath, JSON.stringify([...saves.values()], null, "\t")); - } - static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise { let manifest = {}; try { @@ -894,19 +877,7 @@ export default class Controller { * @internal */ clearSavesOfInstance(instanceId: number) { - const updates = []; - const now = Date.now(); - for (const [id, save] of this.saves) { - if (save.instanceId === instanceId) { - save.isDeleted = true; - save.updatedAtMs = now; - updates.push(save); - this.saves.delete(id); - } - } - if (updates.length) { - this.savesUpdated(updates); - } + this.saves.deleteMany([...this.saves.values()].filter(save => save.instanceId === instanceId)); } /** @@ -1038,7 +1009,6 @@ export default class Controller { } savesUpdated(saves: lib.SaveDetails[]) { - this.savesDirty = true; this.subscriptions.broadcast(new lib.InstanceSaveDetailsUpdatesEvent(saves)); } @@ -1049,11 +1019,6 @@ export default class Controller { modPacksUpdated(modPacks: lib.ModPack[]) { - const now = Date.now(); - for (const modPack of modPacks) { - modPack.updatedAtMs = now; - } - this.modPacksDirty = true; this.subscriptions.broadcast(new lib.ModPackUpdatesEvent(modPacks)); lib.invokeHook(this.plugins, "onModPacksUpdated", modPacks); } diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index ab566a40..7a82b233 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -279,32 +279,30 @@ export default class HostConnection extends BaseConnection { } async handleInstanceSaveDetailsUpdatesEvent(event: lib.InstanceSaveDetailsUpdatesEvent) { - const now = Date.now(); const updates: lib.SaveDetails[] = []; + const deletes: lib.SaveDetails[] = []; for (const save of event.updates) { const existingSave = this._controller.saves.get(save.id); if (existingSave && save.equals(existingSave)) { continue; } - save.updatedAtMs = now; - this._controller.saves.set(save.id, save); updates.push(save); } if (event.instanceId !== undefined) { const updatedSaves = new Set(event.updates.map(s => s.id)); - for (const [id, save] of this._controller.saves) { - if (save.instanceId === event.instanceId && !updatedSaves.has(id)) { - save.isDeleted = true; - save.updatedAtMs = now; - updates.push(save); - this._controller.saves.delete(id); + for (const save of this._controller.saves.values()) { + if (save.instanceId === event.instanceId && !updatedSaves.has(save.id)) { + deletes.push(save); } } } // Hosts eagerly send updates, which means we may get an update // where nothing actualy changed. if (updates.length) { - this._controller.savesUpdated(updates); + this._controller.saves.setMany(updates); + } + if (deletes.length) { + this._controller.saves.deleteMany(deletes); } } diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index e4336bbb..a3a54253 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -261,8 +261,7 @@ async function uploadExport(req: Request, res: Response) { modPack.exportManifest = new lib.ExportManifest(assets); modPack.fillDefaultSettings(settingPrototypes, logger); - res.app.locals.controller.modPacks.set(modPack.id, modPack); - res.app.locals.controller.modPacksUpdated([modPack]); + res.app.locals.controller.modPacks.set(modPack); res.sendStatus(200); } diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index af82a446..9f3aa3fb 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -3,6 +3,7 @@ import fs from "fs-extra"; import { safeOutputFile } from "./file_ops"; import { SubscribableValue } from "./subscriptions"; import { JSONDeserialisable } from "./data/composites"; +import { EventEmitter } from "stream"; type DatastoreKey = string | number; type DatastoreValue = string | number | boolean | object; @@ -119,7 +120,10 @@ export class JsonIdDatastoreProvider< const serialized = this.migrations(rawJson); // Convert to data class objects - return new Map(serialized.map((e) => [e.id, this.dataClass.fromJSON(e)])); + return new Map(serialized.map((e) => { + const v = this.dataClass.fromJSON(e); + return [v.id, v]; + })); } } @@ -127,13 +131,15 @@ export class JsonIdDatastoreProvider< export abstract class BaseDatastore< K extends DatastoreKey, V extends DatastoreValue, -> { +> extends EventEmitter { protected dirty = false; constructor( protected provider: DatastoreProvider = new MemoryDatastoreProvider(), protected data = new Map(), - ) {} + ) { + super(); + } // Save the datastore to the provider async save() { @@ -161,8 +167,9 @@ export abstract class BaseDatastore< // Get a copy of a value from the datastore getCopy(key: K) { - const v = this.data.get(key); - return v === undefined ? v : deepCopy(v); + // const v = this.data.get(key); + // return v === undefined ? v : deepCopy(v); + return this.data.get(key); } // Returns all values in the datastore @@ -180,19 +187,34 @@ export class Datastore< set(key: K, value: V) { this.data.set(key, value); this.dirty = true; + this.emit("update", [key, value]); } // Set many values at once from an array of key value pairs - setMany(pairs: [[K, V]]) { + setMany(pairs: [K, V][]) { this.dirty ||= pairs.length > 0; for (const [k, v] of pairs) { this.data.set(k, v); } + this.emit("update", pairs); } // Delete a value in the datastore delete(key: K) { + const value = this.data.get(key); this.data.delete(key); + this.dirty = true; + this.emit("update", [key, value, true]); + } + + // Delete many values at once from an array of keys + deleteMany(keys: K[]) { + this.dirty ||= keys.length > 0; + for (const key of keys) { + const value = this.data.get(key); + this.data.delete(key); + } + this.emit("update", keys); } } @@ -206,6 +228,7 @@ export class SubscribableDatastore< value.updatedAtMs = Date.now(); this.data.set(value.id, value); this.dirty = true; + this.emit("update", [value]); } // Set many values at once from an array of key value pairs @@ -216,10 +239,25 @@ export class SubscribableDatastore< value.updatedAtMs = nowMs; this.data.set(value.id, value); } + this.emit("update", values); } // Delete a value in the datastore delete(value: V) { this.data.delete(value.id); + value.isDeleted = true; + value.updatedAtMs = Date.now(); + this.emit("update", [value]); + } + + // Delete many values at once from an array of values + deleteMany(values: V[]) { + this.dirty ||= values.length > 0; + for (const value of values) { + value.isDeleted = true; + value.updatedAtMs = Date.now(); + this.data.delete(value.id); + } + this.emit("update", values); } }