From 80b4a96e856cc75d8de341c5c25516fee68f5006 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sun, 19 May 2024 01:24:30 +0100 Subject: [PATCH 01/11] Add datastore to lib --- packages/lib/index.ts | 1 + packages/lib/src/datastore.ts | 215 ++++++++++++++++++++++++++++++++++ 2 files changed, 216 insertions(+) create mode 100644 packages/lib/src/datastore.ts diff --git a/packages/lib/index.ts b/packages/lib/index.ts index 3117bcbc..bec088ad 100644 --- a/packages/lib/index.ts +++ b/packages/lib/index.ts @@ -28,6 +28,7 @@ export * from "./src/stream"; export * from "./src/system_collectors"; export * from "./src/zip_ops"; export * from "./src/subscriptions"; +export * from "./src/datastore"; export { default as ExponentialBackoff } from "./src/ExponentialBackoff"; export { default as ModStore } from "./src/ModStore"; diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts new file mode 100644 index 00000000..fe17b37e --- /dev/null +++ b/packages/lib/src/datastore.ts @@ -0,0 +1,215 @@ +import fs from "fs-extra"; + +import { safeOutputFile } from "./file_ops"; +import { SubscribableValue } from "./subscriptions"; +import { JSONDeserialisable } from "./data/composites"; + +type DatastoreKey = string | number; +type DatastoreValue = string | number | boolean | object; + +// Deep copy required to support the read only property of datastore values +function deepCopy(value: V): V { + if (value instanceof Object) { + return Object.fromEntries( + Object.entries(value).map(([k, v]) => [k, deepCopy(v)]) + ) as V; + } + + return value; +} + +// Abstract class which can provide saving and loading capabilities to a datastore +export abstract class DatastoreProvider< + K extends DatastoreKey, + V extends DatastoreValue, +> { + abstract save(data: Map): Promise + abstract load(): Promise> + + // Load data from the source and return it, this should be passed to the constructor of a datastore + async bootstrap() { + return [this, await this.load()] as const; + } +} + +// Store all data in memory, used as a default and should be avoided +export class MemoryDatastoreProvider< + K extends DatastoreKey, + V extends DatastoreValue, +> extends DatastoreProvider { + async save(data: Map) { + } + + async load() { + return new Map(); + } +} + +// Store all data within a json object file and validate the result against a schema +export class JsonDatastoreProvider< + K extends DatastoreKey, + V extends DatastoreValue, +> extends DatastoreProvider { + constructor( + private filePath: string, + private dataClass: JSONDeserialisable, + private migrations: (rawJson: unknown) => Array<[K, unknown]> = v => v as any, + ) { + super(); + } + + // Save the data to the json file + async save(data: Map) { + await safeOutputFile(this.filePath, JSON.stringify(Object.fromEntries(data), null, "\t")); + } + + // Load the data from the json file + async load() { + // Read the raw json from the file + let rawJson; + try { + rawJson = JSON.parse(await fs.readFile(this.filePath, { encoding: "utf8" })); + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + return new Map(); + } + + // Apply migrations + const serialized = this.migrations(rawJson); + + // Convert to data class objects + return new Map(serialized.map((k, v) => [k, this.dataClass.fromJSON(v)])); + } +} + +// Store all data within a json array file where each element has a unique id +export class JsonIdDatastoreProvider< + K extends DatastoreKey, + V extends DatastoreValue & { id: K }, +> extends DatastoreProvider { + constructor( + private filePath: string, + private dataClass: JSONDeserialisable, + private migrations: (rawJson: unknown) => Array<{ id: K }> = v => v as any, + ) { + super(); + } + + // Save the data to the json file + async save(data: Map) { + await safeOutputFile(this.filePath, JSON.stringify([...data.values()], null, "\t")); + } + + // Load the data from the json file + async load() { + // Read the raw json from the file + let rawJson; + try { + rawJson = JSON.parse(await fs.readFile(this.filePath, { encoding: "utf8" })); + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + return new Map(); + } + + // Apply migrations + const serialized = this.migrations(rawJson); + + // Convert to data class objects + return new Map(serialized.map((e) => [e.id, this.dataClass.fromJSON(e)])); + } +} + +// Implements a similar interface to a map +export abstract class BaseDatastore< + K extends DatastoreKey, + V extends DatastoreValue, +> { + protected dirty = false; + + constructor( + protected provider: DatastoreProvider = new MemoryDatastoreProvider(), + protected data = new Map(), + ) {} + + // Save the datastore to the provider + async save() { + if (this.dirty) { + await this.provider.save(this.data); + this.dirty = false; + } + } + + // Load the datastore from the provider + async load() { + this.data = await this.provider.load(); + this.dirty = false; + } + + // Returns true if the datastore has the value, false otherwise + has(key: K) { + return this.data.has(key); + } + + // Get a value from from the datastore + get(key: K) { + return this.data.get(key) as Readonly ?? undefined; + } + + // Get a copy of a value from the datastore + getCopy(key: K) { + const v = this.data.get(key); + return v === undefined ? v : deepCopy(v); + } + + // Returns all values in the datastore + values() { + return this.data.values() as IterableIterator>; + } +} + +// General key value mapping which can be saved and loaded with the file system +export class Datastore< + K extends DatastoreKey, + V extends DatastoreValue, +> extends BaseDatastore { + // Set the value in the datastore, be careful of race conditions if you await any functions before calling set + set(key: K, value: V) { + this.data.set(key, value); + this.dirty = true; + } + + // Set many values at once from an array of key value pairs + setMany(pairs: [[K, V]]) { + this.dirty ||= pairs.length > 0; + for (const [k, v] of pairs) { + this.data.set(k, v); + } + } +} + +// A special implementation of the datastore to work directly with subscribable values +// TODO broadcast value changes to subscribers +export class SubscribableDatastore< + V extends SubscribableValue, +> extends BaseDatastore { + // Set the value in the datastore, be careful of race conditions if you await any functions before calling set + set(value: V) { + value.updatedAtMs = Date.now(); + this.data.set(value.id, value); + this.dirty = true; + } + + // Set many values at once from an array of key value pairs + setMany(values: V[]) { + const nowMs = Date.now(); + this.dirty ||= values.length > 0; + for (const value of values) { + value.updatedAtMs = nowMs; + this.data.set(value.id, value); + } + } +} From 6b11fb810ad50cc9499bd2d24516e56fb9a0340d Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sun, 19 May 2024 01:41:28 +0100 Subject: [PATCH 02/11] Change mod packs to datastore --- packages/controller/src/ControlConnection.ts | 2 +- packages/controller/src/Controller.ts | 31 +++++--------------- packages/controller/src/routes.ts | 3 +- packages/lib/src/datastore.ts | 10 +++++++ 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 595f900b..5db6fcba 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -412,7 +412,7 @@ export default class ControlConnection extends BaseConnection { async handleModPackDeleteRequest(request: lib.ModPackDeleteRequest) { let { id } = request; - let modPack = this._controller.modPacks.get(id); + let modPack = this._controller.modPacks.getCopy(id); if (!modPack) { throw new lib.RequestError(`Mod pack with ID ${id} does not exist`); } diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index f256da32..d4950775 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -115,7 +115,11 @@ export default class Controller { const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json")); const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json")); const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json")); - const modPacks = await Controller.loadModPacks(path.join(databaseDirectory, "mod-packs.json")); + + const modPacks = new lib.Datastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "mod-packs.json"), lib.ModPack + ).bootstrap()); + const userManager = new UserManager(config); await userManager.load(path.join(databaseDirectory, "users.json")); @@ -153,7 +157,7 @@ export default class Controller { /** Mapping of save id to save details */ public saves = new Map(), /** Mapping of mod pack id to mod pack */ - public modPacks = new Map(), + public modPacks = new lib.Datastore(), /** Mods stored on the controller */ public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()), /** User and roles manager for the cluster */ @@ -566,10 +570,7 @@ export default class Controller { await Controller.saveSaves(path.join(databaseDirectory, "saves.json"), this.saves); } - if (this.modPacksDirty) { - this.modPacksDirty = false; - await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks); - } + await this.modPacks.save(); if (this.userManager.dirty) { await this.userManager.save(path.join(databaseDirectory, "users.json")); @@ -683,24 +684,6 @@ export default class Controller { await lib.safeOutputFile(filePath, JSON.stringify([...saves.values()], null, "\t")); } - - static async loadModPacks(filePath: string): Promise> { - let json; - try { - json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); - } - return new Map(json.map((e: any) => [e.id, lib.ModPack.fromJSON(e)])); - } - - static async saveModPacks(filePath: string, modPacks: Map) { - await lib.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, "\t")); - } - static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise { let manifest = {}; try { diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 51b25bee..e4336bbb 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -218,7 +218,7 @@ async function uploadExport(req: Request, res: Response) { res.sendStatus(400); return; } - let modPack = res.app.locals.controller.modPacks.get(modPackId); + let modPack = res.app.locals.controller.modPacks.getCopy(modPackId); if (!modPack) { res.sendStatus(400); return; @@ -261,6 +261,7 @@ async function uploadExport(req: Request, res: Response) { modPack.exportManifest = new lib.ExportManifest(assets); modPack.fillDefaultSettings(settingPrototypes, logger); + res.app.locals.controller.modPacks.set(modPack.id, modPack); res.app.locals.controller.modPacksUpdated([modPack]); res.sendStatus(200); diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index fe17b37e..af82a446 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -189,6 +189,11 @@ export class Datastore< this.data.set(k, v); } } + + // Delete a value in the datastore + delete(key: K) { + this.data.delete(key); + } } // A special implementation of the datastore to work directly with subscribable values @@ -212,4 +217,9 @@ export class SubscribableDatastore< this.data.set(value.id, value); } } + + // Delete a value in the datastore + delete(value: V) { + this.data.delete(value.id); + } } From 509fab358a369edb05a783e3b5fe8c3b1798e79a Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sun, 19 May 2024 02:28:36 +0100 Subject: [PATCH 03/11] Add saves datastore --- packages/controller/src/ControlConnection.ts | 10 +--- packages/controller/src/Controller.ts | 63 +++++--------------- packages/controller/src/HostConnection.ts | 18 +++--- packages/controller/src/routes.ts | 3 +- packages/lib/src/datastore.ts | 50 ++++++++++++++-- 5 files changed, 70 insertions(+), 74 deletions(-) diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 5db6fcba..20651843 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -397,8 +397,7 @@ export default class ControlConnection extends BaseConnection { if (this._controller.modPacks.has(modPack.id)) { throw new lib.RequestError(`Mod pack with ID ${modPack.id} already exist`); } - this._controller.modPacks.set(modPack.id, modPack); - this._controller.modPacksUpdated([modPack]); + this._controller.modPacks.set(modPack); } async handleModPackUpdateRequest(request: lib.ModPackUpdateRequest) { @@ -406,8 +405,7 @@ export default class ControlConnection extends BaseConnection { if (modPack.id === undefined || !this._controller.modPacks.has(modPack.id)) { throw new lib.RequestError(`Mod pack with ID ${modPack.id} does not exist`); } - this._controller.modPacks.set(modPack.id, modPack); - this._controller.modPacksUpdated([modPack]); + this._controller.modPacks.set(modPack); } async handleModPackDeleteRequest(request: lib.ModPackDeleteRequest) { @@ -416,9 +414,7 @@ export default class ControlConnection extends BaseConnection { if (!modPack) { throw new lib.RequestError(`Mod pack with ID ${id} does not exist`); } - modPack.isDeleted = true; - this._controller.modPacks.delete(id); - this._controller.modPacksUpdated([modPack]); + this._controller.modPacks.delete(modPack); } async handleModGetRequest(request: lib.ModGetRequest) { diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index d4950775..2badf027 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -61,10 +61,6 @@ export default class Controller { hostsDirty = false; /** True if {@link Controller.instances} has changed since last time it was saved */ instancesDirty = false; - /** True if {@link Controller.saves} has changed since last time it was saved */ - savesDirty = false; - /** True if {@link Controller.modPacks} has changed since last time it was saved */ - modPacksDirty = false; httpServer: http.Server | null = null; httpServerCloser: HttpCloser | null = null; @@ -114,9 +110,12 @@ export default class Controller { const systems = await Controller.loadSystems(path.join(databaseDirectory, "systems.json")); const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json")); const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json")); - const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json")); - const modPacks = new lib.Datastore(...await new lib.JsonIdDatastoreProvider( + const saves = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "saves.json"), lib.SaveDetails + ).bootstrap()); + + const modPacks = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( path.join(databaseDirectory, "mod-packs.json"), lib.ModPack ).bootstrap()); @@ -155,9 +154,9 @@ export default class Controller { /** Mapping of instance id to instance info */ public instances = new Map(), /** Mapping of save id to save details */ - public saves = new Map(), + public saves = new lib.SubscribableDatastore(), /** Mapping of mod pack id to mod pack */ - public modPacks = new lib.Datastore(), + public modPacks = new lib.SubscribableDatastore(), /** Mods stored on the controller */ public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()), /** User and roles manager for the cluster */ @@ -183,10 +182,15 @@ export default class Controller { this.subscriptions.handle(lib.SystemInfoUpdateEvent, this.handleSystemInfoSubscription.bind(this)); this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this)); this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this)); + this.subscriptions.handle( lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsSubscription.bind(this) ); + this.saves.on("update", this.savesUpdated.bind(this)); + this.subscriptions.handle(lib.ModPackUpdatesEvent, this.handleModPackSubscription.bind(this)); + this.modPacks.on("update", this.modPacksUpdated.bind(this)); + this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this)); this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this)); } @@ -565,11 +569,7 @@ export default class Controller { await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances); } - if (this.savesDirty) { - this.savesDirty = false; - await Controller.saveSaves(path.join(databaseDirectory, "saves.json"), this.saves); - } - + await this.saves.save(); await this.modPacks.save(); if (this.userManager.dirty) { @@ -667,23 +667,6 @@ export default class Controller { await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t")); } - static async loadSaves(filePath: string): Promise> { - let json: Static[]; - try { - json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); - } - return new Map(json.map(s => lib.SaveDetails.fromJSON(s)).map(s => [s.id, s])); - } - - static async saveSaves(filePath: string, saves: Map) { - await lib.safeOutputFile(filePath, JSON.stringify([...saves.values()], null, "\t")); - } - static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise { let manifest = {}; try { @@ -894,19 +877,7 @@ export default class Controller { * @internal */ clearSavesOfInstance(instanceId: number) { - const updates = []; - const now = Date.now(); - for (const [id, save] of this.saves) { - if (save.instanceId === instanceId) { - save.isDeleted = true; - save.updatedAtMs = now; - updates.push(save); - this.saves.delete(id); - } - } - if (updates.length) { - this.savesUpdated(updates); - } + this.saves.deleteMany([...this.saves.values()].filter(save => save.instanceId === instanceId)); } /** @@ -1038,7 +1009,6 @@ export default class Controller { } savesUpdated(saves: lib.SaveDetails[]) { - this.savesDirty = true; this.subscriptions.broadcast(new lib.InstanceSaveDetailsUpdatesEvent(saves)); } @@ -1049,11 +1019,6 @@ export default class Controller { modPacksUpdated(modPacks: lib.ModPack[]) { - const now = Date.now(); - for (const modPack of modPacks) { - modPack.updatedAtMs = now; - } - this.modPacksDirty = true; this.subscriptions.broadcast(new lib.ModPackUpdatesEvent(modPacks)); lib.invokeHook(this.plugins, "onModPacksUpdated", modPacks); } diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index ab566a40..7a82b233 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -279,32 +279,30 @@ export default class HostConnection extends BaseConnection { } async handleInstanceSaveDetailsUpdatesEvent(event: lib.InstanceSaveDetailsUpdatesEvent) { - const now = Date.now(); const updates: lib.SaveDetails[] = []; + const deletes: lib.SaveDetails[] = []; for (const save of event.updates) { const existingSave = this._controller.saves.get(save.id); if (existingSave && save.equals(existingSave)) { continue; } - save.updatedAtMs = now; - this._controller.saves.set(save.id, save); updates.push(save); } if (event.instanceId !== undefined) { const updatedSaves = new Set(event.updates.map(s => s.id)); - for (const [id, save] of this._controller.saves) { - if (save.instanceId === event.instanceId && !updatedSaves.has(id)) { - save.isDeleted = true; - save.updatedAtMs = now; - updates.push(save); - this._controller.saves.delete(id); + for (const save of this._controller.saves.values()) { + if (save.instanceId === event.instanceId && !updatedSaves.has(save.id)) { + deletes.push(save); } } } // Hosts eagerly send updates, which means we may get an update // where nothing actualy changed. if (updates.length) { - this._controller.savesUpdated(updates); + this._controller.saves.setMany(updates); + } + if (deletes.length) { + this._controller.saves.deleteMany(deletes); } } diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index e4336bbb..a3a54253 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -261,8 +261,7 @@ async function uploadExport(req: Request, res: Response) { modPack.exportManifest = new lib.ExportManifest(assets); modPack.fillDefaultSettings(settingPrototypes, logger); - res.app.locals.controller.modPacks.set(modPack.id, modPack); - res.app.locals.controller.modPacksUpdated([modPack]); + res.app.locals.controller.modPacks.set(modPack); res.sendStatus(200); } diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index af82a446..9f3aa3fb 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -3,6 +3,7 @@ import fs from "fs-extra"; import { safeOutputFile } from "./file_ops"; import { SubscribableValue } from "./subscriptions"; import { JSONDeserialisable } from "./data/composites"; +import { EventEmitter } from "stream"; type DatastoreKey = string | number; type DatastoreValue = string | number | boolean | object; @@ -119,7 +120,10 @@ export class JsonIdDatastoreProvider< const serialized = this.migrations(rawJson); // Convert to data class objects - return new Map(serialized.map((e) => [e.id, this.dataClass.fromJSON(e)])); + return new Map(serialized.map((e) => { + const v = this.dataClass.fromJSON(e); + return [v.id, v]; + })); } } @@ -127,13 +131,15 @@ export class JsonIdDatastoreProvider< export abstract class BaseDatastore< K extends DatastoreKey, V extends DatastoreValue, -> { +> extends EventEmitter { protected dirty = false; constructor( protected provider: DatastoreProvider = new MemoryDatastoreProvider(), protected data = new Map(), - ) {} + ) { + super(); + } // Save the datastore to the provider async save() { @@ -161,8 +167,9 @@ export abstract class BaseDatastore< // Get a copy of a value from the datastore getCopy(key: K) { - const v = this.data.get(key); - return v === undefined ? v : deepCopy(v); + // const v = this.data.get(key); + // return v === undefined ? v : deepCopy(v); + return this.data.get(key); } // Returns all values in the datastore @@ -180,19 +187,34 @@ export class Datastore< set(key: K, value: V) { this.data.set(key, value); this.dirty = true; + this.emit("update", [key, value]); } // Set many values at once from an array of key value pairs - setMany(pairs: [[K, V]]) { + setMany(pairs: [K, V][]) { this.dirty ||= pairs.length > 0; for (const [k, v] of pairs) { this.data.set(k, v); } + this.emit("update", pairs); } // Delete a value in the datastore delete(key: K) { + const value = this.data.get(key); this.data.delete(key); + this.dirty = true; + this.emit("update", [key, value, true]); + } + + // Delete many values at once from an array of keys + deleteMany(keys: K[]) { + this.dirty ||= keys.length > 0; + for (const key of keys) { + const value = this.data.get(key); + this.data.delete(key); + } + this.emit("update", keys); } } @@ -206,6 +228,7 @@ export class SubscribableDatastore< value.updatedAtMs = Date.now(); this.data.set(value.id, value); this.dirty = true; + this.emit("update", [value]); } // Set many values at once from an array of key value pairs @@ -216,10 +239,25 @@ export class SubscribableDatastore< value.updatedAtMs = nowMs; this.data.set(value.id, value); } + this.emit("update", values); } // Delete a value in the datastore delete(value: V) { this.data.delete(value.id); + value.isDeleted = true; + value.updatedAtMs = Date.now(); + this.emit("update", [value]); + } + + // Delete many values at once from an array of values + deleteMany(values: V[]) { + this.dirty ||= values.length > 0; + for (const value of values) { + value.isDeleted = true; + value.updatedAtMs = Date.now(); + this.data.delete(value.id); + } + this.emit("update", values); } } From 609db3102787e97c5ef9336e2985092974618c4a Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Wed, 22 May 2024 16:39:22 +0100 Subject: [PATCH 04/11] Add hosts, instances, systems --- packages/controller/controller.ts | 8 +- packages/controller/src/ControlConnection.ts | 6 +- packages/controller/src/Controller.ts | 194 ++++++------------- packages/controller/src/HostConnection.ts | 31 ++- packages/controller/src/InstanceInfo.ts | 7 + packages/controller/src/routes.ts | 2 +- packages/lib/src/datastore.ts | 82 ++++---- plugins/player_auth/test/plugin.js | 2 + test/integration/routing.js | 2 +- test/mock.js | 4 +- 10 files changed, 135 insertions(+), 203 deletions(-) diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index b4161f65..29692954 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -66,9 +66,9 @@ void new lib.Gauge( if (!controller || !controller.hosts) { return; } - for (let [id, host] of controller.hosts) { + for (const host of controller.hosts.values()) { gauge.labels({ - host_id: String(id), + host_id: String(host.id), host_name: host.name, }).set(1); }; @@ -86,9 +86,9 @@ void new lib.Gauge( if (!controller || !controller.instances) { return; } - for (let [id, instance] of controller.instances) { + for (const instance of controller.instances.values()) { gauge.labels({ - instance_id: String(id), + instance_id: String(instance.id), instance_name: String(instance.config.get("instance.name")), host_id: String(instance.config.get("instance.assigned_host")), }).set(1); diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 20651843..ba59deaf 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -195,7 +195,7 @@ export default class ControlConnection extends BaseConnection { } async handleHostRevokeTokensRequest(request: lib.HostRevokeTokensRequest) { - const host = this._controller.hosts.get(request.hostId); + const host = this._controller.hosts.getMutable(request.hostId); if (!host) { throw new Error(`Unknown host id (${request.hostId})`); } @@ -207,7 +207,7 @@ export default class ControlConnection extends BaseConnection { hostConnection.connector.terminate(); } - this._controller.hostsUpdated([host]); + this._controller.hosts.set(host); } async handleHostListRequest(): Promise { @@ -410,7 +410,7 @@ export default class ControlConnection extends BaseConnection { async handleModPackDeleteRequest(request: lib.ModPackDeleteRequest) { let { id } = request; - let modPack = this._controller.modPacks.getCopy(id); + let modPack = this._controller.modPacks.getMutable(id); if (!modPack) { throw new lib.RequestError(`Mod pack with ID ${id} does not exist`); } diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 2badf027..4c767d9c 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -55,13 +55,6 @@ export default class Controller { config: lib.ControllerConfig; app: Application; - /** True if (@link Controller.systems} has changed since last time it was saved */ - systemsDirty = false; - /** True if {@link Controller.hosts} has changed since last time it was saved */ - hostsDirty = false; - /** True if {@link Controller.instances} has changed since last time it was saved */ - instancesDirty = false; - httpServer: http.Server | null = null; httpServerCloser: HttpCloser | null = null; httpsServer: https.Server | null = null; @@ -107,16 +100,31 @@ export default class Controller { let databaseDirectory = config.get("controller.database_directory"); await fs.ensureDir(databaseDirectory); - const systems = await Controller.loadSystems(path.join(databaseDirectory, "systems.json")); - const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json")); - const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json")); + const systems = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "systems.json"), + lib.SystemInfo.fromJSON + ).bootstrap()); + + const hosts = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "hosts.json"), + HostInfo.fromJSON, + this.migrateHosts, this.finaliseHosts, + ).bootstrap()); + + const instances = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "instances.json"), + json => InstanceInfo.fromJSON(json, "controller"), + this.migrateInstances, this.finaliseInstances, + ).bootstrap()); const saves = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( - path.join(databaseDirectory, "saves.json"), lib.SaveDetails + path.join(databaseDirectory, "saves.json"), + lib.SaveDetails.fromJSON ).bootstrap()); const modPacks = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( - path.join(databaseDirectory, "mod-packs.json"), lib.ModPack + path.join(databaseDirectory, "mod-packs.json"), + lib.ModPack.fromJSON ).bootstrap()); const userManager = new UserManager(config); @@ -148,11 +156,11 @@ export default class Controller { * will restart the controller on non-zero exit codes. */ public canRestart: boolean = false, - public systems = new Map(), + public systems = new lib.SubscribableDatastore(), /** Mapping of host id to host info */ - public hosts = new Map(), + public hosts = new lib.SubscribableDatastore(), /** Mapping of instance id to instance info */ - public instances = new Map(), + public instances = new lib.SubscribableDatastore(), /** Mapping of save id to save details */ public saves = new lib.SubscribableDatastore(), /** Mapping of mod pack id to mod pack */ @@ -182,17 +190,18 @@ export default class Controller { this.subscriptions.handle(lib.SystemInfoUpdateEvent, this.handleSystemInfoSubscription.bind(this)); this.subscriptions.handle(lib.HostUpdatesEvent, this.handleHostSubscription.bind(this)); this.subscriptions.handle(lib.InstanceDetailsUpdatesEvent, this.handleInstanceDetailsSubscription.bind(this)); - this.subscriptions.handle( lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsSubscription.bind(this) ); - this.saves.on("update", this.savesUpdated.bind(this)); - this.subscriptions.handle(lib.ModPackUpdatesEvent, this.handleModPackSubscription.bind(this)); - this.modPacks.on("update", this.modPacksUpdated.bind(this)); - this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this)); this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this)); + + // Handle updates for datastores + this.hosts.on("update", this.hostsUpdated.bind(this)); + this.instances.on("update", this.instanceDetailsUpdated.bind(this)); + this.saves.on("update", this.savesUpdated.bind(this)); + this.modPacks.on("update", this.modPacksUpdated.bind(this)); } async start(args: ControllerArgs) { @@ -509,9 +518,8 @@ export default class Controller { requests.push(lib.gatherSystemInfo("controller", this.canRestart)); const newMetrics = await Promise.all(requests); for (const metric of newMetrics) { - this.systems.set(metric.id, metric); + this.systems.set(metric); } - this.systemsDirty = true; this.subscriptions.broadcast(new lib.SystemInfoUpdateEvent(newMetrics)); } catch (err: any) { logger.error(`Unexpected error updating system infos:\n${err.stack ?? err.message}`); @@ -554,21 +562,10 @@ export default class Controller { } let databaseDirectory = this.config.get("controller.database_directory"); - if (this.systemsDirty) { - this.systemsDirty = false; - await Controller.saveSystems(path.join(databaseDirectory, "systems.json"), this.systems); - } - - if (this.hostsDirty) { - this.hostsDirty = false; - await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts); - } - - if (this.instancesDirty) { - this.instancesDirty = false; - await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances); - } + await this.systems.save(); + await this.hosts.save(); + await this.instances.save(); await this.saves.save(); await this.modPacks.save(); @@ -579,92 +576,42 @@ export default class Controller { await lib.invokeHook(this.plugins, "onSaveData"); } - static async loadSystems(filePath: string): Promise> { - let json: Static[]; - try { - json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); - - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); - } - - return new Map(json.map(m => lib.SystemInfo.fromJSON(m)).map(m => [m.id, m])); - } - - static async saveSystems(filePath: string, systems: Map) { - await lib.safeOutputFile(filePath, JSON.stringify([...systems.values()], null, "\t")); - } - - static async loadHosts(filePath: string): Promise> { - let serialized: [number, Static][]; - try { - serialized = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); - - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); - } + static migrateHosts(rawJson: unknown[]): Static[] { + const serialized = rawJson as Static[]; // TODO: migrate remove after release. if (serialized.length && !(serialized[0] instanceof Array)) { - return new Map(); // Discard old format. + return []; // Discard old format. } - const hosts = new Map(); - for (let [id, json] of serialized) { - const host = HostInfo.fromJSON(json); - if (host.connected) { - host.connected = false; - host.updatedAtMs = Date.now(); - } - hosts.set(id, host); + return serialized; + } + + static finaliseHosts(host: HostInfo): HostInfo { + if (host.connected) { + host.connected = false; + host.updatedAtMs = Date.now(); } - return hosts; + return host; } - static async saveHosts(filePath: string, hosts: Map) { - await lib.safeOutputFile(filePath, JSON.stringify([...hosts.entries()], null, "\t")); + static migrateInstances(rawJson: unknown[]): Static[] { + const serialized = rawJson as Static[]; + return serialized.map(json => { + if (!json.config) { // migrate: from pre Alpha 14 format. + return { config: json as any, status: "running" }; // Use running to force updatedAtMs + } + return json; + }); } - static async loadInstances(filePath: string): Promise> { - logger.info(`Loading ${filePath}`); - - let instances = new Map(); - try { - let serialized = JSON.parse( - await fs.readFile(filePath, { encoding: "utf8" }) - ) as Static[]; - for (let json of serialized) { - if (!json.config) { // migrate: from pre Alpha 14 format. - json = { config: json as any, status: "running" }; // Use running to force updatedAtMs - } - const instance = InstanceInfo.fromJSON(json, "controller"); - const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown"; - if (instance.status !== status) { - instance.status = status; - instance.updatedAtMs = Date.now(); - } - instances.set(instance.id, instance); - } - - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); + static finaliseInstances(instance: InstanceInfo): InstanceInfo { + const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown"; + if (instance.status !== status) { + instance.status = status; + instance.updatedAtMs = Date.now(); } - - return instances; - } - - static async saveInstances(filePath: string, instances: Map) { - let serialized = [...instances.values()]; - await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t")); + return instance; } static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise { @@ -782,11 +729,6 @@ export default class Controller { } hostsUpdated(hosts: HostInfo[]) { - const now = Date.now(); - for (const host of hosts) { - host.updatedAtMs = now; - } - this.hostsDirty = true; let updates = hosts.map(host => host.toHostDetails()); this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates)); } @@ -863,11 +805,9 @@ export default class Controller { instanceConfig.set("factorio.settings", settings); let instance = new InstanceInfo(instanceConfig, "unassigned", undefined, Date.now()); - this.instances.set(instanceId, instance); - this.instancesDirty = true; + this.instances.set(instance); await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance); this.addInstanceHooks(instance); - this.instanceDetailsUpdated([instance]); return instance; } @@ -924,7 +864,6 @@ export default class Controller { // Assign to target instance.config.set("instance.assigned_host", hostId ?? null); - // "fieldChanged" event handler will set this.instancesDirty if (hostId !== undefined && newHostConnection) { await newHostConnection.send( new lib.InstanceAssignInternalRequest(instanceId, instance.config.toRemote("host")) @@ -932,8 +871,7 @@ export default class Controller { } else { instance.status = "unassigned"; } - instance.updatedAtMs = Date.now(); - this.instanceDetailsUpdated([instance]); + this.instances.set(instance); } /** @@ -950,16 +888,13 @@ export default class Controller { if (hostId !== null) { await this.sendTo({ hostId }, new lib.InstanceDeleteInternalRequest(instanceId)); } - this.instances.delete(instanceId); - this.instancesDirty = true; + + let prev = instance.status; + this.instances.delete(instance); this.clearSavesOfInstance(instanceId); this.userManager.clearStatsOfInstance(instanceId); - let prev = instance.status; - instance.status = "deleted"; - instance.updatedAtMs = Date.now(); - this.instanceDetailsUpdated([instance]); await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance, prev); } @@ -988,10 +923,9 @@ export default class Controller { instance.config.on("fieldChanged", (field: string, curr: any, prev: any) => { instance.updatedAtMs = Date.now(); if (field === "instance.name") { - this.instanceDetailsUpdated([instance]); + this.instances.set(instance); // Trigger updated logic } - this.instancesDirty = true; lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, field, curr, prev); }); } diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index 7a82b233..7d31d979 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -42,8 +42,7 @@ export default class HostConnection extends BaseConnection { 0, false, ); - this._controller.hosts.set(this.id, this.info); - this._controller.hostsUpdated([this.info]); + this._controller.hosts.set(this.info); this._checkPluginVersions(); @@ -57,10 +56,9 @@ export default class HostConnection extends BaseConnection { } this.connector.on("close", () => { - const now = Date.now(); // Update status to unknown for instances on this host. const instances: InstanceInfo[] = []; - for (let instance of this._controller.instances.values()) { + for (let instance of this._controller.instances.valuesMutable()) { if (instance.config.get("instance.assigned_host") !== this.id) { continue; } @@ -68,13 +66,12 @@ export default class HostConnection extends BaseConnection { instances.push(instance); let prev = instance.status; instance.status = "unknown"; - instance.updatedAtMs = now; lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev); } - this._controller.instanceDetailsUpdated(instances); + this._controller.instances.setMany(instances); this.info.connected = false; - this._controller.hostsUpdated([this.info]); + this._controller.hosts.set(this.info); }); this.handle(lib.HostInfoUpdateEvent, this.handleHostInfoUpdateEvent.bind(this)); @@ -158,11 +155,11 @@ export default class HostConnection extends BaseConnection { async handleHostInfoUpdateEvent(event: lib.HostInfoUpdateEvent) { this.info.name = event.update.name; this.info.publicAddress = event.update.publicAddress; - this._controller.hostsUpdated([this.info]); + this._controller.hosts.set(this.info); } async handleInstanceStatusChangedEvent(request: lib.InstanceStatusChangedEvent) { - let instance = this._controller.instances.get(request.instanceId); + let instance = this._controller.instances.getMutable(request.instanceId); // It's possible to get updates from an instance that does not exist // or is not assigned to the host it originated from if it was @@ -188,9 +185,8 @@ export default class HostConnection extends BaseConnection { let prev = instance.status; instance.status = request.status; instance.gamePort = request.gamePort; - instance.updatedAtMs = Date.now(); + this._controller.instances.set(instance); logger.verbose(`Instance ${instance.config.get("instance.name")} State: ${instance.status}`); - this._controller.instanceDetailsUpdated([instance]); await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev); } @@ -210,7 +206,7 @@ export default class HostConnection extends BaseConnection { const instanceConfig = new lib.InstanceConfig("controller"); instanceConfig.update(instanceData.config, false, "host"); - let controllerInstance = this._controller.instances.get(instanceConfig.get("instance.id")); + let controllerInstance = this._controller.instances.getMutable(instanceConfig.get("instance.id")); if (controllerInstance) { // Check if this instance is assigned somewhere else. if (controllerInstance.config.get("instance.assigned_host") !== this.id) { @@ -225,9 +221,8 @@ export default class HostConnection extends BaseConnection { let prev = controllerInstance.status; controllerInstance.status = instanceData.status; controllerInstance.gamePort = instanceData.gamePort; - controllerInstance.updatedAtMs = Date.now(); - logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`); instanceUpdates.push(controllerInstance); + logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`); await lib.invokeHook( this._controller.plugins, "onInstanceStatusChanged", controllerInstance, prev ); @@ -242,21 +237,17 @@ export default class HostConnection extends BaseConnection { instanceData.gamePort, Date.now(), ); - this._controller.instances.set(instanceConfig.get("instance.id"), newInstance); - this._controller.instancesDirty = true; + instanceUpdates.push(newInstance); this._controller.addInstanceHooks(newInstance); await this.send( new lib.InstanceAssignInternalRequest( instanceConfig.get("instance.id"), instanceConfig.toRemote("host") ) ); - instanceUpdates.push(newInstance); await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", newInstance); } - if (instanceUpdates.length) { - this._controller.instanceDetailsUpdated(instanceUpdates); - } + this._controller.instances.setMany(instanceUpdates); // Push lists to make sure they are in sync. let adminlist: Set = new Set(); diff --git a/packages/controller/src/InstanceInfo.ts b/packages/controller/src/InstanceInfo.ts index 0534c20d..b88c6b5b 100644 --- a/packages/controller/src/InstanceInfo.ts +++ b/packages/controller/src/InstanceInfo.ts @@ -48,6 +48,13 @@ export default class InstanceInfo { return this.status === "deleted"; } + set isDeleted(value: boolean) { + if (value === false) { + throw new Error("Setting isDeleted to false is not supported, use status instead."); + } + this.status = "deleted"; + } + /** Shorthand for `instance.config.get("instance.id")` */ get id():number { return this.config.get("instance.id"); diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index a3a54253..068bf3e5 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -218,7 +218,7 @@ async function uploadExport(req: Request, res: Response) { res.sendStatus(400); return; } - let modPack = res.app.locals.controller.modPacks.getCopy(modPackId); + let modPack = res.app.locals.controller.modPacks.getMutable(modPackId); if (!modPack) { res.sendStatus(400); return; diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index 9f3aa3fb..3624efee 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -2,23 +2,11 @@ import fs from "fs-extra"; import { safeOutputFile } from "./file_ops"; import { SubscribableValue } from "./subscriptions"; -import { JSONDeserialisable } from "./data/composites"; import { EventEmitter } from "stream"; type DatastoreKey = string | number; type DatastoreValue = string | number | boolean | object; -// Deep copy required to support the read only property of datastore values -function deepCopy(value: V): V { - if (value instanceof Object) { - return Object.fromEntries( - Object.entries(value).map(([k, v]) => [k, deepCopy(v)]) - ) as V; - } - - return value; -} - // Abstract class which can provide saving and loading capabilities to a datastore export abstract class DatastoreProvider< K extends DatastoreKey, @@ -50,11 +38,13 @@ export class MemoryDatastoreProvider< export class JsonDatastoreProvider< K extends DatastoreKey, V extends DatastoreValue, + J, // Intermediate type returned from migrations > extends DatastoreProvider { constructor( private filePath: string, - private dataClass: JSONDeserialisable, - private migrations: (rawJson: unknown) => Array<[K, unknown]> = v => v as any, + private fromJson: (json: J) => V, + private migrations: (rawJson: Record) => Record = v => v as any, + private finalise: (obj: V) => V = v => v, ) { super(); } @@ -81,7 +71,7 @@ export class JsonDatastoreProvider< const serialized = this.migrations(rawJson); // Convert to data class objects - return new Map(serialized.map((k, v) => [k, this.dataClass.fromJSON(v)])); + return new Map(Object.entries(serialized).map(([k, v]) => [k, this.finalise(this.fromJson(v))])); } } @@ -89,11 +79,13 @@ export class JsonDatastoreProvider< export class JsonIdDatastoreProvider< K extends DatastoreKey, V extends DatastoreValue & { id: K }, + J, // Intermediate type returned from migrations > extends DatastoreProvider { constructor( private filePath: string, - private dataClass: JSONDeserialisable, - private migrations: (rawJson: unknown) => Array<{ id: K }> = v => v as any, + private fromJson: (json: J) => V, + private migrations: (rawJson: Array) => Array = v => v as any, + private finalise: (obj: V) => V = v => v, ) { super(); } @@ -121,7 +113,7 @@ export class JsonIdDatastoreProvider< // Convert to data class objects return new Map(serialized.map((e) => { - const v = this.dataClass.fromJSON(e); + const v = this.finalise(this.fromJson(e)); return [v.id, v]; })); } @@ -132,10 +124,10 @@ export abstract class BaseDatastore< K extends DatastoreKey, V extends DatastoreValue, > extends EventEmitter { - protected dirty = false; + private dirty = false; constructor( - protected provider: DatastoreProvider = new MemoryDatastoreProvider(), + private provider: DatastoreProvider = new MemoryDatastoreProvider(), protected data = new Map(), ) { super(); @@ -155,6 +147,14 @@ export abstract class BaseDatastore< this.dirty = false; } + // Set the dirty flag and call update handlers + protected touch(updates: any[]) { + if (updates.length) { + this.dirty = true; + this.emit("update", updates); + } + } + // Returns true if the datastore has the value, false otherwise has(key: K) { return this.data.has(key); @@ -165,10 +165,8 @@ export abstract class BaseDatastore< return this.data.get(key) as Readonly ?? undefined; } - // Get a copy of a value from the datastore - getCopy(key: K) { - // const v = this.data.get(key); - // return v === undefined ? v : deepCopy(v); + // Get a mutable reference to a value in the datastore, call set after use + getMutable(key: K) { return this.data.get(key); } @@ -176,6 +174,11 @@ export abstract class BaseDatastore< values() { return this.data.values() as IterableIterator>; } + + // Returns all values in the datastore as mutable references, call setMany after use + valuesMutable() { + return this.data.values(); + } } // General key value mapping which can be saved and loaded with the file system @@ -186,35 +189,33 @@ export class Datastore< // Set the value in the datastore, be careful of race conditions if you await any functions before calling set set(key: K, value: V) { this.data.set(key, value); - this.dirty = true; - this.emit("update", [key, value]); + this.touch([[key, value]]); } // Set many values at once from an array of key value pairs setMany(pairs: [K, V][]) { - this.dirty ||= pairs.length > 0; for (const [k, v] of pairs) { this.data.set(k, v); } - this.emit("update", pairs); + this.touch(pairs); } // Delete a value in the datastore delete(key: K) { const value = this.data.get(key); this.data.delete(key); - this.dirty = true; - this.emit("update", [key, value, true]); + this.touch([[key, value!, true]]); } // Delete many values at once from an array of keys deleteMany(keys: K[]) { - this.dirty ||= keys.length > 0; + const updates = [] as [K, V, true][]; for (const key of keys) { const value = this.data.get(key); + updates.push([key, value!, true]); this.data.delete(key); } - this.emit("update", keys); + this.touch(updates); } } @@ -225,39 +226,36 @@ export class SubscribableDatastore< > extends BaseDatastore { // Set the value in the datastore, be careful of race conditions if you await any functions before calling set set(value: V) { - value.updatedAtMs = Date.now(); this.data.set(value.id, value); - this.dirty = true; - this.emit("update", [value]); + this.touch([value]); } // Set many values at once from an array of key value pairs setMany(values: V[]) { const nowMs = Date.now(); - this.dirty ||= values.length > 0; for (const value of values) { value.updatedAtMs = nowMs; this.data.set(value.id, value); } - this.emit("update", values); + this.touch(values); } // Delete a value in the datastore delete(value: V) { this.data.delete(value.id); - value.isDeleted = true; value.updatedAtMs = Date.now(); - this.emit("update", [value]); + value.isDeleted = true; + this.touch([value]); } // Delete many values at once from an array of values deleteMany(values: V[]) { - this.dirty ||= values.length > 0; + const nowMs = Date.now(); for (const value of values) { value.isDeleted = true; - value.updatedAtMs = Date.now(); + value.updatedAtMs = nowMs; this.data.delete(value.id); } - this.emit("update", values); + this.touch(values); } } diff --git a/plugins/player_auth/test/plugin.js b/plugins/player_auth/test/plugin.js index 6b90348e..3865fcdb 100644 --- a/plugins/player_auth/test/plugin.js +++ b/plugins/player_auth/test/plugin.js @@ -61,6 +61,8 @@ describe("player_auth", function() { return load; } else if (field === "factorio.settings") { return { name }; + } else if (field === "instance.id") { + return id; } throw new Error(`field ${field} not implemented`); }, diff --git a/test/integration/routing.js b/test/integration/routing.js index 7c5b204c..3b423786 100644 --- a/test/integration/routing.js +++ b/test/integration/routing.js @@ -39,7 +39,7 @@ function connectInstance(controller, host, instanceId) { instanceConfig.set("instance.id", instanceId); instanceConfig.set("instance.assigned_host", host.config.get("host.id")); host.instanceInfos.set(instanceId, { path: "invalid", config: instanceConfig }); - controller.instances.set(instanceId, new InstanceInfo(instanceConfig, "stopped")); + controller.instances.set(new InstanceInfo(instanceConfig, "stopped")); const instance = new Instance(host, instanceSide, "invalid", "invalid", instanceConfig); const instanceConnection = new InstanceConnection(hostSide, host, instance); host.instanceConnections.set(instanceId, instanceConnection); diff --git a/test/mock.js b/test/mock.js index 1c1fb84b..3eaabea2 100644 --- a/test/mock.js +++ b/test/mock.js @@ -182,8 +182,8 @@ class MockController { ["test", ControllerUser.fromJSON({ name: "test", roles: [0, 1] }, this.userManager)], ["player", ControllerUser.fromJSON({ name: "player", roles: [1] }, this.userManager)], ]); - this.instances = new Map(); - this.hosts = new Map(); + this.instances = new lib.Datastore(); + this.hosts = new lib.Datastore(); this.handles = new Map(); } From af50cfd47a56448339eb614df04444c3029a265d Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Wed, 22 May 2024 17:19:05 +0100 Subject: [PATCH 05/11] Consistency changes --- packages/controller/src/Controller.ts | 22 +++++++++++++--------- packages/controller/src/HostConnection.ts | 10 +++------- packages/lib/src/datastore.ts | 8 +++++++- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 4c767d9c..20e5148d 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -198,6 +198,7 @@ export default class Controller { this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this)); // Handle updates for datastores + this.systems.on("update", this.systemsUpdated.bind(this)); this.hosts.on("update", this.hostsUpdated.bind(this)); this.instances.on("update", this.instanceDetailsUpdated.bind(this)); this.saves.on("update", this.savesUpdated.bind(this)); @@ -520,7 +521,6 @@ export default class Controller { for (const metric of newMetrics) { this.systems.set(metric); } - this.subscriptions.broadcast(new lib.SystemInfoUpdateEvent(newMetrics)); } catch (err: any) { logger.error(`Unexpected error updating system infos:\n${err.stack ?? err.message}`); } @@ -561,16 +561,16 @@ export default class Controller { await lib.safeOutputFile(this.configPath, JSON.stringify(this.config, null, "\t")); } - let databaseDirectory = this.config.get("controller.database_directory"); - - await this.systems.save(); - await this.hosts.save(); - await this.instances.save(); - await this.saves.save(); - await this.modPacks.save(); + await Promise.all([ + this.systems.save(), + this.hosts.save(), + this.instances.save(), + this.saves.save(), + this.modPacks.save(), + ]); if (this.userManager.dirty) { - await this.userManager.save(path.join(databaseDirectory, "users.json")); + await this.userManager.save(path.join(this.config.get("controller.database_directory"), "users.json")); } await lib.invokeHook(this.plugins, "onSaveData"); @@ -708,6 +708,10 @@ export default class Controller { } } + systemsUpdated(systems: lib.SystemInfo[]) { + this.subscriptions.broadcast(new lib.SystemInfoUpdateEvent(systems)); + } + async handleSystemInfoSubscription(request: lib.SubscriptionRequest) { const systems = [...this.systems.values()].filter( metric => metric.updatedAtMs > request.lastRequestTimeMs, diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index 7d31d979..71c72e2a 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -288,13 +288,9 @@ export default class HostConnection extends BaseConnection { } } // Hosts eagerly send updates, which means we may get an update - // where nothing actualy changed. - if (updates.length) { - this._controller.saves.setMany(updates); - } - if (deletes.length) { - this._controller.saves.deleteMany(deletes); - } + // where nothing actually changed. + this._controller.saves.setMany(updates); + this._controller.saves.deleteMany(deletes); } async handleLogMessageEvent(event: lib.LogMessageEvent) { diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index 3624efee..d03b85ac 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -3,6 +3,8 @@ import fs from "fs-extra"; import { safeOutputFile } from "./file_ops"; import { SubscribableValue } from "./subscriptions"; import { EventEmitter } from "stream"; +import { ControllerConfig } from "./config"; +import path from "path"; type DatastoreKey = string | number; type DatastoreValue = string | number | boolean | object; @@ -133,6 +135,11 @@ export abstract class BaseDatastore< super(); } + // Get the file path based on the controller config + static getFilePath(config: ControllerConfig, file: string) { + return path.resolve(config.get("controller.database_directory"), file); + } + // Save the datastore to the provider async save() { if (this.dirty) { @@ -220,7 +227,6 @@ export class Datastore< } // A special implementation of the datastore to work directly with subscribable values -// TODO broadcast value changes to subscribers export class SubscribableDatastore< V extends SubscribableValue, > extends BaseDatastore { From 94a07b5af449dff796ebd6a981dd875fd7cea854 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Wed, 22 May 2024 17:42:25 +0100 Subject: [PATCH 06/11] Give restore abilty to memory provider --- packages/lib/src/datastore.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index d03b85ac..3a44cf39 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -28,11 +28,18 @@ export class MemoryDatastoreProvider< K extends DatastoreKey, V extends DatastoreValue, > extends DatastoreProvider { + constructor( + private value: Map = new Map() + ) { + super(); + } + async save(data: Map) { + this.value = new Map(data); } async load() { - return new Map(); + return new Map(this.value); } } From d696b8b52cf5260e05c567f9029a759029820d56 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Fri, 24 May 2024 18:29:08 +0100 Subject: [PATCH 07/11] Change abstract class name --- packages/lib/src/datastore.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index 3a44cf39..d07f1318 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -51,7 +51,7 @@ export class JsonDatastoreProvider< > extends DatastoreProvider { constructor( private filePath: string, - private fromJson: (json: J) => V, + private fromJson: (json: J) => V = v => v as any, private migrations: (rawJson: Record) => Record = v => v as any, private finalise: (obj: V) => V = v => v, ) { @@ -92,7 +92,7 @@ export class JsonIdDatastoreProvider< > extends DatastoreProvider { constructor( private filePath: string, - private fromJson: (json: J) => V, + private fromJson: (json: J) => V = v => v as any, private migrations: (rawJson: Array) => Array = v => v as any, private finalise: (obj: V) => V = v => v, ) { @@ -129,7 +129,7 @@ export class JsonIdDatastoreProvider< } // Implements a similar interface to a map -export abstract class BaseDatastore< +export abstract class Datastore< K extends DatastoreKey, V extends DatastoreValue, > extends EventEmitter { @@ -196,10 +196,10 @@ export abstract class BaseDatastore< } // General key value mapping which can be saved and loaded with the file system -export class Datastore< +export class KeyValueDatastore< K extends DatastoreKey, V extends DatastoreValue, -> extends BaseDatastore { +> extends Datastore { // Set the value in the datastore, be careful of race conditions if you await any functions before calling set set(key: K, value: V) { this.data.set(key, value); @@ -236,7 +236,7 @@ export class Datastore< // A special implementation of the datastore to work directly with subscribable values export class SubscribableDatastore< V extends SubscribableValue, -> extends BaseDatastore { +> extends Datastore { // Set the value in the datastore, be careful of race conditions if you await any functions before calling set set(value: V) { this.data.set(value.id, value); From eb658f86c0cabe58a61f95a70ed909fb86423163 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Fri, 24 May 2024 22:10:04 +0100 Subject: [PATCH 08/11] Add unit tests --- packages/lib/src/datastore.ts | 1 + test/lib/datastore.js | 646 ++++++++++++++++++++++++++++++++++ test/mock.js | 4 +- 3 files changed, 649 insertions(+), 2 deletions(-) create mode 100644 test/lib/datastore.js diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index d07f1318..42bc4303 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -240,6 +240,7 @@ export class SubscribableDatastore< // Set the value in the datastore, be careful of race conditions if you await any functions before calling set set(value: V) { this.data.set(value.id, value); + value.updatedAtMs = Date.now(); this.touch([value]); } diff --git a/test/lib/datastore.js b/test/lib/datastore.js new file mode 100644 index 00000000..4c14c678 --- /dev/null +++ b/test/lib/datastore.js @@ -0,0 +1,646 @@ +"use strict"; +const events = require("events"); +const assert = require("assert").strict; +const lib = require("@clusterio/lib"); +const fs = require("fs-extra"); +const path = require("path"); + +class MockDatastoreProvider extends lib.DatastoreProvider { + constructor() { + super(); + this.methodCalls = []; + this.value = new Map(); + this.methodCalls.push(["constructor"]); + } + + async save(data) { + this.methodCalls.push(["save", data]); + } + + async load() { + this.methodCalls.push(["load"]); + return this.value; + } +} + +class MockDatastore extends lib.Datastore { + getData() { + return this.data; // Protected member + } + + callTouch(updates = [0]) { + this.touch(updates); // Protected method + } +} + +describe("lib/datastore", function() { + const baseDir = path.join("temp", "test", "datastore"); + const testFiles = path.join("test", "file"); + before(async function() { + await fs.emptyDir(baseDir); + }); + + describe("class DatastoreProvider", function() { + let datastoreProvider; + beforeEach(function() { + datastoreProvider = new MockDatastoreProvider(); + }); + + it("can be constructed", function() { + assert.deepEqual(datastoreProvider.methodCalls[0], ["constructor"]); + }); + + it("can be bootstrapped", async function() { + const rtn = await datastoreProvider.bootstrap(); + assert.equal(rtn[0], datastoreProvider); + assert.equal(rtn[1], datastoreProvider.value); + assert.deepEqual(datastoreProvider.methodCalls[1], ["load"]); + }); + }); + + describe("class MemoryDatastoreProvider", function() { + let datastoreProvider, initialData; + beforeEach(function() { + initialData = new Map([ + ["foo", "fooValue"], ["bar", "barValue"], ["baz", "bazValue"], + ]); + datastoreProvider = new lib.MemoryDatastoreProvider(initialData); + }); + + describe("constructor", function() { + it("has a default value", async function() { + const [, value] = await new lib.MemoryDatastoreProvider().bootstrap(); + assert.notEqual(value, undefined); + assert.notEqual(value, null); + }); + it("can have a value given", async function() { + const expectedValue = new Map([["foo", "bar"]]); + const [, value] = await new lib.MemoryDatastoreProvider(expectedValue).bootstrap(); + assert.deepEqual(value, expectedValue); + }); + }); + + describe("save", function() { + it("can save data", async function() { + const newData = new Map([["foo2", "fooValue2"]]); + await datastoreProvider.save(newData); + const loadedData = await datastoreProvider.load(); + assert.deepEqual(loadedData, newData); + }); + }); + + describe("load", function() { + it("can load data", async function() { + const loadedData = await datastoreProvider.load(); + assert.deepEqual(loadedData, initialData); + }); + }); + }); + + describe("class JsonDatastoreProvider", function() { + let datastoreProvider, initialData, dataPath; + before(async function() { + dataPath = path.join(baseDir, "data.json"); + initialData = Object.entries(JSON.parse(await fs.readFile( + path.join(testFiles, "json", "load_map.json"), { encoding: "utf8" } + ))); + }); + + beforeEach(async function() { + await fs.copy(path.join(testFiles, "json", "load_map.json"), dataPath); + datastoreProvider = new lib.JsonDatastoreProvider(dataPath); + }); + + describe("constructor", function() { + it("has default values", async function() { + datastoreProvider = new lib.JsonDatastoreProvider(dataPath); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData)); + }); + it("can accept a fromJson method", async function() { + datastoreProvider = new lib.JsonDatastoreProvider( + dataPath, + json => [json] + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData.map(([k, v]) => [k, [v]]))); + }); + it("can accept a migration method", async function() { + datastoreProvider = new lib.JsonDatastoreProvider( + dataPath, + json => json, // Nop + rawJson => ({ c: "foo", ...rawJson }) + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map([...initialData, ["c", "foo"]])); + }); + it("can accept a finalise method", async function() { + datastoreProvider = new lib.JsonDatastoreProvider( + dataPath, + json => json, // Nop + rawJson => rawJson, // Nop + obj => [obj] + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData.map(([k, v]) => [k, [v]]))); + }); + }); + + describe("save", function() { + it("can save data", async function() { + const newData = new Map([["foo", "bar"]]); + await datastoreProvider.save(newData); + const loadedData = await datastoreProvider.load(); + assert.deepEqual(loadedData, newData); + }); + }); + + describe("load", function() { + it("can load data", async function() { + const loadedData = await datastoreProvider.load(); + assert.deepEqual(loadedData, new Map(initialData)); + }); + it("has a default value", async function() { + datastoreProvider = new lib.JsonDatastoreProvider(path.join(baseDir, "not_exists.json")); + const loadedData = await datastoreProvider.load(); + assert.equal(loadedData.size, 0); + }); + }); + }); + + describe("class JsonIdDatastoreProvider", function() { + let datastoreProvider, initialData, dataPath; + before(async function() { + dataPath = path.join(baseDir, "data.json"); + initialData = JSON.parse(await fs.readFile( + path.join(testFiles, "json", "load_array_map.json"), { encoding: "utf8" } + )).map(v => [v.id, v]); + }); + + beforeEach(async function() { + await fs.copy(path.join(testFiles, "json", "load_array_map.json"), dataPath); + datastoreProvider = new lib.JsonIdDatastoreProvider(dataPath); + }); + + describe("constructor", function() { + it("has default values", async function() { + datastoreProvider = new lib.JsonIdDatastoreProvider(dataPath); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData)); + }); + it("can accept a fromJson method", async function() { + datastoreProvider = new lib.JsonIdDatastoreProvider( + dataPath, + json => ({...json, foo: true}) + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData.map(v => ([v[0], {...v[1], foo: true}])))); + }); + it("can accept a migration method", async function() { + datastoreProvider = new lib.JsonIdDatastoreProvider( + dataPath, + json => json, // Nop + rawJson => ([{ id: "c" }, ...rawJson]) + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map([...initialData, ["c", { id: "c" }]])); + }); + it("can accept a finalise method", async function() { + datastoreProvider = new lib.JsonIdDatastoreProvider( + dataPath, + json => json, // Nop + rawJson => rawJson, // Nop + obj => ({...obj, foo: true}) + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData.map(v => ([v[0], {...v[1], foo: true}])))); + }); + }); + + describe("save", function() { + it("can save data", async function() { + const newData = new Map([["foo", { id: "foo" }]]); + await datastoreProvider.save(newData); + const loadedData = await datastoreProvider.load(); + assert.deepEqual(loadedData, newData); + }); + }); + + describe("load", function() { + it("can load data", async function() { + const loadedData = await datastoreProvider.load(); + assert.equal(loadedData.size, initialData.length); + assert.deepEqual(loadedData, new Map(initialData)); + }); + it("has a default value", async function() { + datastoreProvider = new lib.JsonIdDatastoreProvider(path.join(baseDir, "not_exists.json")); + const loadedData = await datastoreProvider.load(); + assert.equal(loadedData.size, 0); + }); + }); + }); + + describe("class Datastore", function() { + let datastoreProvider, datastore; + beforeEach(function() { + datastoreProvider = new MockDatastoreProvider(); + datastoreProvider.value = new Map([ + ["foo", "fooValue"], ["bar", "barValue"], ["baz", "bazValue"], + ]); + datastore = new MockDatastore(datastoreProvider, datastoreProvider.value); + }); + + describe("constructor", function() { + it("has a default value", function() { + datastore = new MockDatastore(datastoreProvider); + assert.deepEqual(datastore.getData(), new Map()); + }); + it("accepts a starting value", function() { + datastore = new MockDatastore(datastoreProvider, datastoreProvider.value); + assert.deepEqual(datastore.getData(), datastoreProvider.value); + }); + it("accepts the return of bootstrap", async function() { + datastore = new MockDatastore(...await datastoreProvider.bootstrap()); + assert.deepEqual(datastore.getData(), datastoreProvider.value); + }); + }); + + describe("getFilePath", function() { + it("returns the correct filepath", function() { + const mockConfig = { + get(field) { + if (field === "controller.database_directory") { + return "databaseDir"; + } + throw new Error(`field ${field} not implemented`); + }, + }; + + assert.equal( + lib.Datastore.getFilePath(mockConfig, "db1.json"), + path.resolve("databaseDir", "db1.json") + ); + assert.equal( + lib.Datastore.getFilePath(mockConfig, "db2.txt"), + path.resolve("databaseDir", "db2.txt") + ); + assert.equal( + lib.Datastore.getFilePath(mockConfig, path.join("foo", "db3.json")), + path.resolve("databaseDir", "foo", "db3.json") + ); + }); + }); + + describe("save", function() { + it("should not save when clean", async function() { + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 1); + }); + it("should save when dirty", async function() { + datastore.callTouch(); // Make the data dirty + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], ["save", datastore.getData()]); + }); + it("should not save multiple times", async function() { + datastore.callTouch(); // Make the data dirty + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], ["save", datastore.getData()]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + }); + }); + + describe("load", function() { + it("should load new data", async function() { + datastoreProvider.value = new Map(); + assert.notEqual(datastore.getData(), datastoreProvider.value); + await datastore.load(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], ["load"]); + assert.deepEqual(datastore.getData(), datastoreProvider.value); + }); + it("should not save after a load", async function() { + datastore.callTouch(); // Make the data dirty + await datastore.load(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], ["load"]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + }); + }); + + describe("touch", function() { + it("should emit update events", function() { + const eventsRaised = []; + datastore.on("update", updates => eventsRaised.push(updates)); + + let updates = ["foo"]; + datastore.callTouch(updates); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], updates); + + updates = ["foo", "bar"]; + datastore.callTouch(updates); + assert.equal(eventsRaised.length, 2); + assert.deepEqual(eventsRaised[1], updates); + }); + }); + + describe("has", function() { + it("returns true correctly", function() { + assert.equal(datastore.has("foo"), true); + assert.equal(datastore.has("bar"), true); + assert.equal(datastore.has("baz"), true); + }); + it("returns false correctly", function() { + assert.equal(datastore.has("missing1"), false); + assert.equal(datastore.has("missing2"), false); + assert.equal(datastore.has(1), false); + assert.equal(datastore.has(2), false); + }); + }); + + describe("get", function() { + it("returns the correct value", function() { + assert.equal(datastore.get("foo"), "fooValue"); + assert.equal(datastore.get("bar"), "barValue"); + assert.equal(datastore.get("baz"), "bazValue"); + }); + it("returns undefined correctly", function() { + assert.equal(datastore.get("missing1"), undefined); + assert.equal(datastore.get("missing2"), undefined); + assert.equal(datastore.get(1), undefined); + assert.equal(datastore.get(2), undefined); + }); + }); + + describe("getMutable", function() { + it("returns the correct value", function() { + assert.equal(datastore.get("foo"), "fooValue"); + assert.equal(datastore.get("bar"), "barValue"); + assert.equal(datastore.get("baz"), "bazValue"); + }); + it("returns undefined correctly", function() { + assert.equal(datastore.get("missing1"), undefined); + assert.equal(datastore.get("missing2"), undefined); + assert.equal(datastore.get(1), undefined); + assert.equal(datastore.get(2), undefined); + }); + }); + + describe("values", function() { + it("returns all values", function() { + assert.deepEqual([...datastore.values()], [...datastoreProvider.value.values()]); + }); + }); + + describe("valuesMutable", function() { + it("returns all values", function() { + assert.deepEqual([...datastore.values()], [...datastoreProvider.value.values()]); + }); + }); + }); + + describe("class KeyValueDatastore", function() { + let datastoreProvider, datastore, eventsRaised; + beforeEach(function() { + eventsRaised = []; + datastoreProvider = new MockDatastoreProvider(); + datastoreProvider.value = new Map([ + ["foo", "fooValue"], ["bar", "barValue"], ["baz", "bazValue"], + ]); + datastore = new lib.KeyValueDatastore(datastoreProvider, datastoreProvider.value); + datastore.on("update", updates => eventsRaised.push(updates)); + }); + + describe("set", function() { + it("can set a value", function() { + datastore.set("test", "testValue"); + assert.equal(datastore.get("test"), "testValue"); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [["test", "testValue"]]); + }); + it("allows saving after a value is set", async function() { + datastore.set("test", "testValue"); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries(), ["test", "testValue"]])] + ); + }); + }); + + describe("setMany", function() { + it("does not save with zero changes", async function() { + datastore.setMany([]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 1); + }); + it("can set many values at once", async function() { + const updates = [["test1", "testValue1"], ["test2", "testValue2"]]; + datastore.setMany(updates); + assert.equal(datastore.get("test1"), "testValue1"); + assert.equal(datastore.get("test2"), "testValue2"); + await datastore.save(); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], updates); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries(), ...updates])] + ); + }); + }); + + describe("delete", function() { + it("can delete a value", function() { + datastore.delete("foo"); + assert.equal(datastore.get("foo"), undefined); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [["foo", "fooValue", true]]); + }); + it("allows saving after a value is deleted", async function() { + datastore.delete("foo"); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => k !== "foo"))] + ); + }); + it("does not break when the key does not exist", function() { + datastore.delete("test"); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [["test", undefined, true]]); + }); + }); + + describe("deleteMany", function() { + it("does not save with zero changes", async function() { + datastore.deleteMany([]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 1); + }); + it("can delete many values at once", async function() { + const keys = ["foo", "bar"]; + datastore.deleteMany(keys); + assert.equal(datastore.get("foo"), undefined); + assert.equal(datastore.get("bar"), undefined); + await datastore.save(); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [["foo", "fooValue", true], ["bar", "barValue", true]]); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => !keys.includes(k)))] + ); + }); + }); + }); + + describe("class SubscribableDatastore", function() { + let datastoreProvider, datastore, eventsRaised; + beforeEach(function() { + eventsRaised = []; + datastoreProvider = new MockDatastoreProvider(); + datastoreProvider.value = new Map([ + ["foo", {id: "foo"}], ["bar", {id: "bar"}], ["baz", {id: "baz"}], + ]); + datastore = new lib.SubscribableDatastore(datastoreProvider, datastoreProvider.value); + datastore.on("update", updates => eventsRaised.push(updates)); + }); + + describe("set", function() { + it("can set a value", function() { + const value = { id: "test" }; + datastore.set(value); + assert.equal(datastore.get(value.id), value); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [value]); + }); + it("allows saving after a value is set", async function() { + const value = { id: "test" }; + datastore.set(value); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries(), [value.id, value]])] + ); + }); + it("sets the updatedAtMs property", function() { + const value = { id: "test" }; + datastore.set(value); + assert.notEqual(value.updatedAtMs, undefined); + assert(value.updatedAtMs > 0, "updatedAtMs greater than zero"); + }); + }); + + describe("setMany", function() { + it("does not save with zero changes", async function() { + datastore.setMany([]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 1); + }); + it("can set many values at once", async function() { + const updates = [{ id: "test1" }, { id: "test2" }]; + datastore.setMany(updates); + assert.equal(datastore.get(updates[0].id), updates[0]); + assert.equal(datastore.get(updates[1].id), updates[1]); + await datastore.save(); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], updates); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries(), ...updates.map(v => [v.id, v])])] + ); + }); + it("sets the updatedAtMs property", function() { + const updates = [{ id: "test1" }, { id: "test2" }]; + datastore.setMany(updates); + assert.notEqual(updates[0].updatedAtMs, undefined); + assert.notEqual(updates[1].updatedAtMs, undefined); + assert(updates[0].updatedAtMs > 0, "updatedAtMs greater than zero"); + assert(updates[1].updatedAtMs > 0, "updatedAtMs greater than zero"); + }); + }); + + describe("delete", function() { + it("can delete a value", function() { + const value = datastore.get("foo"); + datastore.delete(value); + assert.equal(datastore.get(value.id), undefined); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [value]); + }); + it("allows saving after a value is deleted", async function() { + const value = datastore.get("foo"); + datastore.delete(value); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => k !== value.id))] + ); + }); + it("does not break when the key does not exist", function() { + const value = { id: "test" }; + datastore.delete(value); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [value]); + }); + it("sets the updatedAtMs property", function() { + const value = { id: "test" }; + datastore.delete(value); + assert.notEqual(value.updatedAtMs, undefined); + assert(value.updatedAtMs > 0, "updatedAtMs greater than zero"); + }); + it("sets the isDeleted property", function() { + const value = { id: "test" }; + datastore.delete(value); + assert.equal(value.isDeleted, true); + }); + }); + + describe("deleteMany", function() { + it("does not save with zero changes", async function() { + datastore.deleteMany([]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 1); + }); + it("can delete many values at once", async function() { + const values = [{id: "foo"}, {id: "bar"}]; + datastore.deleteMany(values); + assert.equal(datastore.get(values[0].id), undefined); + assert.equal(datastore.get(values[1].id), undefined); + await datastore.save(); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], values); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries()] + .filter((_, k) => !values.some(v => v.id === k)))] + ); + }); + it("sets the updatedAtMs property", function() { + const values = [{id: "foo"}, {id: "bar"}]; + datastore.deleteMany(values); + assert.notEqual(values[0].updatedAtMs, undefined); + assert.notEqual(values[1].updatedAtMs, undefined); + assert(values[0].updatedAtMs > 0, "updatedAtMs greater than zero"); + assert(values[1].updatedAtMs > 0, "updatedAtMs greater than zero"); + }); + it("sets the isDeleted property", function() { + const values = [{id: "foo"}, {id: "bar"}]; + datastore.deleteMany(values); + assert.equal(values[0].isDeleted, true); + assert.equal(values[1].isDeleted, true); + }); + }); + }); +}); diff --git a/test/mock.js b/test/mock.js index 3eaabea2..fd47c72d 100644 --- a/test/mock.js +++ b/test/mock.js @@ -182,8 +182,8 @@ class MockController { ["test", ControllerUser.fromJSON({ name: "test", roles: [0, 1] }, this.userManager)], ["player", ControllerUser.fromJSON({ name: "player", roles: [1] }, this.userManager)], ]); - this.instances = new lib.Datastore(); - this.hosts = new lib.Datastore(); + this.instances = new lib.KeyValueDatastore(); + this.hosts = new lib.KeyValueDatastore(); this.handles = new Map(); } From 222a8abdb8d662c98073b86f953a721a10011746 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sat, 25 May 2024 00:32:14 +0100 Subject: [PATCH 09/11] Update change log --- CHANGELOG.md | 4 ++++ test/lib/datastore.js | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 099eb639..22a3639a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,10 @@ Many thanks to the following for contributing to this release: ## Version 2.0.0-alpha.19 +### Features + +- Added Datastore and Datastore Provider classes to the library to support savable Maps [#629](https://github.com/clusterio/clusterio/pull/629). + ### Changes - Fixed IPC errors reporting undefined when no stack trace was present [#624](https://github.com/clusterio/clusterio/pull/624). diff --git a/test/lib/datastore.js b/test/lib/datastore.js index 4c14c678..0a690163 100644 --- a/test/lib/datastore.js +++ b/test/lib/datastore.js @@ -539,6 +539,9 @@ describe("lib/datastore", function() { datastore.set(value); assert.notEqual(value.updatedAtMs, undefined); assert(value.updatedAtMs > 0, "updatedAtMs greater than zero"); + const prev = value.updatedAtMs; + datastore.set(value); + assert(value.updatedAtMs > prev, "updatedAtMs must increase"); }); }); @@ -568,6 +571,10 @@ describe("lib/datastore", function() { assert.notEqual(updates[1].updatedAtMs, undefined); assert(updates[0].updatedAtMs > 0, "updatedAtMs greater than zero"); assert(updates[1].updatedAtMs > 0, "updatedAtMs greater than zero"); + const prev = [updates[0].updatedAtMs, updates[1].updatedAtMs]; + datastore.setMany(updates); + assert(updates[0].updatedAtMs > prev[0], "updatedAtMs must increase"); + assert(updates[1].updatedAtMs > prev[1], "updatedAtMs must increase"); }); }); From e90451ead8ec3399899c235e4d29de41493d9d00 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sat, 25 May 2024 01:20:58 +0100 Subject: [PATCH 10/11] Fix broken test --- test/lib/datastore.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/lib/datastore.js b/test/lib/datastore.js index 0a690163..eae2a3a1 100644 --- a/test/lib/datastore.js +++ b/test/lib/datastore.js @@ -534,12 +534,13 @@ describe("lib/datastore", function() { ["save", new Map([...datastoreProvider.value.entries(), [value.id, value]])] ); }); - it("sets the updatedAtMs property", function() { + it("sets the updatedAtMs property", async function() { const value = { id: "test" }; datastore.set(value); assert.notEqual(value.updatedAtMs, undefined); assert(value.updatedAtMs > 0, "updatedAtMs greater than zero"); const prev = value.updatedAtMs; + await lib.wait(); datastore.set(value); assert(value.updatedAtMs > prev, "updatedAtMs must increase"); }); @@ -564,7 +565,7 @@ describe("lib/datastore", function() { ["save", new Map([...datastoreProvider.value.entries(), ...updates.map(v => [v.id, v])])] ); }); - it("sets the updatedAtMs property", function() { + it("sets the updatedAtMs property", async function() { const updates = [{ id: "test1" }, { id: "test2" }]; datastore.setMany(updates); assert.notEqual(updates[0].updatedAtMs, undefined); @@ -572,6 +573,7 @@ describe("lib/datastore", function() { assert(updates[0].updatedAtMs > 0, "updatedAtMs greater than zero"); assert(updates[1].updatedAtMs > 0, "updatedAtMs greater than zero"); const prev = [updates[0].updatedAtMs, updates[1].updatedAtMs]; + await lib.wait(); datastore.setMany(updates); assert(updates[0].updatedAtMs > prev[0], "updatedAtMs must increase"); assert(updates[1].updatedAtMs > prev[1], "updatedAtMs must increase"); From 8b823a571d257bdb8807908ebe63c82148f908b4 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sat, 25 May 2024 18:17:50 +0100 Subject: [PATCH 11/11] Coverage 100% --- test/file/json/invalid.json | 1 + test/lib/datastore.js | 24 ++++++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 test/file/json/invalid.json diff --git a/test/file/json/invalid.json b/test/file/json/invalid.json new file mode 100644 index 00000000..004e1e20 --- /dev/null +++ b/test/file/json/invalid.json @@ -0,0 +1 @@ +{,} diff --git a/test/lib/datastore.js b/test/lib/datastore.js index eae2a3a1..3e0f0802 100644 --- a/test/lib/datastore.js +++ b/test/lib/datastore.js @@ -169,6 +169,10 @@ describe("lib/datastore", function() { const loadedData = await datastoreProvider.load(); assert.equal(loadedData.size, 0); }); + it("throws for invalid file contents", async function() { + await fs.copy(path.join(testFiles, "json", "invalid.json"), dataPath); + await assert.rejects(() => datastoreProvider.load(dataPath)); + }); }); }); @@ -245,6 +249,10 @@ describe("lib/datastore", function() { const loadedData = await datastoreProvider.load(); assert.equal(loadedData.size, 0); }); + it("throws for invalid file contents", async function() { + await fs.copy(path.join(testFiles, "json", "invalid.json"), dataPath); + await assert.rejects(() => datastoreProvider.load(dataPath)); + }); }); }); @@ -386,15 +394,15 @@ describe("lib/datastore", function() { describe("getMutable", function() { it("returns the correct value", function() { - assert.equal(datastore.get("foo"), "fooValue"); - assert.equal(datastore.get("bar"), "barValue"); - assert.equal(datastore.get("baz"), "bazValue"); + assert.equal(datastore.getMutable("foo"), "fooValue"); + assert.equal(datastore.getMutable("bar"), "barValue"); + assert.equal(datastore.getMutable("baz"), "bazValue"); }); it("returns undefined correctly", function() { - assert.equal(datastore.get("missing1"), undefined); - assert.equal(datastore.get("missing2"), undefined); - assert.equal(datastore.get(1), undefined); - assert.equal(datastore.get(2), undefined); + assert.equal(datastore.getMutable("missing1"), undefined); + assert.equal(datastore.getMutable("missing2"), undefined); + assert.equal(datastore.getMutable(1), undefined); + assert.equal(datastore.getMutable(2), undefined); }); }); @@ -406,7 +414,7 @@ describe("lib/datastore", function() { describe("valuesMutable", function() { it("returns all values", function() { - assert.deepEqual([...datastore.values()], [...datastoreProvider.value.values()]); + assert.deepEqual([...datastore.valuesMutable()], [...datastoreProvider.value.values()]); }); }); });