diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ef76ed2..b388eaa6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,7 @@ Many thanks to the following for contributing to this release: ### Features +- Added Datastore and Datastore Provider classes to the library to support savable Maps [#629](https://github.com/clusterio/clusterio/pull/629). - Added clean script to speed up publishing and debugging of build steps [#640](https://github.com/clusterio/clusterio/pull/640). - Added sub-tile option on plugin page headers [#642](https://github.com/clusterio/clusterio/pull/642). - Greatly improved plugin and module template generation [#641](https://github.com/clusterio/clusterio/pull/641). diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 4b59b18c..b4b9d4e2 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -66,9 +66,9 @@ void new lib.Gauge( if (!controller || !controller.hosts) { return; } - for (let [id, host] of controller.hosts) { + for (const host of controller.hosts.values()) { gauge.labels({ - host_id: String(id), + host_id: String(host.id), host_name: host.name, }).set(1); }; @@ -86,9 +86,9 @@ void new lib.Gauge( if (!controller || !controller.instances) { return; } - for (let [id, instance] of controller.instances) { + for (const instance of controller.instances.values()) { gauge.labels({ - instance_id: String(id), + instance_id: String(instance.id), instance_name: String(instance.config.get("instance.name")), host_id: String(instance.config.get("instance.assigned_host")), }).set(1); diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 595f900b..ba59deaf 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -195,7 +195,7 @@ export default class ControlConnection extends BaseConnection { } async handleHostRevokeTokensRequest(request: lib.HostRevokeTokensRequest) { - const host = this._controller.hosts.get(request.hostId); + const host = this._controller.hosts.getMutable(request.hostId); if (!host) { throw new Error(`Unknown host id (${request.hostId})`); } @@ -207,7 +207,7 @@ export default class ControlConnection extends BaseConnection { hostConnection.connector.terminate(); } - this._controller.hostsUpdated([host]); + this._controller.hosts.set(host); } async handleHostListRequest(): Promise { @@ -397,8 +397,7 @@ export default class ControlConnection extends BaseConnection { if (this._controller.modPacks.has(modPack.id)) { throw new lib.RequestError(`Mod pack with ID ${modPack.id} already exist`); } - this._controller.modPacks.set(modPack.id, modPack); - this._controller.modPacksUpdated([modPack]); + this._controller.modPacks.set(modPack); } async handleModPackUpdateRequest(request: lib.ModPackUpdateRequest) { @@ -406,19 +405,16 @@ export default class ControlConnection extends BaseConnection { if (modPack.id === undefined || !this._controller.modPacks.has(modPack.id)) { throw new lib.RequestError(`Mod pack with ID ${modPack.id} does not exist`); } - this._controller.modPacks.set(modPack.id, modPack); - this._controller.modPacksUpdated([modPack]); + this._controller.modPacks.set(modPack); } async handleModPackDeleteRequest(request: lib.ModPackDeleteRequest) { let { id } = request; - let modPack = this._controller.modPacks.get(id); + let modPack = this._controller.modPacks.getMutable(id); if (!modPack) { throw new lib.RequestError(`Mod pack with ID ${id} does not exist`); } - modPack.isDeleted = true; - this._controller.modPacks.delete(id); - this._controller.modPacksUpdated([modPack]); + this._controller.modPacks.delete(modPack); } async handleModGetRequest(request: lib.ModGetRequest) { diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 530ad456..4a18a9cb 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -55,17 +55,6 @@ export default class Controller { config: lib.ControllerConfig; app: Application; - /** True if (@link Controller.systems} has changed since last time it was saved */ - systemsDirty = false; - /** True if {@link Controller.hosts} has changed since last time it was saved */ - hostsDirty = false; - /** True if {@link Controller.instances} has changed since last time it was saved */ - instancesDirty = false; - /** True if {@link Controller.saves} has changed since last time it was saved */ - savesDirty = false; - /** True if {@link Controller.modPacks} has changed since last time it was saved */ - modPacksDirty = false; - httpServer: http.Server | null = null; httpServerCloser: HttpCloser | null = null; httpsServer: https.Server | null = null; @@ -111,11 +100,33 @@ export default class Controller { let databaseDirectory = config.get("controller.database_directory"); await fs.ensureDir(databaseDirectory); - const systems = await Controller.loadSystems(path.join(databaseDirectory, "systems.json")); - const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json")); - const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json")); - const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json")); - const modPacks = await Controller.loadModPacks(path.join(databaseDirectory, "mod-packs.json")); + const systems = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "systems.json"), + lib.SystemInfo.fromJSON + ).bootstrap()); + + const hosts = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "hosts.json"), + HostInfo.fromJSON, + this.migrateHosts, this.finaliseHosts, + ).bootstrap()); + + const instances = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "instances.json"), + json => InstanceInfo.fromJSON(json, "controller"), + this.migrateInstances, this.finaliseInstances, + ).bootstrap()); + + const saves = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "saves.json"), + lib.SaveDetails.fromJSON + ).bootstrap()); + + const modPacks = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider( + path.join(databaseDirectory, "mod-packs.json"), + lib.ModPack.fromJSON + ).bootstrap()); + const userManager = new UserManager(config); await userManager.load(path.join(databaseDirectory, "users.json")); @@ -145,15 +156,15 @@ export default class Controller { * will restart the controller on non-zero exit codes. */ public canRestart: boolean = false, - public systems = new Map(), + public systems = new lib.SubscribableDatastore(), /** Mapping of host id to host info */ - public hosts = new Map(), + public hosts = new lib.SubscribableDatastore(), /** Mapping of instance id to instance info */ - public instances = new Map(), + public instances = new lib.SubscribableDatastore(), /** Mapping of save id to save details */ - public saves = new Map(), + public saves = new lib.SubscribableDatastore(), /** Mapping of mod pack id to mod pack */ - public modPacks = new Map(), + public modPacks = new lib.SubscribableDatastore(), /** Mods stored on the controller */ public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()), /** User and roles manager for the cluster */ @@ -185,6 +196,13 @@ export default class Controller { this.subscriptions.handle(lib.ModPackUpdatesEvent, this.handleModPackSubscription.bind(this)); this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this)); this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this)); + + // Handle updates for datastores + this.systems.on("update", this.systemsUpdated.bind(this)); + this.hosts.on("update", this.hostsUpdated.bind(this)); + this.instances.on("update", this.instanceDetailsUpdated.bind(this)); + this.saves.on("update", this.savesUpdated.bind(this)); + this.modPacks.on("update", this.modPacksUpdated.bind(this)); } async start(args: ControllerArgs) { @@ -501,10 +519,8 @@ export default class Controller { requests.push(lib.gatherSystemInfo("controller", this.canRestart)); const newMetrics = await Promise.all(requests); for (const metric of newMetrics) { - this.systems.set(metric.id, metric); + this.systems.set(metric); } - this.systemsDirty = true; - this.subscriptions.broadcast(new lib.SystemInfoUpdateEvent(newMetrics)); } catch (err: any) { logger.error(`Unexpected error updating system infos:\n${err.stack ?? err.message}`); } @@ -545,160 +561,57 @@ export default class Controller { await lib.safeOutputFile(this.configPath, JSON.stringify(this.config, null, "\t")); } - let databaseDirectory = this.config.get("controller.database_directory"); - if (this.systemsDirty) { - this.systemsDirty = false; - await Controller.saveSystems(path.join(databaseDirectory, "systems.json"), this.systems); - } - - if (this.hostsDirty) { - this.hostsDirty = false; - await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts); - } - - if (this.instancesDirty) { - this.instancesDirty = false; - await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances); - } - - if (this.savesDirty) { - this.savesDirty = false; - await Controller.saveSaves(path.join(databaseDirectory, "saves.json"), this.saves); - } - - if (this.modPacksDirty) { - this.modPacksDirty = false; - await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks); - } + await Promise.all([ + this.systems.save(), + this.hosts.save(), + this.instances.save(), + this.saves.save(), + this.modPacks.save(), + ]); if (this.userManager.dirty) { - await this.userManager.save(path.join(databaseDirectory, "users.json")); + await this.userManager.save(path.join(this.config.get("controller.database_directory"), "users.json")); } await lib.invokeHook(this.plugins, "onSaveData"); } - static async loadSystems(filePath: string): Promise> { - let json: Static[]; - try { - json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); - - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); - } - - return new Map(json.map(m => lib.SystemInfo.fromJSON(m)).map(m => [m.id, m])); - } - - static async saveSystems(filePath: string, systems: Map) { - await lib.safeOutputFile(filePath, JSON.stringify([...systems.values()], null, "\t")); - } - - static async loadHosts(filePath: string): Promise> { - let serialized: [number, Static][]; - try { - serialized = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); - - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); - } + static migrateHosts(rawJson: unknown[]): Static[] { + const serialized = rawJson as Static[]; // TODO: migrate remove after release. if (serialized.length && !(serialized[0] instanceof Array)) { - return new Map(); // Discard old format. + return []; // Discard old format. } - const hosts = new Map(); - for (let [id, json] of serialized) { - const host = HostInfo.fromJSON(json); - if (host.connected) { - host.connected = false; - host.updatedAtMs = Date.now(); - } - hosts.set(id, host); + return serialized; + } + + static finaliseHosts(host: HostInfo): HostInfo { + if (host.connected) { + host.connected = false; + host.updatedAtMs = Date.now(); } - return hosts; + return host; } - static async saveHosts(filePath: string, hosts: Map) { - await lib.safeOutputFile(filePath, JSON.stringify([...hosts.entries()], null, "\t")); + static migrateInstances(rawJson: unknown[]): Static[] { + const serialized = rawJson as Static[]; + return serialized.map(json => { + if (!json.config) { // migrate: from pre Alpha 14 format. + return { config: json as any, status: "running" }; // Use running to force updatedAtMs + } + return json; + }); } - static async loadInstances(filePath: string): Promise> { - logger.info(`Loading ${filePath}`); - - let instances = new Map(); - try { - let serialized = JSON.parse( - await fs.readFile(filePath, { encoding: "utf8" }) - ) as Static[]; - for (let json of serialized) { - if (!json.config) { // migrate: from pre Alpha 14 format. - json = { config: json as any, status: "running" }; // Use running to force updatedAtMs - } - const instance = InstanceInfo.fromJSON(json, "controller"); - const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown"; - if (instance.status !== status) { - instance.status = status; - instance.updatedAtMs = Date.now(); - } - instances.set(instance.id, instance); - } - - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); + static finaliseInstances(instance: InstanceInfo): InstanceInfo { + const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown"; + if (instance.status !== status) { + instance.status = status; + instance.updatedAtMs = Date.now(); } - - return instances; - } - - static async saveInstances(filePath: string, instances: Map) { - let serialized = [...instances.values()]; - await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t")); - } - - static async loadSaves(filePath: string): Promise> { - let json: Static[]; - try { - json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); - } - return new Map(json.map(s => lib.SaveDetails.fromJSON(s)).map(s => [s.id, s])); - } - - static async saveSaves(filePath: string, saves: Map) { - await lib.safeOutputFile(filePath, JSON.stringify([...saves.values()], null, "\t")); - } - - - static async loadModPacks(filePath: string): Promise> { - let json; - try { - json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" })); - } catch (err: any) { - if (err.code !== "ENOENT") { - throw err; - } - return new Map(); - } - return new Map(json.map((e: any) => [e.id, lib.ModPack.fromJSON(e)])); - } - - static async saveModPacks(filePath: string, modPacks: Map) { - await lib.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, "\t")); + return instance; } static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise { @@ -795,6 +708,10 @@ export default class Controller { } } + systemsUpdated(systems: lib.SystemInfo[]) { + this.subscriptions.broadcast(new lib.SystemInfoUpdateEvent(systems)); + } + async handleSystemInfoSubscription(request: lib.SubscriptionRequest) { const systems = [...this.systems.values()].filter( metric => metric.updatedAtMs > request.lastRequestTimeMs, @@ -816,11 +733,6 @@ export default class Controller { } hostsUpdated(hosts: HostInfo[]) { - const now = Date.now(); - for (const host of hosts) { - host.updatedAtMs = now; - } - this.hostsDirty = true; let updates = hosts.map(host => host.toHostDetails()); this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates)); } @@ -897,11 +809,9 @@ export default class Controller { instanceConfig.set("factorio.settings", settings); let instance = new InstanceInfo(instanceConfig, "unassigned", undefined, undefined, Date.now()); - this.instances.set(instanceId, instance); - this.instancesDirty = true; + this.instances.set(instance); await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance); this.addInstanceHooks(instance); - this.instanceDetailsUpdated([instance]); return instance; } @@ -911,19 +821,7 @@ export default class Controller { * @internal */ clearSavesOfInstance(instanceId: number) { - const updates = []; - const now = Date.now(); - for (const [id, save] of this.saves) { - if (save.instanceId === instanceId) { - save.isDeleted = true; - save.updatedAtMs = now; - updates.push(save); - this.saves.delete(id); - } - } - if (updates.length) { - this.savesUpdated(updates); - } + this.saves.deleteMany([...this.saves.values()].filter(save => save.instanceId === instanceId)); } /** @@ -970,7 +868,6 @@ export default class Controller { // Assign to target instance.config.set("instance.assigned_host", hostId ?? null); - // "fieldChanged" event handler will set this.instancesDirty if (hostId !== undefined && newHostConnection) { await newHostConnection.send( new lib.InstanceAssignInternalRequest(instanceId, instance.config.toRemote("host")) @@ -978,8 +875,7 @@ export default class Controller { } else { instance.status = "unassigned"; } - instance.updatedAtMs = Date.now(); - this.instanceDetailsUpdated([instance]); + this.instances.set(instance); } /** @@ -996,16 +892,13 @@ export default class Controller { if (hostId !== null) { await this.sendTo({ hostId }, new lib.InstanceDeleteInternalRequest(instanceId)); } - this.instances.delete(instanceId); - this.instancesDirty = true; + + let prev = instance.status; + this.instances.delete(instance); this.clearSavesOfInstance(instanceId); this.userManager.clearStatsOfInstance(instanceId); - let prev = instance.status; - instance.status = "deleted"; - instance.updatedAtMs = Date.now(); - this.instanceDetailsUpdated([instance]); await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance, prev); } @@ -1034,10 +927,9 @@ export default class Controller { instance.config.on("fieldChanged", (field: string, curr: any, prev: any) => { instance.updatedAtMs = Date.now(); if (field === "instance.name") { - this.instanceDetailsUpdated([instance]); + this.instances.set(instance); // Trigger updated logic } - this.instancesDirty = true; lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, field, curr, prev); }); } @@ -1060,7 +952,6 @@ export default class Controller { } savesUpdated(saves: lib.SaveDetails[]) { - this.savesDirty = true; this.subscriptions.broadcast(new lib.InstanceSaveDetailsUpdatesEvent(saves)); } @@ -1071,11 +962,6 @@ export default class Controller { modPacksUpdated(modPacks: lib.ModPack[]) { - const now = Date.now(); - for (const modPack of modPacks) { - modPack.updatedAtMs = now; - } - this.modPacksDirty = true; this.subscriptions.broadcast(new lib.ModPackUpdatesEvent(modPacks)); lib.invokeHook(this.plugins, "onModPacksUpdated", modPacks); } diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index 7540452c..7dda1d44 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -42,8 +42,7 @@ export default class HostConnection extends BaseConnection { 0, false, ); - this._controller.hosts.set(this.id, this.info); - this._controller.hostsUpdated([this.info]); + this._controller.hosts.set(this.info); this._checkPluginVersions(); @@ -57,10 +56,9 @@ export default class HostConnection extends BaseConnection { } this.connector.on("close", () => { - const now = Date.now(); // Update status to unknown for instances on this host. const instances: InstanceInfo[] = []; - for (let instance of this._controller.instances.values()) { + for (let instance of this._controller.instances.valuesMutable()) { if (instance.config.get("instance.assigned_host") !== this.id) { continue; } @@ -68,13 +66,12 @@ export default class HostConnection extends BaseConnection { instances.push(instance); let prev = instance.status; instance.status = "unknown"; - instance.updatedAtMs = now; lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev); } - this._controller.instanceDetailsUpdated(instances); + this._controller.instances.setMany(instances); this.info.connected = false; - this._controller.hostsUpdated([this.info]); + this._controller.hosts.set(this.info); }); this.handle(lib.HostInfoUpdateEvent, this.handleHostInfoUpdateEvent.bind(this)); @@ -158,11 +155,11 @@ export default class HostConnection extends BaseConnection { async handleHostInfoUpdateEvent(event: lib.HostInfoUpdateEvent) { this.info.name = event.update.name; this.info.publicAddress = event.update.publicAddress; - this._controller.hostsUpdated([this.info]); + this._controller.hosts.set(this.info); } async handleInstanceStatusChangedEvent(request: lib.InstanceStatusChangedEvent) { - let instance = this._controller.instances.get(request.instanceId); + let instance = this._controller.instances.getMutable(request.instanceId); // It's possible to get updates from an instance that does not exist // or is not assigned to the host it originated from if it was @@ -189,9 +186,8 @@ export default class HostConnection extends BaseConnection { instance.status = request.status; instance.gamePort = request.gamePort; instance.factorioVersion = request.factorioVersion; - instance.updatedAtMs = Date.now(); + this._controller.instances.set(instance); logger.verbose(`Instance ${instance.config.get("instance.name")} State: ${instance.status}`); - this._controller.instanceDetailsUpdated([instance]); await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev); } @@ -211,7 +207,7 @@ export default class HostConnection extends BaseConnection { const instanceConfig = new lib.InstanceConfig("controller"); instanceConfig.update(instanceData.config, false, "host"); - let controllerInstance = this._controller.instances.get(instanceConfig.get("instance.id")); + let controllerInstance = this._controller.instances.getMutable(instanceConfig.get("instance.id")); if (controllerInstance) { // Check if this instance is assigned somewhere else. if (controllerInstance.config.get("instance.assigned_host") !== this.id) { @@ -227,9 +223,8 @@ export default class HostConnection extends BaseConnection { controllerInstance.status = instanceData.status; controllerInstance.gamePort = instanceData.gamePort; controllerInstance.factorioVersion = instanceData.factorioVersion; - controllerInstance.updatedAtMs = Date.now(); - logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`); instanceUpdates.push(controllerInstance); + logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`); await lib.invokeHook( this._controller.plugins, "onInstanceStatusChanged", controllerInstance, prev ); @@ -245,21 +240,17 @@ export default class HostConnection extends BaseConnection { instanceData.factorioVersion, Date.now(), ); - this._controller.instances.set(instanceConfig.get("instance.id"), newInstance); - this._controller.instancesDirty = true; + instanceUpdates.push(newInstance); this._controller.addInstanceHooks(newInstance); await this.send( new lib.InstanceAssignInternalRequest( instanceConfig.get("instance.id"), instanceConfig.toRemote("host") ) ); - instanceUpdates.push(newInstance); await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", newInstance); } - if (instanceUpdates.length) { - this._controller.instanceDetailsUpdated(instanceUpdates); - } + this._controller.instances.setMany(instanceUpdates); // Push lists to make sure they are in sync. let adminlist: Set = new Set(); @@ -282,33 +273,27 @@ export default class HostConnection extends BaseConnection { } async handleInstanceSaveDetailsUpdatesEvent(event: lib.InstanceSaveDetailsUpdatesEvent) { - const now = Date.now(); const updates: lib.SaveDetails[] = []; + const deletes: lib.SaveDetails[] = []; for (const save of event.updates) { const existingSave = this._controller.saves.get(save.id); if (existingSave && save.equals(existingSave)) { continue; } - save.updatedAtMs = now; - this._controller.saves.set(save.id, save); updates.push(save); } if (event.instanceId !== undefined) { const updatedSaves = new Set(event.updates.map(s => s.id)); - for (const [id, save] of this._controller.saves) { - if (save.instanceId === event.instanceId && !updatedSaves.has(id)) { - save.isDeleted = true; - save.updatedAtMs = now; - updates.push(save); - this._controller.saves.delete(id); + for (const save of this._controller.saves.values()) { + if (save.instanceId === event.instanceId && !updatedSaves.has(save.id)) { + deletes.push(save); } } } // Hosts eagerly send updates, which means we may get an update - // where nothing actualy changed. - if (updates.length) { - this._controller.savesUpdated(updates); - } + // where nothing actually changed. + this._controller.saves.setMany(updates); + this._controller.saves.deleteMany(deletes); } async handleLogMessageEvent(event: lib.LogMessageEvent) { diff --git a/packages/controller/src/InstanceInfo.ts b/packages/controller/src/InstanceInfo.ts index ab664511..f04c1a3f 100644 --- a/packages/controller/src/InstanceInfo.ts +++ b/packages/controller/src/InstanceInfo.ts @@ -52,6 +52,13 @@ export default class InstanceInfo { return this.status === "deleted"; } + set isDeleted(value: boolean) { + if (value === false) { + throw new Error("Setting isDeleted to false is not supported, use status instead."); + } + this.status = "deleted"; + } + /** Shorthand for `instance.config.get("instance.id")` */ get id():number { return this.config.get("instance.id"); diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 51b25bee..068bf3e5 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -218,7 +218,7 @@ async function uploadExport(req: Request, res: Response) { res.sendStatus(400); return; } - let modPack = res.app.locals.controller.modPacks.get(modPackId); + let modPack = res.app.locals.controller.modPacks.getMutable(modPackId); if (!modPack) { res.sendStatus(400); return; @@ -261,7 +261,7 @@ async function uploadExport(req: Request, res: Response) { modPack.exportManifest = new lib.ExportManifest(assets); modPack.fillDefaultSettings(settingPrototypes, logger); - res.app.locals.controller.modPacksUpdated([modPack]); + res.app.locals.controller.modPacks.set(modPack); res.sendStatus(200); } diff --git a/packages/lib/index.ts b/packages/lib/index.ts index 3117bcbc..bec088ad 100644 --- a/packages/lib/index.ts +++ b/packages/lib/index.ts @@ -28,6 +28,7 @@ export * from "./src/stream"; export * from "./src/system_collectors"; export * from "./src/zip_ops"; export * from "./src/subscriptions"; +export * from "./src/datastore"; export { default as ExponentialBackoff } from "./src/ExponentialBackoff"; export { default as ModStore } from "./src/ModStore"; diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts new file mode 100644 index 00000000..42bc4303 --- /dev/null +++ b/packages/lib/src/datastore.ts @@ -0,0 +1,275 @@ +import fs from "fs-extra"; + +import { safeOutputFile } from "./file_ops"; +import { SubscribableValue } from "./subscriptions"; +import { EventEmitter } from "stream"; +import { ControllerConfig } from "./config"; +import path from "path"; + +type DatastoreKey = string | number; +type DatastoreValue = string | number | boolean | object; + +// Abstract class which can provide saving and loading capabilities to a datastore +export abstract class DatastoreProvider< + K extends DatastoreKey, + V extends DatastoreValue, +> { + abstract save(data: Map): Promise + abstract load(): Promise> + + // Load data from the source and return it, this should be passed to the constructor of a datastore + async bootstrap() { + return [this, await this.load()] as const; + } +} + +// Store all data in memory, used as a default and should be avoided +export class MemoryDatastoreProvider< + K extends DatastoreKey, + V extends DatastoreValue, +> extends DatastoreProvider { + constructor( + private value: Map = new Map() + ) { + super(); + } + + async save(data: Map) { + this.value = new Map(data); + } + + async load() { + return new Map(this.value); + } +} + +// Store all data within a json object file and validate the result against a schema +export class JsonDatastoreProvider< + K extends DatastoreKey, + V extends DatastoreValue, + J, // Intermediate type returned from migrations +> extends DatastoreProvider { + constructor( + private filePath: string, + private fromJson: (json: J) => V = v => v as any, + private migrations: (rawJson: Record) => Record = v => v as any, + private finalise: (obj: V) => V = v => v, + ) { + super(); + } + + // Save the data to the json file + async save(data: Map) { + await safeOutputFile(this.filePath, JSON.stringify(Object.fromEntries(data), null, "\t")); + } + + // Load the data from the json file + async load() { + // Read the raw json from the file + let rawJson; + try { + rawJson = JSON.parse(await fs.readFile(this.filePath, { encoding: "utf8" })); + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + return new Map(); + } + + // Apply migrations + const serialized = this.migrations(rawJson); + + // Convert to data class objects + return new Map(Object.entries(serialized).map(([k, v]) => [k, this.finalise(this.fromJson(v))])); + } +} + +// Store all data within a json array file where each element has a unique id +export class JsonIdDatastoreProvider< + K extends DatastoreKey, + V extends DatastoreValue & { id: K }, + J, // Intermediate type returned from migrations +> extends DatastoreProvider { + constructor( + private filePath: string, + private fromJson: (json: J) => V = v => v as any, + private migrations: (rawJson: Array) => Array = v => v as any, + private finalise: (obj: V) => V = v => v, + ) { + super(); + } + + // Save the data to the json file + async save(data: Map) { + await safeOutputFile(this.filePath, JSON.stringify([...data.values()], null, "\t")); + } + + // Load the data from the json file + async load() { + // Read the raw json from the file + let rawJson; + try { + rawJson = JSON.parse(await fs.readFile(this.filePath, { encoding: "utf8" })); + } catch (err: any) { + if (err.code !== "ENOENT") { + throw err; + } + return new Map(); + } + + // Apply migrations + const serialized = this.migrations(rawJson); + + // Convert to data class objects + return new Map(serialized.map((e) => { + const v = this.finalise(this.fromJson(e)); + return [v.id, v]; + })); + } +} + +// Implements a similar interface to a map +export abstract class Datastore< + K extends DatastoreKey, + V extends DatastoreValue, +> extends EventEmitter { + private dirty = false; + + constructor( + private provider: DatastoreProvider = new MemoryDatastoreProvider(), + protected data = new Map(), + ) { + super(); + } + + // Get the file path based on the controller config + static getFilePath(config: ControllerConfig, file: string) { + return path.resolve(config.get("controller.database_directory"), file); + } + + // Save the datastore to the provider + async save() { + if (this.dirty) { + await this.provider.save(this.data); + this.dirty = false; + } + } + + // Load the datastore from the provider + async load() { + this.data = await this.provider.load(); + this.dirty = false; + } + + // Set the dirty flag and call update handlers + protected touch(updates: any[]) { + if (updates.length) { + this.dirty = true; + this.emit("update", updates); + } + } + + // Returns true if the datastore has the value, false otherwise + has(key: K) { + return this.data.has(key); + } + + // Get a value from from the datastore + get(key: K) { + return this.data.get(key) as Readonly ?? undefined; + } + + // Get a mutable reference to a value in the datastore, call set after use + getMutable(key: K) { + return this.data.get(key); + } + + // Returns all values in the datastore + values() { + return this.data.values() as IterableIterator>; + } + + // Returns all values in the datastore as mutable references, call setMany after use + valuesMutable() { + return this.data.values(); + } +} + +// General key value mapping which can be saved and loaded with the file system +export class KeyValueDatastore< + K extends DatastoreKey, + V extends DatastoreValue, +> extends Datastore { + // Set the value in the datastore, be careful of race conditions if you await any functions before calling set + set(key: K, value: V) { + this.data.set(key, value); + this.touch([[key, value]]); + } + + // Set many values at once from an array of key value pairs + setMany(pairs: [K, V][]) { + for (const [k, v] of pairs) { + this.data.set(k, v); + } + this.touch(pairs); + } + + // Delete a value in the datastore + delete(key: K) { + const value = this.data.get(key); + this.data.delete(key); + this.touch([[key, value!, true]]); + } + + // Delete many values at once from an array of keys + deleteMany(keys: K[]) { + const updates = [] as [K, V, true][]; + for (const key of keys) { + const value = this.data.get(key); + updates.push([key, value!, true]); + this.data.delete(key); + } + this.touch(updates); + } +} + +// A special implementation of the datastore to work directly with subscribable values +export class SubscribableDatastore< + V extends SubscribableValue, +> extends Datastore { + // Set the value in the datastore, be careful of race conditions if you await any functions before calling set + set(value: V) { + this.data.set(value.id, value); + value.updatedAtMs = Date.now(); + this.touch([value]); + } + + // Set many values at once from an array of key value pairs + setMany(values: V[]) { + const nowMs = Date.now(); + for (const value of values) { + value.updatedAtMs = nowMs; + this.data.set(value.id, value); + } + this.touch(values); + } + + // Delete a value in the datastore + delete(value: V) { + this.data.delete(value.id); + value.updatedAtMs = Date.now(); + value.isDeleted = true; + this.touch([value]); + } + + // Delete many values at once from an array of values + deleteMany(values: V[]) { + const nowMs = Date.now(); + for (const value of values) { + value.isDeleted = true; + value.updatedAtMs = nowMs; + this.data.delete(value.id); + } + this.touch(values); + } +} diff --git a/plugins/player_auth/test/plugin.js b/plugins/player_auth/test/plugin.js index 6b90348e..3865fcdb 100644 --- a/plugins/player_auth/test/plugin.js +++ b/plugins/player_auth/test/plugin.js @@ -61,6 +61,8 @@ describe("player_auth", function() { return load; } else if (field === "factorio.settings") { return { name }; + } else if (field === "instance.id") { + return id; } throw new Error(`field ${field} not implemented`); }, diff --git a/test/file/json/invalid.json b/test/file/json/invalid.json new file mode 100644 index 00000000..004e1e20 --- /dev/null +++ b/test/file/json/invalid.json @@ -0,0 +1 @@ +{,} diff --git a/test/integration/routing.js b/test/integration/routing.js index 7c5b204c..3b423786 100644 --- a/test/integration/routing.js +++ b/test/integration/routing.js @@ -39,7 +39,7 @@ function connectInstance(controller, host, instanceId) { instanceConfig.set("instance.id", instanceId); instanceConfig.set("instance.assigned_host", host.config.get("host.id")); host.instanceInfos.set(instanceId, { path: "invalid", config: instanceConfig }); - controller.instances.set(instanceId, new InstanceInfo(instanceConfig, "stopped")); + controller.instances.set(new InstanceInfo(instanceConfig, "stopped")); const instance = new Instance(host, instanceSide, "invalid", "invalid", instanceConfig); const instanceConnection = new InstanceConnection(hostSide, host, instance); host.instanceConnections.set(instanceId, instanceConnection); diff --git a/test/lib/datastore.js b/test/lib/datastore.js new file mode 100644 index 00000000..3e0f0802 --- /dev/null +++ b/test/lib/datastore.js @@ -0,0 +1,663 @@ +"use strict"; +const events = require("events"); +const assert = require("assert").strict; +const lib = require("@clusterio/lib"); +const fs = require("fs-extra"); +const path = require("path"); + +class MockDatastoreProvider extends lib.DatastoreProvider { + constructor() { + super(); + this.methodCalls = []; + this.value = new Map(); + this.methodCalls.push(["constructor"]); + } + + async save(data) { + this.methodCalls.push(["save", data]); + } + + async load() { + this.methodCalls.push(["load"]); + return this.value; + } +} + +class MockDatastore extends lib.Datastore { + getData() { + return this.data; // Protected member + } + + callTouch(updates = [0]) { + this.touch(updates); // Protected method + } +} + +describe("lib/datastore", function() { + const baseDir = path.join("temp", "test", "datastore"); + const testFiles = path.join("test", "file"); + before(async function() { + await fs.emptyDir(baseDir); + }); + + describe("class DatastoreProvider", function() { + let datastoreProvider; + beforeEach(function() { + datastoreProvider = new MockDatastoreProvider(); + }); + + it("can be constructed", function() { + assert.deepEqual(datastoreProvider.methodCalls[0], ["constructor"]); + }); + + it("can be bootstrapped", async function() { + const rtn = await datastoreProvider.bootstrap(); + assert.equal(rtn[0], datastoreProvider); + assert.equal(rtn[1], datastoreProvider.value); + assert.deepEqual(datastoreProvider.methodCalls[1], ["load"]); + }); + }); + + describe("class MemoryDatastoreProvider", function() { + let datastoreProvider, initialData; + beforeEach(function() { + initialData = new Map([ + ["foo", "fooValue"], ["bar", "barValue"], ["baz", "bazValue"], + ]); + datastoreProvider = new lib.MemoryDatastoreProvider(initialData); + }); + + describe("constructor", function() { + it("has a default value", async function() { + const [, value] = await new lib.MemoryDatastoreProvider().bootstrap(); + assert.notEqual(value, undefined); + assert.notEqual(value, null); + }); + it("can have a value given", async function() { + const expectedValue = new Map([["foo", "bar"]]); + const [, value] = await new lib.MemoryDatastoreProvider(expectedValue).bootstrap(); + assert.deepEqual(value, expectedValue); + }); + }); + + describe("save", function() { + it("can save data", async function() { + const newData = new Map([["foo2", "fooValue2"]]); + await datastoreProvider.save(newData); + const loadedData = await datastoreProvider.load(); + assert.deepEqual(loadedData, newData); + }); + }); + + describe("load", function() { + it("can load data", async function() { + const loadedData = await datastoreProvider.load(); + assert.deepEqual(loadedData, initialData); + }); + }); + }); + + describe("class JsonDatastoreProvider", function() { + let datastoreProvider, initialData, dataPath; + before(async function() { + dataPath = path.join(baseDir, "data.json"); + initialData = Object.entries(JSON.parse(await fs.readFile( + path.join(testFiles, "json", "load_map.json"), { encoding: "utf8" } + ))); + }); + + beforeEach(async function() { + await fs.copy(path.join(testFiles, "json", "load_map.json"), dataPath); + datastoreProvider = new lib.JsonDatastoreProvider(dataPath); + }); + + describe("constructor", function() { + it("has default values", async function() { + datastoreProvider = new lib.JsonDatastoreProvider(dataPath); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData)); + }); + it("can accept a fromJson method", async function() { + datastoreProvider = new lib.JsonDatastoreProvider( + dataPath, + json => [json] + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData.map(([k, v]) => [k, [v]]))); + }); + it("can accept a migration method", async function() { + datastoreProvider = new lib.JsonDatastoreProvider( + dataPath, + json => json, // Nop + rawJson => ({ c: "foo", ...rawJson }) + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map([...initialData, ["c", "foo"]])); + }); + it("can accept a finalise method", async function() { + datastoreProvider = new lib.JsonDatastoreProvider( + dataPath, + json => json, // Nop + rawJson => rawJson, // Nop + obj => [obj] + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData.map(([k, v]) => [k, [v]]))); + }); + }); + + describe("save", function() { + it("can save data", async function() { + const newData = new Map([["foo", "bar"]]); + await datastoreProvider.save(newData); + const loadedData = await datastoreProvider.load(); + assert.deepEqual(loadedData, newData); + }); + }); + + describe("load", function() { + it("can load data", async function() { + const loadedData = await datastoreProvider.load(); + assert.deepEqual(loadedData, new Map(initialData)); + }); + it("has a default value", async function() { + datastoreProvider = new lib.JsonDatastoreProvider(path.join(baseDir, "not_exists.json")); + const loadedData = await datastoreProvider.load(); + assert.equal(loadedData.size, 0); + }); + it("throws for invalid file contents", async function() { + await fs.copy(path.join(testFiles, "json", "invalid.json"), dataPath); + await assert.rejects(() => datastoreProvider.load(dataPath)); + }); + }); + }); + + describe("class JsonIdDatastoreProvider", function() { + let datastoreProvider, initialData, dataPath; + before(async function() { + dataPath = path.join(baseDir, "data.json"); + initialData = JSON.parse(await fs.readFile( + path.join(testFiles, "json", "load_array_map.json"), { encoding: "utf8" } + )).map(v => [v.id, v]); + }); + + beforeEach(async function() { + await fs.copy(path.join(testFiles, "json", "load_array_map.json"), dataPath); + datastoreProvider = new lib.JsonIdDatastoreProvider(dataPath); + }); + + describe("constructor", function() { + it("has default values", async function() { + datastoreProvider = new lib.JsonIdDatastoreProvider(dataPath); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData)); + }); + it("can accept a fromJson method", async function() { + datastoreProvider = new lib.JsonIdDatastoreProvider( + dataPath, + json => ({...json, foo: true}) + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData.map(v => ([v[0], {...v[1], foo: true}])))); + }); + it("can accept a migration method", async function() { + datastoreProvider = new lib.JsonIdDatastoreProvider( + dataPath, + json => json, // Nop + rawJson => ([{ id: "c" }, ...rawJson]) + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map([...initialData, ["c", { id: "c" }]])); + }); + it("can accept a finalise method", async function() { + datastoreProvider = new lib.JsonIdDatastoreProvider( + dataPath, + json => json, // Nop + rawJson => rawJson, // Nop + obj => ({...obj, foo: true}) + ); + + const [, value] = await datastoreProvider.bootstrap(); + assert.deepEqual(value, new Map(initialData.map(v => ([v[0], {...v[1], foo: true}])))); + }); + }); + + describe("save", function() { + it("can save data", async function() { + const newData = new Map([["foo", { id: "foo" }]]); + await datastoreProvider.save(newData); + const loadedData = await datastoreProvider.load(); + assert.deepEqual(loadedData, newData); + }); + }); + + describe("load", function() { + it("can load data", async function() { + const loadedData = await datastoreProvider.load(); + assert.equal(loadedData.size, initialData.length); + assert.deepEqual(loadedData, new Map(initialData)); + }); + it("has a default value", async function() { + datastoreProvider = new lib.JsonIdDatastoreProvider(path.join(baseDir, "not_exists.json")); + const loadedData = await datastoreProvider.load(); + assert.equal(loadedData.size, 0); + }); + it("throws for invalid file contents", async function() { + await fs.copy(path.join(testFiles, "json", "invalid.json"), dataPath); + await assert.rejects(() => datastoreProvider.load(dataPath)); + }); + }); + }); + + describe("class Datastore", function() { + let datastoreProvider, datastore; + beforeEach(function() { + datastoreProvider = new MockDatastoreProvider(); + datastoreProvider.value = new Map([ + ["foo", "fooValue"], ["bar", "barValue"], ["baz", "bazValue"], + ]); + datastore = new MockDatastore(datastoreProvider, datastoreProvider.value); + }); + + describe("constructor", function() { + it("has a default value", function() { + datastore = new MockDatastore(datastoreProvider); + assert.deepEqual(datastore.getData(), new Map()); + }); + it("accepts a starting value", function() { + datastore = new MockDatastore(datastoreProvider, datastoreProvider.value); + assert.deepEqual(datastore.getData(), datastoreProvider.value); + }); + it("accepts the return of bootstrap", async function() { + datastore = new MockDatastore(...await datastoreProvider.bootstrap()); + assert.deepEqual(datastore.getData(), datastoreProvider.value); + }); + }); + + describe("getFilePath", function() { + it("returns the correct filepath", function() { + const mockConfig = { + get(field) { + if (field === "controller.database_directory") { + return "databaseDir"; + } + throw new Error(`field ${field} not implemented`); + }, + }; + + assert.equal( + lib.Datastore.getFilePath(mockConfig, "db1.json"), + path.resolve("databaseDir", "db1.json") + ); + assert.equal( + lib.Datastore.getFilePath(mockConfig, "db2.txt"), + path.resolve("databaseDir", "db2.txt") + ); + assert.equal( + lib.Datastore.getFilePath(mockConfig, path.join("foo", "db3.json")), + path.resolve("databaseDir", "foo", "db3.json") + ); + }); + }); + + describe("save", function() { + it("should not save when clean", async function() { + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 1); + }); + it("should save when dirty", async function() { + datastore.callTouch(); // Make the data dirty + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], ["save", datastore.getData()]); + }); + it("should not save multiple times", async function() { + datastore.callTouch(); // Make the data dirty + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], ["save", datastore.getData()]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + }); + }); + + describe("load", function() { + it("should load new data", async function() { + datastoreProvider.value = new Map(); + assert.notEqual(datastore.getData(), datastoreProvider.value); + await datastore.load(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], ["load"]); + assert.deepEqual(datastore.getData(), datastoreProvider.value); + }); + it("should not save after a load", async function() { + datastore.callTouch(); // Make the data dirty + await datastore.load(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], ["load"]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + }); + }); + + describe("touch", function() { + it("should emit update events", function() { + const eventsRaised = []; + datastore.on("update", updates => eventsRaised.push(updates)); + + let updates = ["foo"]; + datastore.callTouch(updates); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], updates); + + updates = ["foo", "bar"]; + datastore.callTouch(updates); + assert.equal(eventsRaised.length, 2); + assert.deepEqual(eventsRaised[1], updates); + }); + }); + + describe("has", function() { + it("returns true correctly", function() { + assert.equal(datastore.has("foo"), true); + assert.equal(datastore.has("bar"), true); + assert.equal(datastore.has("baz"), true); + }); + it("returns false correctly", function() { + assert.equal(datastore.has("missing1"), false); + assert.equal(datastore.has("missing2"), false); + assert.equal(datastore.has(1), false); + assert.equal(datastore.has(2), false); + }); + }); + + describe("get", function() { + it("returns the correct value", function() { + assert.equal(datastore.get("foo"), "fooValue"); + assert.equal(datastore.get("bar"), "barValue"); + assert.equal(datastore.get("baz"), "bazValue"); + }); + it("returns undefined correctly", function() { + assert.equal(datastore.get("missing1"), undefined); + assert.equal(datastore.get("missing2"), undefined); + assert.equal(datastore.get(1), undefined); + assert.equal(datastore.get(2), undefined); + }); + }); + + describe("getMutable", function() { + it("returns the correct value", function() { + assert.equal(datastore.getMutable("foo"), "fooValue"); + assert.equal(datastore.getMutable("bar"), "barValue"); + assert.equal(datastore.getMutable("baz"), "bazValue"); + }); + it("returns undefined correctly", function() { + assert.equal(datastore.getMutable("missing1"), undefined); + assert.equal(datastore.getMutable("missing2"), undefined); + assert.equal(datastore.getMutable(1), undefined); + assert.equal(datastore.getMutable(2), undefined); + }); + }); + + describe("values", function() { + it("returns all values", function() { + assert.deepEqual([...datastore.values()], [...datastoreProvider.value.values()]); + }); + }); + + describe("valuesMutable", function() { + it("returns all values", function() { + assert.deepEqual([...datastore.valuesMutable()], [...datastoreProvider.value.values()]); + }); + }); + }); + + describe("class KeyValueDatastore", function() { + let datastoreProvider, datastore, eventsRaised; + beforeEach(function() { + eventsRaised = []; + datastoreProvider = new MockDatastoreProvider(); + datastoreProvider.value = new Map([ + ["foo", "fooValue"], ["bar", "barValue"], ["baz", "bazValue"], + ]); + datastore = new lib.KeyValueDatastore(datastoreProvider, datastoreProvider.value); + datastore.on("update", updates => eventsRaised.push(updates)); + }); + + describe("set", function() { + it("can set a value", function() { + datastore.set("test", "testValue"); + assert.equal(datastore.get("test"), "testValue"); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [["test", "testValue"]]); + }); + it("allows saving after a value is set", async function() { + datastore.set("test", "testValue"); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries(), ["test", "testValue"]])] + ); + }); + }); + + describe("setMany", function() { + it("does not save with zero changes", async function() { + datastore.setMany([]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 1); + }); + it("can set many values at once", async function() { + const updates = [["test1", "testValue1"], ["test2", "testValue2"]]; + datastore.setMany(updates); + assert.equal(datastore.get("test1"), "testValue1"); + assert.equal(datastore.get("test2"), "testValue2"); + await datastore.save(); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], updates); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries(), ...updates])] + ); + }); + }); + + describe("delete", function() { + it("can delete a value", function() { + datastore.delete("foo"); + assert.equal(datastore.get("foo"), undefined); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [["foo", "fooValue", true]]); + }); + it("allows saving after a value is deleted", async function() { + datastore.delete("foo"); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => k !== "foo"))] + ); + }); + it("does not break when the key does not exist", function() { + datastore.delete("test"); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [["test", undefined, true]]); + }); + }); + + describe("deleteMany", function() { + it("does not save with zero changes", async function() { + datastore.deleteMany([]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 1); + }); + it("can delete many values at once", async function() { + const keys = ["foo", "bar"]; + datastore.deleteMany(keys); + assert.equal(datastore.get("foo"), undefined); + assert.equal(datastore.get("bar"), undefined); + await datastore.save(); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [["foo", "fooValue", true], ["bar", "barValue", true]]); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => !keys.includes(k)))] + ); + }); + }); + }); + + describe("class SubscribableDatastore", function() { + let datastoreProvider, datastore, eventsRaised; + beforeEach(function() { + eventsRaised = []; + datastoreProvider = new MockDatastoreProvider(); + datastoreProvider.value = new Map([ + ["foo", {id: "foo"}], ["bar", {id: "bar"}], ["baz", {id: "baz"}], + ]); + datastore = new lib.SubscribableDatastore(datastoreProvider, datastoreProvider.value); + datastore.on("update", updates => eventsRaised.push(updates)); + }); + + describe("set", function() { + it("can set a value", function() { + const value = { id: "test" }; + datastore.set(value); + assert.equal(datastore.get(value.id), value); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [value]); + }); + it("allows saving after a value is set", async function() { + const value = { id: "test" }; + datastore.set(value); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries(), [value.id, value]])] + ); + }); + it("sets the updatedAtMs property", async function() { + const value = { id: "test" }; + datastore.set(value); + assert.notEqual(value.updatedAtMs, undefined); + assert(value.updatedAtMs > 0, "updatedAtMs greater than zero"); + const prev = value.updatedAtMs; + await lib.wait(); + datastore.set(value); + assert(value.updatedAtMs > prev, "updatedAtMs must increase"); + }); + }); + + describe("setMany", function() { + it("does not save with zero changes", async function() { + datastore.setMany([]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 1); + }); + it("can set many values at once", async function() { + const updates = [{ id: "test1" }, { id: "test2" }]; + datastore.setMany(updates); + assert.equal(datastore.get(updates[0].id), updates[0]); + assert.equal(datastore.get(updates[1].id), updates[1]); + await datastore.save(); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], updates); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries(), ...updates.map(v => [v.id, v])])] + ); + }); + it("sets the updatedAtMs property", async function() { + const updates = [{ id: "test1" }, { id: "test2" }]; + datastore.setMany(updates); + assert.notEqual(updates[0].updatedAtMs, undefined); + assert.notEqual(updates[1].updatedAtMs, undefined); + assert(updates[0].updatedAtMs > 0, "updatedAtMs greater than zero"); + assert(updates[1].updatedAtMs > 0, "updatedAtMs greater than zero"); + const prev = [updates[0].updatedAtMs, updates[1].updatedAtMs]; + await lib.wait(); + datastore.setMany(updates); + assert(updates[0].updatedAtMs > prev[0], "updatedAtMs must increase"); + assert(updates[1].updatedAtMs > prev[1], "updatedAtMs must increase"); + }); + }); + + describe("delete", function() { + it("can delete a value", function() { + const value = datastore.get("foo"); + datastore.delete(value); + assert.equal(datastore.get(value.id), undefined); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [value]); + }); + it("allows saving after a value is deleted", async function() { + const value = datastore.get("foo"); + datastore.delete(value); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => k !== value.id))] + ); + }); + it("does not break when the key does not exist", function() { + const value = { id: "test" }; + datastore.delete(value); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], [value]); + }); + it("sets the updatedAtMs property", function() { + const value = { id: "test" }; + datastore.delete(value); + assert.notEqual(value.updatedAtMs, undefined); + assert(value.updatedAtMs > 0, "updatedAtMs greater than zero"); + }); + it("sets the isDeleted property", function() { + const value = { id: "test" }; + datastore.delete(value); + assert.equal(value.isDeleted, true); + }); + }); + + describe("deleteMany", function() { + it("does not save with zero changes", async function() { + datastore.deleteMany([]); + await datastore.save(); + assert.equal(datastoreProvider.methodCalls.length, 1); + }); + it("can delete many values at once", async function() { + const values = [{id: "foo"}, {id: "bar"}]; + datastore.deleteMany(values); + assert.equal(datastore.get(values[0].id), undefined); + assert.equal(datastore.get(values[1].id), undefined); + await datastore.save(); + assert.equal(eventsRaised.length, 1); + assert.deepEqual(eventsRaised[0], values); + assert.equal(datastoreProvider.methodCalls.length, 2); + assert.deepEqual(datastoreProvider.methodCalls[1], + ["save", new Map([...datastoreProvider.value.entries()] + .filter((_, k) => !values.some(v => v.id === k)))] + ); + }); + it("sets the updatedAtMs property", function() { + const values = [{id: "foo"}, {id: "bar"}]; + datastore.deleteMany(values); + assert.notEqual(values[0].updatedAtMs, undefined); + assert.notEqual(values[1].updatedAtMs, undefined); + assert(values[0].updatedAtMs > 0, "updatedAtMs greater than zero"); + assert(values[1].updatedAtMs > 0, "updatedAtMs greater than zero"); + }); + it("sets the isDeleted property", function() { + const values = [{id: "foo"}, {id: "bar"}]; + datastore.deleteMany(values); + assert.equal(values[0].isDeleted, true); + assert.equal(values[1].isDeleted, true); + }); + }); + }); +}); diff --git a/test/mock.js b/test/mock.js index 365b3000..d1c6b30d 100644 --- a/test/mock.js +++ b/test/mock.js @@ -183,8 +183,8 @@ class MockController { ["test", ControllerUser.fromJSON({ name: "test", roles: [0, 1] }, this.userManager)], ["player", ControllerUser.fromJSON({ name: "player", roles: [1] }, this.userManager)], ]); - this.instances = new Map(); - this.hosts = new Map(); + this.instances = new lib.KeyValueDatastore(); + this.hosts = new lib.KeyValueDatastore(); this.handles = new Map(); }