Merge pull request #629 from Cooldude2606/feature/datastore

Impliment datastores to reduce boilerplate

CHANGELOG.md

@@ -27,6 +27,7 @@ Many thanks to the following for contributing to this release:
 
 ### Features
 
+- Added Datastore and Datastore Provider classes to the library to support savable Maps [#629](https://github.com/clusterio/clusterio/pull/629).
 - Added clean script to speed up publishing and debugging of build steps [#640](https://github.com/clusterio/clusterio/pull/640).
 - Added sub-tile option on plugin page headers [#642](https://github.com/clusterio/clusterio/pull/642).
 - Greatly improved plugin and module template generation [#641](https://github.com/clusterio/clusterio/pull/641).

packages/controller/controller.ts

@@ -66,9 +66,9 @@ void new lib.Gauge(
 			if (!controller || !controller.hosts) {
 				return;
 			}
-			for (let [id, host] of controller.hosts) {
+			for (const host of controller.hosts.values()) {
 				gauge.labels({
-					host_id: String(id),
+					host_id: String(host.id),
 					host_name: host.name,
 				}).set(1);
 			};
@@ -86,9 +86,9 @@ void new lib.Gauge(
 			if (!controller || !controller.instances) {
 				return;
 			}
-			for (let [id, instance] of controller.instances) {
+			for (const instance of controller.instances.values()) {
 				gauge.labels({
-					instance_id: String(id),
+					instance_id: String(instance.id),
 					instance_name: String(instance.config.get("instance.name")),
 					host_id: String(instance.config.get("instance.assigned_host")),
 				}).set(1);

packages/controller/src/ControlConnection.ts

@@ -195,7 +195,7 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleHostRevokeTokensRequest(request: lib.HostRevokeTokensRequest) {
-		const host = this._controller.hosts.get(request.hostId);
+		const host = this._controller.hosts.getMutable(request.hostId);
 		if (!host) {
 			throw new Error(`Unknown host id (${request.hostId})`);
 		}
@@ -207,7 +207,7 @@ export default class ControlConnection extends BaseConnection {
 			hostConnection.connector.terminate();
 		}
 
-		this._controller.hostsUpdated([host]);
+		this._controller.hosts.set(host);
 	}
 
 	async handleHostListRequest(): Promise<lib.HostDetails[]> {
@@ -397,8 +397,7 @@ export default class ControlConnection extends BaseConnection {
 		if (this._controller.modPacks.has(modPack.id)) {
 			throw new lib.RequestError(`Mod pack with ID ${modPack.id} already exist`);
 		}
-		this._controller.modPacks.set(modPack.id, modPack);
-		this._controller.modPacksUpdated([modPack]);
+		this._controller.modPacks.set(modPack);
 	}
 
 	async handleModPackUpdateRequest(request: lib.ModPackUpdateRequest) {
@@ -406,19 +405,16 @@ export default class ControlConnection extends BaseConnection {
 		if (modPack.id === undefined || !this._controller.modPacks.has(modPack.id)) {
 			throw new lib.RequestError(`Mod pack with ID ${modPack.id} does not exist`);
 		}
-		this._controller.modPacks.set(modPack.id, modPack);
-		this._controller.modPacksUpdated([modPack]);
+		this._controller.modPacks.set(modPack);
 	}
 
 	async handleModPackDeleteRequest(request: lib.ModPackDeleteRequest) {
 		let { id } = request;
-		let modPack = this._controller.modPacks.get(id);
+		let modPack = this._controller.modPacks.getMutable(id);
 		if (!modPack) {
 			throw new lib.RequestError(`Mod pack with ID ${id} does not exist`);
 		}
-		modPack.isDeleted = true;
-		this._controller.modPacks.delete(id);
-		this._controller.modPacksUpdated([modPack]);
+		this._controller.modPacks.delete(modPack);
 	}
 
 	async handleModGetRequest(request: lib.ModGetRequest) {

packages/controller/src/Controller.ts

@@ -55,17 +55,6 @@ export default class Controller {
 	config: lib.ControllerConfig;
 	app: Application;
 
-	/** True if (@link Controller.systems} has changed since last time it was saved */
-	systemsDirty = false;
-	/** True if {@link Controller.hosts} has changed since last time it was saved */
-	hostsDirty = false;
-	/** True if {@link Controller.instances} has changed since last time it was saved */
-	instancesDirty = false;
-	/** True if {@link Controller.saves} has changed since last time it was saved */
-	savesDirty = false;
-	/** True if {@link Controller.modPacks} has changed since last time it was saved */
-	modPacksDirty = false;
-
 	httpServer: http.Server | null = null;
 	httpServerCloser: HttpCloser | null = null;
 	httpsServer: https.Server | null = null;
@@ -111,11 +100,33 @@ export default class Controller {
 		let databaseDirectory = config.get("controller.database_directory");
 		await fs.ensureDir(databaseDirectory);
 
-		const systems = await Controller.loadSystems(path.join(databaseDirectory, "systems.json"));
-		const hosts = await Controller.loadHosts(path.join(databaseDirectory, "hosts.json"));
-		const instances = await Controller.loadInstances(path.join(databaseDirectory, "instances.json"));
-		const saves = await Controller.loadSaves(path.join(databaseDirectory, "saves.json"));
-		const modPacks = await Controller.loadModPacks(path.join(databaseDirectory, "mod-packs.json"));
+		const systems = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "systems.json"),
+			lib.SystemInfo.fromJSON
+		).bootstrap());
+
+		const hosts = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "hosts.json"),
+			HostInfo.fromJSON,
+			this.migrateHosts, this.finaliseHosts,
+		).bootstrap());
+
+		const instances = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "instances.json"),
+			json => InstanceInfo.fromJSON(json, "controller"),
+			this.migrateInstances, this.finaliseInstances,
+		).bootstrap());
+
+		const saves = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "saves.json"),
+			lib.SaveDetails.fromJSON
+		).bootstrap());
+
+		const modPacks = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "mod-packs.json"),
+			lib.ModPack.fromJSON
+		).bootstrap());
+
 		const userManager = new UserManager(config);
 		await userManager.load(path.join(databaseDirectory, "users.json"));
 
@@ -145,15 +156,15 @@ export default class Controller {
 		 * will restart the controller on non-zero exit codes.
 		 */
 		public canRestart: boolean = false,
-		public systems = new Map<lib.SystemInfo["id"], lib.SystemInfo>(),
+		public systems = new lib.SubscribableDatastore<lib.SystemInfo>(),
 		/** Mapping of host id to host info */
-		public hosts = new Map<number, HostInfo>(),
+		public hosts = new lib.SubscribableDatastore<HostInfo>(),
 		/** Mapping of instance id to instance info */
-		public instances = new Map<number, InstanceInfo>(),
+		public instances = new lib.SubscribableDatastore<InstanceInfo>(),
 		/** Mapping of save id to save details */
-		public saves = new Map<string, lib.SaveDetails>(),
+		public saves = new lib.SubscribableDatastore<lib.SaveDetails>(),
 		/** Mapping of mod pack id to mod pack */
-		public modPacks = new Map<number, lib.ModPack>(),
+		public modPacks = new lib.SubscribableDatastore<lib.ModPack>(),
 		/** Mods stored on the controller */
 		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
 		/** User and roles manager for the cluster */
@@ -185,6 +196,13 @@ export default class Controller {
 		this.subscriptions.handle(lib.ModPackUpdatesEvent, this.handleModPackSubscription.bind(this));
 		this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this));
 		this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this));
+
+		// Handle updates for datastores
+		this.systems.on("update", this.systemsUpdated.bind(this));
+		this.hosts.on("update", this.hostsUpdated.bind(this));
+		this.instances.on("update", this.instanceDetailsUpdated.bind(this));
+		this.saves.on("update", this.savesUpdated.bind(this));
+		this.modPacks.on("update", this.modPacksUpdated.bind(this));
 	}
 
 	async start(args: ControllerArgs) {
@@ -501,10 +519,8 @@ export default class Controller {
 			requests.push(lib.gatherSystemInfo("controller", this.canRestart));
 			const newMetrics = await Promise.all(requests);
 			for (const metric of newMetrics) {
-				this.systems.set(metric.id, metric);
+				this.systems.set(metric);
 			}
-			this.systemsDirty = true;
-			this.subscriptions.broadcast(new lib.SystemInfoUpdateEvent(newMetrics));
 		} catch (err: any) {
 			logger.error(`Unexpected error updating system infos:\n${err.stack ?? err.message}`);
 		}
@@ -545,160 +561,57 @@ export default class Controller {
 			await lib.safeOutputFile(this.configPath, JSON.stringify(this.config, null, "\t"));
 		}
 
-		let databaseDirectory = this.config.get("controller.database_directory");
-		if (this.systemsDirty) {
-			this.systemsDirty = false;
-			await Controller.saveSystems(path.join(databaseDirectory, "systems.json"), this.systems);
-		}
-
-		if (this.hostsDirty) {
-			this.hostsDirty = false;
-			await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts);
-		}
-
-		if (this.instancesDirty) {
-			this.instancesDirty = false;
-			await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances);
-		}
-
-		if (this.savesDirty) {
-			this.savesDirty = false;
-			await Controller.saveSaves(path.join(databaseDirectory, "saves.json"), this.saves);
-		}
-
-		if (this.modPacksDirty) {
-			this.modPacksDirty = false;
-			await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks);
-		}
+		await Promise.all([
+			this.systems.save(),
+			this.hosts.save(),
+			this.instances.save(),
+			this.saves.save(),
+			this.modPacks.save(),
+		]);
 
 		if (this.userManager.dirty) {
-			await this.userManager.save(path.join(databaseDirectory, "users.json"));
+			await this.userManager.save(path.join(this.config.get("controller.database_directory"), "users.json"));
 		}
 
 		await lib.invokeHook(this.plugins, "onSaveData");
 	}
 
-	static async loadSystems(filePath: string): Promise<Map<lib.SystemInfo["id"], lib.SystemInfo>> {
-		let json: Static<typeof lib.SystemInfo.jsonSchema>[];
-		try {
-			json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-			return new Map();
-		}
-
-		return new Map(json.map(m => lib.SystemInfo.fromJSON(m)).map(m => [m.id, m]));
-	}
-
-	static async saveSystems(filePath: string, systems: Map<lib.SystemInfo["id"], lib.SystemInfo>) {
-		await lib.safeOutputFile(filePath, JSON.stringify([...systems.values()], null, "\t"));
-	}
-
-	static async loadHosts(filePath: string): Promise<Map<number, HostInfo>> {
-		let serialized: [number, Static<typeof HostInfo.jsonSchema>][];
-		try {
-			serialized = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-			return new Map();
-		}
+	static migrateHosts(rawJson: unknown[]): Static<typeof HostInfo.jsonSchema>[] {
+		const serialized = rawJson as Static<typeof HostInfo.jsonSchema>[];
 
 		// TODO: migrate remove after release.
 		if (serialized.length && !(serialized[0] instanceof Array)) {
-			return new Map(); // Discard old format.
+			return []; // Discard old format.
 		}
 
-		const hosts = new Map();
-		for (let [id, json] of serialized) {
-			const host = HostInfo.fromJSON(json);
-			if (host.connected) {
-				host.connected = false;
-				host.updatedAtMs = Date.now();
-			}
-			hosts.set(id, host);
+		return serialized;
+	}
+
+	static finaliseHosts(host: HostInfo): HostInfo {
+		if (host.connected) {
+			host.connected = false;
+			host.updatedAtMs = Date.now();
 		}
-		return hosts;
+		return host;
 	}
 
-	static async saveHosts(filePath: string, hosts: Map<number, HostInfo>) {
-		await lib.safeOutputFile(filePath, JSON.stringify([...hosts.entries()], null, "\t"));
+	static migrateInstances(rawJson: unknown[]): Static<typeof InstanceInfo.jsonSchema>[] {
+		const serialized = rawJson as Static<typeof InstanceInfo.jsonSchema>[];
+		return serialized.map(json => {
+			if (!json.config) { // migrate: from pre Alpha 14 format.
+				return { config: json as any, status: "running" }; // Use running to force updatedAtMs
+			}
+			return json;
+		});
 	}
 
-	static async loadInstances(filePath: string): Promise<Map<number, InstanceInfo>> {
-		logger.info(`Loading ${filePath}`);
-
-		let instances = new Map();
-		try {
-			let serialized = JSON.parse(
-				await fs.readFile(filePath, { encoding: "utf8" })
-			) as Static<typeof InstanceInfo.jsonSchema>[];
-			for (let json of serialized) {
-				if (!json.config) { // migrate: from pre Alpha 14 format.
-					json = { config: json as any, status: "running" }; // Use running to force updatedAtMs
-				}
-				const instance = InstanceInfo.fromJSON(json, "controller");
-				const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown";
-				if (instance.status !== status) {
-					instance.status = status;
-					instance.updatedAtMs = Date.now();
-				}
-				instances.set(instance.id, instance);
-			}
-
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-			return new Map();
+	static finaliseInstances(instance: InstanceInfo): InstanceInfo {
+		const status = instance.config.get("instance.assigned_host") === null ? "unassigned" : "unknown";
+		if (instance.status !== status) {
+			instance.status = status;
+			instance.updatedAtMs = Date.now();
 		}
-
-		return instances;
-	}
-
-	static async saveInstances(filePath: string, instances: Map<number, InstanceInfo>) {
-		let serialized = [...instances.values()];
-		await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t"));
-	}
-
-	static async loadSaves(filePath: string): Promise<Map<string, lib.SaveDetails>> {
-		let json: Static<typeof lib.SaveDetails.jsonSchema>[];
-		try {
-			json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-			return new Map();
-		}
-		return new Map(json.map(s => lib.SaveDetails.fromJSON(s)).map(s => [s.id, s]));
-	}
-
-	static async saveSaves(filePath: string, saves: Map<string, lib.SaveDetails>) {
-		await lib.safeOutputFile(filePath, JSON.stringify([...saves.values()], null, "\t"));
-	}
-
-
-	static async loadModPacks(filePath: string): Promise<Map<number, lib.ModPack>> {
-		let json;
-		try {
-			json = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-			return new Map();
-		}
-		return new Map(json.map((e: any) => [e.id, lib.ModPack.fromJSON(e)]));
-	}
-
-	static async saveModPacks(filePath: string, modPacks: Map<number, lib.ModPack>) {
-		await lib.safeOutputFile(filePath, JSON.stringify([...modPacks.values()], null, "\t"));
+		return instance;
 	}
 
 	static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
@@ -795,6 +708,10 @@ export default class Controller {
 		}
 	}
 
+	systemsUpdated(systems: lib.SystemInfo[]) {
+		this.subscriptions.broadcast(new lib.SystemInfoUpdateEvent(systems));
+	}
+
 	async handleSystemInfoSubscription(request: lib.SubscriptionRequest) {
 		const systems = [...this.systems.values()].filter(
 			metric => metric.updatedAtMs > request.lastRequestTimeMs,
@@ -816,11 +733,6 @@ export default class Controller {
 	}
 
 	hostsUpdated(hosts: HostInfo[]) {
-		const now = Date.now();
-		for (const host of hosts) {
-			host.updatedAtMs = now;
-		}
-		this.hostsDirty = true;
 		let updates = hosts.map(host => host.toHostDetails());
 		this.subscriptions.broadcast(new lib.HostUpdatesEvent(updates));
 	}
@@ -897,11 +809,9 @@ export default class Controller {
 		instanceConfig.set("factorio.settings", settings);
 
 		let instance = new InstanceInfo(instanceConfig, "unassigned", undefined, undefined, Date.now());
-		this.instances.set(instanceId, instance);
-		this.instancesDirty = true;
+		this.instances.set(instance);
 		await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance);
 		this.addInstanceHooks(instance);
-		this.instanceDetailsUpdated([instance]);
 		return instance;
 	}
 
@@ -911,19 +821,7 @@ export default class Controller {
 	 * @internal
 	 */
 	clearSavesOfInstance(instanceId: number) {
-		const updates = [];
-		const now = Date.now();
-		for (const [id, save] of this.saves) {
-			if (save.instanceId === instanceId) {
-				save.isDeleted = true;
-				save.updatedAtMs = now;
-				updates.push(save);
-				this.saves.delete(id);
-			}
-		}
-		if (updates.length) {
-			this.savesUpdated(updates);
-		}
+		this.saves.deleteMany([...this.saves.values()].filter(save => save.instanceId === instanceId));
 	}
 
 	/**
@@ -970,7 +868,6 @@ export default class Controller {
 
 		// Assign to target
 		instance.config.set("instance.assigned_host", hostId ?? null);
-		// "fieldChanged" event handler will set this.instancesDirty
 		if (hostId !== undefined && newHostConnection) {
 			await newHostConnection.send(
 				new lib.InstanceAssignInternalRequest(instanceId, instance.config.toRemote("host"))
@@ -978,8 +875,7 @@ export default class Controller {
 		} else {
 			instance.status = "unassigned";
 		}
-		instance.updatedAtMs = Date.now();
-		this.instanceDetailsUpdated([instance]);
+		this.instances.set(instance);
 	}
 
 	/**
@@ -996,16 +892,13 @@ export default class Controller {
 		if (hostId !== null) {
 			await this.sendTo({ hostId }, new lib.InstanceDeleteInternalRequest(instanceId));
 		}
-		this.instances.delete(instanceId);
-		this.instancesDirty = true;
+
+		let prev = instance.status;
+		this.instances.delete(instance);
 
 		this.clearSavesOfInstance(instanceId);
 		this.userManager.clearStatsOfInstance(instanceId);
 
-		let prev = instance.status;
-		instance.status = "deleted";
-		instance.updatedAtMs = Date.now();
-		this.instanceDetailsUpdated([instance]);
 		await lib.invokeHook(this.plugins, "onInstanceStatusChanged", instance, prev);
 	}
 
@@ -1034,10 +927,9 @@ export default class Controller {
 		instance.config.on("fieldChanged", (field: string, curr: any, prev: any) => {
 			instance.updatedAtMs = Date.now();
 			if (field === "instance.name") {
-				this.instanceDetailsUpdated([instance]);
+				this.instances.set(instance); // Trigger updated logic
 			}
 
-			this.instancesDirty = true;
 			lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, field, curr, prev);
 		});
 	}
@@ -1060,7 +952,6 @@ export default class Controller {
 	}
 
 	savesUpdated(saves: lib.SaveDetails[]) {
-		this.savesDirty = true;
 		this.subscriptions.broadcast(new lib.InstanceSaveDetailsUpdatesEvent(saves));
 	}
 
@@ -1071,11 +962,6 @@ export default class Controller {
 
 
 	modPacksUpdated(modPacks: lib.ModPack[]) {
-		const now = Date.now();
-		for (const modPack of modPacks) {
-			modPack.updatedAtMs = now;
-		}
-		this.modPacksDirty = true;
 		this.subscriptions.broadcast(new lib.ModPackUpdatesEvent(modPacks));
 		lib.invokeHook(this.plugins, "onModPacksUpdated", modPacks);
 	}

packages/controller/src/HostConnection.ts

@@ -42,8 +42,7 @@ export default class HostConnection extends BaseConnection {
 			0,
 			false,
 		);
-		this._controller.hosts.set(this.id, this.info);
-		this._controller.hostsUpdated([this.info]);
+		this._controller.hosts.set(this.info);
 
 		this._checkPluginVersions();
 
@@ -57,10 +56,9 @@ export default class HostConnection extends BaseConnection {
 		}
 
 		this.connector.on("close", () => {
-			const now = Date.now();
 			// Update status to unknown for instances on this host.
 			const instances: InstanceInfo[] = [];
-			for (let instance of this._controller.instances.values()) {
+			for (let instance of this._controller.instances.valuesMutable()) {
 				if (instance.config.get("instance.assigned_host") !== this.id) {
 					continue;
 				}
@@ -68,13 +66,12 @@ export default class HostConnection extends BaseConnection {
 				instances.push(instance);
 				let prev = instance.status;
 				instance.status = "unknown";
-				instance.updatedAtMs = now;
 				lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev);
 			}
-			this._controller.instanceDetailsUpdated(instances);
+			this._controller.instances.setMany(instances);
 
 			this.info.connected = false;
-			this._controller.hostsUpdated([this.info]);
+			this._controller.hosts.set(this.info);
 		});
 
 		this.handle(lib.HostInfoUpdateEvent, this.handleHostInfoUpdateEvent.bind(this));
@@ -158,11 +155,11 @@ export default class HostConnection extends BaseConnection {
 	async handleHostInfoUpdateEvent(event: lib.HostInfoUpdateEvent) {
 		this.info.name = event.update.name;
 		this.info.publicAddress = event.update.publicAddress;
-		this._controller.hostsUpdated([this.info]);
+		this._controller.hosts.set(this.info);
 	}
 
 	async handleInstanceStatusChangedEvent(request: lib.InstanceStatusChangedEvent) {
-		let instance = this._controller.instances.get(request.instanceId);
+		let instance = this._controller.instances.getMutable(request.instanceId);
 
 		// It's possible to get updates from an instance that does not exist
 		// or is not assigned to the host it originated from if it was
@@ -189,9 +186,8 @@ export default class HostConnection extends BaseConnection {
 		instance.status = request.status;
 		instance.gamePort = request.gamePort;
 		instance.factorioVersion = request.factorioVersion;
-		instance.updatedAtMs = Date.now();
+		this._controller.instances.set(instance);
 		logger.verbose(`Instance ${instance.config.get("instance.name")} State: ${instance.status}`);
-		this._controller.instanceDetailsUpdated([instance]);
 		await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", instance, prev);
 	}
 
@@ -211,7 +207,7 @@ export default class HostConnection extends BaseConnection {
 			const instanceConfig = new lib.InstanceConfig("controller");
 			instanceConfig.update(instanceData.config, false, "host");
 
-			let controllerInstance = this._controller.instances.get(instanceConfig.get("instance.id"));
+			let controllerInstance = this._controller.instances.getMutable(instanceConfig.get("instance.id"));
 			if (controllerInstance) {
 				// Check if this instance is assigned somewhere else.
 				if (controllerInstance.config.get("instance.assigned_host") !== this.id) {
@@ -227,9 +223,8 @@ export default class HostConnection extends BaseConnection {
 					controllerInstance.status = instanceData.status;
 					controllerInstance.gamePort = instanceData.gamePort;
 					controllerInstance.factorioVersion = instanceData.factorioVersion;
-					controllerInstance.updatedAtMs = Date.now();
-					logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`);
 					instanceUpdates.push(controllerInstance);
+					logger.verbose(`Instance ${instanceConfig.get("instance.name")} State: ${instanceData.status}`);
 					await lib.invokeHook(
 						this._controller.plugins, "onInstanceStatusChanged", controllerInstance, prev
 					);
@@ -245,21 +240,17 @@ export default class HostConnection extends BaseConnection {
 				instanceData.factorioVersion,
 				Date.now(),
 			);
-			this._controller.instances.set(instanceConfig.get("instance.id"), newInstance);
-			this._controller.instancesDirty = true;
+			instanceUpdates.push(newInstance);
 			this._controller.addInstanceHooks(newInstance);
 			await this.send(
 				new lib.InstanceAssignInternalRequest(
 					instanceConfig.get("instance.id"), instanceConfig.toRemote("host")
 				)
 			);
-			instanceUpdates.push(newInstance);
 			await lib.invokeHook(this._controller.plugins, "onInstanceStatusChanged", newInstance);
 		}
 
-		if (instanceUpdates.length) {
-			this._controller.instanceDetailsUpdated(instanceUpdates);
-		}
+		this._controller.instances.setMany(instanceUpdates);
 
 		// Push lists to make sure they are in sync.
 		let adminlist: Set<string> = new Set();
@@ -282,33 +273,27 @@ export default class HostConnection extends BaseConnection {
 	}
 
 	async handleInstanceSaveDetailsUpdatesEvent(event: lib.InstanceSaveDetailsUpdatesEvent) {
-		const now = Date.now();
 		const updates: lib.SaveDetails[] = [];
+		const deletes: lib.SaveDetails[] = [];
 		for (const save of event.updates) {
 			const existingSave = this._controller.saves.get(save.id);
 			if (existingSave && save.equals(existingSave)) {
 				continue;
 			}
-			save.updatedAtMs = now;
-			this._controller.saves.set(save.id, save);
 			updates.push(save);
 		}
 		if (event.instanceId !== undefined) {
 			const updatedSaves = new Set(event.updates.map(s => s.id));
-			for (const [id, save] of this._controller.saves) {
-				if (save.instanceId === event.instanceId && !updatedSaves.has(id)) {
-					save.isDeleted = true;
-					save.updatedAtMs = now;
-					updates.push(save);
-					this._controller.saves.delete(id);
+			for (const save of this._controller.saves.values()) {
+				if (save.instanceId === event.instanceId && !updatedSaves.has(save.id)) {
+					deletes.push(save);
 				}
 			}
 		}
 		// Hosts eagerly send updates, which means we may get an update
-		// where nothing actualy changed.
-		if (updates.length) {
-			this._controller.savesUpdated(updates);
-		}
+		// where nothing actually changed.
+		this._controller.saves.setMany(updates);
+		this._controller.saves.deleteMany(deletes);
 	}
 
 	async handleLogMessageEvent(event: lib.LogMessageEvent) {

packages/controller/src/InstanceInfo.ts

@@ -52,6 +52,13 @@ export default class InstanceInfo {
 		return this.status === "deleted";
 	}
 
+	set isDeleted(value: boolean) {
+		if (value === false) {
+			throw new Error("Setting isDeleted to false is not supported, use status instead.");
+		}
+		this.status = "deleted";
+	}
+
 	/** Shorthand for `instance.config.get("instance.id")` */
 	get id():number {
 		return this.config.get("instance.id");

packages/controller/src/routes.ts

@@ -218,7 +218,7 @@ async function uploadExport(req: Request, res: Response) {
 		res.sendStatus(400);
 		return;
 	}
-	let modPack = res.app.locals.controller.modPacks.get(modPackId);
+	let modPack = res.app.locals.controller.modPacks.getMutable(modPackId);
 	if (!modPack) {
 		res.sendStatus(400);
 		return;
@@ -261,7 +261,7 @@ async function uploadExport(req: Request, res: Response) {
 
 	modPack.exportManifest = new lib.ExportManifest(assets);
 	modPack.fillDefaultSettings(settingPrototypes, logger);
-	res.app.locals.controller.modPacksUpdated([modPack]);
+	res.app.locals.controller.modPacks.set(modPack);
 
 	res.sendStatus(200);
 }

packages/lib/index.ts

@@ -28,6 +28,7 @@ export * from "./src/stream";
 export * from "./src/system_collectors";
 export * from "./src/zip_ops";
 export * from "./src/subscriptions";
+export * from "./src/datastore";
 
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as ModStore } from "./src/ModStore";

packages/lib/src/datastore.ts

@@ -0,0 +1,275 @@
+import fs from "fs-extra";
+
+import { safeOutputFile } from "./file_ops";
+import { SubscribableValue } from "./subscriptions";
+import { EventEmitter } from "stream";
+import { ControllerConfig } from "./config";
+import path from "path";
+
+type DatastoreKey = string | number;
+type DatastoreValue = string | number | boolean | object;
+
+// Abstract class which can provide saving and loading capabilities to a datastore
+export abstract class DatastoreProvider<
+	K extends DatastoreKey,
+	V extends DatastoreValue,
+> {
+	abstract save(data: Map<K, V>): Promise<void>
+	abstract load(): Promise<Map<K, V>>
+
+	// Load data from the source and return it, this should be passed to the constructor of a datastore
+	async bootstrap() {
+		return [this, await this.load()] as const;
+	}
+}
+
+// Store all data in memory, used as a default and should be avoided
+export class MemoryDatastoreProvider<
+	K extends DatastoreKey,
+	V extends DatastoreValue,
+> extends DatastoreProvider<K, V> {
+	constructor(
+		private value: Map<K, V> = new Map()
+	) {
+		super();
+	}
+
+	async save(data: Map<K, V>) {
+		this.value = new Map(data);
+	}
+
+	async load() {
+		return new Map(this.value);
+	}
+}
+
+// Store all data within a json object file and validate the result against a schema
+export class JsonDatastoreProvider<
+	K extends DatastoreKey,
+	V extends DatastoreValue,
+	J, // Intermediate type returned from migrations
+> extends DatastoreProvider<K, V> {
+	constructor(
+		private filePath: string,
+		private fromJson: (json: J) => V = v => v as any,
+		private migrations: (rawJson: Record<DatastoreKey, unknown>) => Record<DatastoreKey, J> = v => v as any,
+		private finalise: (obj: V) => V = v => v,
+	) {
+		super();
+	}
+
+	// Save the data to the json file
+	async save(data: Map<K, V>) {
+		await safeOutputFile(this.filePath, JSON.stringify(Object.fromEntries(data), null, "\t"));
+	}
+
+	// Load the data from the json file
+	async load() {
+		// Read the raw json from the file
+		let rawJson;
+		try {
+			rawJson = JSON.parse(await fs.readFile(this.filePath, { encoding: "utf8" }));
+		} catch (err: any) {
+			if (err.code !== "ENOENT") {
+				throw err;
+			}
+			return new Map();
+		}
+
+		// Apply migrations
+		const serialized = this.migrations(rawJson);
+
+		// Convert to data class objects
+		return new Map(Object.entries(serialized).map(([k, v]) => [k, this.finalise(this.fromJson(v))]));
+	}
+}
+
+// Store all data within a json array file where each element has a unique id
+export class JsonIdDatastoreProvider<
+	K extends DatastoreKey,
+	V extends DatastoreValue & { id: K },
+	J, // Intermediate type returned from migrations
+> extends DatastoreProvider<K, V> {
+	constructor(
+		private filePath: string,
+		private fromJson: (json: J) => V = v => v as any,
+		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as any,
+		private finalise: (obj: V) => V = v => v,
+	) {
+		super();
+	}
+
+	// Save the data to the json file
+	async save(data: Map<K, V>) {
+		await safeOutputFile(this.filePath, JSON.stringify([...data.values()], null, "\t"));
+	}
+
+	// Load the data from the json file
+	async load() {
+		// Read the raw json from the file
+		let rawJson;
+		try {
+			rawJson = JSON.parse(await fs.readFile(this.filePath, { encoding: "utf8" }));
+		} catch (err: any) {
+			if (err.code !== "ENOENT") {
+				throw err;
+			}
+			return new Map();
+		}
+
+		// Apply migrations
+		const serialized = this.migrations(rawJson);
+
+		// Convert to data class objects
+		return new Map(serialized.map((e) => {
+			const v = this.finalise(this.fromJson(e));
+			return [v.id, v];
+		}));
+	}
+}
+
+// Implements a similar interface to a map
+export abstract class Datastore<
+	K extends DatastoreKey,
+	V extends DatastoreValue,
+> extends EventEmitter {
+	private dirty = false;
+
+	constructor(
+		private provider: DatastoreProvider<K, V> = new MemoryDatastoreProvider(),
+		protected data = new Map<K, V>(),
+	) {
+		super();
+	}
+
+	// Get the file path based on the controller config
+	static getFilePath(config: ControllerConfig, file: string) {
+		return path.resolve(config.get("controller.database_directory"), file);
+	}
+
+	// Save the datastore to the provider
+	async save() {
+		if (this.dirty) {
+			await this.provider.save(this.data);
+			this.dirty = false;
+		}
+	}
+
+	// Load the datastore from the provider
+	async load() {
+		this.data = await this.provider.load();
+		this.dirty = false;
+	}
+
+	// Set the dirty flag and call update handlers
+	protected touch(updates: any[]) {
+		if (updates.length) {
+			this.dirty = true;
+			this.emit("update", updates);
+		}
+	}
+
+	// Returns true if the datastore has the value, false otherwise
+	has(key: K) {
+		return this.data.has(key);
+	}
+
+	// Get a value from from the datastore
+	get(key: K) {
+		return this.data.get(key) as Readonly<V> ?? undefined;
+	}
+
+	// Get a mutable reference to a value in the datastore, call set after use
+	getMutable(key: K) {
+		return this.data.get(key);
+	}
+
+	// Returns all values in the datastore
+	values() {
+		return this.data.values() as IterableIterator<Readonly<V>>;
+	}
+
+	// Returns all values in the datastore as mutable references, call setMany after use
+	valuesMutable() {
+		return this.data.values();
+	}
+}
+
+// General key value mapping which can be saved and loaded with the file system
+export class KeyValueDatastore<
+	K extends DatastoreKey,
+	V extends DatastoreValue,
+> extends Datastore<K, V> {
+	// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
+	set(key: K, value: V) {
+		this.data.set(key, value);
+		this.touch([[key, value]]);
+	}
+
+	// Set many values at once from an array of key value pairs
+	setMany(pairs: [K, V][]) {
+		for (const [k, v] of pairs) {
+			this.data.set(k, v);
+		}
+		this.touch(pairs);
+	}
+
+	// Delete a value in the datastore
+	delete(key: K) {
+		const value = this.data.get(key);
+		this.data.delete(key);
+		this.touch([[key, value!, true]]);
+	}
+
+	// Delete many values at once from an array of keys
+	deleteMany(keys: K[]) {
+		const updates = [] as [K, V, true][];
+		for (const key of keys) {
+			const value = this.data.get(key);
+			updates.push([key, value!, true]);
+			this.data.delete(key);
+		}
+		this.touch(updates);
+	}
+}
+
+// A special implementation of the datastore to work directly with subscribable values
+export class SubscribableDatastore<
+	V extends SubscribableValue,
+> extends Datastore<V["id"], V> {
+	// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
+	set(value: V) {
+		this.data.set(value.id, value);
+		value.updatedAtMs = Date.now();
+		this.touch([value]);
+	}
+
+	// Set many values at once from an array of key value pairs
+	setMany(values: V[]) {
+		const nowMs = Date.now();
+		for (const value of values) {
+			value.updatedAtMs = nowMs;
+			this.data.set(value.id, value);
+		}
+		this.touch(values);
+	}
+
+	// Delete a value in the datastore
+	delete(value: V) {
+		this.data.delete(value.id);
+		value.updatedAtMs = Date.now();
+		value.isDeleted = true;
+		this.touch([value]);
+	}
+
+	// Delete many values at once from an array of values
+	deleteMany(values: V[]) {
+		const nowMs = Date.now();
+		for (const value of values) {
+			value.isDeleted = true;
+			value.updatedAtMs = nowMs;
+			this.data.delete(value.id);
+		}
+		this.touch(values);
+	}
+}

plugins/player_auth/test/plugin.js

@@ -61,6 +61,8 @@ describe("player_auth", function() {
 										return load;
 									} else if (field === "factorio.settings") {
 										return { name };
+									} else if (field === "instance.id") {
+										return id;
 									}
 									throw new Error(`field ${field} not implemented`);
 								},

test/file/json/invalid.json

@@ -0,0 +1 @@
+{,}

test/integration/routing.js

@@ -39,7 +39,7 @@ function connectInstance(controller, host, instanceId) {
 	instanceConfig.set("instance.id", instanceId);
 	instanceConfig.set("instance.assigned_host", host.config.get("host.id"));
 	host.instanceInfos.set(instanceId, { path: "invalid", config: instanceConfig });
-	controller.instances.set(instanceId, new InstanceInfo(instanceConfig, "stopped"));
+	controller.instances.set(new InstanceInfo(instanceConfig, "stopped"));
 	const instance = new Instance(host, instanceSide, "invalid", "invalid", instanceConfig);
 	const instanceConnection = new InstanceConnection(hostSide, host, instance);
 	host.instanceConnections.set(instanceId, instanceConnection);

test/lib/datastore.js

@@ -0,0 +1,663 @@
+"use strict";
+const events = require("events");
+const assert = require("assert").strict;
+const lib = require("@clusterio/lib");
+const fs = require("fs-extra");
+const path = require("path");
+
+class MockDatastoreProvider extends lib.DatastoreProvider {
+	constructor() {
+		super();
+		this.methodCalls = [];
+		this.value = new Map();
+		this.methodCalls.push(["constructor"]);
+	}
+
+	async save(data) {
+		this.methodCalls.push(["save", data]);
+	}
+
+	async load() {
+		this.methodCalls.push(["load"]);
+		return this.value;
+	}
+}
+
+class MockDatastore extends lib.Datastore {
+	getData() {
+		return this.data; // Protected member
+	}
+
+	callTouch(updates = [0]) {
+		this.touch(updates); // Protected method
+	}
+}
+
+describe("lib/datastore", function() {
+	const baseDir = path.join("temp", "test", "datastore");
+	const testFiles = path.join("test", "file");
+	before(async function() {
+		await fs.emptyDir(baseDir);
+	});
+
+	describe("class DatastoreProvider", function() {
+		let datastoreProvider;
+		beforeEach(function() {
+			datastoreProvider = new MockDatastoreProvider();
+		});
+
+		it("can be constructed", function() {
+			assert.deepEqual(datastoreProvider.methodCalls[0], ["constructor"]);
+		});
+
+		it("can be bootstrapped", async function() {
+			const rtn = await datastoreProvider.bootstrap();
+			assert.equal(rtn[0], datastoreProvider);
+			assert.equal(rtn[1], datastoreProvider.value);
+			assert.deepEqual(datastoreProvider.methodCalls[1], ["load"]);
+		});
+	});
+
+	describe("class MemoryDatastoreProvider", function() {
+		let datastoreProvider, initialData;
+		beforeEach(function() {
+			initialData = new Map([
+				["foo", "fooValue"], ["bar", "barValue"], ["baz", "bazValue"],
+			]);
+			datastoreProvider = new lib.MemoryDatastoreProvider(initialData);
+		});
+
+		describe("constructor", function() {
+			it("has a default value", async function() {
+				const [, value] = await new lib.MemoryDatastoreProvider().bootstrap();
+				assert.notEqual(value, undefined);
+				assert.notEqual(value, null);
+			});
+			it("can have a value given", async function() {
+				const expectedValue = new Map([["foo", "bar"]]);
+				const [, value] = await new lib.MemoryDatastoreProvider(expectedValue).bootstrap();
+				assert.deepEqual(value, expectedValue);
+			});
+		});
+
+		describe("save", function() {
+			it("can save data", async function() {
+				const newData = new Map([["foo2", "fooValue2"]]);
+				await datastoreProvider.save(newData);
+				const loadedData = await datastoreProvider.load();
+				assert.deepEqual(loadedData, newData);
+			});
+		});
+
+		describe("load", function() {
+			it("can load data", async function() {
+				const loadedData = await datastoreProvider.load();
+				assert.deepEqual(loadedData, initialData);
+			});
+		});
+	});
+
+	describe("class JsonDatastoreProvider", function() {
+		let datastoreProvider, initialData, dataPath;
+		before(async function() {
+			dataPath = path.join(baseDir, "data.json");
+			initialData = Object.entries(JSON.parse(await fs.readFile(
+				path.join(testFiles, "json", "load_map.json"), { encoding: "utf8" }
+			)));
+		});
+
+		beforeEach(async function() {
+			await fs.copy(path.join(testFiles, "json", "load_map.json"), dataPath);
+			datastoreProvider = new lib.JsonDatastoreProvider(dataPath);
+		});
+
+		describe("constructor", function() {
+			it("has default values", async function() {
+				datastoreProvider = new lib.JsonDatastoreProvider(dataPath);
+
+				const [, value] = await datastoreProvider.bootstrap();
+				assert.deepEqual(value, new Map(initialData));
+			});
+			it("can accept a fromJson method", async function() {
+				datastoreProvider = new lib.JsonDatastoreProvider(
+					dataPath,
+					json => [json]
+				);
+
+				const [, value] = await datastoreProvider.bootstrap();
+				assert.deepEqual(value, new Map(initialData.map(([k, v]) => [k, [v]])));
+			});
+			it("can accept a migration method", async function() {
+				datastoreProvider = new lib.JsonDatastoreProvider(
+					dataPath,
+					json => json, // Nop
+					rawJson => ({ c: "foo", ...rawJson })
+				);
+
+				const [, value] = await datastoreProvider.bootstrap();
+				assert.deepEqual(value, new Map([...initialData, ["c", "foo"]]));
+			});
+			it("can accept a finalise method", async function() {
+				datastoreProvider = new lib.JsonDatastoreProvider(
+					dataPath,
+					json => json, // Nop
+					rawJson => rawJson, // Nop
+					obj => [obj]
+				);
+
+				const [, value] = await datastoreProvider.bootstrap();
+				assert.deepEqual(value, new Map(initialData.map(([k, v]) => [k, [v]])));
+			});
+		});
+
+		describe("save", function() {
+			it("can save data", async function() {
+				const newData = new Map([["foo", "bar"]]);
+				await datastoreProvider.save(newData);
+				const loadedData = await datastoreProvider.load();
+				assert.deepEqual(loadedData, newData);
+			});
+		});
+
+		describe("load", function() {
+			it("can load data", async function() {
+				const loadedData = await datastoreProvider.load();
+				assert.deepEqual(loadedData, new Map(initialData));
+			});
+			it("has a default value", async function() {
+				datastoreProvider = new lib.JsonDatastoreProvider(path.join(baseDir, "not_exists.json"));
+				const loadedData = await datastoreProvider.load();
+				assert.equal(loadedData.size, 0);
+			});
+			it("throws for invalid file contents", async function() {
+				await fs.copy(path.join(testFiles, "json", "invalid.json"), dataPath);
+				await assert.rejects(() => datastoreProvider.load(dataPath));
+			});
+		});
+	});
+
+	describe("class JsonIdDatastoreProvider", function() {
+		let datastoreProvider, initialData, dataPath;
+		before(async function() {
+			dataPath = path.join(baseDir, "data.json");
+			initialData = JSON.parse(await fs.readFile(
+				path.join(testFiles, "json", "load_array_map.json"), { encoding: "utf8" }
+			)).map(v => [v.id, v]);
+		});
+
+		beforeEach(async function() {
+			await fs.copy(path.join(testFiles, "json", "load_array_map.json"), dataPath);
+			datastoreProvider = new lib.JsonIdDatastoreProvider(dataPath);
+		});
+
+		describe("constructor", function() {
+			it("has default values", async function() {
+				datastoreProvider = new lib.JsonIdDatastoreProvider(dataPath);
+
+				const [, value] = await datastoreProvider.bootstrap();
+				assert.deepEqual(value, new Map(initialData));
+			});
+			it("can accept a fromJson method", async function() {
+				datastoreProvider = new lib.JsonIdDatastoreProvider(
+					dataPath,
+					json => ({...json, foo: true})
+				);
+
+				const [, value] = await datastoreProvider.bootstrap();
+				assert.deepEqual(value, new Map(initialData.map(v => ([v[0], {...v[1], foo: true}]))));
+			});
+			it("can accept a migration method", async function() {
+				datastoreProvider = new lib.JsonIdDatastoreProvider(
+					dataPath,
+					json => json, // Nop
+					rawJson => ([{ id: "c" }, ...rawJson])
+				);
+
+				const [, value] = await datastoreProvider.bootstrap();
+				assert.deepEqual(value, new Map([...initialData, ["c", { id: "c" }]]));
+			});
+			it("can accept a finalise method", async function() {
+				datastoreProvider = new lib.JsonIdDatastoreProvider(
+					dataPath,
+					json => json, // Nop
+					rawJson => rawJson, // Nop
+					obj => ({...obj, foo: true})
+				);
+
+				const [, value] = await datastoreProvider.bootstrap();
+				assert.deepEqual(value, new Map(initialData.map(v => ([v[0], {...v[1], foo: true}]))));
+			});
+		});
+
+		describe("save", function() {
+			it("can save data", async function() {
+				const newData = new Map([["foo", { id: "foo" }]]);
+				await datastoreProvider.save(newData);
+				const loadedData = await datastoreProvider.load();
+				assert.deepEqual(loadedData, newData);
+			});
+		});
+
+		describe("load", function() {
+			it("can load data", async function() {
+				const loadedData = await datastoreProvider.load();
+				assert.equal(loadedData.size, initialData.length);
+				assert.deepEqual(loadedData, new Map(initialData));
+			});
+			it("has a default value", async function() {
+				datastoreProvider = new lib.JsonIdDatastoreProvider(path.join(baseDir, "not_exists.json"));
+				const loadedData = await datastoreProvider.load();
+				assert.equal(loadedData.size, 0);
+			});
+			it("throws for invalid file contents", async function() {
+				await fs.copy(path.join(testFiles, "json", "invalid.json"), dataPath);
+				await assert.rejects(() => datastoreProvider.load(dataPath));
+			});
+		});
+	});
+
+	describe("class Datastore", function() {
+		let datastoreProvider, datastore;
+		beforeEach(function() {
+			datastoreProvider = new MockDatastoreProvider();
+			datastoreProvider.value = new Map([
+				["foo", "fooValue"], ["bar", "barValue"], ["baz", "bazValue"],
+			]);
+			datastore = new MockDatastore(datastoreProvider, datastoreProvider.value);
+		});
+
+		describe("constructor", function() {
+			it("has a default value", function() {
+				datastore = new MockDatastore(datastoreProvider);
+				assert.deepEqual(datastore.getData(), new Map());
+			});
+			it("accepts a starting value", function() {
+				datastore = new MockDatastore(datastoreProvider, datastoreProvider.value);
+				assert.deepEqual(datastore.getData(), datastoreProvider.value);
+			});
+			it("accepts the return of bootstrap", async function() {
+				datastore = new MockDatastore(...await datastoreProvider.bootstrap());
+				assert.deepEqual(datastore.getData(), datastoreProvider.value);
+			});
+		});
+
+		describe("getFilePath", function() {
+			it("returns the correct filepath", function() {
+				const mockConfig = {
+					get(field) {
+						if (field === "controller.database_directory") {
+							return "databaseDir";
+						}
+						throw new Error(`field ${field} not implemented`);
+					},
+				};
+
+				assert.equal(
+					lib.Datastore.getFilePath(mockConfig, "db1.json"),
+					path.resolve("databaseDir", "db1.json")
+				);
+				assert.equal(
+					lib.Datastore.getFilePath(mockConfig, "db2.txt"),
+					path.resolve("databaseDir", "db2.txt")
+				);
+				assert.equal(
+					lib.Datastore.getFilePath(mockConfig, path.join("foo", "db3.json")),
+					path.resolve("databaseDir", "foo", "db3.json")
+				);
+			});
+		});
+
+		describe("save", function() {
+			it("should not save when clean", async function() {
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 1);
+			});
+			it("should save when dirty", async function() {
+				datastore.callTouch(); // Make the data dirty
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1], ["save", datastore.getData()]);
+			});
+			it("should not save multiple times", async function() {
+				datastore.callTouch(); // Make the data dirty
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1], ["save", datastore.getData()]);
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+			});
+		});
+
+		describe("load", function() {
+			it("should load new data", async function() {
+				datastoreProvider.value = new Map();
+				assert.notEqual(datastore.getData(), datastoreProvider.value);
+				await datastore.load();
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1], ["load"]);
+				assert.deepEqual(datastore.getData(), datastoreProvider.value);
+			});
+			it("should not save after a load", async function() {
+				datastore.callTouch(); // Make the data dirty
+				await datastore.load();
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1], ["load"]);
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+			});
+		});
+
+		describe("touch", function() {
+			it("should emit update events", function() {
+				const eventsRaised = [];
+				datastore.on("update", updates => eventsRaised.push(updates));
+
+				let updates = ["foo"];
+				datastore.callTouch(updates);
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], updates);
+
+				updates = ["foo", "bar"];
+				datastore.callTouch(updates);
+				assert.equal(eventsRaised.length, 2);
+				assert.deepEqual(eventsRaised[1], updates);
+			});
+		});
+
+		describe("has", function() {
+			it("returns true correctly", function() {
+				assert.equal(datastore.has("foo"), true);
+				assert.equal(datastore.has("bar"), true);
+				assert.equal(datastore.has("baz"), true);
+			});
+			it("returns false correctly", function() {
+				assert.equal(datastore.has("missing1"), false);
+				assert.equal(datastore.has("missing2"), false);
+				assert.equal(datastore.has(1), false);
+				assert.equal(datastore.has(2), false);
+			});
+		});
+
+		describe("get", function() {
+			it("returns the correct value", function() {
+				assert.equal(datastore.get("foo"), "fooValue");
+				assert.equal(datastore.get("bar"), "barValue");
+				assert.equal(datastore.get("baz"), "bazValue");
+			});
+			it("returns undefined correctly", function() {
+				assert.equal(datastore.get("missing1"), undefined);
+				assert.equal(datastore.get("missing2"), undefined);
+				assert.equal(datastore.get(1), undefined);
+				assert.equal(datastore.get(2), undefined);
+			});
+		});
+
+		describe("getMutable", function() {
+			it("returns the correct value", function() {
+				assert.equal(datastore.getMutable("foo"), "fooValue");
+				assert.equal(datastore.getMutable("bar"), "barValue");
+				assert.equal(datastore.getMutable("baz"), "bazValue");
+			});
+			it("returns undefined correctly", function() {
+				assert.equal(datastore.getMutable("missing1"), undefined);
+				assert.equal(datastore.getMutable("missing2"), undefined);
+				assert.equal(datastore.getMutable(1), undefined);
+				assert.equal(datastore.getMutable(2), undefined);
+			});
+		});
+
+		describe("values", function() {
+			it("returns all values", function() {
+				assert.deepEqual([...datastore.values()], [...datastoreProvider.value.values()]);
+			});
+		});
+
+		describe("valuesMutable", function() {
+			it("returns all values", function() {
+				assert.deepEqual([...datastore.valuesMutable()], [...datastoreProvider.value.values()]);
+			});
+		});
+	});
+
+	describe("class KeyValueDatastore", function() {
+		let datastoreProvider, datastore, eventsRaised;
+		beforeEach(function() {
+			eventsRaised = [];
+			datastoreProvider = new MockDatastoreProvider();
+			datastoreProvider.value = new Map([
+				["foo", "fooValue"], ["bar", "barValue"], ["baz", "bazValue"],
+			]);
+			datastore = new lib.KeyValueDatastore(datastoreProvider, datastoreProvider.value);
+			datastore.on("update", updates => eventsRaised.push(updates));
+		});
+
+		describe("set", function() {
+			it("can set a value", function() {
+				datastore.set("test", "testValue");
+				assert.equal(datastore.get("test"), "testValue");
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], [["test", "testValue"]]);
+			});
+			it("allows saving after a value is set", async function() {
+				datastore.set("test", "testValue");
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1],
+					["save", new Map([...datastoreProvider.value.entries(), ["test", "testValue"]])]
+				);
+			});
+		});
+
+		describe("setMany", function() {
+			it("does not save with zero changes", async function() {
+				datastore.setMany([]);
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 1);
+			});
+			it("can set many values at once", async function() {
+				const updates = [["test1", "testValue1"], ["test2", "testValue2"]];
+				datastore.setMany(updates);
+				assert.equal(datastore.get("test1"), "testValue1");
+				assert.equal(datastore.get("test2"), "testValue2");
+				await datastore.save();
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], updates);
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1],
+					["save", new Map([...datastoreProvider.value.entries(), ...updates])]
+				);
+			});
+		});
+
+		describe("delete", function() {
+			it("can delete a value", function() {
+				datastore.delete("foo");
+				assert.equal(datastore.get("foo"), undefined);
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], [["foo", "fooValue", true]]);
+			});
+			it("allows saving after a value is deleted", async function() {
+				datastore.delete("foo");
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1],
+					["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => k !== "foo"))]
+				);
+			});
+			it("does not break when the key does not exist", function() {
+				datastore.delete("test");
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], [["test", undefined, true]]);
+			});
+		});
+
+		describe("deleteMany", function() {
+			it("does not save with zero changes", async function() {
+				datastore.deleteMany([]);
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 1);
+			});
+			it("can delete many values at once", async function() {
+				const keys = ["foo", "bar"];
+				datastore.deleteMany(keys);
+				assert.equal(datastore.get("foo"), undefined);
+				assert.equal(datastore.get("bar"), undefined);
+				await datastore.save();
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], [["foo", "fooValue", true], ["bar", "barValue", true]]);
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1],
+					["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => !keys.includes(k)))]
+				);
+			});
+		});
+	});
+
+	describe("class SubscribableDatastore", function() {
+		let datastoreProvider, datastore, eventsRaised;
+		beforeEach(function() {
+			eventsRaised = [];
+			datastoreProvider = new MockDatastoreProvider();
+			datastoreProvider.value = new Map([
+				["foo", {id: "foo"}], ["bar", {id: "bar"}], ["baz", {id: "baz"}],
+			]);
+			datastore = new lib.SubscribableDatastore(datastoreProvider, datastoreProvider.value);
+			datastore.on("update", updates => eventsRaised.push(updates));
+		});
+
+		describe("set", function() {
+			it("can set a value", function() {
+				const value = { id: "test" };
+				datastore.set(value);
+				assert.equal(datastore.get(value.id), value);
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], [value]);
+			});
+			it("allows saving after a value is set", async function() {
+				const value = { id: "test" };
+				datastore.set(value);
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1],
+					["save", new Map([...datastoreProvider.value.entries(), [value.id, value]])]
+				);
+			});
+			it("sets the updatedAtMs property", async function() {
+				const value = { id: "test" };
+				datastore.set(value);
+				assert.notEqual(value.updatedAtMs, undefined);
+				assert(value.updatedAtMs > 0, "updatedAtMs greater than zero");
+				const prev = value.updatedAtMs;
+				await lib.wait();
+				datastore.set(value);
+				assert(value.updatedAtMs > prev, "updatedAtMs must increase");
+			});
+		});
+
+		describe("setMany", function() {
+			it("does not save with zero changes", async function() {
+				datastore.setMany([]);
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 1);
+			});
+			it("can set many values at once", async function() {
+				const updates = [{ id: "test1" }, { id: "test2" }];
+				datastore.setMany(updates);
+				assert.equal(datastore.get(updates[0].id), updates[0]);
+				assert.equal(datastore.get(updates[1].id), updates[1]);
+				await datastore.save();
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], updates);
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1],
+					["save", new Map([...datastoreProvider.value.entries(), ...updates.map(v => [v.id, v])])]
+				);
+			});
+			it("sets the updatedAtMs property", async function() {
+				const updates = [{ id: "test1" }, { id: "test2" }];
+				datastore.setMany(updates);
+				assert.notEqual(updates[0].updatedAtMs, undefined);
+				assert.notEqual(updates[1].updatedAtMs, undefined);
+				assert(updates[0].updatedAtMs > 0, "updatedAtMs greater than zero");
+				assert(updates[1].updatedAtMs > 0, "updatedAtMs greater than zero");
+				const prev = [updates[0].updatedAtMs, updates[1].updatedAtMs];
+				await lib.wait();
+				datastore.setMany(updates);
+				assert(updates[0].updatedAtMs > prev[0], "updatedAtMs must increase");
+				assert(updates[1].updatedAtMs > prev[1], "updatedAtMs must increase");
+			});
+		});
+
+		describe("delete", function() {
+			it("can delete a value", function() {
+				const value = datastore.get("foo");
+				datastore.delete(value);
+				assert.equal(datastore.get(value.id), undefined);
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], [value]);
+			});
+			it("allows saving after a value is deleted", async function() {
+				const value = datastore.get("foo");
+				datastore.delete(value);
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1],
+					["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => k !== value.id))]
+				);
+			});
+			it("does not break when the key does not exist", function() {
+				const value = { id: "test" };
+				datastore.delete(value);
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], [value]);
+			});
+			it("sets the updatedAtMs property", function() {
+				const value = { id: "test" };
+				datastore.delete(value);
+				assert.notEqual(value.updatedAtMs, undefined);
+				assert(value.updatedAtMs > 0, "updatedAtMs greater than zero");
+			});
+			it("sets the isDeleted property", function() {
+				const value = { id: "test" };
+				datastore.delete(value);
+				assert.equal(value.isDeleted, true);
+			});
+		});
+
+		describe("deleteMany", function() {
+			it("does not save with zero changes", async function() {
+				datastore.deleteMany([]);
+				await datastore.save();
+				assert.equal(datastoreProvider.methodCalls.length, 1);
+			});
+			it("can delete many values at once", async function() {
+				const values = [{id: "foo"}, {id: "bar"}];
+				datastore.deleteMany(values);
+				assert.equal(datastore.get(values[0].id), undefined);
+				assert.equal(datastore.get(values[1].id), undefined);
+				await datastore.save();
+				assert.equal(eventsRaised.length, 1);
+				assert.deepEqual(eventsRaised[0], values);
+				assert.equal(datastoreProvider.methodCalls.length, 2);
+				assert.deepEqual(datastoreProvider.methodCalls[1],
+					["save", new Map([...datastoreProvider.value.entries()]
+						.filter((_, k) => !values.some(v => v.id === k)))]
+				);
+			});
+			it("sets the updatedAtMs property", function() {
+				const values = [{id: "foo"}, {id: "bar"}];
+				datastore.deleteMany(values);
+				assert.notEqual(values[0].updatedAtMs, undefined);
+				assert.notEqual(values[1].updatedAtMs, undefined);
+				assert(values[0].updatedAtMs > 0, "updatedAtMs greater than zero");
+				assert(values[1].updatedAtMs > 0, "updatedAtMs greater than zero");
+			});
+			it("sets the isDeleted property", function() {
+				const values = [{id: "foo"}, {id: "bar"}];
+				datastore.deleteMany(values);
+				assert.equal(values[0].isDeleted, true);
+				assert.equal(values[1].isDeleted, true);
+			});
+		});
+	});
+});

test/mock.js

@@ -183,8 +183,8 @@ class MockController {
 			["test", ControllerUser.fromJSON({ name: "test", roles: [0, 1] }, this.userManager)],
 			["player", ControllerUser.fromJSON({ name: "player", roles: [1] }, this.userManager)],
 		]);
-		this.instances = new Map();
-		this.hosts = new Map();
+		this.instances = new lib.KeyValueDatastore();
+		this.hosts = new lib.KeyValueDatastore();
 		this.handles = new Map();
 	}