Implement autosaving of data held in memory

Add controller.autosave_interval config that specifies the frequency
which controller config, hosts, instances, users, mod packs and plugin
specific data which is held in memory should be automatically flushed to
disk if they have changed since the last time they were saved.  This
prevents data loss in the event of a server crash.

Plugins can use the onSaveData hook to save its own data at the same
time as the controller does and this obsoletes making an autosave
interval in the plugin.

Resolves #523.

packages/controller/src/BaseControllerPlugin.ts

@@ -40,6 +40,19 @@ export default class BaseControllerPlugin {
 	 */
 	async init() { }
 
+	/**
+	 * Called when the controller saves data in memory to disk
+	 *
+	 * Invoked on the configured controller.autosave_interval by the
+	 * controller and intended to be used to flush any in memory data that
+	 * has changed to disk.
+	 *
+	 * This will also be called during graceful shutdown after {@link
+	 * BaseControllerPlugin.onShutdown} have been invoked and all links have
+	 * been disconnected.
+	 */
+	async onSaveData() { }
+
 	/**
 	 * Called when the status of an instance changes
 	 *

packages/controller/src/ControlConnection.ts

@@ -665,6 +665,7 @@ export default class ControlConnection extends BaseConnection {
 		// Start at 5 to leave space for future default roles
 		let id = Math.max(5, lastId+1);
 		this._controller.userManager.roles.set(id, new lib.Role({ id, ...request }));
+		this._controller.userManager.dirty = true;
 		return id;
 	}
 
@@ -699,6 +700,7 @@ export default class ControlConnection extends BaseConnection {
 		}
 
 		this._controller.userManager.roles.delete(id);
+		this._controller.userManager.dirty = true;
 		for (let user of this._controller.userManager.users.values()) {
 			user.roles.delete(role);
 			this._controller.userPermissionsUpdated(user);

packages/controller/src/Controller.ts

@@ -59,6 +59,13 @@ export default class Controller {
 	/** User and roles manager for the cluster */
 	userManager: UserManager;
 
+	/** True if {@link Controller.hosts} has changed since last time it was saved */
+	hostsDirty = false;
+	/** True if {@link Controller.instances} has changed since last time it was saved */
+	instancesDirty = false;
+	/** True if {@link Controller.modPacks} has changed since last time it was saved */
+	modPacksDirty = false;
+
 	httpServer: http.Server | null = null;
 	httpServerCloser: HttpCloser | null = null;
 	httpsServer: https.Server | null = null;
@@ -85,6 +92,8 @@ export default class Controller {
 
 	devMiddleware: any | null = null;
 
+	autosaveInterval?: ReturnType<typeof setInterval>;
+
 	logDirectory: string = "";
 	clusterLogIndex: lib.LogIndex | null = null;
 	clusterLogBuildInterval: ReturnType<typeof setInterval> | null = null;
@@ -182,6 +191,9 @@ export default class Controller {
 		this.mods = await Controller.loadModInfos(modsDirectory);
 
 		this.config.on("fieldChanged", (group, field, prev) => {
+			if (group.name === "controller" && field === "autosave_interval") {
+				this.onAutosaveIntervalChanged();
+			}
 			lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", group, field, prev);
 		});
 		for (let instance of this.instances!.values()) {
@@ -253,6 +265,8 @@ export default class Controller {
 			logger.info(`Listening for HTTPS on port ${(this.httpsServer.address() as AddressInfo).port}`);
 		}
 
+		this.onAutosaveIntervalChanged();
+
 		logger.info("Started controller");
 		this._state = "running";
 	}
@@ -317,34 +331,19 @@ export default class Controller {
 			clearInterval(this.clusterLogBuildInterval);
 		}
 
+		if (this.autosaveInterval) {
+			clearInterval(this.autosaveInterval);
+			this.autosaveInterval = undefined;
+		}
+
 		if (this.clusterLogIndex) {
 			await this.clusterLogIndex.save();
 		}
 
-		logger.info("Saving config");
-		await lib.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, "\t"));
-
 		if (this.devMiddleware) {
 			await new Promise((resolve, reject) => { this.devMiddleware.close(resolve); });
 		}
 
-		let databaseDirectory = this.config.get("controller.database_directory");
-		if (this.hosts) {
-			await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts);
-		}
-
-		if (this.instances) {
-			await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances);
-		}
-
-		if (this.modPacks) {
-			await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks);
-		}
-
-		if (this.userManager) {
-			await this.userManager.save(path.join(databaseDirectory, "users.json"));
-		}
-
 		await lib.invokeHook(this.plugins, "onShutdown");
 
 		if (this.wsServer) {
@@ -361,9 +360,85 @@ export default class Controller {
 		}
 		await Promise.all(stopTasks);
 
+		logger.info("Saving data");
+		await this.saveData();
 		logger.info("Goodbye");
 	}
 
+	onAutosaveIntervalChanged() {
+		if (this.autosaveInterval) {
+			clearInterval(this.autosaveInterval);
+			this.autosaveInterval = undefined;
+		}
+		const autosaveIntervalSeconds = this.config.get("controller.autosave_interval");
+		if (autosaveIntervalSeconds > 0) {
+			this.autosaveInterval = setInterval(this.saveData.bind(this), autosaveIntervalSeconds * 1000);
+		}
+	}
+
+	private _saveDataCurrentlyRunning = false;
+	private _saveDataCurrentlyWaiting: (() => void)[] = [];
+
+	/**
+	 * Save all data currently in memory to disk
+	 *
+	 * Note: If a save is currently in progress this will wait until that
+	 * completes and start another save.
+	 */
+	async saveData() {
+		if (this._saveDataCurrentlyRunning) {
+			const waitForCompletion = () => new Promise<void>(resolve => {
+				this._saveDataCurrentlyWaiting.push(resolve);
+			});
+			await waitForCompletion();
+			if (this._saveDataCurrentlyRunning) {
+				await waitForCompletion();
+				return;
+			}
+		}
+
+		this._saveDataCurrentlyRunning = true;
+		try {
+			await this._saveDataInternal();
+		} catch (err: any) {
+			logger.error(`Unexpected error during saveData:\n${err.stack}`);
+		}
+		this._saveDataCurrentlyRunning = false;
+		for (const callback of this._saveDataCurrentlyWaiting) {
+			callback();
+		}
+		this._saveDataCurrentlyWaiting.length = 0;
+	}
+
+	async _saveDataInternal() {
+		if (this.config.dirty) {
+			this.config.dirty = false;
+			await lib.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, "\t"));
+		}
+
+		let databaseDirectory = this.config.get("controller.database_directory");
+		if (this.hosts && this.hostsDirty) {
+			this.hostsDirty = false;
+			await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts);
+		}
+
+		if (this.instances && this.instancesDirty) {
+			this.instancesDirty = false;
+			await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances);
+		}
+
+		if (this.modPacks && this.modPacksDirty) {
+			this.modPacksDirty = false;
+			await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks);
+		}
+
+		if (this.userManager && this.userManager.dirty) {
+			await this.userManager.save(path.join(databaseDirectory, "users.json"));
+		}
+
+		await lib.invokeHook(this.plugins, "onSaveData");
+	}
+
 	static async loadHosts(filePath: string): Promise<Map<number, HostInfo>> {
 		let serialized: [number, HostInfo][];
 		try {
@@ -766,6 +841,7 @@ export default class Controller {
 				this.instanceUpdated(instance);
 			}
 
+			this.instancesDirty = true;
 			lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, group, field, prev);
 		});
 
@@ -773,6 +849,7 @@ export default class Controller {
 	}
 
 	instanceUpdated(instance: InstanceInfo) {
+		this.instancesDirty = true;
 		let assigned_host: number|null|undefined = instance.config.get("instance.assigned_host");
 		if (assigned_host === null) {
 			assigned_host = undefined;
@@ -799,6 +876,7 @@ export default class Controller {
 	}
 
 	modPackUpdated(modPack: lib.ModPack) {
+		this.modPacksDirty = true;
 		this.subscriptions.broadcast(new lib.ModPackUpdateEvent(modPack));
 		lib.invokeHook(this.plugins, "onModPackUpdated", modPack);
 	}
@@ -809,6 +887,7 @@ export default class Controller {
 	}
 
 	userUpdated(user: lib.User) {
+		this.userManager.dirty = true;
 		this.subscriptions.broadcast(new lib.UserUpdateEvent(
 			new lib.RawUser(
 				user.name,

packages/controller/src/HostConnection.ts

@@ -57,6 +57,7 @@ export default class HostConnection extends BaseConnection {
 			plugins: registerData.plugins,
 			token_valid_after: currentHostInfo?.token_valid_after,
 		});
+		this._controller.hostsDirty = true;
 
 		for (let event of ["connect", "drop", "resume", "close"] as const) {
 			// eslint-disable-next-line no-loop-func

packages/controller/src/UserManager.ts

@@ -10,6 +10,7 @@ import * as lib from "@clusterio/lib";
 export default class UserManager {
 	roles: Map<number, lib.Role> = new Map();
 	users: Map<string, lib.User> = new Map();
+	dirty = false;
 
 	constructor(
 		private _config: lib.ControllerConfig
@@ -41,10 +42,6 @@ export default class UserManager {
 	}
 
 	async save(filePath:string): Promise<void> {
-		if (this.roles.size === 0 || this.users.size === 0) {
-			return;
-		}
-
 		let serializedRoles = [];
 		for (let role of this.roles.values()) {
 			serializedRoles.push(role.serialize());
@@ -59,6 +56,7 @@ export default class UserManager {
 			users: serializedUsers,
 			roles: serializedRoles,
 		};
+		this.dirty = false;
 		await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t"));
 	}
 
@@ -80,6 +78,7 @@ export default class UserManager {
 
 		let user = new lib.User({ name, roles }, this.roles);
 		this.users.set(name, user);
+		this.dirty = true;
 		return user;
 	}