Implement autosaving of data held in memory

Add controller.autosave_interval config that specifies the frequency
which controller config, hosts, instances, users, mod packs and plugin
specific data which is held in memory should be automatically flushed to
disk if they have changed since the last time they were saved.  This
prevents data loss in the event of a server crash.

Plugins can use the onSaveData hook to save its own data at the same
time as the controller does and this obsoletes making an autosave
interval in the plugin.

Resolves #523.

.eslintrc.js

@@ -137,7 +137,11 @@ module.exports = {
 		"func-call-spacing": "error",
 		"func-name-matching": "error",
 		"func-names": "off",
-		"func-style": ["error", "declaration"],
+		"func-style": ["error", "declaration", {
+			// Allow arrow functions as it better expresses referencing this
+			// of the containing scope, which is unweilding in plain functions.
+			"allowArrowFunctions": true,
+		}],
 		"function-call-argument-newline": "off",
 		"function-paren-newline": "off",
 		"generator-star-spacing": ["error", { "before": false, "after": true }],

packages/controller/src/BaseControllerPlugin.ts

@@ -40,6 +40,19 @@ export default class BaseControllerPlugin {
 	 */
 	async init() { }
 
+	/**
+	 * Called when the controller saves data in memory to disk
+	 *
+	 * Invoked on the configured controller.autosave_interval by the
+	 * controller and intended to be used to flush any in memory data that
+	 * has changed to disk.
+	 *
+	 * This will also be called during graceful shutdown after {@link
+	 * BaseControllerPlugin.onShutdown} have been invoked and all links have
+	 * been disconnected.
+	 */
+	async onSaveData() { }
+
 	/**
 	 * Called when the status of an instance changes
 	 *

packages/controller/src/ControlConnection.ts

@@ -665,6 +665,7 @@ export default class ControlConnection extends BaseConnection {
 		// Start at 5 to leave space for future default roles
 		let id = Math.max(5, lastId+1);
 		this._controller.userManager.roles.set(id, new lib.Role({ id, ...request }));
+		this._controller.userManager.dirty = true;
 		return id;
 	}
 
@@ -699,6 +700,7 @@ export default class ControlConnection extends BaseConnection {
 		}
 
 		this._controller.userManager.roles.delete(id);
+		this._controller.userManager.dirty = true;
 		for (let user of this._controller.userManager.users.values()) {
 			user.roles.delete(role);
 			this._controller.userPermissionsUpdated(user);

packages/controller/src/Controller.ts

@@ -59,6 +59,13 @@ export default class Controller {
 	/** User and roles manager for the cluster */
 	userManager: UserManager;
 
+	/** True if {@link Controller.hosts} has changed since last time it was saved */
+	hostsDirty = false;
+	/** True if {@link Controller.instances} has changed since last time it was saved */
+	instancesDirty = false;
+	/** True if {@link Controller.modPacks} has changed since last time it was saved */
+	modPacksDirty = false;
+
 	httpServer: http.Server | null = null;
 	httpServerCloser: HttpCloser | null = null;
 	httpsServer: https.Server | null = null;
@@ -85,6 +92,8 @@ export default class Controller {
 
 	devMiddleware: any | null = null;
 
+	autosaveInterval?: ReturnType<typeof setInterval>;
+
 	logDirectory: string = "";
 	clusterLogIndex: lib.LogIndex | null = null;
 	clusterLogBuildInterval: ReturnType<typeof setInterval> | null = null;
@@ -182,6 +191,9 @@ export default class Controller {
 		this.mods = await Controller.loadModInfos(modsDirectory);
 
 		this.config.on("fieldChanged", (group, field, prev) => {
+			if (group.name === "controller" && field === "autosave_interval") {
+				this.onAutosaveIntervalChanged();
+			}
 			lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", group, field, prev);
 		});
 		for (let instance of this.instances!.values()) {
@@ -253,6 +265,8 @@ export default class Controller {
 			logger.info(`Listening for HTTPS on port ${(this.httpsServer.address() as AddressInfo).port}`);
 		}
 
+		this.onAutosaveIntervalChanged();
+
 		logger.info("Started controller");
 		this._state = "running";
 	}
@@ -317,34 +331,19 @@ export default class Controller {
 			clearInterval(this.clusterLogBuildInterval);
 		}
 
+		if (this.autosaveInterval) {
+			clearInterval(this.autosaveInterval);
+			this.autosaveInterval = undefined;
+		}
+
 		if (this.clusterLogIndex) {
 			await this.clusterLogIndex.save();
 		}
 
-		logger.info("Saving config");
-		await lib.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, "\t"));
-
 		if (this.devMiddleware) {
 			await new Promise((resolve, reject) => { this.devMiddleware.close(resolve); });
 		}
 
-		let databaseDirectory = this.config.get("controller.database_directory");
-		if (this.hosts) {
-			await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts);
-		}
-
-		if (this.instances) {
-			await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances);
-		}
-
-		if (this.modPacks) {
-			await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks);
-		}
-
-		if (this.userManager) {
-			await this.userManager.save(path.join(databaseDirectory, "users.json"));
-		}
-
 		await lib.invokeHook(this.plugins, "onShutdown");
 
 		if (this.wsServer) {
@@ -361,9 +360,85 @@ export default class Controller {
 		}
 		await Promise.all(stopTasks);
 
+		logger.info("Saving data");
+		await this.saveData();
 		logger.info("Goodbye");
 	}
 
+	onAutosaveIntervalChanged() {
+		if (this.autosaveInterval) {
+			clearInterval(this.autosaveInterval);
+			this.autosaveInterval = undefined;
+		}
+		const autosaveIntervalSeconds = this.config.get("controller.autosave_interval");
+		if (autosaveIntervalSeconds > 0) {
+			this.autosaveInterval = setInterval(this.saveData.bind(this), autosaveIntervalSeconds * 1000);
+		}
+	}
+
+	private _saveDataCurrentlyRunning = false;
+	private _saveDataCurrentlyWaiting: (() => void)[] = [];
+
+	/**
+	 * Save all data currently in memory to disk
+	 *
+	 * Note: If a save is currently in progress this will wait until that
+	 * completes and start another save.
+	 */
+	async saveData() {
+		if (this._saveDataCurrentlyRunning) {
+			const waitForCompletion = () => new Promise<void>(resolve => {
+				this._saveDataCurrentlyWaiting.push(resolve);
+			});
+			await waitForCompletion();
+			if (this._saveDataCurrentlyRunning) {
+				await waitForCompletion();
+				return;
+			}
+		}
+
+		this._saveDataCurrentlyRunning = true;
+		try {
+			await this._saveDataInternal();
+		} catch (err: any) {
+			logger.error(`Unexpected error during saveData:\n${err.stack}`);
+		}
+		this._saveDataCurrentlyRunning = false;
+		for (const callback of this._saveDataCurrentlyWaiting) {
+			callback();
+		}
+		this._saveDataCurrentlyWaiting.length = 0;
+	}
+
+	async _saveDataInternal() {
+		if (this.config.dirty) {
+			this.config.dirty = false;
+			await lib.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, "\t"));
+		}
+
+		let databaseDirectory = this.config.get("controller.database_directory");
+		if (this.hosts && this.hostsDirty) {
+			this.hostsDirty = false;
+			await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts);
+		}
+
+		if (this.instances && this.instancesDirty) {
+			this.instancesDirty = false;
+			await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances);
+		}
+
+		if (this.modPacks && this.modPacksDirty) {
+			this.modPacksDirty = false;
+			await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks);
+		}
+
+		if (this.userManager && this.userManager.dirty) {
+			await this.userManager.save(path.join(databaseDirectory, "users.json"));
+		}
+
+		await lib.invokeHook(this.plugins, "onSaveData");
+	}
+
 	static async loadHosts(filePath: string): Promise<Map<number, HostInfo>> {
 		let serialized: [number, HostInfo][];
 		try {
@@ -766,6 +841,7 @@ export default class Controller {
 				this.instanceUpdated(instance);
 			}
 
+			this.instancesDirty = true;
 			lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, group, field, prev);
 		});
 
@@ -773,6 +849,7 @@ export default class Controller {
 	}
 
 	instanceUpdated(instance: InstanceInfo) {
+		this.instancesDirty = true;
 		let assigned_host: number|null|undefined = instance.config.get("instance.assigned_host");
 		if (assigned_host === null) {
 			assigned_host = undefined;
@@ -799,6 +876,7 @@ export default class Controller {
 	}
 
 	modPackUpdated(modPack: lib.ModPack) {
+		this.modPacksDirty = true;
 		this.subscriptions.broadcast(new lib.ModPackUpdateEvent(modPack));
 		lib.invokeHook(this.plugins, "onModPackUpdated", modPack);
 	}
@@ -809,6 +887,7 @@ export default class Controller {
 	}
 
 	userUpdated(user: lib.User) {
+		this.userManager.dirty = true;
 		this.subscriptions.broadcast(new lib.UserUpdateEvent(
 			new lib.RawUser(
 				user.name,

packages/controller/src/HostConnection.ts

@@ -57,6 +57,7 @@ export default class HostConnection extends BaseConnection {
 			plugins: registerData.plugins,
 			token_valid_after: currentHostInfo?.token_valid_after,
 		});
+		this._controller.hostsDirty = true;
 
 		for (let event of ["connect", "drop", "resume", "close"] as const) {
 			// eslint-disable-next-line no-loop-func

packages/controller/src/UserManager.ts

@@ -10,6 +10,7 @@ import * as lib from "@clusterio/lib";
 export default class UserManager {
 	roles: Map<number, lib.Role> = new Map();
 	users: Map<string, lib.User> = new Map();
+	dirty = false;
 
 	constructor(
 		private _config: lib.ControllerConfig
@@ -41,10 +42,6 @@ export default class UserManager {
 	}
 
 	async save(filePath:string): Promise<void> {
-		if (this.roles.size === 0 || this.users.size === 0) {
-			return;
-		}
-
 		let serializedRoles = [];
 		for (let role of this.roles.values()) {
 			serializedRoles.push(role.serialize());
@@ -59,6 +56,7 @@ export default class UserManager {
 			users: serializedUsers,
 			roles: serializedRoles,
 		};
+		this.dirty = false;
 		await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t"));
 	}
 
@@ -80,6 +78,7 @@ export default class UserManager {
 
 		let user = new lib.User({ name, roles }, this.roles);
 		this.users.set(name, user);
+		this.dirty = true;
 		return user;
 	}
 

packages/lib/src/config/classes.ts

@@ -85,6 +85,9 @@ export class Config extends events.EventEmitter {
 	_groups = new Map<string, ConfigGroup>();
 	_unknownGroups = new Map<string, SerializedGroup>();
 
+	/** Set to true when a field in the config is changed. */
+	dirty = false;
+
 	/**
 	 * Create a new instance of the given config
 	 *
@@ -566,6 +569,7 @@ export class ConfigGroup {
 		let prev = this._fields.get(name);
 		this._fields.set(name, value);
 		if (this.config && !isDeepStrictEqual(value, prev)) {
+			this.config.dirty = true;
 			this.config.emit("fieldChanged", this, name, prev);
 		}
 	}
@@ -604,6 +608,7 @@ export class ConfigGroup {
 		}
 		this._fields.set(name, updated);
 		if (this.config && !isDeepStrictEqual(updated, prev)) {
+			this.config.dirty = true;
 			this.config.emit("fieldChanged", this, name, prev);
 		}
 	}
@@ -663,6 +668,7 @@ export class ConfigGroup {
 			let prev = this._fields.get(name);
 			this._fields.set(name, value);
 			if (notify && this.config && !isDeepStrictEqual(value, prev)) {
+				this.config.dirty = true;
 				this.config.emit("fieldChanged", this, name, prev);
 			}
 		}

packages/lib/src/config/definitions.ts

@@ -31,6 +31,13 @@ ControllerGroup.define({
 	type: "string",
 	initial_value: "database",
 });
+ControllerGroup.define({
+	name: "autosave_interval",
+	title: "Autosave Interval",
+	description: "Interval in seconds to autosave data in memory to disk.",
+	type: "number",
+	initial_value: 60,
+});
 ControllerGroup.define({
 	name: "http_port",
 	title: "HTTP Port",
@@ -157,6 +164,7 @@ export class ControllerConfig extends classes.Config {
 	get(name: "controller.proxy_stream_timeout"): number;
 	get(name: "controller.default_mod_pack_id"): number | null;
 	get(name: "controller.default_role_id"): number | null;
+	get(name: "controller.autosave_interval"): number;
 	get(name: string): unknown;
 	get(name: string) {
 		return super.get(name);

plugins/inventory_sync/controller.ts

@@ -38,16 +38,11 @@ async function saveDatabase(
 export class ControllerPlugin extends BaseControllerPlugin {
 	acquiredPlayers!: Map<string, { instanceId: number, expires?: number }>;
 	playerDatastore!: Map<string, IpcPlayerData>;
-	autosaveId!: ReturnType<typeof setInterval>;
+	playerDatastoreDirty = false;
 
 	async init() {
 		this.acquiredPlayers = new Map();
 		this.playerDatastore = await loadDatabase(this.controller.config, this.logger);
-		this.autosaveId = setInterval(() => {
-			saveDatabase(this.controller.config, this.playerDatastore, this.logger).catch(err => {
-				this.logger.error(`Unexpected error autosaving player data:\n${err.stack}`);
-			});
-		}, this.controller.config.get("inventory_sync.autosave_interval") as number * 1000);
 
 		this.controller.handle(msg.AcquireRequest, this.handleAcquireRequest.bind(this));
 		this.controller.handle(msg.ReleaseRequest, this.handleReleaseRequest.bind(this));
@@ -161,6 +156,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		if (store) {
 			this.logger.verbose(`Received player data for ${playerName} from ${instanceName}`);
 			this.playerDatastore.set(playerName, playerData);
+			this.playerDatastoreDirty = true;
 		}
 	}
 
@@ -179,9 +175,11 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		return new msg.DownloadRequest.Response(this.playerDatastore.get(playerName) || null);
 	}
 
-	async onShutdown() {
-		clearInterval(this.autosaveId);
-		await saveDatabase(this.controller.config, this.playerDatastore, this.logger);
+	async onSaveData() {
+		if (this.playerDatastoreDirty) {
+			this.playerDatastoreDirty = false;
+			await saveDatabase(this.controller.config, this.playerDatastore, this.logger);
+		}
 	}
 
 	async handleDatabaseStatsRequest() {

plugins/inventory_sync/info.ts

@@ -4,13 +4,6 @@ import * as messages from "./messages";
 class ControllerConfigGroup extends lib.PluginConfigGroup { }
 ControllerConfigGroup.defaultAccess = ["controller", "host", "control"];
 ControllerConfigGroup.groupName = "inventory_sync";
-ControllerConfigGroup.define({
-	name: "autosave_interval",
-	title: "Autosave Interval",
-	description: "Interval the player data is autosaved at in seconds.",
-	type: "number",
-	initial_value: 600, // 10 minutes
-});
 ControllerConfigGroup.define({
 	name: "player_lock_timeout",
 	title: "Player Lock Timeout",

plugins/research_sync/controller.ts

@@ -51,6 +51,7 @@ async function saveTechnologies(
 
 export class ControllerPlugin extends BaseControllerPlugin {
 	technologies!: Map<string, Technology>;
+	technologiesDirty = true;
 	progressRateLimiter!: lib.RateLimiter;
 	progressBroadcastId!: ReturnType<typeof setInterval> | null;
 	progressToBroadcast!: Set<string>;
@@ -70,9 +71,15 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		this.controller.handle(SyncTechnologiesRequest, this.handleSyncTechnologiesRequest.bind(this));
 	}
 
+	async onSaveData() {
+		if (this.technologiesDirty) {
+			this.technologiesDirty = false;
+			await saveTechnologies(this.controller.config, this.technologies, this.logger);
+		}
+	}
+
 	async onShutdown() {
 		this.progressRateLimiter.cancel();
-		await saveTechnologies(this.controller.config, this.technologies, this.logger);
 	}
 
 	broadcastProgress() {
@@ -96,6 +103,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		if (!tech) {
 			tech = { level, progress: 0, researched: false };
 			this.technologies.set(name, tech);
+			this.technologiesDirty = true;
 
 		// Ignore contribution to already researched technologies
 		} else if (tech.level > level || tech.level === level && tech.researched) {
@@ -126,6 +134,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 			this.controller.sendTo("allInstances", new FinishedEvent(name, tech.level));
 		}
+		this.technologiesDirty = true;
 	}
 
 	async handleFinishedEvent(event: FinishedEvent) {
@@ -134,6 +143,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		if (!tech || tech.level <= level) {
 			this.progressToBroadcast.delete(name);
 			this.technologies.set(name, { level, progress: null, researched: true });
+			this.technologiesDirty = true;
 		}
 	}
 
@@ -151,6 +161,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			let tech = this.technologies.get(name);
 			if (!tech) {
 				this.technologies.set(name, { level, progress, researched });
+				this.technologiesDirty = true;
 				if (progress) {
 					this.progressToBroadcast.add(name);
 				} else if (researched || baseLevel(name) !== level) {
@@ -176,10 +187,12 @@ export class ControllerPlugin extends BaseControllerPlugin {
 					} else {
 						this.progressToBroadcast.delete(name);
 					}
+					this.technologiesDirty = true;
 
 				} else if (tech.progress && progress && tech.progress < progress) {
 					tech.progress = progress;
 					this.progressToBroadcast.add(name);
+					this.technologiesDirty = true;
 				}
 			}
 		}

plugins/subspace_storage/controller.ts

@@ -75,9 +75,9 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	itemUpdateRateLimiter!: lib.RateLimiter;
 	itemsLastUpdate!: Map<string, number>;
 	subscribedControlLinks!: Set<ControlConnection>;
-	autosaveId!: ReturnType<typeof setInterval>;
 	doleMagicId!: ReturnType<typeof setInterval>;
 	neuralDole!: dole.NeuralDole;
+	storageDirty = false;
 
 	async init() {
 		this.items = await loadDatabase(this.controller.config, this.logger);
@@ -92,11 +92,6 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			},
 		});
 		this.itemsLastUpdate = new Map(this.items.getEntries());
-		this.autosaveId = setInterval(() => {
-			saveDatabase(this.controller.config, this.items, this.logger).catch(err => {
-				this.logger.error(`Unexpected error autosaving items:\n${err.stack}`);
-			});
-		}, this.controller.config.get("subspace_storage.autosave_interval") as number * 1000);
 
 		this.neuralDole = new dole.NeuralDole({ items: this.items });
 		this.doleMagicId = setInterval(() => {
@@ -117,6 +112,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 	updateStorage() {
 		this.itemUpdateRateLimiter.activate();
+		this.storageDirty = true;
 	}
 
 	broadcastStorage() {
@@ -254,8 +250,13 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 	async onShutdown() {
 		this.itemUpdateRateLimiter.cancel();
-		clearInterval(this.autosaveId);
 		clearInterval(this.doleMagicId);
-		await saveDatabase(this.controller.config, this.items, this.logger);
+	}
+
+	async onSaveData() {
+		if (this.storageDirty) {
+			this.storageDirty = false;
+			await saveDatabase(this.controller.config, this.items, this.logger);
+		}
 	}
 }

plugins/subspace_storage/info.ts

@@ -4,13 +4,6 @@ import * as messages from "./messages";
 class ControllerConfigGroup extends lib.PluginConfigGroup {}
 ControllerConfigGroup.defaultAccess = ["controller", "host", "control"];
 ControllerConfigGroup.groupName = "subspace_storage";
-ControllerConfigGroup.define({
-	name: "autosave_interval",
-	title: "Autosave Interval",
-	description: "Interval the storage is autosaved at in seconds.",
-	type: "number",
-	initial_value: 60,
-});
 ControllerConfigGroup.define({
 	name: "division_method",
 	title: "Division Method",

test/host.js

@@ -20,6 +20,7 @@ describe("Host testing", function() {
 			await referenceConfig.init();
 			referenceConfig.set("instance.id", 1);
 			referenceConfig.set("instance.name", "test");
+			referenceConfig.dirty = false;
 
 			assert.deepEqual(instanceInfos, new Map([
 				[1, {