Implement autosaving of data held in memory

Add controller.autosave_interval config that specifies the frequency
which controller config, hosts, instances, users, mod packs and plugin
specific data which is held in memory should be automatically flushed to
disk if they have changed since the last time they were saved.  This
prevents data loss in the event of a server crash.

Plugins can use the onSaveData hook to save its own data at the same
time as the controller does and this obsoletes making an autosave
interval in the plugin.

Resolves #523.

plugins/inventory_sync/controller.ts

@@ -38,16 +38,11 @@ async function saveDatabase(
 export class ControllerPlugin extends BaseControllerPlugin {
 	acquiredPlayers!: Map<string, { instanceId: number, expires?: number }>;
 	playerDatastore!: Map<string, IpcPlayerData>;
-	autosaveId!: ReturnType<typeof setInterval>;
+	playerDatastoreDirty = false;
 
 	async init() {
 		this.acquiredPlayers = new Map();
 		this.playerDatastore = await loadDatabase(this.controller.config, this.logger);
-		this.autosaveId = setInterval(() => {
-			saveDatabase(this.controller.config, this.playerDatastore, this.logger).catch(err => {
-				this.logger.error(`Unexpected error autosaving player data:\n${err.stack}`);
-			});
-		}, this.controller.config.get("inventory_sync.autosave_interval") as number * 1000);
 
 		this.controller.handle(msg.AcquireRequest, this.handleAcquireRequest.bind(this));
 		this.controller.handle(msg.ReleaseRequest, this.handleReleaseRequest.bind(this));
@@ -161,6 +156,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		if (store) {
 			this.logger.verbose(`Received player data for ${playerName} from ${instanceName}`);
 			this.playerDatastore.set(playerName, playerData);
+			this.playerDatastoreDirty = true;
 		}
 	}
 
@@ -179,9 +175,11 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		return new msg.DownloadRequest.Response(this.playerDatastore.get(playerName) || null);
 	}
 
-	async onShutdown() {
-		clearInterval(this.autosaveId);
-		await saveDatabase(this.controller.config, this.playerDatastore, this.logger);
+	async onSaveData() {
+		if (this.playerDatastoreDirty) {
+			this.playerDatastoreDirty = false;
+			await saveDatabase(this.controller.config, this.playerDatastore, this.logger);
+		}
 	}
 
 	async handleDatabaseStatsRequest() {

plugins/inventory_sync/info.ts

@@ -4,13 +4,6 @@ import * as messages from "./messages";
 class ControllerConfigGroup extends lib.PluginConfigGroup { }
 ControllerConfigGroup.defaultAccess = ["controller", "host", "control"];
 ControllerConfigGroup.groupName = "inventory_sync";
-ControllerConfigGroup.define({
-	name: "autosave_interval",
-	title: "Autosave Interval",
-	description: "Interval the player data is autosaved at in seconds.",
-	type: "number",
-	initial_value: 600, // 10 minutes
-});
 ControllerConfigGroup.define({
 	name: "player_lock_timeout",
 	title: "Player Lock Timeout",

plugins/research_sync/controller.ts

@@ -51,6 +51,7 @@ async function saveTechnologies(
 
 export class ControllerPlugin extends BaseControllerPlugin {
 	technologies!: Map<string, Technology>;
+	technologiesDirty = true;
 	progressRateLimiter!: lib.RateLimiter;
 	progressBroadcastId!: ReturnType<typeof setInterval> | null;
 	progressToBroadcast!: Set<string>;
@@ -70,9 +71,15 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		this.controller.handle(SyncTechnologiesRequest, this.handleSyncTechnologiesRequest.bind(this));
 	}
 
+	async onSaveData() {
+		if (this.technologiesDirty) {
+			this.technologiesDirty = false;
+			await saveTechnologies(this.controller.config, this.technologies, this.logger);
+		}
+	}
+
 	async onShutdown() {
 		this.progressRateLimiter.cancel();
-		await saveTechnologies(this.controller.config, this.technologies, this.logger);
 	}
 
 	broadcastProgress() {
@@ -96,6 +103,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		if (!tech) {
 			tech = { level, progress: 0, researched: false };
 			this.technologies.set(name, tech);
+			this.technologiesDirty = true;
 
 		// Ignore contribution to already researched technologies
 		} else if (tech.level > level || tech.level === level && tech.researched) {
@@ -126,6 +134,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 			this.controller.sendTo("allInstances", new FinishedEvent(name, tech.level));
 		}
+		this.technologiesDirty = true;
 	}
 
 	async handleFinishedEvent(event: FinishedEvent) {
@@ -134,6 +143,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 		if (!tech || tech.level <= level) {
 			this.progressToBroadcast.delete(name);
 			this.technologies.set(name, { level, progress: null, researched: true });
+			this.technologiesDirty = true;
 		}
 	}
 
@@ -151,6 +161,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			let tech = this.technologies.get(name);
 			if (!tech) {
 				this.technologies.set(name, { level, progress, researched });
+				this.technologiesDirty = true;
 				if (progress) {
 					this.progressToBroadcast.add(name);
 				} else if (researched || baseLevel(name) !== level) {
@@ -176,10 +187,12 @@ export class ControllerPlugin extends BaseControllerPlugin {
 					} else {
 						this.progressToBroadcast.delete(name);
 					}
+					this.technologiesDirty = true;
 
 				} else if (tech.progress && progress && tech.progress < progress) {
 					tech.progress = progress;
 					this.progressToBroadcast.add(name);
+					this.technologiesDirty = true;
 				}
 			}
 		}

plugins/subspace_storage/controller.ts

@@ -75,9 +75,9 @@ export class ControllerPlugin extends BaseControllerPlugin {
 	itemUpdateRateLimiter!: lib.RateLimiter;
 	itemsLastUpdate!: Map<string, number>;
 	subscribedControlLinks!: Set<ControlConnection>;
-	autosaveId!: ReturnType<typeof setInterval>;
 	doleMagicId!: ReturnType<typeof setInterval>;
 	neuralDole!: dole.NeuralDole;
+	storageDirty = false;
 
 	async init() {
 		this.items = await loadDatabase(this.controller.config, this.logger);
@@ -92,11 +92,6 @@ export class ControllerPlugin extends BaseControllerPlugin {
 			},
 		});
 		this.itemsLastUpdate = new Map(this.items.getEntries());
-		this.autosaveId = setInterval(() => {
-			saveDatabase(this.controller.config, this.items, this.logger).catch(err => {
-				this.logger.error(`Unexpected error autosaving items:\n${err.stack}`);
-			});
-		}, this.controller.config.get("subspace_storage.autosave_interval") as number * 1000);
 
 		this.neuralDole = new dole.NeuralDole({ items: this.items });
 		this.doleMagicId = setInterval(() => {
@@ -117,6 +112,7 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 	updateStorage() {
 		this.itemUpdateRateLimiter.activate();
+		this.storageDirty = true;
 	}
 
 	broadcastStorage() {
@@ -254,8 +250,13 @@ export class ControllerPlugin extends BaseControllerPlugin {
 
 	async onShutdown() {
 		this.itemUpdateRateLimiter.cancel();
-		clearInterval(this.autosaveId);
 		clearInterval(this.doleMagicId);
-		await saveDatabase(this.controller.config, this.items, this.logger);
+	}
+
+	async onSaveData() {
+		if (this.storageDirty) {
+			this.storageDirty = false;
+			await saveDatabase(this.controller.config, this.items, this.logger);
+		}
 	}
 }

plugins/subspace_storage/info.ts

@@ -4,13 +4,6 @@ import * as messages from "./messages";
 class ControllerConfigGroup extends lib.PluginConfigGroup {}
 ControllerConfigGroup.defaultAccess = ["controller", "host", "control"];
 ControllerConfigGroup.groupName = "subspace_storage";
-ControllerConfigGroup.define({
-	name: "autosave_interval",
-	title: "Autosave Interval",
-	description: "Interval the storage is autosaved at in seconds.",
-	type: "number",
-	initial_value: 60,
-});
 ControllerConfigGroup.define({
 	name: "division_method",
 	title: "Division Method",