From ffe5040644fd20f8490cc5eb7dac225dac4fe563 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 27 Nov 2023 04:57:02 +0100 Subject: [PATCH] Implement autosaving of data held in memory Add controller.autosave_interval config that specifies the frequency which controller config, hosts, instances, users, mod packs and plugin specific data which is held in memory should be automatically flushed to disk if they have changed since the last time they were saved. This prevents data loss in the event of a server crash. Plugins can use the onSaveData hook to save its own data at the same time as the controller does and this obsoletes making an autosave interval in the plugin. Resolves #523. --- .eslintrc.js | 6 +- .../controller/src/BaseControllerPlugin.ts | 13 ++ packages/controller/src/ControlConnection.ts | 2 + packages/controller/src/Controller.ts | 119 +++++++++++++++--- packages/controller/src/HostConnection.ts | 1 + packages/controller/src/UserManager.ts | 7 +- packages/lib/src/config/classes.ts | 6 + packages/lib/src/config/definitions.ts | 8 ++ plugins/inventory_sync/controller.ts | 16 ++- plugins/inventory_sync/info.ts | 7 -- plugins/research_sync/controller.ts | 15 ++- plugins/subspace_storage/controller.ts | 17 +-- plugins/subspace_storage/info.ts | 7 -- test/host.js | 1 + 14 files changed, 168 insertions(+), 57 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index e3b50a27..b316b1ed 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -137,7 +137,11 @@ module.exports = { "func-call-spacing": "error", "func-name-matching": "error", "func-names": "off", - "func-style": ["error", "declaration"], + "func-style": ["error", "declaration", { + // Allow arrow functions as it better expresses referencing this + // of the containing scope, which is unweilding in plain functions. + "allowArrowFunctions": true, + }], "function-call-argument-newline": "off", "function-paren-newline": "off", "generator-star-spacing": ["error", { "before": false, "after": true }], diff --git a/packages/controller/src/BaseControllerPlugin.ts b/packages/controller/src/BaseControllerPlugin.ts index 1beb1728..659001d2 100644 --- a/packages/controller/src/BaseControllerPlugin.ts +++ b/packages/controller/src/BaseControllerPlugin.ts @@ -40,6 +40,19 @@ export default class BaseControllerPlugin { */ async init() { } + /** + * Called when the controller saves data in memory to disk + * + * Invoked on the configured controller.autosave_interval by the + * controller and intended to be used to flush any in memory data that + * has changed to disk. + * + * This will also be called during graceful shutdown after {@link + * BaseControllerPlugin.onShutdown} have been invoked and all links have + * been disconnected. + */ + async onSaveData() { } + /** * Called when the status of an instance changes * diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index b416b0f0..eee48d8d 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -665,6 +665,7 @@ export default class ControlConnection extends BaseConnection { // Start at 5 to leave space for future default roles let id = Math.max(5, lastId+1); this._controller.userManager.roles.set(id, new lib.Role({ id, ...request })); + this._controller.userManager.dirty = true; return id; } @@ -699,6 +700,7 @@ export default class ControlConnection extends BaseConnection { } this._controller.userManager.roles.delete(id); + this._controller.userManager.dirty = true; for (let user of this._controller.userManager.users.values()) { user.roles.delete(role); this._controller.userPermissionsUpdated(user); diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 8d26c24e..10bfacf6 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -59,6 +59,13 @@ export default class Controller { /** User and roles manager for the cluster */ userManager: UserManager; + /** True if {@link Controller.hosts} has changed since last time it was saved */ + hostsDirty = false; + /** True if {@link Controller.instances} has changed since last time it was saved */ + instancesDirty = false; + /** True if {@link Controller.modPacks} has changed since last time it was saved */ + modPacksDirty = false; + httpServer: http.Server | null = null; httpServerCloser: HttpCloser | null = null; httpsServer: https.Server | null = null; @@ -85,6 +92,8 @@ export default class Controller { devMiddleware: any | null = null; + autosaveInterval?: ReturnType; + logDirectory: string = ""; clusterLogIndex: lib.LogIndex | null = null; clusterLogBuildInterval: ReturnType | null = null; @@ -182,6 +191,9 @@ export default class Controller { this.mods = await Controller.loadModInfos(modsDirectory); this.config.on("fieldChanged", (group, field, prev) => { + if (group.name === "controller" && field === "autosave_interval") { + this.onAutosaveIntervalChanged(); + } lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", group, field, prev); }); for (let instance of this.instances!.values()) { @@ -253,6 +265,8 @@ export default class Controller { logger.info(`Listening for HTTPS on port ${(this.httpsServer.address() as AddressInfo).port}`); } + this.onAutosaveIntervalChanged(); + logger.info("Started controller"); this._state = "running"; } @@ -317,34 +331,19 @@ export default class Controller { clearInterval(this.clusterLogBuildInterval); } + if (this.autosaveInterval) { + clearInterval(this.autosaveInterval); + this.autosaveInterval = undefined; + } + if (this.clusterLogIndex) { await this.clusterLogIndex.save(); } - logger.info("Saving config"); - await lib.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, "\t")); - if (this.devMiddleware) { await new Promise((resolve, reject) => { this.devMiddleware.close(resolve); }); } - let databaseDirectory = this.config.get("controller.database_directory"); - if (this.hosts) { - await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts); - } - - if (this.instances) { - await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances); - } - - if (this.modPacks) { - await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks); - } - - if (this.userManager) { - await this.userManager.save(path.join(databaseDirectory, "users.json")); - } - await lib.invokeHook(this.plugins, "onShutdown"); if (this.wsServer) { @@ -361,9 +360,85 @@ export default class Controller { } await Promise.all(stopTasks); + logger.info("Saving data"); + await this.saveData(); logger.info("Goodbye"); } + onAutosaveIntervalChanged() { + if (this.autosaveInterval) { + clearInterval(this.autosaveInterval); + this.autosaveInterval = undefined; + } + const autosaveIntervalSeconds = this.config.get("controller.autosave_interval"); + if (autosaveIntervalSeconds > 0) { + this.autosaveInterval = setInterval(this.saveData.bind(this), autosaveIntervalSeconds * 1000); + } + } + + private _saveDataCurrentlyRunning = false; + private _saveDataCurrentlyWaiting: (() => void)[] = []; + + /** + * Save all data currently in memory to disk + * + * Note: If a save is currently in progress this will wait until that + * completes and start another save. + */ + async saveData() { + if (this._saveDataCurrentlyRunning) { + const waitForCompletion = () => new Promise(resolve => { + this._saveDataCurrentlyWaiting.push(resolve); + }); + await waitForCompletion(); + if (this._saveDataCurrentlyRunning) { + await waitForCompletion(); + return; + } + } + + this._saveDataCurrentlyRunning = true; + try { + await this._saveDataInternal(); + } catch (err: any) { + logger.error(`Unexpected error during saveData:\n${err.stack}`); + } + this._saveDataCurrentlyRunning = false; + for (const callback of this._saveDataCurrentlyWaiting) { + callback(); + } + this._saveDataCurrentlyWaiting.length = 0; + } + + async _saveDataInternal() { + if (this.config.dirty) { + this.config.dirty = false; + await lib.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, "\t")); + } + + let databaseDirectory = this.config.get("controller.database_directory"); + if (this.hosts && this.hostsDirty) { + this.hostsDirty = false; + await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts); + } + + if (this.instances && this.instancesDirty) { + this.instancesDirty = false; + await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances); + } + + if (this.modPacks && this.modPacksDirty) { + this.modPacksDirty = false; + await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks); + } + + if (this.userManager && this.userManager.dirty) { + await this.userManager.save(path.join(databaseDirectory, "users.json")); + } + + await lib.invokeHook(this.plugins, "onSaveData"); + } + static async loadHosts(filePath: string): Promise> { let serialized: [number, HostInfo][]; try { @@ -766,6 +841,7 @@ export default class Controller { this.instanceUpdated(instance); } + this.instancesDirty = true; lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, group, field, prev); }); @@ -773,6 +849,7 @@ export default class Controller { } instanceUpdated(instance: InstanceInfo) { + this.instancesDirty = true; let assigned_host: number|null|undefined = instance.config.get("instance.assigned_host"); if (assigned_host === null) { assigned_host = undefined; @@ -799,6 +876,7 @@ export default class Controller { } modPackUpdated(modPack: lib.ModPack) { + this.modPacksDirty = true; this.subscriptions.broadcast(new lib.ModPackUpdateEvent(modPack)); lib.invokeHook(this.plugins, "onModPackUpdated", modPack); } @@ -809,6 +887,7 @@ export default class Controller { } userUpdated(user: lib.User) { + this.userManager.dirty = true; this.subscriptions.broadcast(new lib.UserUpdateEvent( new lib.RawUser( user.name, diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index 5d4f3b13..5c62f151 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -57,6 +57,7 @@ export default class HostConnection extends BaseConnection { plugins: registerData.plugins, token_valid_after: currentHostInfo?.token_valid_after, }); + this._controller.hostsDirty = true; for (let event of ["connect", "drop", "resume", "close"] as const) { // eslint-disable-next-line no-loop-func diff --git a/packages/controller/src/UserManager.ts b/packages/controller/src/UserManager.ts index e1348a07..eadaef9a 100644 --- a/packages/controller/src/UserManager.ts +++ b/packages/controller/src/UserManager.ts @@ -10,6 +10,7 @@ import * as lib from "@clusterio/lib"; export default class UserManager { roles: Map = new Map(); users: Map = new Map(); + dirty = false; constructor( private _config: lib.ControllerConfig @@ -41,10 +42,6 @@ export default class UserManager { } async save(filePath:string): Promise { - if (this.roles.size === 0 || this.users.size === 0) { - return; - } - let serializedRoles = []; for (let role of this.roles.values()) { serializedRoles.push(role.serialize()); @@ -59,6 +56,7 @@ export default class UserManager { users: serializedUsers, roles: serializedRoles, }; + this.dirty = false; await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t")); } @@ -80,6 +78,7 @@ export default class UserManager { let user = new lib.User({ name, roles }, this.roles); this.users.set(name, user); + this.dirty = true; return user; } diff --git a/packages/lib/src/config/classes.ts b/packages/lib/src/config/classes.ts index a3ce2fa4..c809a34c 100644 --- a/packages/lib/src/config/classes.ts +++ b/packages/lib/src/config/classes.ts @@ -85,6 +85,9 @@ export class Config extends events.EventEmitter { _groups = new Map(); _unknownGroups = new Map(); + /** Set to true when a field in the config is changed. */ + dirty = false; + /** * Create a new instance of the given config * @@ -566,6 +569,7 @@ export class ConfigGroup { let prev = this._fields.get(name); this._fields.set(name, value); if (this.config && !isDeepStrictEqual(value, prev)) { + this.config.dirty = true; this.config.emit("fieldChanged", this, name, prev); } } @@ -604,6 +608,7 @@ export class ConfigGroup { } this._fields.set(name, updated); if (this.config && !isDeepStrictEqual(updated, prev)) { + this.config.dirty = true; this.config.emit("fieldChanged", this, name, prev); } } @@ -663,6 +668,7 @@ export class ConfigGroup { let prev = this._fields.get(name); this._fields.set(name, value); if (notify && this.config && !isDeepStrictEqual(value, prev)) { + this.config.dirty = true; this.config.emit("fieldChanged", this, name, prev); } } diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 91a4277f..282e5b60 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -31,6 +31,13 @@ ControllerGroup.define({ type: "string", initial_value: "database", }); +ControllerGroup.define({ + name: "autosave_interval", + title: "Autosave Interval", + description: "Interval in seconds to autosave data in memory to disk.", + type: "number", + initial_value: 60, +}); ControllerGroup.define({ name: "http_port", title: "HTTP Port", @@ -157,6 +164,7 @@ export class ControllerConfig extends classes.Config { get(name: "controller.proxy_stream_timeout"): number; get(name: "controller.default_mod_pack_id"): number | null; get(name: "controller.default_role_id"): number | null; + get(name: "controller.autosave_interval"): number; get(name: string): unknown; get(name: string) { return super.get(name); diff --git a/plugins/inventory_sync/controller.ts b/plugins/inventory_sync/controller.ts index 3b399315..e5d4d8ea 100644 --- a/plugins/inventory_sync/controller.ts +++ b/plugins/inventory_sync/controller.ts @@ -38,16 +38,11 @@ async function saveDatabase( export class ControllerPlugin extends BaseControllerPlugin { acquiredPlayers!: Map; playerDatastore!: Map; - autosaveId!: ReturnType; + playerDatastoreDirty = false; async init() { this.acquiredPlayers = new Map(); this.playerDatastore = await loadDatabase(this.controller.config, this.logger); - this.autosaveId = setInterval(() => { - saveDatabase(this.controller.config, this.playerDatastore, this.logger).catch(err => { - this.logger.error(`Unexpected error autosaving player data:\n${err.stack}`); - }); - }, this.controller.config.get("inventory_sync.autosave_interval") as number * 1000); this.controller.handle(msg.AcquireRequest, this.handleAcquireRequest.bind(this)); this.controller.handle(msg.ReleaseRequest, this.handleReleaseRequest.bind(this)); @@ -161,6 +156,7 @@ export class ControllerPlugin extends BaseControllerPlugin { if (store) { this.logger.verbose(`Received player data for ${playerName} from ${instanceName}`); this.playerDatastore.set(playerName, playerData); + this.playerDatastoreDirty = true; } } @@ -179,9 +175,11 @@ export class ControllerPlugin extends BaseControllerPlugin { return new msg.DownloadRequest.Response(this.playerDatastore.get(playerName) || null); } - async onShutdown() { - clearInterval(this.autosaveId); - await saveDatabase(this.controller.config, this.playerDatastore, this.logger); + async onSaveData() { + if (this.playerDatastoreDirty) { + this.playerDatastoreDirty = false; + await saveDatabase(this.controller.config, this.playerDatastore, this.logger); + } } async handleDatabaseStatsRequest() { diff --git a/plugins/inventory_sync/info.ts b/plugins/inventory_sync/info.ts index d3090271..62f693bd 100644 --- a/plugins/inventory_sync/info.ts +++ b/plugins/inventory_sync/info.ts @@ -4,13 +4,6 @@ import * as messages from "./messages"; class ControllerConfigGroup extends lib.PluginConfigGroup { } ControllerConfigGroup.defaultAccess = ["controller", "host", "control"]; ControllerConfigGroup.groupName = "inventory_sync"; -ControllerConfigGroup.define({ - name: "autosave_interval", - title: "Autosave Interval", - description: "Interval the player data is autosaved at in seconds.", - type: "number", - initial_value: 600, // 10 minutes -}); ControllerConfigGroup.define({ name: "player_lock_timeout", title: "Player Lock Timeout", diff --git a/plugins/research_sync/controller.ts b/plugins/research_sync/controller.ts index ea8174b4..79decf82 100644 --- a/plugins/research_sync/controller.ts +++ b/plugins/research_sync/controller.ts @@ -51,6 +51,7 @@ async function saveTechnologies( export class ControllerPlugin extends BaseControllerPlugin { technologies!: Map; + technologiesDirty = true; progressRateLimiter!: lib.RateLimiter; progressBroadcastId!: ReturnType | null; progressToBroadcast!: Set; @@ -70,9 +71,15 @@ export class ControllerPlugin extends BaseControllerPlugin { this.controller.handle(SyncTechnologiesRequest, this.handleSyncTechnologiesRequest.bind(this)); } + async onSaveData() { + if (this.technologiesDirty) { + this.technologiesDirty = false; + await saveTechnologies(this.controller.config, this.technologies, this.logger); + } + } + async onShutdown() { this.progressRateLimiter.cancel(); - await saveTechnologies(this.controller.config, this.technologies, this.logger); } broadcastProgress() { @@ -96,6 +103,7 @@ export class ControllerPlugin extends BaseControllerPlugin { if (!tech) { tech = { level, progress: 0, researched: false }; this.technologies.set(name, tech); + this.technologiesDirty = true; // Ignore contribution to already researched technologies } else if (tech.level > level || tech.level === level && tech.researched) { @@ -126,6 +134,7 @@ export class ControllerPlugin extends BaseControllerPlugin { this.controller.sendTo("allInstances", new FinishedEvent(name, tech.level)); } + this.technologiesDirty = true; } async handleFinishedEvent(event: FinishedEvent) { @@ -134,6 +143,7 @@ export class ControllerPlugin extends BaseControllerPlugin { if (!tech || tech.level <= level) { this.progressToBroadcast.delete(name); this.technologies.set(name, { level, progress: null, researched: true }); + this.technologiesDirty = true; } } @@ -151,6 +161,7 @@ export class ControllerPlugin extends BaseControllerPlugin { let tech = this.technologies.get(name); if (!tech) { this.technologies.set(name, { level, progress, researched }); + this.technologiesDirty = true; if (progress) { this.progressToBroadcast.add(name); } else if (researched || baseLevel(name) !== level) { @@ -176,10 +187,12 @@ export class ControllerPlugin extends BaseControllerPlugin { } else { this.progressToBroadcast.delete(name); } + this.technologiesDirty = true; } else if (tech.progress && progress && tech.progress < progress) { tech.progress = progress; this.progressToBroadcast.add(name); + this.technologiesDirty = true; } } } diff --git a/plugins/subspace_storage/controller.ts b/plugins/subspace_storage/controller.ts index 65c219cd..b08fe064 100644 --- a/plugins/subspace_storage/controller.ts +++ b/plugins/subspace_storage/controller.ts @@ -75,9 +75,9 @@ export class ControllerPlugin extends BaseControllerPlugin { itemUpdateRateLimiter!: lib.RateLimiter; itemsLastUpdate!: Map; subscribedControlLinks!: Set; - autosaveId!: ReturnType; doleMagicId!: ReturnType; neuralDole!: dole.NeuralDole; + storageDirty = false; async init() { this.items = await loadDatabase(this.controller.config, this.logger); @@ -92,11 +92,6 @@ export class ControllerPlugin extends BaseControllerPlugin { }, }); this.itemsLastUpdate = new Map(this.items.getEntries()); - this.autosaveId = setInterval(() => { - saveDatabase(this.controller.config, this.items, this.logger).catch(err => { - this.logger.error(`Unexpected error autosaving items:\n${err.stack}`); - }); - }, this.controller.config.get("subspace_storage.autosave_interval") as number * 1000); this.neuralDole = new dole.NeuralDole({ items: this.items }); this.doleMagicId = setInterval(() => { @@ -117,6 +112,7 @@ export class ControllerPlugin extends BaseControllerPlugin { updateStorage() { this.itemUpdateRateLimiter.activate(); + this.storageDirty = true; } broadcastStorage() { @@ -254,8 +250,13 @@ export class ControllerPlugin extends BaseControllerPlugin { async onShutdown() { this.itemUpdateRateLimiter.cancel(); - clearInterval(this.autosaveId); clearInterval(this.doleMagicId); - await saveDatabase(this.controller.config, this.items, this.logger); + } + + async onSaveData() { + if (this.storageDirty) { + this.storageDirty = false; + await saveDatabase(this.controller.config, this.items, this.logger); + } } } diff --git a/plugins/subspace_storage/info.ts b/plugins/subspace_storage/info.ts index c54da3c2..04c47cd9 100644 --- a/plugins/subspace_storage/info.ts +++ b/plugins/subspace_storage/info.ts @@ -4,13 +4,6 @@ import * as messages from "./messages"; class ControllerConfigGroup extends lib.PluginConfigGroup {} ControllerConfigGroup.defaultAccess = ["controller", "host", "control"]; ControllerConfigGroup.groupName = "subspace_storage"; -ControllerConfigGroup.define({ - name: "autosave_interval", - title: "Autosave Interval", - description: "Interval the storage is autosaved at in seconds.", - type: "number", - initial_value: 60, -}); ControllerConfigGroup.define({ name: "division_method", title: "Division Method", diff --git a/test/host.js b/test/host.js index 9aee450c..60cd63fe 100644 --- a/test/host.js +++ b/test/host.js @@ -20,6 +20,7 @@ describe("Host testing", function() { await referenceConfig.init(); referenceConfig.set("instance.id", 1); referenceConfig.set("instance.name", "test"); + referenceConfig.dirty = false; assert.deepEqual(instanceInfos, new Map([ [1, {