diff --git a/.eslintrc.js b/.eslintrc.js index e3b50a27..b316b1ed 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -137,7 +137,11 @@ module.exports = { "func-call-spacing": "error", "func-name-matching": "error", "func-names": "off", - "func-style": ["error", "declaration"], + "func-style": ["error", "declaration", { + // Allow arrow functions as it better expresses referencing this + // of the containing scope, which is unweilding in plain functions. + "allowArrowFunctions": true, + }], "function-call-argument-newline": "off", "function-paren-newline": "off", "generator-star-spacing": ["error", { "before": false, "after": true }], diff --git a/packages/controller/src/BaseControllerPlugin.ts b/packages/controller/src/BaseControllerPlugin.ts index 1beb1728..659001d2 100644 --- a/packages/controller/src/BaseControllerPlugin.ts +++ b/packages/controller/src/BaseControllerPlugin.ts @@ -40,6 +40,19 @@ export default class BaseControllerPlugin { */ async init() { } + /** + * Called when the controller saves data in memory to disk + * + * Invoked on the configured controller.autosave_interval by the + * controller and intended to be used to flush any in memory data that + * has changed to disk. + * + * This will also be called during graceful shutdown after {@link + * BaseControllerPlugin.onShutdown} have been invoked and all links have + * been disconnected. + */ + async onSaveData() { } + /** * Called when the status of an instance changes * diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index b416b0f0..eee48d8d 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -665,6 +665,7 @@ export default class ControlConnection extends BaseConnection { // Start at 5 to leave space for future default roles let id = Math.max(5, lastId+1); this._controller.userManager.roles.set(id, new lib.Role({ id, ...request })); + this._controller.userManager.dirty = true; return id; } @@ -699,6 +700,7 @@ export default class ControlConnection extends BaseConnection { } this._controller.userManager.roles.delete(id); + this._controller.userManager.dirty = true; for (let user of this._controller.userManager.users.values()) { user.roles.delete(role); this._controller.userPermissionsUpdated(user); diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 8d26c24e..10bfacf6 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -59,6 +59,13 @@ export default class Controller { /** User and roles manager for the cluster */ userManager: UserManager; + /** True if {@link Controller.hosts} has changed since last time it was saved */ + hostsDirty = false; + /** True if {@link Controller.instances} has changed since last time it was saved */ + instancesDirty = false; + /** True if {@link Controller.modPacks} has changed since last time it was saved */ + modPacksDirty = false; + httpServer: http.Server | null = null; httpServerCloser: HttpCloser | null = null; httpsServer: https.Server | null = null; @@ -85,6 +92,8 @@ export default class Controller { devMiddleware: any | null = null; + autosaveInterval?: ReturnType; + logDirectory: string = ""; clusterLogIndex: lib.LogIndex | null = null; clusterLogBuildInterval: ReturnType | null = null; @@ -182,6 +191,9 @@ export default class Controller { this.mods = await Controller.loadModInfos(modsDirectory); this.config.on("fieldChanged", (group, field, prev) => { + if (group.name === "controller" && field === "autosave_interval") { + this.onAutosaveIntervalChanged(); + } lib.invokeHook(this.plugins, "onControllerConfigFieldChanged", group, field, prev); }); for (let instance of this.instances!.values()) { @@ -253,6 +265,8 @@ export default class Controller { logger.info(`Listening for HTTPS on port ${(this.httpsServer.address() as AddressInfo).port}`); } + this.onAutosaveIntervalChanged(); + logger.info("Started controller"); this._state = "running"; } @@ -317,34 +331,19 @@ export default class Controller { clearInterval(this.clusterLogBuildInterval); } + if (this.autosaveInterval) { + clearInterval(this.autosaveInterval); + this.autosaveInterval = undefined; + } + if (this.clusterLogIndex) { await this.clusterLogIndex.save(); } - logger.info("Saving config"); - await lib.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, "\t")); - if (this.devMiddleware) { await new Promise((resolve, reject) => { this.devMiddleware.close(resolve); }); } - let databaseDirectory = this.config.get("controller.database_directory"); - if (this.hosts) { - await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts); - } - - if (this.instances) { - await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances); - } - - if (this.modPacks) { - await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks); - } - - if (this.userManager) { - await this.userManager.save(path.join(databaseDirectory, "users.json")); - } - await lib.invokeHook(this.plugins, "onShutdown"); if (this.wsServer) { @@ -361,9 +360,85 @@ export default class Controller { } await Promise.all(stopTasks); + logger.info("Saving data"); + await this.saveData(); logger.info("Goodbye"); } + onAutosaveIntervalChanged() { + if (this.autosaveInterval) { + clearInterval(this.autosaveInterval); + this.autosaveInterval = undefined; + } + const autosaveIntervalSeconds = this.config.get("controller.autosave_interval"); + if (autosaveIntervalSeconds > 0) { + this.autosaveInterval = setInterval(this.saveData.bind(this), autosaveIntervalSeconds * 1000); + } + } + + private _saveDataCurrentlyRunning = false; + private _saveDataCurrentlyWaiting: (() => void)[] = []; + + /** + * Save all data currently in memory to disk + * + * Note: If a save is currently in progress this will wait until that + * completes and start another save. + */ + async saveData() { + if (this._saveDataCurrentlyRunning) { + const waitForCompletion = () => new Promise(resolve => { + this._saveDataCurrentlyWaiting.push(resolve); + }); + await waitForCompletion(); + if (this._saveDataCurrentlyRunning) { + await waitForCompletion(); + return; + } + } + + this._saveDataCurrentlyRunning = true; + try { + await this._saveDataInternal(); + } catch (err: any) { + logger.error(`Unexpected error during saveData:\n${err.stack}`); + } + this._saveDataCurrentlyRunning = false; + for (const callback of this._saveDataCurrentlyWaiting) { + callback(); + } + this._saveDataCurrentlyWaiting.length = 0; + } + + async _saveDataInternal() { + if (this.config.dirty) { + this.config.dirty = false; + await lib.safeOutputFile(this.configPath, JSON.stringify(this.config.serialize(), null, "\t")); + } + + let databaseDirectory = this.config.get("controller.database_directory"); + if (this.hosts && this.hostsDirty) { + this.hostsDirty = false; + await Controller.saveHosts(path.join(databaseDirectory, "hosts.json"), this.hosts); + } + + if (this.instances && this.instancesDirty) { + this.instancesDirty = false; + await Controller.saveInstances(path.join(databaseDirectory, "instances.json"), this.instances); + } + + if (this.modPacks && this.modPacksDirty) { + this.modPacksDirty = false; + await Controller.saveModPacks(path.join(databaseDirectory, "mod-packs.json"), this.modPacks); + } + + if (this.userManager && this.userManager.dirty) { + await this.userManager.save(path.join(databaseDirectory, "users.json")); + } + + await lib.invokeHook(this.plugins, "onSaveData"); + } + static async loadHosts(filePath: string): Promise> { let serialized: [number, HostInfo][]; try { @@ -766,6 +841,7 @@ export default class Controller { this.instanceUpdated(instance); } + this.instancesDirty = true; lib.invokeHook(this.plugins, "onInstanceConfigFieldChanged", instance, group, field, prev); }); @@ -773,6 +849,7 @@ export default class Controller { } instanceUpdated(instance: InstanceInfo) { + this.instancesDirty = true; let assigned_host: number|null|undefined = instance.config.get("instance.assigned_host"); if (assigned_host === null) { assigned_host = undefined; @@ -799,6 +876,7 @@ export default class Controller { } modPackUpdated(modPack: lib.ModPack) { + this.modPacksDirty = true; this.subscriptions.broadcast(new lib.ModPackUpdateEvent(modPack)); lib.invokeHook(this.plugins, "onModPackUpdated", modPack); } @@ -809,6 +887,7 @@ export default class Controller { } userUpdated(user: lib.User) { + this.userManager.dirty = true; this.subscriptions.broadcast(new lib.UserUpdateEvent( new lib.RawUser( user.name, diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index 5d4f3b13..5c62f151 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -57,6 +57,7 @@ export default class HostConnection extends BaseConnection { plugins: registerData.plugins, token_valid_after: currentHostInfo?.token_valid_after, }); + this._controller.hostsDirty = true; for (let event of ["connect", "drop", "resume", "close"] as const) { // eslint-disable-next-line no-loop-func diff --git a/packages/controller/src/UserManager.ts b/packages/controller/src/UserManager.ts index e1348a07..eadaef9a 100644 --- a/packages/controller/src/UserManager.ts +++ b/packages/controller/src/UserManager.ts @@ -10,6 +10,7 @@ import * as lib from "@clusterio/lib"; export default class UserManager { roles: Map = new Map(); users: Map = new Map(); + dirty = false; constructor( private _config: lib.ControllerConfig @@ -41,10 +42,6 @@ export default class UserManager { } async save(filePath:string): Promise { - if (this.roles.size === 0 || this.users.size === 0) { - return; - } - let serializedRoles = []; for (let role of this.roles.values()) { serializedRoles.push(role.serialize()); @@ -59,6 +56,7 @@ export default class UserManager { users: serializedUsers, roles: serializedRoles, }; + this.dirty = false; await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t")); } @@ -80,6 +78,7 @@ export default class UserManager { let user = new lib.User({ name, roles }, this.roles); this.users.set(name, user); + this.dirty = true; return user; } diff --git a/packages/lib/src/config/classes.ts b/packages/lib/src/config/classes.ts index a3ce2fa4..c809a34c 100644 --- a/packages/lib/src/config/classes.ts +++ b/packages/lib/src/config/classes.ts @@ -85,6 +85,9 @@ export class Config extends events.EventEmitter { _groups = new Map(); _unknownGroups = new Map(); + /** Set to true when a field in the config is changed. */ + dirty = false; + /** * Create a new instance of the given config * @@ -566,6 +569,7 @@ export class ConfigGroup { let prev = this._fields.get(name); this._fields.set(name, value); if (this.config && !isDeepStrictEqual(value, prev)) { + this.config.dirty = true; this.config.emit("fieldChanged", this, name, prev); } } @@ -604,6 +608,7 @@ export class ConfigGroup { } this._fields.set(name, updated); if (this.config && !isDeepStrictEqual(updated, prev)) { + this.config.dirty = true; this.config.emit("fieldChanged", this, name, prev); } } @@ -663,6 +668,7 @@ export class ConfigGroup { let prev = this._fields.get(name); this._fields.set(name, value); if (notify && this.config && !isDeepStrictEqual(value, prev)) { + this.config.dirty = true; this.config.emit("fieldChanged", this, name, prev); } } diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 91a4277f..282e5b60 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -31,6 +31,13 @@ ControllerGroup.define({ type: "string", initial_value: "database", }); +ControllerGroup.define({ + name: "autosave_interval", + title: "Autosave Interval", + description: "Interval in seconds to autosave data in memory to disk.", + type: "number", + initial_value: 60, +}); ControllerGroup.define({ name: "http_port", title: "HTTP Port", @@ -157,6 +164,7 @@ export class ControllerConfig extends classes.Config { get(name: "controller.proxy_stream_timeout"): number; get(name: "controller.default_mod_pack_id"): number | null; get(name: "controller.default_role_id"): number | null; + get(name: "controller.autosave_interval"): number; get(name: string): unknown; get(name: string) { return super.get(name); diff --git a/plugins/inventory_sync/controller.ts b/plugins/inventory_sync/controller.ts index 3b399315..e5d4d8ea 100644 --- a/plugins/inventory_sync/controller.ts +++ b/plugins/inventory_sync/controller.ts @@ -38,16 +38,11 @@ async function saveDatabase( export class ControllerPlugin extends BaseControllerPlugin { acquiredPlayers!: Map; playerDatastore!: Map; - autosaveId!: ReturnType; + playerDatastoreDirty = false; async init() { this.acquiredPlayers = new Map(); this.playerDatastore = await loadDatabase(this.controller.config, this.logger); - this.autosaveId = setInterval(() => { - saveDatabase(this.controller.config, this.playerDatastore, this.logger).catch(err => { - this.logger.error(`Unexpected error autosaving player data:\n${err.stack}`); - }); - }, this.controller.config.get("inventory_sync.autosave_interval") as number * 1000); this.controller.handle(msg.AcquireRequest, this.handleAcquireRequest.bind(this)); this.controller.handle(msg.ReleaseRequest, this.handleReleaseRequest.bind(this)); @@ -161,6 +156,7 @@ export class ControllerPlugin extends BaseControllerPlugin { if (store) { this.logger.verbose(`Received player data for ${playerName} from ${instanceName}`); this.playerDatastore.set(playerName, playerData); + this.playerDatastoreDirty = true; } } @@ -179,9 +175,11 @@ export class ControllerPlugin extends BaseControllerPlugin { return new msg.DownloadRequest.Response(this.playerDatastore.get(playerName) || null); } - async onShutdown() { - clearInterval(this.autosaveId); - await saveDatabase(this.controller.config, this.playerDatastore, this.logger); + async onSaveData() { + if (this.playerDatastoreDirty) { + this.playerDatastoreDirty = false; + await saveDatabase(this.controller.config, this.playerDatastore, this.logger); + } } async handleDatabaseStatsRequest() { diff --git a/plugins/inventory_sync/info.ts b/plugins/inventory_sync/info.ts index d3090271..62f693bd 100644 --- a/plugins/inventory_sync/info.ts +++ b/plugins/inventory_sync/info.ts @@ -4,13 +4,6 @@ import * as messages from "./messages"; class ControllerConfigGroup extends lib.PluginConfigGroup { } ControllerConfigGroup.defaultAccess = ["controller", "host", "control"]; ControllerConfigGroup.groupName = "inventory_sync"; -ControllerConfigGroup.define({ - name: "autosave_interval", - title: "Autosave Interval", - description: "Interval the player data is autosaved at in seconds.", - type: "number", - initial_value: 600, // 10 minutes -}); ControllerConfigGroup.define({ name: "player_lock_timeout", title: "Player Lock Timeout", diff --git a/plugins/research_sync/controller.ts b/plugins/research_sync/controller.ts index ea8174b4..79decf82 100644 --- a/plugins/research_sync/controller.ts +++ b/plugins/research_sync/controller.ts @@ -51,6 +51,7 @@ async function saveTechnologies( export class ControllerPlugin extends BaseControllerPlugin { technologies!: Map; + technologiesDirty = true; progressRateLimiter!: lib.RateLimiter; progressBroadcastId!: ReturnType | null; progressToBroadcast!: Set; @@ -70,9 +71,15 @@ export class ControllerPlugin extends BaseControllerPlugin { this.controller.handle(SyncTechnologiesRequest, this.handleSyncTechnologiesRequest.bind(this)); } + async onSaveData() { + if (this.technologiesDirty) { + this.technologiesDirty = false; + await saveTechnologies(this.controller.config, this.technologies, this.logger); + } + } + async onShutdown() { this.progressRateLimiter.cancel(); - await saveTechnologies(this.controller.config, this.technologies, this.logger); } broadcastProgress() { @@ -96,6 +103,7 @@ export class ControllerPlugin extends BaseControllerPlugin { if (!tech) { tech = { level, progress: 0, researched: false }; this.technologies.set(name, tech); + this.technologiesDirty = true; // Ignore contribution to already researched technologies } else if (tech.level > level || tech.level === level && tech.researched) { @@ -126,6 +134,7 @@ export class ControllerPlugin extends BaseControllerPlugin { this.controller.sendTo("allInstances", new FinishedEvent(name, tech.level)); } + this.technologiesDirty = true; } async handleFinishedEvent(event: FinishedEvent) { @@ -134,6 +143,7 @@ export class ControllerPlugin extends BaseControllerPlugin { if (!tech || tech.level <= level) { this.progressToBroadcast.delete(name); this.technologies.set(name, { level, progress: null, researched: true }); + this.technologiesDirty = true; } } @@ -151,6 +161,7 @@ export class ControllerPlugin extends BaseControllerPlugin { let tech = this.technologies.get(name); if (!tech) { this.technologies.set(name, { level, progress, researched }); + this.technologiesDirty = true; if (progress) { this.progressToBroadcast.add(name); } else if (researched || baseLevel(name) !== level) { @@ -176,10 +187,12 @@ export class ControllerPlugin extends BaseControllerPlugin { } else { this.progressToBroadcast.delete(name); } + this.technologiesDirty = true; } else if (tech.progress && progress && tech.progress < progress) { tech.progress = progress; this.progressToBroadcast.add(name); + this.technologiesDirty = true; } } } diff --git a/plugins/subspace_storage/controller.ts b/plugins/subspace_storage/controller.ts index 65c219cd..b08fe064 100644 --- a/plugins/subspace_storage/controller.ts +++ b/plugins/subspace_storage/controller.ts @@ -75,9 +75,9 @@ export class ControllerPlugin extends BaseControllerPlugin { itemUpdateRateLimiter!: lib.RateLimiter; itemsLastUpdate!: Map; subscribedControlLinks!: Set; - autosaveId!: ReturnType; doleMagicId!: ReturnType; neuralDole!: dole.NeuralDole; + storageDirty = false; async init() { this.items = await loadDatabase(this.controller.config, this.logger); @@ -92,11 +92,6 @@ export class ControllerPlugin extends BaseControllerPlugin { }, }); this.itemsLastUpdate = new Map(this.items.getEntries()); - this.autosaveId = setInterval(() => { - saveDatabase(this.controller.config, this.items, this.logger).catch(err => { - this.logger.error(`Unexpected error autosaving items:\n${err.stack}`); - }); - }, this.controller.config.get("subspace_storage.autosave_interval") as number * 1000); this.neuralDole = new dole.NeuralDole({ items: this.items }); this.doleMagicId = setInterval(() => { @@ -117,6 +112,7 @@ export class ControllerPlugin extends BaseControllerPlugin { updateStorage() { this.itemUpdateRateLimiter.activate(); + this.storageDirty = true; } broadcastStorage() { @@ -254,8 +250,13 @@ export class ControllerPlugin extends BaseControllerPlugin { async onShutdown() { this.itemUpdateRateLimiter.cancel(); - clearInterval(this.autosaveId); clearInterval(this.doleMagicId); - await saveDatabase(this.controller.config, this.items, this.logger); + } + + async onSaveData() { + if (this.storageDirty) { + this.storageDirty = false; + await saveDatabase(this.controller.config, this.items, this.logger); + } } } diff --git a/plugins/subspace_storage/info.ts b/plugins/subspace_storage/info.ts index c54da3c2..04c47cd9 100644 --- a/plugins/subspace_storage/info.ts +++ b/plugins/subspace_storage/info.ts @@ -4,13 +4,6 @@ import * as messages from "./messages"; class ControllerConfigGroup extends lib.PluginConfigGroup {} ControllerConfigGroup.defaultAccess = ["controller", "host", "control"]; ControllerConfigGroup.groupName = "subspace_storage"; -ControllerConfigGroup.define({ - name: "autosave_interval", - title: "Autosave Interval", - description: "Interval the storage is autosaved at in seconds.", - type: "number", - initial_value: 60, -}); ControllerConfigGroup.define({ name: "division_method", title: "Division Method", diff --git a/test/host.js b/test/host.js index 9aee450c..60cd63fe 100644 --- a/test/host.js +++ b/test/host.js @@ -20,6 +20,7 @@ describe("Host testing", function() { await referenceConfig.init(); referenceConfig.set("instance.id", 1); referenceConfig.set("instance.name", "test"); + referenceConfig.dirty = false; assert.deepEqual(instanceInfos, new Map([ [1, {