diff --git a/docs/configuration.md b/docs/configuration.md index af288ee0..90465063 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -549,6 +549,23 @@ Does not affect start or stop of the instance while the host is running. Defaults to false. +### instance.restart_after_crash + +If enabled the host starts this instance again when the Factorio server exits unexpectedly after running for at least 10 minutes. +A crash sooner than that is logged as an error and the instance is left stopped, as it is likely to crash again. + +Defaults to false. + + +### instance.crash_retry_attempts + +Number of times the host restarts this instance after a crash before giving up and logging an error. +The count resets when a user starts or restarts the instance, or the host is restarted. +Set to 0 for no limit. + +Defaults to 3. + + ### instance.exclude_from_start_all If enabled this instance will be excluded from the "Start all" button operation in the web interface. diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index aca92062..deee9e6a 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -351,6 +351,8 @@ export default class Host extends lib.Link { _shuttingDown = false; /** Instances waiting on {@link Instance.fromConfig} */ _startingInstances = new Set(); + /** Restarts after a crash per instance ID since the last manual start */ + _crashRetryAttempts = new Map(); startedAtMs = Date.now(); diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 2048cdf8..67d1265c 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -44,6 +44,9 @@ const transitoryStatuses: ReadonlySet = new Set([ "starting", "stopping", "creating_save", "exporting_data", ]); +// Crashes sooner than this after starting are not restarted to avoid boot loops. +const crashRestartMinUptimeMs = 10 * 60e3; + const instanceRconCommandDuration = new lib.Histogram( "clusterio_instance_rcon_command_duration_seconds", "Histogram of the RCON command duration from request to response.", @@ -194,6 +197,8 @@ export default class Instance extends lib.Link { _playerAutosaveSlot = 1; _expectedUserUpdates: { action: string, name: string, reason: string }[] = []; _settleWaiters: (() => void)[] = []; + _startedAtMs: number | null = null; + _killRequested = false; constructor( host: Host, @@ -594,6 +599,9 @@ end`.replace(/\r?\n/g, " "); notifyStatus(status: Instance["_status"]) { this._status = status; + if (status === "running") { + this._startedAtMs = Date.now(); + } if (!transitoryStatuses.has(status)) { const waiters = this._settleWaiters; this._settleWaiters = []; @@ -643,6 +651,7 @@ end`.replace(/\r?\n/g, " "); } notifyExit() { + const crashed = this._status === "running" && !this._killRequested; this._loadedSave = null; this._autoPause = false; this.notifyStatus("stopped"); @@ -668,6 +677,29 @@ end`.replace(/\r?\n/g, " "); this._recordPlayerLeave(player, "server_quit"); } this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`)); + + if (crashed && this.config.get("instance.restart_after_crash") && !this._host._shuttingDown) { + this._restartAfterCrash(Date.now() - this._startedAtMs!); + } + } + + _restartAfterCrash(uptimeMs: number) { + if (uptimeMs < crashRestartMinUptimeMs) { + this.logger.error(`Instance ${this.name} crashed within 10 minutes of starting, not restarting it`); + return; + } + const limit = this.config.get("instance.crash_retry_attempts"); + const attempts = this._host._crashRetryAttempts.get(this.id) ?? 0; + if (limit > 0 && attempts >= limit) { + this.logger.error(`Instance ${this.name} reached the limit of ${limit} crash restarts, not restarting it`); + return; + } + this._host._crashRetryAttempts.set(this.id, attempts + 1); + const of = limit > 0 ? ` of ${limit}` : ""; + this.logger.warn(`Restarting instance ${this.name} after crash, attempt ${attempts + 1}${of}`); + this._host._connectInstance(this.id) + .then(connection => connection.instance._startSave()) + .catch(err => this.logger.error(`Error restarting instance ${this.name} after crash:\n${err.stack}`)); } static async _readStats(statsPath: string) { @@ -1358,6 +1390,7 @@ end`.replace(/\r?\n/g, " "); if (this._status === "stopped") { return; } + this._killRequested = true; await this.server.kill(true); } @@ -1395,7 +1428,11 @@ end`.replace(/\r?\n/g, " "); } async handleInstanceStartRequest(request: lib.InstanceStartRequest) { - let saveName = request.save; + this._host._crashRetryAttempts.delete(this.id); + await this._startSave(request.save); + } + + async _startSave(saveName?: string) { if (saveName !== undefined) { checkRequestSaveName(saveName); } @@ -1424,6 +1461,7 @@ end`.replace(/\r?\n/g, " "); } async handleInstanceLoadScenarioRequest(request: lib.InstanceLoadScenarioRequest) { + this._host._crashRetryAttempts.delete(this.id); if (this.config.get("factorio.enable_save_patching")) { this.notifyExit(); throw new lib.ConflictError("Load scenario cannot be used with save patching enabled"); diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 7ed641d8..f003c220 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -520,6 +520,8 @@ export interface InstanceConfigFields { "instance.assigned_host": number | null; "instance.auto_start": boolean; "instance.exclude_from_start_all": boolean; + "instance.restart_after_crash": boolean; + "instance.crash_retry_attempts": number; "instance.is_template": boolean; "instance.template_id": number | null; "instance.inherited_fields": Record; @@ -633,6 +635,16 @@ export class InstanceConfig extends classes.Config { type: "boolean", initialValue: false, }, + "instance.restart_after_crash": { + description: "Start this instance again if it crashes after running for at least 10 minutes", + type: "boolean", + initialValue: false, + }, + "instance.crash_retry_attempts": { + description: "Restarts after a crash before giving up, 0 for no limit. Reset when the instance is started", + type: "number", + initialValue: 3, + }, "instance.is_template": { description: "Mark this instance as a template other instances can inherit config from. " + diff --git a/test/host/Instance.js b/test/host/Instance.js index 2d6893ed..e3247a89 100644 --- a/test/host/Instance.js +++ b/test/host/Instance.js @@ -95,7 +95,9 @@ describe("class Instance", function() { instanceConfig.set("instance.name", "foo"); src = addr({ instanceId: instanceConfig.get("instance.id") }); connector = new MockConnector(src, dst); - instance = new Instance({}, connector, "dir", instanceConfig, new MockServer()); + instance = new Instance( + { _crashRetryAttempts: new Map() }, connector, "dir", instanceConfig, new MockServer() + ); }); describe(".name", function() { @@ -308,6 +310,79 @@ describe("class Instance", function() { }); }); + describe("restart after crash", function() { + let errors; + let starts; + beforeEach(function() { + errors = []; + starts = []; + instance.logger = new MockLogger(); + instance.logger.error = message => errors.push(message); + instance._saveStats = async () => {}; + instance._host._connectInstance = async id => { + starts.push(id); + return { instance: { _startSave: async () => {} } }; + }; + instance.config.set("instance.restart_after_crash", true); + }); + function crash(uptimeMinutes) { + instance.notifyStatus("running"); + instance._startedAtMs = Date.now() - uptimeMinutes * 60e3; + instance.notifyExit(); + } + + it("should restart after running for 10 minutes until the attempts run out", function() { + for (let i = 0; i < 4; i++) { + crash(11); + } + assert.deepEqual(starts, [instance.id, instance.id, instance.id]); + assert.deepEqual(errors, ["Instance foo reached the limit of 3 crash restarts, not restarting it"]); + }); + it("should restart without limit when attempts is 0", function() { + instance.config.set("instance.crash_retry_attempts", 0); + for (let i = 0; i < 5; i++) { + crash(11); + } + assert.equal(starts.length, 5); + assert.deepEqual(errors, []); + }); + it("should not restart a crash within 10 minutes of starting", function() { + crash(9); + assert.deepEqual(starts, []); + assert.deepEqual(errors, ["Instance foo crashed within 10 minutes of starting, not restarting it"]); + }); + it("should not restart when disabled", function() { + instance.config.set("instance.restart_after_crash", false); + crash(11); + assert.deepEqual(starts, []); + assert.deepEqual(errors, []); + }); + it("should not restart after a stop or kill", async function() { + instance.server.kill = async () => {}; + instance.notifyStatus("running"); + instance._startedAtMs = 0; + await instance.handleInstanceKillRequest(); + instance.notifyExit(); + instance.notifyStatus("stopping"); + instance.notifyExit(); + assert.deepEqual(starts, []); + assert.deepEqual(errors, []); + }); + it("should not restart while the host shuts down", function() { + instance._host._shuttingDown = true; + crash(11); + assert.deepEqual(starts, []); + }); + it("should reset the attempts on a manual start", async function() { + instance.config.set("instance.crash_retry_attempts", 1); + instance._startSave = async () => {}; + crash(11); + await instance.handleInstanceStartRequest(new lib.InstanceStartRequest()); + crash(11); + assert.equal(starts.length, 2); + }); + }); + describe(".checkModPackVersion()", function() { function modPack(factorioVersion) { return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion });