From 565e931a963a3428da7c2f473b30fb1a80cedbdb Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Sat, 21 Nov 2026 15:55:28 +0000 Subject: [PATCH 1/2] Restart instances that crash after running for 10 minutes Adds instance.restart_after_crash (default off) and instance.crash_retry_attempts (default 3, 0 for no limit). The host only restarts an instance that ran for at least 10 minutes before exiting unexpectedly, so a boot loop stops after one crash. A user stop, kill or restart resets the count; it is not persisted across host restarts. Closes #1251 Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 17 ++++++ packages/host/src/Host.ts | 2 + packages/host/src/Instance.ts | 36 ++++++++++++ packages/lib/src/config/definitions.ts | 12 ++++ test/host/Instance.js | 76 +++++++++++++++++++++++++- 5 files changed, 142 insertions(+), 1 deletion(-) diff --git a/docs/configuration.md b/docs/configuration.md index af288ee0..49b7d7c2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -549,6 +549,23 @@ Does not affect start or stop of the instance while the host is running. Defaults to false. +### instance.restart_after_crash + +If enabled the host starts this instance again when the Factorio server exits unexpectedly after running for at least 10 minutes. +A crash sooner than that is logged as an error and the instance is left stopped, as it is likely to crash again. + +Defaults to false. + + +### instance.crash_retry_attempts + +Number of times the host restarts this instance after a crash before giving up and logging an error. +The count resets when the instance is stopped or restarted by a user, or the host is restarted. +Set to 0 for no limit. + +Defaults to 3. + + ### instance.exclude_from_start_all If enabled this instance will be excluded from the "Start all" button operation in the web interface. diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index aca92062..f46e5555 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -351,6 +351,8 @@ export default class Host extends lib.Link { _shuttingDown = false; /** Instances waiting on {@link Instance.fromConfig} */ _startingInstances = new Set(); + /** Restarts after a crash per instance ID since the last manual stop */ + _crashRetryAttempts = new Map(); startedAtMs = Date.now(); diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 2048cdf8..61fea792 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -44,6 +44,9 @@ const transitoryStatuses: ReadonlySet = new Set([ "starting", "stopping", "creating_save", "exporting_data", ]); +// Crashes sooner than this after starting are not restarted to avoid boot loops. +const crashRestartMinUptimeMs = 10 * 60e3; + const instanceRconCommandDuration = new lib.Histogram( "clusterio_instance_rcon_command_duration_seconds", "Histogram of the RCON command duration from request to response.", @@ -194,6 +197,8 @@ export default class Instance extends lib.Link { _playerAutosaveSlot = 1; _expectedUserUpdates: { action: string, name: string, reason: string }[] = []; _settleWaiters: (() => void)[] = []; + _runningSinceMs: number | null = null; + _killRequested = false; constructor( host: Host, @@ -594,6 +599,9 @@ end`.replace(/\r?\n/g, " "); notifyStatus(status: Instance["_status"]) { this._status = status; + if (status === "running") { + this._runningSinceMs = Date.now(); + } if (!transitoryStatuses.has(status)) { const waiters = this._settleWaiters; this._settleWaiters = []; @@ -643,6 +651,7 @@ end`.replace(/\r?\n/g, " "); } notifyExit() { + const crashed = this._status === "running" && !this._killRequested; this._loadedSave = null; this._autoPause = false; this.notifyStatus("stopped"); @@ -668,6 +677,29 @@ end`.replace(/\r?\n/g, " "); this._recordPlayerLeave(player, "server_quit"); } this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`)); + + if (crashed && this.config.get("instance.restart_after_crash") && !this._host._shuttingDown) { + this._restartAfterCrash(Date.now() - this._runningSinceMs!); + } + } + + _restartAfterCrash(uptimeMs: number) { + if (uptimeMs < crashRestartMinUptimeMs) { + this.logger.error(`Instance ${this.name} crashed within 10 minutes of starting, not restarting it`); + return; + } + const limit = this.config.get("instance.crash_retry_attempts"); + const attempts = this._host._crashRetryAttempts.get(this.id) ?? 0; + if (limit > 0 && attempts >= limit) { + this.logger.error(`Instance ${this.name} reached the limit of ${limit} crash restarts, not restarting it`); + return; + } + this._host._crashRetryAttempts.set(this.id, attempts + 1); + const of = limit > 0 ? ` of ${limit}` : ""; + this.logger.warn(`Restarting instance ${this.name} after crash, attempt ${attempts + 1}${of}`); + this._host._connectInstance(this.id) + .then(connection => connection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest())) + .catch(err => this.logger.error(`Error restarting instance ${this.name} after crash:\n${err.stack}`)); } static async _readStats(statsPath: string) { @@ -1358,6 +1390,7 @@ end`.replace(/\r?\n/g, " "); if (this._status === "stopped") { return; } + this._killRequested = true; await this.server.kill(true); } @@ -1540,6 +1573,7 @@ end`.replace(/\r?\n/g, " "); } async handleInstanceRestartRequest(request: lib.InstanceRestartRequest) { + this._host._crashRetryAttempts.delete(this.id); await this._stopWhenSettled(); // Stopping invalidates this instance object, a new one must be created by the host // To do this we simulate a waking an instance and sending a start request @@ -1548,10 +1582,12 @@ end`.replace(/\r?\n/g, " "); } async handleInstanceStopRequest() { + this._host._crashRetryAttempts.delete(this.id); await this._stopWhenSettled(); } async handleInstanceKillRequest() { + this._host._crashRetryAttempts.delete(this.id); await this.kill(); } diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 7ed641d8..b4e3a396 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -520,6 +520,8 @@ export interface InstanceConfigFields { "instance.assigned_host": number | null; "instance.auto_start": boolean; "instance.exclude_from_start_all": boolean; + "instance.restart_after_crash": boolean; + "instance.crash_retry_attempts": number; "instance.is_template": boolean; "instance.template_id": number | null; "instance.inherited_fields": Record; @@ -633,6 +635,16 @@ export class InstanceConfig extends classes.Config { type: "boolean", initialValue: false, }, + "instance.restart_after_crash": { + description: "Start this instance again if it crashes after running for at least 10 minutes", + type: "boolean", + initialValue: false, + }, + "instance.crash_retry_attempts": { + description: "Restarts after a crash before giving up, 0 for no limit. Reset when the instance is stopped", + type: "number", + initialValue: 3, + }, "instance.is_template": { description: "Mark this instance as a template other instances can inherit config from. " + diff --git a/test/host/Instance.js b/test/host/Instance.js index 2d6893ed..9840cad0 100644 --- a/test/host/Instance.js +++ b/test/host/Instance.js @@ -95,7 +95,9 @@ describe("class Instance", function() { instanceConfig.set("instance.name", "foo"); src = addr({ instanceId: instanceConfig.get("instance.id") }); connector = new MockConnector(src, dst); - instance = new Instance({}, connector, "dir", instanceConfig, new MockServer()); + instance = new Instance( + { _crashRetryAttempts: new Map() }, connector, "dir", instanceConfig, new MockServer() + ); }); describe(".name", function() { @@ -308,6 +310,78 @@ describe("class Instance", function() { }); }); + describe("restart after crash", function() { + let errors; + let starts; + beforeEach(function() { + errors = []; + starts = []; + instance.logger = new MockLogger(); + instance.logger.error = message => errors.push(message); + instance._saveStats = async () => {}; + instance._host._connectInstance = async id => { + starts.push(id); + return { instance: { handleInstanceStartRequest: async () => {} } }; + }; + instance.config.set("instance.restart_after_crash", true); + }); + function crash(uptimeMinutes) { + instance.notifyStatus("running"); + instance._runningSinceMs = Date.now() - uptimeMinutes * 60e3; + instance.notifyExit(); + } + + it("should restart after running for 10 minutes until the attempts run out", function() { + for (let i = 0; i < 4; i++) { + crash(11); + } + assert.deepEqual(starts, [instance.id, instance.id, instance.id]); + assert.deepEqual(errors, ["Instance foo reached the limit of 3 crash restarts, not restarting it"]); + }); + it("should restart without limit when attempts is 0", function() { + instance.config.set("instance.crash_retry_attempts", 0); + for (let i = 0; i < 5; i++) { + crash(11); + } + assert.equal(starts.length, 5); + assert.deepEqual(errors, []); + }); + it("should not restart a crash within 10 minutes of starting", function() { + crash(9); + assert.deepEqual(starts, []); + assert.deepEqual(errors, ["Instance foo crashed within 10 minutes of starting, not restarting it"]); + }); + it("should not restart when disabled", function() { + instance.config.set("instance.restart_after_crash", false); + crash(11); + assert.deepEqual(starts, []); + assert.deepEqual(errors, []); + }); + it("should not restart after a stop or kill", async function() { + instance.server.kill = async () => {}; + instance.notifyStatus("running"); + instance._runningSinceMs = 0; + await instance.handleInstanceKillRequest(); + instance.notifyExit(); + instance.notifyStatus("stopping"); + instance.notifyExit(); + assert.deepEqual(starts, []); + assert.deepEqual(errors, []); + }); + it("should not restart while the host shuts down", function() { + instance._host._shuttingDown = true; + crash(11); + assert.deepEqual(starts, []); + }); + it("should reset the attempts on a manual stop", async function() { + instance.config.set("instance.crash_retry_attempts", 1); + crash(11); + await instance.handleInstanceStopRequest(); + crash(11); + assert.equal(starts.length, 2); + }); + }); + describe(".checkModPackVersion()", function() { function modPack(factorioVersion) { return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion }); From 079340e5ddd3de66c71a28587f4720eec4120bb1 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Sat, 21 Nov 2026 16:29:03 +0000 Subject: [PATCH 2/2] Reset crash restart attempts on a manual start A stop is always followed by a start, so the start is the one place that needs it. The crash restart calls _startSave directly to keep its count. Rename _runningSinceMs to _startedAtMs to match InstanceDetails. Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 2 +- packages/host/src/Host.ts | 2 +- packages/host/src/Instance.ts | 18 ++++++++++-------- packages/lib/src/config/definitions.ts | 2 +- test/host/Instance.js | 11 ++++++----- 5 files changed, 19 insertions(+), 16 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 49b7d7c2..90465063 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -560,7 +560,7 @@ Defaults to false. ### instance.crash_retry_attempts Number of times the host restarts this instance after a crash before giving up and logging an error. -The count resets when the instance is stopped or restarted by a user, or the host is restarted. +The count resets when a user starts or restarts the instance, or the host is restarted. Set to 0 for no limit. Defaults to 3. diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index f46e5555..deee9e6a 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -351,7 +351,7 @@ export default class Host extends lib.Link { _shuttingDown = false; /** Instances waiting on {@link Instance.fromConfig} */ _startingInstances = new Set(); - /** Restarts after a crash per instance ID since the last manual stop */ + /** Restarts after a crash per instance ID since the last manual start */ _crashRetryAttempts = new Map(); startedAtMs = Date.now(); diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 61fea792..67d1265c 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -197,7 +197,7 @@ export default class Instance extends lib.Link { _playerAutosaveSlot = 1; _expectedUserUpdates: { action: string, name: string, reason: string }[] = []; _settleWaiters: (() => void)[] = []; - _runningSinceMs: number | null = null; + _startedAtMs: number | null = null; _killRequested = false; constructor( @@ -600,7 +600,7 @@ end`.replace(/\r?\n/g, " "); notifyStatus(status: Instance["_status"]) { this._status = status; if (status === "running") { - this._runningSinceMs = Date.now(); + this._startedAtMs = Date.now(); } if (!transitoryStatuses.has(status)) { const waiters = this._settleWaiters; @@ -679,7 +679,7 @@ end`.replace(/\r?\n/g, " "); this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`)); if (crashed && this.config.get("instance.restart_after_crash") && !this._host._shuttingDown) { - this._restartAfterCrash(Date.now() - this._runningSinceMs!); + this._restartAfterCrash(Date.now() - this._startedAtMs!); } } @@ -698,7 +698,7 @@ end`.replace(/\r?\n/g, " "); const of = limit > 0 ? ` of ${limit}` : ""; this.logger.warn(`Restarting instance ${this.name} after crash, attempt ${attempts + 1}${of}`); this._host._connectInstance(this.id) - .then(connection => connection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest())) + .then(connection => connection.instance._startSave()) .catch(err => this.logger.error(`Error restarting instance ${this.name} after crash:\n${err.stack}`)); } @@ -1428,7 +1428,11 @@ end`.replace(/\r?\n/g, " "); } async handleInstanceStartRequest(request: lib.InstanceStartRequest) { - let saveName = request.save; + this._host._crashRetryAttempts.delete(this.id); + await this._startSave(request.save); + } + + async _startSave(saveName?: string) { if (saveName !== undefined) { checkRequestSaveName(saveName); } @@ -1457,6 +1461,7 @@ end`.replace(/\r?\n/g, " "); } async handleInstanceLoadScenarioRequest(request: lib.InstanceLoadScenarioRequest) { + this._host._crashRetryAttempts.delete(this.id); if (this.config.get("factorio.enable_save_patching")) { this.notifyExit(); throw new lib.ConflictError("Load scenario cannot be used with save patching enabled"); @@ -1573,7 +1578,6 @@ end`.replace(/\r?\n/g, " "); } async handleInstanceRestartRequest(request: lib.InstanceRestartRequest) { - this._host._crashRetryAttempts.delete(this.id); await this._stopWhenSettled(); // Stopping invalidates this instance object, a new one must be created by the host // To do this we simulate a waking an instance and sending a start request @@ -1582,12 +1586,10 @@ end`.replace(/\r?\n/g, " "); } async handleInstanceStopRequest() { - this._host._crashRetryAttempts.delete(this.id); await this._stopWhenSettled(); } async handleInstanceKillRequest() { - this._host._crashRetryAttempts.delete(this.id); await this.kill(); } diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index b4e3a396..f003c220 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -641,7 +641,7 @@ export class InstanceConfig extends classes.Config { initialValue: false, }, "instance.crash_retry_attempts": { - description: "Restarts after a crash before giving up, 0 for no limit. Reset when the instance is stopped", + description: "Restarts after a crash before giving up, 0 for no limit. Reset when the instance is started", type: "number", initialValue: 3, }, diff --git a/test/host/Instance.js b/test/host/Instance.js index 9840cad0..e3247a89 100644 --- a/test/host/Instance.js +++ b/test/host/Instance.js @@ -321,13 +321,13 @@ describe("class Instance", function() { instance._saveStats = async () => {}; instance._host._connectInstance = async id => { starts.push(id); - return { instance: { handleInstanceStartRequest: async () => {} } }; + return { instance: { _startSave: async () => {} } }; }; instance.config.set("instance.restart_after_crash", true); }); function crash(uptimeMinutes) { instance.notifyStatus("running"); - instance._runningSinceMs = Date.now() - uptimeMinutes * 60e3; + instance._startedAtMs = Date.now() - uptimeMinutes * 60e3; instance.notifyExit(); } @@ -360,7 +360,7 @@ describe("class Instance", function() { it("should not restart after a stop or kill", async function() { instance.server.kill = async () => {}; instance.notifyStatus("running"); - instance._runningSinceMs = 0; + instance._startedAtMs = 0; await instance.handleInstanceKillRequest(); instance.notifyExit(); instance.notifyStatus("stopping"); @@ -373,10 +373,11 @@ describe("class Instance", function() { crash(11); assert.deepEqual(starts, []); }); - it("should reset the attempts on a manual stop", async function() { + it("should reset the attempts on a manual start", async function() { instance.config.set("instance.crash_retry_attempts", 1); + instance._startSave = async () => {}; crash(11); - await instance.handleInstanceStopRequest(); + await instance.handleInstanceStartRequest(new lib.InstanceStartRequest()); crash(11); assert.equal(starts.length, 2); });