Restart instances that crash after running for 10 minutes

Adds instance.restart_after_crash (default off) and
instance.crash_retry_attempts (default 3, 0 for no limit). The host only
restarts an instance that ran for at least 10 minutes before exiting
unexpectedly, so a boot loop stops after one crash. A user stop, kill or
restart resets the count; it is not persisted across host restarts.

Closes #1251

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bbassieandClaude Opus 5.5 committed 2026-11-21 15:55:28 +00:00
1 parent 7a2aade25b
commit 565e931a96
5 files changed
+142 -1

No files matched your search

+17
View File
@@ -549,6 +549,23 @@ Does not affect start or stop of the instance while the host is running.
Defaults to false. Defaults to false.
### instance.restart_after_crash
If enabled the host starts this instance again when the Factorio server exits unexpectedly after running for at least 10 minutes.
A crash sooner than that is logged as an error and the instance is left stopped, as it is likely to crash again.
Defaults to false.
### instance.crash_retry_attempts
Number of times the host restarts this instance after a crash before giving up and logging an error.
The count resets when the instance is stopped or restarted by a user, or the host is restarted.
Set to 0 for no limit.
Defaults to 3.
### instance.exclude_from_start_all ### instance.exclude_from_start_all
If enabled this instance will be excluded from the "Start all" button operation in the web interface. If enabled this instance will be excluded from the "Start all" button operation in the web interface.
+2
View File
@@ -351,6 +351,8 @@ export default class Host extends lib.Link {
_shuttingDown = false; _shuttingDown = false;
/** Instances waiting on {@link Instance.fromConfig} */ /** Instances waiting on {@link Instance.fromConfig} */
_startingInstances = new Set<number>(); _startingInstances = new Set<number>();
/** Restarts after a crash per instance ID since the last manual stop */
_crashRetryAttempts = new Map<number, number>();
startedAtMs = Date.now(); startedAtMs = Date.now();
+36
View File
@@ -44,6 +44,9 @@ const transitoryStatuses: ReadonlySet<Instance["_status"]> = new Set([
"starting", "stopping", "creating_save", "exporting_data", "starting", "stopping", "creating_save", "exporting_data",
]); ]);
// Crashes sooner than this after starting are not restarted to avoid boot loops.
const crashRestartMinUptimeMs = 10 * 60e3;
const instanceRconCommandDuration = new lib.Histogram( const instanceRconCommandDuration = new lib.Histogram(
"clusterio_instance_rcon_command_duration_seconds", "clusterio_instance_rcon_command_duration_seconds",
"Histogram of the RCON command duration from request to response.", "Histogram of the RCON command duration from request to response.",
@@ -194,6 +197,8 @@ export default class Instance extends lib.Link {
_playerAutosaveSlot = 1; _playerAutosaveSlot = 1;
_expectedUserUpdates: { action: string, name: string, reason: string }[] = []; _expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
_settleWaiters: (() => void)[] = []; _settleWaiters: (() => void)[] = [];
_runningSinceMs: number | null = null;
_killRequested = false;
constructor( constructor(
host: Host, host: Host,
@@ -594,6 +599,9 @@ end`.replace(/\r?\n/g, " ");
notifyStatus(status: Instance["_status"]) { notifyStatus(status: Instance["_status"]) {
this._status = status; this._status = status;
if (status === "running") {
this._runningSinceMs = Date.now();
}
if (!transitoryStatuses.has(status)) { if (!transitoryStatuses.has(status)) {
const waiters = this._settleWaiters; const waiters = this._settleWaiters;
this._settleWaiters = []; this._settleWaiters = [];
@@ -643,6 +651,7 @@ end`.replace(/\r?\n/g, " ");
} }
notifyExit() { notifyExit() {
const crashed = this._status === "running" && !this._killRequested;
this._loadedSave = null; this._loadedSave = null;
this._autoPause = false; this._autoPause = false;
this.notifyStatus("stopped"); this.notifyStatus("stopped");
@@ -668,6 +677,29 @@ end`.replace(/\r?\n/g, " ");
this._recordPlayerLeave(player, "server_quit"); this._recordPlayerLeave(player, "server_quit");
} }
this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`)); this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`));
if (crashed && this.config.get("instance.restart_after_crash") && !this._host._shuttingDown) {
this._restartAfterCrash(Date.now() - this._runningSinceMs!);
}
}
_restartAfterCrash(uptimeMs: number) {
if (uptimeMs < crashRestartMinUptimeMs) {
this.logger.error(`Instance ${this.name} crashed within 10 minutes of starting, not restarting it`);
return;
}
const limit = this.config.get("instance.crash_retry_attempts");
const attempts = this._host._crashRetryAttempts.get(this.id) ?? 0;
if (limit > 0 && attempts >= limit) {
this.logger.error(`Instance ${this.name} reached the limit of ${limit} crash restarts, not restarting it`);
return;
}
this._host._crashRetryAttempts.set(this.id, attempts + 1);
const of = limit > 0 ? ` of ${limit}` : "";
this.logger.warn(`Restarting instance ${this.name} after crash, attempt ${attempts + 1}${of}`);
this._host._connectInstance(this.id)
.then(connection => connection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest()))
.catch(err => this.logger.error(`Error restarting instance ${this.name} after crash:\n${err.stack}`));
} }
static async _readStats(statsPath: string) { static async _readStats(statsPath: string) {
@@ -1358,6 +1390,7 @@ end`.replace(/\r?\n/g, " ");
if (this._status === "stopped") { if (this._status === "stopped") {
return; return;
} }
this._killRequested = true;
await this.server.kill(true); await this.server.kill(true);
} }
@@ -1540,6 +1573,7 @@ end`.replace(/\r?\n/g, " ");
} }
async handleInstanceRestartRequest(request: lib.InstanceRestartRequest) { async handleInstanceRestartRequest(request: lib.InstanceRestartRequest) {
this._host._crashRetryAttempts.delete(this.id);
await this._stopWhenSettled(); await this._stopWhenSettled();
// Stopping invalidates this instance object, a new one must be created by the host // Stopping invalidates this instance object, a new one must be created by the host
// To do this we simulate a waking an instance and sending a start request // To do this we simulate a waking an instance and sending a start request
@@ -1548,10 +1582,12 @@ end`.replace(/\r?\n/g, " ");
} }
async handleInstanceStopRequest() { async handleInstanceStopRequest() {
this._host._crashRetryAttempts.delete(this.id);
await this._stopWhenSettled(); await this._stopWhenSettled();
} }
async handleInstanceKillRequest() { async handleInstanceKillRequest() {
this._host._crashRetryAttempts.delete(this.id);
await this.kill(); await this.kill();
} }
+12
View File
@@ -520,6 +520,8 @@ export interface InstanceConfigFields {
"instance.assigned_host": number | null; "instance.assigned_host": number | null;
"instance.auto_start": boolean; "instance.auto_start": boolean;
"instance.exclude_from_start_all": boolean; "instance.exclude_from_start_all": boolean;
"instance.restart_after_crash": boolean;
"instance.crash_retry_attempts": number;
"instance.is_template": boolean; "instance.is_template": boolean;
"instance.template_id": number | null; "instance.template_id": number | null;
"instance.inherited_fields": Record<string, true | string[]>; "instance.inherited_fields": Record<string, true | string[]>;
@@ -633,6 +635,16 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
type: "boolean", type: "boolean",
initialValue: false, initialValue: false,
}, },
"instance.restart_after_crash": {
description: "Start this instance again if it crashes after running for at least 10 minutes",
type: "boolean",
initialValue: false,
},
"instance.crash_retry_attempts": {
description: "Restarts after a crash before giving up, 0 for no limit. Reset when the instance is stopped",
type: "number",
initialValue: 3,
},
"instance.is_template": { "instance.is_template": {
description: description:
"Mark this instance as a template other instances can inherit config from. " + "Mark this instance as a template other instances can inherit config from. " +
+75 -1
View File
@@ -95,7 +95,9 @@ describe("class Instance", function() {
instanceConfig.set("instance.name", "foo"); instanceConfig.set("instance.name", "foo");
src = addr({ instanceId: instanceConfig.get("instance.id") }); src = addr({ instanceId: instanceConfig.get("instance.id") });
connector = new MockConnector(src, dst); connector = new MockConnector(src, dst);
instance = new Instance({}, connector, "dir", instanceConfig, new MockServer()); instance = new Instance(
{ _crashRetryAttempts: new Map() }, connector, "dir", instanceConfig, new MockServer()
);
}); });
describe(".name", function() { describe(".name", function() {
@@ -308,6 +310,78 @@ describe("class Instance", function() {
}); });
}); });
describe("restart after crash", function() {
let errors;
let starts;
beforeEach(function() {
errors = [];
starts = [];
instance.logger = new MockLogger();
instance.logger.error = message => errors.push(message);
instance._saveStats = async () => {};
instance._host._connectInstance = async id => {
starts.push(id);
return { instance: { handleInstanceStartRequest: async () => {} } };
};
instance.config.set("instance.restart_after_crash", true);
});
function crash(uptimeMinutes) {
instance.notifyStatus("running");
instance._runningSinceMs = Date.now() - uptimeMinutes * 60e3;
instance.notifyExit();
}
it("should restart after running for 10 minutes until the attempts run out", function() {
for (let i = 0; i < 4; i++) {
crash(11);
}
assert.deepEqual(starts, [instance.id, instance.id, instance.id]);
assert.deepEqual(errors, ["Instance foo reached the limit of 3 crash restarts, not restarting it"]);
});
it("should restart without limit when attempts is 0", function() {
instance.config.set("instance.crash_retry_attempts", 0);
for (let i = 0; i < 5; i++) {
crash(11);
}
assert.equal(starts.length, 5);
assert.deepEqual(errors, []);
});
it("should not restart a crash within 10 minutes of starting", function() {
crash(9);
assert.deepEqual(starts, []);
assert.deepEqual(errors, ["Instance foo crashed within 10 minutes of starting, not restarting it"]);
});
it("should not restart when disabled", function() {
instance.config.set("instance.restart_after_crash", false);
crash(11);
assert.deepEqual(starts, []);
assert.deepEqual(errors, []);
});
it("should not restart after a stop or kill", async function() {
instance.server.kill = async () => {};
instance.notifyStatus("running");
instance._runningSinceMs = 0;
await instance.handleInstanceKillRequest();
instance.notifyExit();
instance.notifyStatus("stopping");
instance.notifyExit();
assert.deepEqual(starts, []);
assert.deepEqual(errors, []);
});
it("should not restart while the host shuts down", function() {
instance._host._shuttingDown = true;
crash(11);
assert.deepEqual(starts, []);
});
it("should reset the attempts on a manual stop", async function() {
instance.config.set("instance.crash_retry_attempts", 1);
crash(11);
await instance.handleInstanceStopRequest();
crash(11);
assert.equal(starts.length, 2);
});
});
describe(".checkModPackVersion()", function() { describe(".checkModPackVersion()", function() {
function modPack(factorioVersion) { function modPack(factorioVersion) {
return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion }); return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion });