Restart instances that crash after running for 10 minutes

Adds instance.restart_after_crash (default off) and
instance.crash_retry_attempts (default 3, 0 for no limit). The host only
restarts an instance that ran for at least 10 minutes before exiting
unexpectedly, so a boot loop stops after one crash. A user stop, kill or
restart resets the count; it is not persisted across host restarts.

Closes #1251

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bbassieandClaude Opus 5.5 committed 2026-11-21 15:55:28 +00:00
1 parent 7a2aade25b
commit 565e931a96
5 files changed
+142 -1

No files matched your search

+36
View File
@@ -44,6 +44,9 @@ const transitoryStatuses: ReadonlySet<Instance["_status"]> = new Set([
"starting", "stopping", "creating_save", "exporting_data",
]);
// Crashes sooner than this after starting are not restarted to avoid boot loops.
const crashRestartMinUptimeMs = 10 * 60e3;
const instanceRconCommandDuration = new lib.Histogram(
"clusterio_instance_rcon_command_duration_seconds",
"Histogram of the RCON command duration from request to response.",
@@ -194,6 +197,8 @@ export default class Instance extends lib.Link {
_playerAutosaveSlot = 1;
_expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
_settleWaiters: (() => void)[] = [];
_runningSinceMs: number | null = null;
_killRequested = false;
constructor(
host: Host,
@@ -594,6 +599,9 @@ end`.replace(/\r?\n/g, " ");
notifyStatus(status: Instance["_status"]) {
this._status = status;
if (status === "running") {
this._runningSinceMs = Date.now();
}
if (!transitoryStatuses.has(status)) {
const waiters = this._settleWaiters;
this._settleWaiters = [];
@@ -643,6 +651,7 @@ end`.replace(/\r?\n/g, " ");
}
notifyExit() {
const crashed = this._status === "running" && !this._killRequested;
this._loadedSave = null;
this._autoPause = false;
this.notifyStatus("stopped");
@@ -668,6 +677,29 @@ end`.replace(/\r?\n/g, " ");
this._recordPlayerLeave(player, "server_quit");
}
this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`));
if (crashed && this.config.get("instance.restart_after_crash") && !this._host._shuttingDown) {
this._restartAfterCrash(Date.now() - this._runningSinceMs!);
}
}
_restartAfterCrash(uptimeMs: number) {
if (uptimeMs < crashRestartMinUptimeMs) {
this.logger.error(`Instance ${this.name} crashed within 10 minutes of starting, not restarting it`);
return;
}
const limit = this.config.get("instance.crash_retry_attempts");
const attempts = this._host._crashRetryAttempts.get(this.id) ?? 0;
if (limit > 0 && attempts >= limit) {
this.logger.error(`Instance ${this.name} reached the limit of ${limit} crash restarts, not restarting it`);
return;
}
this._host._crashRetryAttempts.set(this.id, attempts + 1);
const of = limit > 0 ? ` of ${limit}` : "";
this.logger.warn(`Restarting instance ${this.name} after crash, attempt ${attempts + 1}${of}`);
this._host._connectInstance(this.id)
.then(connection => connection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest()))
.catch(err => this.logger.error(`Error restarting instance ${this.name} after crash:\n${err.stack}`));
}
static async _readStats(statsPath: string) {
@@ -1358,6 +1390,7 @@ end`.replace(/\r?\n/g, " ");
if (this._status === "stopped") {
return;
}
this._killRequested = true;
await this.server.kill(true);
}
@@ -1540,6 +1573,7 @@ end`.replace(/\r?\n/g, " ");
}
async handleInstanceRestartRequest(request: lib.InstanceRestartRequest) {
this._host._crashRetryAttempts.delete(this.id);
await this._stopWhenSettled();
// Stopping invalidates this instance object, a new one must be created by the host
// To do this we simulate a waking an instance and sending a start request
@@ -1548,10 +1582,12 @@ end`.replace(/\r?\n/g, " ");
}
async handleInstanceStopRequest() {
this._host._crashRetryAttempts.delete(this.id);
await this._stopWhenSettled();
}
async handleInstanceKillRequest() {
this._host._crashRetryAttempts.delete(this.id);
await this.kill();
}