mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-22 08:13:12 +09:00
Restart instances that crash after running for 10 minutes
Adds instance.restart_after_crash (default off) and instance.crash_retry_attempts (default 3, 0 for no limit). The host only restarts an instance that ran for at least 10 minutes before exiting unexpectedly, so a boot loop stops after one crash. A user stop, kill or restart resets the count; it is not persisted across host restarts. Closes #1251 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a2aade25b
commit
565e931a96
5 files changed
+142
-1
No files matched your search
@@ -351,6 +351,8 @@ export default class Host extends lib.Link {
|
||||
_shuttingDown = false;
|
||||
/** Instances waiting on {@link Instance.fromConfig} */
|
||||
_startingInstances = new Set<number>();
|
||||
/** Restarts after a crash per instance ID since the last manual stop */
|
||||
_crashRetryAttempts = new Map<number, number>();
|
||||
|
||||
startedAtMs = Date.now();
|
||||
|
||||
|
||||
@@ -44,6 +44,9 @@ const transitoryStatuses: ReadonlySet<Instance["_status"]> = new Set([
|
||||
"starting", "stopping", "creating_save", "exporting_data",
|
||||
]);
|
||||
|
||||
// Crashes sooner than this after starting are not restarted to avoid boot loops.
|
||||
const crashRestartMinUptimeMs = 10 * 60e3;
|
||||
|
||||
const instanceRconCommandDuration = new lib.Histogram(
|
||||
"clusterio_instance_rcon_command_duration_seconds",
|
||||
"Histogram of the RCON command duration from request to response.",
|
||||
@@ -194,6 +197,8 @@ export default class Instance extends lib.Link {
|
||||
_playerAutosaveSlot = 1;
|
||||
_expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
|
||||
_settleWaiters: (() => void)[] = [];
|
||||
_runningSinceMs: number | null = null;
|
||||
_killRequested = false;
|
||||
|
||||
constructor(
|
||||
host: Host,
|
||||
@@ -594,6 +599,9 @@ end`.replace(/\r?\n/g, " ");
|
||||
|
||||
notifyStatus(status: Instance["_status"]) {
|
||||
this._status = status;
|
||||
if (status === "running") {
|
||||
this._runningSinceMs = Date.now();
|
||||
}
|
||||
if (!transitoryStatuses.has(status)) {
|
||||
const waiters = this._settleWaiters;
|
||||
this._settleWaiters = [];
|
||||
@@ -643,6 +651,7 @@ end`.replace(/\r?\n/g, " ");
|
||||
}
|
||||
|
||||
notifyExit() {
|
||||
const crashed = this._status === "running" && !this._killRequested;
|
||||
this._loadedSave = null;
|
||||
this._autoPause = false;
|
||||
this.notifyStatus("stopped");
|
||||
@@ -668,6 +677,29 @@ end`.replace(/\r?\n/g, " ");
|
||||
this._recordPlayerLeave(player, "server_quit");
|
||||
}
|
||||
this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`));
|
||||
|
||||
if (crashed && this.config.get("instance.restart_after_crash") && !this._host._shuttingDown) {
|
||||
this._restartAfterCrash(Date.now() - this._runningSinceMs!);
|
||||
}
|
||||
}
|
||||
|
||||
_restartAfterCrash(uptimeMs: number) {
|
||||
if (uptimeMs < crashRestartMinUptimeMs) {
|
||||
this.logger.error(`Instance ${this.name} crashed within 10 minutes of starting, not restarting it`);
|
||||
return;
|
||||
}
|
||||
const limit = this.config.get("instance.crash_retry_attempts");
|
||||
const attempts = this._host._crashRetryAttempts.get(this.id) ?? 0;
|
||||
if (limit > 0 && attempts >= limit) {
|
||||
this.logger.error(`Instance ${this.name} reached the limit of ${limit} crash restarts, not restarting it`);
|
||||
return;
|
||||
}
|
||||
this._host._crashRetryAttempts.set(this.id, attempts + 1);
|
||||
const of = limit > 0 ? ` of ${limit}` : "";
|
||||
this.logger.warn(`Restarting instance ${this.name} after crash, attempt ${attempts + 1}${of}`);
|
||||
this._host._connectInstance(this.id)
|
||||
.then(connection => connection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest()))
|
||||
.catch(err => this.logger.error(`Error restarting instance ${this.name} after crash:\n${err.stack}`));
|
||||
}
|
||||
|
||||
static async _readStats(statsPath: string) {
|
||||
@@ -1358,6 +1390,7 @@ end`.replace(/\r?\n/g, " ");
|
||||
if (this._status === "stopped") {
|
||||
return;
|
||||
}
|
||||
this._killRequested = true;
|
||||
await this.server.kill(true);
|
||||
}
|
||||
|
||||
@@ -1540,6 +1573,7 @@ end`.replace(/\r?\n/g, " ");
|
||||
}
|
||||
|
||||
async handleInstanceRestartRequest(request: lib.InstanceRestartRequest) {
|
||||
this._host._crashRetryAttempts.delete(this.id);
|
||||
await this._stopWhenSettled();
|
||||
// Stopping invalidates this instance object, a new one must be created by the host
|
||||
// To do this we simulate a waking an instance and sending a start request
|
||||
@@ -1548,10 +1582,12 @@ end`.replace(/\r?\n/g, " ");
|
||||
}
|
||||
|
||||
async handleInstanceStopRequest() {
|
||||
this._host._crashRetryAttempts.delete(this.id);
|
||||
await this._stopWhenSettled();
|
||||
}
|
||||
|
||||
async handleInstanceKillRequest() {
|
||||
this._host._crashRetryAttempts.delete(this.id);
|
||||
await this.kill();
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user