mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-22 08:13:12 +09:00
Restart instances that crash after running for 10 minutes
Adds instance.restart_after_crash (default off) and instance.crash_retry_attempts (default 3, 0 for no limit). The host only restarts an instance that ran for at least 10 minutes before exiting unexpectedly, so a boot loop stops after one crash. A user stop, kill or restart resets the count; it is not persisted across host restarts. Closes #1251 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a2aade25b
commit
565e931a96
5 files changed
+142
-1
No files matched your search
@@ -549,6 +549,23 @@ Does not affect start or stop of the instance while the host is running.
|
||||
Defaults to false.
|
||||
|
||||
|
||||
### instance.restart_after_crash
|
||||
|
||||
If enabled the host starts this instance again when the Factorio server exits unexpectedly after running for at least 10 minutes.
|
||||
A crash sooner than that is logged as an error and the instance is left stopped, as it is likely to crash again.
|
||||
|
||||
Defaults to false.
|
||||
|
||||
|
||||
### instance.crash_retry_attempts
|
||||
|
||||
Number of times the host restarts this instance after a crash before giving up and logging an error.
|
||||
The count resets when the instance is stopped or restarted by a user, or the host is restarted.
|
||||
Set to 0 for no limit.
|
||||
|
||||
Defaults to 3.
|
||||
|
||||
|
||||
### instance.exclude_from_start_all
|
||||
|
||||
If enabled this instance will be excluded from the "Start all" button operation in the web interface.
|
||||
|
||||
@@ -351,6 +351,8 @@ export default class Host extends lib.Link {
|
||||
_shuttingDown = false;
|
||||
/** Instances waiting on {@link Instance.fromConfig} */
|
||||
_startingInstances = new Set<number>();
|
||||
/** Restarts after a crash per instance ID since the last manual stop */
|
||||
_crashRetryAttempts = new Map<number, number>();
|
||||
|
||||
startedAtMs = Date.now();
|
||||
|
||||
|
||||
@@ -44,6 +44,9 @@ const transitoryStatuses: ReadonlySet<Instance["_status"]> = new Set([
|
||||
"starting", "stopping", "creating_save", "exporting_data",
|
||||
]);
|
||||
|
||||
// Crashes sooner than this after starting are not restarted to avoid boot loops.
|
||||
const crashRestartMinUptimeMs = 10 * 60e3;
|
||||
|
||||
const instanceRconCommandDuration = new lib.Histogram(
|
||||
"clusterio_instance_rcon_command_duration_seconds",
|
||||
"Histogram of the RCON command duration from request to response.",
|
||||
@@ -194,6 +197,8 @@ export default class Instance extends lib.Link {
|
||||
_playerAutosaveSlot = 1;
|
||||
_expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
|
||||
_settleWaiters: (() => void)[] = [];
|
||||
_runningSinceMs: number | null = null;
|
||||
_killRequested = false;
|
||||
|
||||
constructor(
|
||||
host: Host,
|
||||
@@ -594,6 +599,9 @@ end`.replace(/\r?\n/g, " ");
|
||||
|
||||
notifyStatus(status: Instance["_status"]) {
|
||||
this._status = status;
|
||||
if (status === "running") {
|
||||
this._runningSinceMs = Date.now();
|
||||
}
|
||||
if (!transitoryStatuses.has(status)) {
|
||||
const waiters = this._settleWaiters;
|
||||
this._settleWaiters = [];
|
||||
@@ -643,6 +651,7 @@ end`.replace(/\r?\n/g, " ");
|
||||
}
|
||||
|
||||
notifyExit() {
|
||||
const crashed = this._status === "running" && !this._killRequested;
|
||||
this._loadedSave = null;
|
||||
this._autoPause = false;
|
||||
this.notifyStatus("stopped");
|
||||
@@ -668,6 +677,29 @@ end`.replace(/\r?\n/g, " ");
|
||||
this._recordPlayerLeave(player, "server_quit");
|
||||
}
|
||||
this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`));
|
||||
|
||||
if (crashed && this.config.get("instance.restart_after_crash") && !this._host._shuttingDown) {
|
||||
this._restartAfterCrash(Date.now() - this._runningSinceMs!);
|
||||
}
|
||||
}
|
||||
|
||||
_restartAfterCrash(uptimeMs: number) {
|
||||
if (uptimeMs < crashRestartMinUptimeMs) {
|
||||
this.logger.error(`Instance ${this.name} crashed within 10 minutes of starting, not restarting it`);
|
||||
return;
|
||||
}
|
||||
const limit = this.config.get("instance.crash_retry_attempts");
|
||||
const attempts = this._host._crashRetryAttempts.get(this.id) ?? 0;
|
||||
if (limit > 0 && attempts >= limit) {
|
||||
this.logger.error(`Instance ${this.name} reached the limit of ${limit} crash restarts, not restarting it`);
|
||||
return;
|
||||
}
|
||||
this._host._crashRetryAttempts.set(this.id, attempts + 1);
|
||||
const of = limit > 0 ? ` of ${limit}` : "";
|
||||
this.logger.warn(`Restarting instance ${this.name} after crash, attempt ${attempts + 1}${of}`);
|
||||
this._host._connectInstance(this.id)
|
||||
.then(connection => connection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest()))
|
||||
.catch(err => this.logger.error(`Error restarting instance ${this.name} after crash:\n${err.stack}`));
|
||||
}
|
||||
|
||||
static async _readStats(statsPath: string) {
|
||||
@@ -1358,6 +1390,7 @@ end`.replace(/\r?\n/g, " ");
|
||||
if (this._status === "stopped") {
|
||||
return;
|
||||
}
|
||||
this._killRequested = true;
|
||||
await this.server.kill(true);
|
||||
}
|
||||
|
||||
@@ -1540,6 +1573,7 @@ end`.replace(/\r?\n/g, " ");
|
||||
}
|
||||
|
||||
async handleInstanceRestartRequest(request: lib.InstanceRestartRequest) {
|
||||
this._host._crashRetryAttempts.delete(this.id);
|
||||
await this._stopWhenSettled();
|
||||
// Stopping invalidates this instance object, a new one must be created by the host
|
||||
// To do this we simulate a waking an instance and sending a start request
|
||||
@@ -1548,10 +1582,12 @@ end`.replace(/\r?\n/g, " ");
|
||||
}
|
||||
|
||||
async handleInstanceStopRequest() {
|
||||
this._host._crashRetryAttempts.delete(this.id);
|
||||
await this._stopWhenSettled();
|
||||
}
|
||||
|
||||
async handleInstanceKillRequest() {
|
||||
this._host._crashRetryAttempts.delete(this.id);
|
||||
await this.kill();
|
||||
}
|
||||
|
||||
|
||||
@@ -520,6 +520,8 @@ export interface InstanceConfigFields {
|
||||
"instance.assigned_host": number | null;
|
||||
"instance.auto_start": boolean;
|
||||
"instance.exclude_from_start_all": boolean;
|
||||
"instance.restart_after_crash": boolean;
|
||||
"instance.crash_retry_attempts": number;
|
||||
"instance.is_template": boolean;
|
||||
"instance.template_id": number | null;
|
||||
"instance.inherited_fields": Record<string, true | string[]>;
|
||||
@@ -633,6 +635,16 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
|
||||
type: "boolean",
|
||||
initialValue: false,
|
||||
},
|
||||
"instance.restart_after_crash": {
|
||||
description: "Start this instance again if it crashes after running for at least 10 minutes",
|
||||
type: "boolean",
|
||||
initialValue: false,
|
||||
},
|
||||
"instance.crash_retry_attempts": {
|
||||
description: "Restarts after a crash before giving up, 0 for no limit. Reset when the instance is stopped",
|
||||
type: "number",
|
||||
initialValue: 3,
|
||||
},
|
||||
"instance.is_template": {
|
||||
description:
|
||||
"Mark this instance as a template other instances can inherit config from. " +
|
||||
|
||||
+75
-1
@@ -95,7 +95,9 @@ describe("class Instance", function() {
|
||||
instanceConfig.set("instance.name", "foo");
|
||||
src = addr({ instanceId: instanceConfig.get("instance.id") });
|
||||
connector = new MockConnector(src, dst);
|
||||
instance = new Instance({}, connector, "dir", instanceConfig, new MockServer());
|
||||
instance = new Instance(
|
||||
{ _crashRetryAttempts: new Map() }, connector, "dir", instanceConfig, new MockServer()
|
||||
);
|
||||
});
|
||||
|
||||
describe(".name", function() {
|
||||
@@ -308,6 +310,78 @@ describe("class Instance", function() {
|
||||
});
|
||||
});
|
||||
|
||||
describe("restart after crash", function() {
|
||||
let errors;
|
||||
let starts;
|
||||
beforeEach(function() {
|
||||
errors = [];
|
||||
starts = [];
|
||||
instance.logger = new MockLogger();
|
||||
instance.logger.error = message => errors.push(message);
|
||||
instance._saveStats = async () => {};
|
||||
instance._host._connectInstance = async id => {
|
||||
starts.push(id);
|
||||
return { instance: { handleInstanceStartRequest: async () => {} } };
|
||||
};
|
||||
instance.config.set("instance.restart_after_crash", true);
|
||||
});
|
||||
function crash(uptimeMinutes) {
|
||||
instance.notifyStatus("running");
|
||||
instance._runningSinceMs = Date.now() - uptimeMinutes * 60e3;
|
||||
instance.notifyExit();
|
||||
}
|
||||
|
||||
it("should restart after running for 10 minutes until the attempts run out", function() {
|
||||
for (let i = 0; i < 4; i++) {
|
||||
crash(11);
|
||||
}
|
||||
assert.deepEqual(starts, [instance.id, instance.id, instance.id]);
|
||||
assert.deepEqual(errors, ["Instance foo reached the limit of 3 crash restarts, not restarting it"]);
|
||||
});
|
||||
it("should restart without limit when attempts is 0", function() {
|
||||
instance.config.set("instance.crash_retry_attempts", 0);
|
||||
for (let i = 0; i < 5; i++) {
|
||||
crash(11);
|
||||
}
|
||||
assert.equal(starts.length, 5);
|
||||
assert.deepEqual(errors, []);
|
||||
});
|
||||
it("should not restart a crash within 10 minutes of starting", function() {
|
||||
crash(9);
|
||||
assert.deepEqual(starts, []);
|
||||
assert.deepEqual(errors, ["Instance foo crashed within 10 minutes of starting, not restarting it"]);
|
||||
});
|
||||
it("should not restart when disabled", function() {
|
||||
instance.config.set("instance.restart_after_crash", false);
|
||||
crash(11);
|
||||
assert.deepEqual(starts, []);
|
||||
assert.deepEqual(errors, []);
|
||||
});
|
||||
it("should not restart after a stop or kill", async function() {
|
||||
instance.server.kill = async () => {};
|
||||
instance.notifyStatus("running");
|
||||
instance._runningSinceMs = 0;
|
||||
await instance.handleInstanceKillRequest();
|
||||
instance.notifyExit();
|
||||
instance.notifyStatus("stopping");
|
||||
instance.notifyExit();
|
||||
assert.deepEqual(starts, []);
|
||||
assert.deepEqual(errors, []);
|
||||
});
|
||||
it("should not restart while the host shuts down", function() {
|
||||
instance._host._shuttingDown = true;
|
||||
crash(11);
|
||||
assert.deepEqual(starts, []);
|
||||
});
|
||||
it("should reset the attempts on a manual stop", async function() {
|
||||
instance.config.set("instance.crash_retry_attempts", 1);
|
||||
crash(11);
|
||||
await instance.handleInstanceStopRequest();
|
||||
crash(11);
|
||||
assert.equal(starts.length, 2);
|
||||
});
|
||||
});
|
||||
|
||||
describe(".checkModPackVersion()", function() {
|
||||
function modPack(factorioVersion) {
|
||||
return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion });
|
||||
|
||||
Reference in new issue
Block a user