mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-22 08:13:12 +09:00
Restart instances that crash after running for 10 minutes
Adds instance.restart_after_crash (default off) and instance.crash_retry_attempts (default 3, 0 for no limit). The host only restarts an instance that ran for at least 10 minutes before exiting unexpectedly, so a boot loop stops after one crash. A user stop, kill or restart resets the count; it is not persisted across host restarts. Closes #1251 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
7a2aade25b
commit
565e931a96
5 files changed
+142
-1
No files matched your search
@@ -549,6 +549,23 @@ Does not affect start or stop of the instance while the host is running.
|
|||||||
Defaults to false.
|
Defaults to false.
|
||||||
|
|
||||||
|
|
||||||
|
### instance.restart_after_crash
|
||||||
|
|
||||||
|
If enabled the host starts this instance again when the Factorio server exits unexpectedly after running for at least 10 minutes.
|
||||||
|
A crash sooner than that is logged as an error and the instance is left stopped, as it is likely to crash again.
|
||||||
|
|
||||||
|
Defaults to false.
|
||||||
|
|
||||||
|
|
||||||
|
### instance.crash_retry_attempts
|
||||||
|
|
||||||
|
Number of times the host restarts this instance after a crash before giving up and logging an error.
|
||||||
|
The count resets when the instance is stopped or restarted by a user, or the host is restarted.
|
||||||
|
Set to 0 for no limit.
|
||||||
|
|
||||||
|
Defaults to 3.
|
||||||
|
|
||||||
|
|
||||||
### instance.exclude_from_start_all
|
### instance.exclude_from_start_all
|
||||||
|
|
||||||
If enabled this instance will be excluded from the "Start all" button operation in the web interface.
|
If enabled this instance will be excluded from the "Start all" button operation in the web interface.
|
||||||
|
|||||||
@@ -351,6 +351,8 @@ export default class Host extends lib.Link {
|
|||||||
_shuttingDown = false;
|
_shuttingDown = false;
|
||||||
/** Instances waiting on {@link Instance.fromConfig} */
|
/** Instances waiting on {@link Instance.fromConfig} */
|
||||||
_startingInstances = new Set<number>();
|
_startingInstances = new Set<number>();
|
||||||
|
/** Restarts after a crash per instance ID since the last manual stop */
|
||||||
|
_crashRetryAttempts = new Map<number, number>();
|
||||||
|
|
||||||
startedAtMs = Date.now();
|
startedAtMs = Date.now();
|
||||||
|
|
||||||
|
|||||||
@@ -44,6 +44,9 @@ const transitoryStatuses: ReadonlySet<Instance["_status"]> = new Set([
|
|||||||
"starting", "stopping", "creating_save", "exporting_data",
|
"starting", "stopping", "creating_save", "exporting_data",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
// Crashes sooner than this after starting are not restarted to avoid boot loops.
|
||||||
|
const crashRestartMinUptimeMs = 10 * 60e3;
|
||||||
|
|
||||||
const instanceRconCommandDuration = new lib.Histogram(
|
const instanceRconCommandDuration = new lib.Histogram(
|
||||||
"clusterio_instance_rcon_command_duration_seconds",
|
"clusterio_instance_rcon_command_duration_seconds",
|
||||||
"Histogram of the RCON command duration from request to response.",
|
"Histogram of the RCON command duration from request to response.",
|
||||||
@@ -194,6 +197,8 @@ export default class Instance extends lib.Link {
|
|||||||
_playerAutosaveSlot = 1;
|
_playerAutosaveSlot = 1;
|
||||||
_expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
|
_expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
|
||||||
_settleWaiters: (() => void)[] = [];
|
_settleWaiters: (() => void)[] = [];
|
||||||
|
_runningSinceMs: number | null = null;
|
||||||
|
_killRequested = false;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
host: Host,
|
host: Host,
|
||||||
@@ -594,6 +599,9 @@ end`.replace(/\r?\n/g, " ");
|
|||||||
|
|
||||||
notifyStatus(status: Instance["_status"]) {
|
notifyStatus(status: Instance["_status"]) {
|
||||||
this._status = status;
|
this._status = status;
|
||||||
|
if (status === "running") {
|
||||||
|
this._runningSinceMs = Date.now();
|
||||||
|
}
|
||||||
if (!transitoryStatuses.has(status)) {
|
if (!transitoryStatuses.has(status)) {
|
||||||
const waiters = this._settleWaiters;
|
const waiters = this._settleWaiters;
|
||||||
this._settleWaiters = [];
|
this._settleWaiters = [];
|
||||||
@@ -643,6 +651,7 @@ end`.replace(/\r?\n/g, " ");
|
|||||||
}
|
}
|
||||||
|
|
||||||
notifyExit() {
|
notifyExit() {
|
||||||
|
const crashed = this._status === "running" && !this._killRequested;
|
||||||
this._loadedSave = null;
|
this._loadedSave = null;
|
||||||
this._autoPause = false;
|
this._autoPause = false;
|
||||||
this.notifyStatus("stopped");
|
this.notifyStatus("stopped");
|
||||||
@@ -668,6 +677,29 @@ end`.replace(/\r?\n/g, " ");
|
|||||||
this._recordPlayerLeave(player, "server_quit");
|
this._recordPlayerLeave(player, "server_quit");
|
||||||
}
|
}
|
||||||
this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`));
|
this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`));
|
||||||
|
|
||||||
|
if (crashed && this.config.get("instance.restart_after_crash") && !this._host._shuttingDown) {
|
||||||
|
this._restartAfterCrash(Date.now() - this._runningSinceMs!);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_restartAfterCrash(uptimeMs: number) {
|
||||||
|
if (uptimeMs < crashRestartMinUptimeMs) {
|
||||||
|
this.logger.error(`Instance ${this.name} crashed within 10 minutes of starting, not restarting it`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const limit = this.config.get("instance.crash_retry_attempts");
|
||||||
|
const attempts = this._host._crashRetryAttempts.get(this.id) ?? 0;
|
||||||
|
if (limit > 0 && attempts >= limit) {
|
||||||
|
this.logger.error(`Instance ${this.name} reached the limit of ${limit} crash restarts, not restarting it`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
this._host._crashRetryAttempts.set(this.id, attempts + 1);
|
||||||
|
const of = limit > 0 ? ` of ${limit}` : "";
|
||||||
|
this.logger.warn(`Restarting instance ${this.name} after crash, attempt ${attempts + 1}${of}`);
|
||||||
|
this._host._connectInstance(this.id)
|
||||||
|
.then(connection => connection.instance.handleInstanceStartRequest(new lib.InstanceStartRequest()))
|
||||||
|
.catch(err => this.logger.error(`Error restarting instance ${this.name} after crash:\n${err.stack}`));
|
||||||
}
|
}
|
||||||
|
|
||||||
static async _readStats(statsPath: string) {
|
static async _readStats(statsPath: string) {
|
||||||
@@ -1358,6 +1390,7 @@ end`.replace(/\r?\n/g, " ");
|
|||||||
if (this._status === "stopped") {
|
if (this._status === "stopped") {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
this._killRequested = true;
|
||||||
await this.server.kill(true);
|
await this.server.kill(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1540,6 +1573,7 @@ end`.replace(/\r?\n/g, " ");
|
|||||||
}
|
}
|
||||||
|
|
||||||
async handleInstanceRestartRequest(request: lib.InstanceRestartRequest) {
|
async handleInstanceRestartRequest(request: lib.InstanceRestartRequest) {
|
||||||
|
this._host._crashRetryAttempts.delete(this.id);
|
||||||
await this._stopWhenSettled();
|
await this._stopWhenSettled();
|
||||||
// Stopping invalidates this instance object, a new one must be created by the host
|
// Stopping invalidates this instance object, a new one must be created by the host
|
||||||
// To do this we simulate a waking an instance and sending a start request
|
// To do this we simulate a waking an instance and sending a start request
|
||||||
@@ -1548,10 +1582,12 @@ end`.replace(/\r?\n/g, " ");
|
|||||||
}
|
}
|
||||||
|
|
||||||
async handleInstanceStopRequest() {
|
async handleInstanceStopRequest() {
|
||||||
|
this._host._crashRetryAttempts.delete(this.id);
|
||||||
await this._stopWhenSettled();
|
await this._stopWhenSettled();
|
||||||
}
|
}
|
||||||
|
|
||||||
async handleInstanceKillRequest() {
|
async handleInstanceKillRequest() {
|
||||||
|
this._host._crashRetryAttempts.delete(this.id);
|
||||||
await this.kill();
|
await this.kill();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -520,6 +520,8 @@ export interface InstanceConfigFields {
|
|||||||
"instance.assigned_host": number | null;
|
"instance.assigned_host": number | null;
|
||||||
"instance.auto_start": boolean;
|
"instance.auto_start": boolean;
|
||||||
"instance.exclude_from_start_all": boolean;
|
"instance.exclude_from_start_all": boolean;
|
||||||
|
"instance.restart_after_crash": boolean;
|
||||||
|
"instance.crash_retry_attempts": number;
|
||||||
"instance.is_template": boolean;
|
"instance.is_template": boolean;
|
||||||
"instance.template_id": number | null;
|
"instance.template_id": number | null;
|
||||||
"instance.inherited_fields": Record<string, true | string[]>;
|
"instance.inherited_fields": Record<string, true | string[]>;
|
||||||
@@ -633,6 +635,16 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
|
|||||||
type: "boolean",
|
type: "boolean",
|
||||||
initialValue: false,
|
initialValue: false,
|
||||||
},
|
},
|
||||||
|
"instance.restart_after_crash": {
|
||||||
|
description: "Start this instance again if it crashes after running for at least 10 minutes",
|
||||||
|
type: "boolean",
|
||||||
|
initialValue: false,
|
||||||
|
},
|
||||||
|
"instance.crash_retry_attempts": {
|
||||||
|
description: "Restarts after a crash before giving up, 0 for no limit. Reset when the instance is stopped",
|
||||||
|
type: "number",
|
||||||
|
initialValue: 3,
|
||||||
|
},
|
||||||
"instance.is_template": {
|
"instance.is_template": {
|
||||||
description:
|
description:
|
||||||
"Mark this instance as a template other instances can inherit config from. " +
|
"Mark this instance as a template other instances can inherit config from. " +
|
||||||
|
|||||||
+75
-1
@@ -95,7 +95,9 @@ describe("class Instance", function() {
|
|||||||
instanceConfig.set("instance.name", "foo");
|
instanceConfig.set("instance.name", "foo");
|
||||||
src = addr({ instanceId: instanceConfig.get("instance.id") });
|
src = addr({ instanceId: instanceConfig.get("instance.id") });
|
||||||
connector = new MockConnector(src, dst);
|
connector = new MockConnector(src, dst);
|
||||||
instance = new Instance({}, connector, "dir", instanceConfig, new MockServer());
|
instance = new Instance(
|
||||||
|
{ _crashRetryAttempts: new Map() }, connector, "dir", instanceConfig, new MockServer()
|
||||||
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
describe(".name", function() {
|
describe(".name", function() {
|
||||||
@@ -308,6 +310,78 @@ describe("class Instance", function() {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("restart after crash", function() {
|
||||||
|
let errors;
|
||||||
|
let starts;
|
||||||
|
beforeEach(function() {
|
||||||
|
errors = [];
|
||||||
|
starts = [];
|
||||||
|
instance.logger = new MockLogger();
|
||||||
|
instance.logger.error = message => errors.push(message);
|
||||||
|
instance._saveStats = async () => {};
|
||||||
|
instance._host._connectInstance = async id => {
|
||||||
|
starts.push(id);
|
||||||
|
return { instance: { handleInstanceStartRequest: async () => {} } };
|
||||||
|
};
|
||||||
|
instance.config.set("instance.restart_after_crash", true);
|
||||||
|
});
|
||||||
|
function crash(uptimeMinutes) {
|
||||||
|
instance.notifyStatus("running");
|
||||||
|
instance._runningSinceMs = Date.now() - uptimeMinutes * 60e3;
|
||||||
|
instance.notifyExit();
|
||||||
|
}
|
||||||
|
|
||||||
|
it("should restart after running for 10 minutes until the attempts run out", function() {
|
||||||
|
for (let i = 0; i < 4; i++) {
|
||||||
|
crash(11);
|
||||||
|
}
|
||||||
|
assert.deepEqual(starts, [instance.id, instance.id, instance.id]);
|
||||||
|
assert.deepEqual(errors, ["Instance foo reached the limit of 3 crash restarts, not restarting it"]);
|
||||||
|
});
|
||||||
|
it("should restart without limit when attempts is 0", function() {
|
||||||
|
instance.config.set("instance.crash_retry_attempts", 0);
|
||||||
|
for (let i = 0; i < 5; i++) {
|
||||||
|
crash(11);
|
||||||
|
}
|
||||||
|
assert.equal(starts.length, 5);
|
||||||
|
assert.deepEqual(errors, []);
|
||||||
|
});
|
||||||
|
it("should not restart a crash within 10 minutes of starting", function() {
|
||||||
|
crash(9);
|
||||||
|
assert.deepEqual(starts, []);
|
||||||
|
assert.deepEqual(errors, ["Instance foo crashed within 10 minutes of starting, not restarting it"]);
|
||||||
|
});
|
||||||
|
it("should not restart when disabled", function() {
|
||||||
|
instance.config.set("instance.restart_after_crash", false);
|
||||||
|
crash(11);
|
||||||
|
assert.deepEqual(starts, []);
|
||||||
|
assert.deepEqual(errors, []);
|
||||||
|
});
|
||||||
|
it("should not restart after a stop or kill", async function() {
|
||||||
|
instance.server.kill = async () => {};
|
||||||
|
instance.notifyStatus("running");
|
||||||
|
instance._runningSinceMs = 0;
|
||||||
|
await instance.handleInstanceKillRequest();
|
||||||
|
instance.notifyExit();
|
||||||
|
instance.notifyStatus("stopping");
|
||||||
|
instance.notifyExit();
|
||||||
|
assert.deepEqual(starts, []);
|
||||||
|
assert.deepEqual(errors, []);
|
||||||
|
});
|
||||||
|
it("should not restart while the host shuts down", function() {
|
||||||
|
instance._host._shuttingDown = true;
|
||||||
|
crash(11);
|
||||||
|
assert.deepEqual(starts, []);
|
||||||
|
});
|
||||||
|
it("should reset the attempts on a manual stop", async function() {
|
||||||
|
instance.config.set("instance.crash_retry_attempts", 1);
|
||||||
|
crash(11);
|
||||||
|
await instance.handleInstanceStopRequest();
|
||||||
|
crash(11);
|
||||||
|
assert.equal(starts.length, 2);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
describe(".checkModPackVersion()", function() {
|
describe(".checkModPackVersion()", function() {
|
||||||
function modPack(factorioVersion) {
|
function modPack(factorioVersion) {
|
||||||
return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion });
|
return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion });
|
||||||
|
|||||||
Reference in new issue
Block a user