Merge pull request #1252 from bbassie/feature/instance-crash-restart

Restart instances that crash after running for 10 minutes
This commit is contained in:
Cooldude2606 authored and GitHub committed 2026-11-21 17:21:45 +00:00
commit 516dc1b945
5 files changed
+146 -2

No files matched your search

+17
View File
@@ -549,6 +549,23 @@ Does not affect start or stop of the instance while the host is running.
Defaults to false.
### instance.restart_after_crash
If enabled the host starts this instance again when the Factorio server exits unexpectedly after running for at least 10 minutes.
A crash sooner than that is logged as an error and the instance is left stopped, as it is likely to crash again.
Defaults to false.
### instance.crash_retry_attempts
Number of times the host restarts this instance after a crash before giving up and logging an error.
The count resets when a user starts or restarts the instance, or the host is restarted.
Set to 0 for no limit.
Defaults to 3.
### instance.exclude_from_start_all
If enabled this instance will be excluded from the "Start all" button operation in the web interface.
+2
View File
@@ -351,6 +351,8 @@ export default class Host extends lib.Link {
_shuttingDown = false;
/** Instances waiting on {@link Instance.fromConfig} */
_startingInstances = new Set<number>();
/** Restarts after a crash per instance ID since the last manual start */
_crashRetryAttempts = new Map<number, number>();
startedAtMs = Date.now();
+39 -1
View File
@@ -44,6 +44,9 @@ const transitoryStatuses: ReadonlySet<Instance["_status"]> = new Set([
"starting", "stopping", "creating_save", "exporting_data",
]);
// Crashes sooner than this after starting are not restarted to avoid boot loops.
const crashRestartMinUptimeMs = 10 * 60e3;
const instanceRconCommandDuration = new lib.Histogram(
"clusterio_instance_rcon_command_duration_seconds",
"Histogram of the RCON command duration from request to response.",
@@ -194,6 +197,8 @@ export default class Instance extends lib.Link {
_playerAutosaveSlot = 1;
_expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
_settleWaiters: (() => void)[] = [];
_startedAtMs: number | null = null;
_killRequested = false;
constructor(
host: Host,
@@ -594,6 +599,9 @@ end`.replace(/\r?\n/g, " ");
notifyStatus(status: Instance["_status"]) {
this._status = status;
if (status === "running") {
this._startedAtMs = Date.now();
}
if (!transitoryStatuses.has(status)) {
const waiters = this._settleWaiters;
this._settleWaiters = [];
@@ -643,6 +651,7 @@ end`.replace(/\r?\n/g, " ");
}
notifyExit() {
const crashed = this._status === "running" && !this._killRequested;
this._loadedSave = null;
this._autoPause = false;
this.notifyStatus("stopped");
@@ -668,6 +677,29 @@ end`.replace(/\r?\n/g, " ");
this._recordPlayerLeave(player, "server_quit");
}
this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`));
if (crashed && this.config.get("instance.restart_after_crash") && !this._host._shuttingDown) {
this._restartAfterCrash(Date.now() - this._startedAtMs!);
}
}
_restartAfterCrash(uptimeMs: number) {
if (uptimeMs < crashRestartMinUptimeMs) {
this.logger.error(`Instance ${this.name} crashed within 10 minutes of starting, not restarting it`);
return;
}
const limit = this.config.get("instance.crash_retry_attempts");
const attempts = this._host._crashRetryAttempts.get(this.id) ?? 0;
if (limit > 0 && attempts >= limit) {
this.logger.error(`Instance ${this.name} reached the limit of ${limit} crash restarts, not restarting it`);
return;
}
this._host._crashRetryAttempts.set(this.id, attempts + 1);
const of = limit > 0 ? ` of ${limit}` : "";
this.logger.warn(`Restarting instance ${this.name} after crash, attempt ${attempts + 1}${of}`);
this._host._connectInstance(this.id)
.then(connection => connection.instance._startSave())
.catch(err => this.logger.error(`Error restarting instance ${this.name} after crash:\n${err.stack}`));
}
static async _readStats(statsPath: string) {
@@ -1358,6 +1390,7 @@ end`.replace(/\r?\n/g, " ");
if (this._status === "stopped") {
return;
}
this._killRequested = true;
await this.server.kill(true);
}
@@ -1395,7 +1428,11 @@ end`.replace(/\r?\n/g, " ");
}
async handleInstanceStartRequest(request: lib.InstanceStartRequest) {
let saveName = request.save;
this._host._crashRetryAttempts.delete(this.id);
await this._startSave(request.save);
}
async _startSave(saveName?: string) {
if (saveName !== undefined) {
checkRequestSaveName(saveName);
}
@@ -1424,6 +1461,7 @@ end`.replace(/\r?\n/g, " ");
}
async handleInstanceLoadScenarioRequest(request: lib.InstanceLoadScenarioRequest) {
this._host._crashRetryAttempts.delete(this.id);
if (this.config.get("factorio.enable_save_patching")) {
this.notifyExit();
throw new lib.ConflictError("Load scenario cannot be used with save patching enabled");
+12
View File
@@ -520,6 +520,8 @@ export interface InstanceConfigFields {
"instance.assigned_host": number | null;
"instance.auto_start": boolean;
"instance.exclude_from_start_all": boolean;
"instance.restart_after_crash": boolean;
"instance.crash_retry_attempts": number;
"instance.is_template": boolean;
"instance.template_id": number | null;
"instance.inherited_fields": Record<string, true | string[]>;
@@ -633,6 +635,16 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
type: "boolean",
initialValue: false,
},
"instance.restart_after_crash": {
description: "Start this instance again if it crashes after running for at least 10 minutes",
type: "boolean",
initialValue: false,
},
"instance.crash_retry_attempts": {
description: "Restarts after a crash before giving up, 0 for no limit. Reset when the instance is started",
type: "number",
initialValue: 3,
},
"instance.is_template": {
description:
"Mark this instance as a template other instances can inherit config from. " +
+76 -1
View File
@@ -95,7 +95,9 @@ describe("class Instance", function() {
instanceConfig.set("instance.name", "foo");
src = addr({ instanceId: instanceConfig.get("instance.id") });
connector = new MockConnector(src, dst);
instance = new Instance({}, connector, "dir", instanceConfig, new MockServer());
instance = new Instance(
{ _crashRetryAttempts: new Map() }, connector, "dir", instanceConfig, new MockServer()
);
});
describe(".name", function() {
@@ -308,6 +310,79 @@ describe("class Instance", function() {
});
});
describe("restart after crash", function() {
let errors;
let starts;
beforeEach(function() {
errors = [];
starts = [];
instance.logger = new MockLogger();
instance.logger.error = message => errors.push(message);
instance._saveStats = async () => {};
instance._host._connectInstance = async id => {
starts.push(id);
return { instance: { _startSave: async () => {} } };
};
instance.config.set("instance.restart_after_crash", true);
});
function crash(uptimeMinutes) {
instance.notifyStatus("running");
instance._startedAtMs = Date.now() - uptimeMinutes * 60e3;
instance.notifyExit();
}
it("should restart after running for 10 minutes until the attempts run out", function() {
for (let i = 0; i < 4; i++) {
crash(11);
}
assert.deepEqual(starts, [instance.id, instance.id, instance.id]);
assert.deepEqual(errors, ["Instance foo reached the limit of 3 crash restarts, not restarting it"]);
});
it("should restart without limit when attempts is 0", function() {
instance.config.set("instance.crash_retry_attempts", 0);
for (let i = 0; i < 5; i++) {
crash(11);
}
assert.equal(starts.length, 5);
assert.deepEqual(errors, []);
});
it("should not restart a crash within 10 minutes of starting", function() {
crash(9);
assert.deepEqual(starts, []);
assert.deepEqual(errors, ["Instance foo crashed within 10 minutes of starting, not restarting it"]);
});
it("should not restart when disabled", function() {
instance.config.set("instance.restart_after_crash", false);
crash(11);
assert.deepEqual(starts, []);
assert.deepEqual(errors, []);
});
it("should not restart after a stop or kill", async function() {
instance.server.kill = async () => {};
instance.notifyStatus("running");
instance._startedAtMs = 0;
await instance.handleInstanceKillRequest();
instance.notifyExit();
instance.notifyStatus("stopping");
instance.notifyExit();
assert.deepEqual(starts, []);
assert.deepEqual(errors, []);
});
it("should not restart while the host shuts down", function() {
instance._host._shuttingDown = true;
crash(11);
assert.deepEqual(starts, []);
});
it("should reset the attempts on a manual start", async function() {
instance.config.set("instance.crash_retry_attempts", 1);
instance._startSave = async () => {};
crash(11);
await instance.handleInstanceStartRequest(new lib.InstanceStartRequest());
crash(11);
assert.equal(starts.length, 2);
});
});
describe(".checkModPackVersion()", function() {
function modPack(factorioVersion) {
return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion });