Merge pull request #1252 from bbassie/feature/instance-crash-restart

Restart instances that crash after running for 10 minutes
This commit is contained in:
Cooldude2606 authored and GitHub committed 2026-11-21 17:21:45 +00:00
commit 516dc1b945
5 files changed
+146 -2

No files matched your search

+17
View File
@@ -549,6 +549,23 @@ Does not affect start or stop of the instance while the host is running.
Defaults to false. Defaults to false.
### instance.restart_after_crash
If enabled the host starts this instance again when the Factorio server exits unexpectedly after running for at least 10 minutes.
A crash sooner than that is logged as an error and the instance is left stopped, as it is likely to crash again.
Defaults to false.
### instance.crash_retry_attempts
Number of times the host restarts this instance after a crash before giving up and logging an error.
The count resets when a user starts or restarts the instance, or the host is restarted.
Set to 0 for no limit.
Defaults to 3.
### instance.exclude_from_start_all ### instance.exclude_from_start_all
If enabled this instance will be excluded from the "Start all" button operation in the web interface. If enabled this instance will be excluded from the "Start all" button operation in the web interface.
+2
View File
@@ -351,6 +351,8 @@ export default class Host extends lib.Link {
_shuttingDown = false; _shuttingDown = false;
/** Instances waiting on {@link Instance.fromConfig} */ /** Instances waiting on {@link Instance.fromConfig} */
_startingInstances = new Set<number>(); _startingInstances = new Set<number>();
/** Restarts after a crash per instance ID since the last manual start */
_crashRetryAttempts = new Map<number, number>();
startedAtMs = Date.now(); startedAtMs = Date.now();
+39 -1
View File
@@ -44,6 +44,9 @@ const transitoryStatuses: ReadonlySet<Instance["_status"]> = new Set([
"starting", "stopping", "creating_save", "exporting_data", "starting", "stopping", "creating_save", "exporting_data",
]); ]);
// Crashes sooner than this after starting are not restarted to avoid boot loops.
const crashRestartMinUptimeMs = 10 * 60e3;
const instanceRconCommandDuration = new lib.Histogram( const instanceRconCommandDuration = new lib.Histogram(
"clusterio_instance_rcon_command_duration_seconds", "clusterio_instance_rcon_command_duration_seconds",
"Histogram of the RCON command duration from request to response.", "Histogram of the RCON command duration from request to response.",
@@ -194,6 +197,8 @@ export default class Instance extends lib.Link {
_playerAutosaveSlot = 1; _playerAutosaveSlot = 1;
_expectedUserUpdates: { action: string, name: string, reason: string }[] = []; _expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
_settleWaiters: (() => void)[] = []; _settleWaiters: (() => void)[] = [];
_startedAtMs: number | null = null;
_killRequested = false;
constructor( constructor(
host: Host, host: Host,
@@ -594,6 +599,9 @@ end`.replace(/\r?\n/g, " ");
notifyStatus(status: Instance["_status"]) { notifyStatus(status: Instance["_status"]) {
this._status = status; this._status = status;
if (status === "running") {
this._startedAtMs = Date.now();
}
if (!transitoryStatuses.has(status)) { if (!transitoryStatuses.has(status)) {
const waiters = this._settleWaiters; const waiters = this._settleWaiters;
this._settleWaiters = []; this._settleWaiters = [];
@@ -643,6 +651,7 @@ end`.replace(/\r?\n/g, " ");
} }
notifyExit() { notifyExit() {
const crashed = this._status === "running" && !this._killRequested;
this._loadedSave = null; this._loadedSave = null;
this._autoPause = false; this._autoPause = false;
this.notifyStatus("stopped"); this.notifyStatus("stopped");
@@ -668,6 +677,29 @@ end`.replace(/\r?\n/g, " ");
this._recordPlayerLeave(player, "server_quit"); this._recordPlayerLeave(player, "server_quit");
} }
this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`)); this._saveStats().catch(err => this.logger.error(`Error saving stats:\n${err.stack}`));
if (crashed && this.config.get("instance.restart_after_crash") && !this._host._shuttingDown) {
this._restartAfterCrash(Date.now() - this._startedAtMs!);
}
}
_restartAfterCrash(uptimeMs: number) {
if (uptimeMs < crashRestartMinUptimeMs) {
this.logger.error(`Instance ${this.name} crashed within 10 minutes of starting, not restarting it`);
return;
}
const limit = this.config.get("instance.crash_retry_attempts");
const attempts = this._host._crashRetryAttempts.get(this.id) ?? 0;
if (limit > 0 && attempts >= limit) {
this.logger.error(`Instance ${this.name} reached the limit of ${limit} crash restarts, not restarting it`);
return;
}
this._host._crashRetryAttempts.set(this.id, attempts + 1);
const of = limit > 0 ? ` of ${limit}` : "";
this.logger.warn(`Restarting instance ${this.name} after crash, attempt ${attempts + 1}${of}`);
this._host._connectInstance(this.id)
.then(connection => connection.instance._startSave())
.catch(err => this.logger.error(`Error restarting instance ${this.name} after crash:\n${err.stack}`));
} }
static async _readStats(statsPath: string) { static async _readStats(statsPath: string) {
@@ -1358,6 +1390,7 @@ end`.replace(/\r?\n/g, " ");
if (this._status === "stopped") { if (this._status === "stopped") {
return; return;
} }
this._killRequested = true;
await this.server.kill(true); await this.server.kill(true);
} }
@@ -1395,7 +1428,11 @@ end`.replace(/\r?\n/g, " ");
} }
async handleInstanceStartRequest(request: lib.InstanceStartRequest) { async handleInstanceStartRequest(request: lib.InstanceStartRequest) {
let saveName = request.save; this._host._crashRetryAttempts.delete(this.id);
await this._startSave(request.save);
}
async _startSave(saveName?: string) {
if (saveName !== undefined) { if (saveName !== undefined) {
checkRequestSaveName(saveName); checkRequestSaveName(saveName);
} }
@@ -1424,6 +1461,7 @@ end`.replace(/\r?\n/g, " ");
} }
async handleInstanceLoadScenarioRequest(request: lib.InstanceLoadScenarioRequest) { async handleInstanceLoadScenarioRequest(request: lib.InstanceLoadScenarioRequest) {
this._host._crashRetryAttempts.delete(this.id);
if (this.config.get("factorio.enable_save_patching")) { if (this.config.get("factorio.enable_save_patching")) {
this.notifyExit(); this.notifyExit();
throw new lib.ConflictError("Load scenario cannot be used with save patching enabled"); throw new lib.ConflictError("Load scenario cannot be used with save patching enabled");
+12
View File
@@ -520,6 +520,8 @@ export interface InstanceConfigFields {
"instance.assigned_host": number | null; "instance.assigned_host": number | null;
"instance.auto_start": boolean; "instance.auto_start": boolean;
"instance.exclude_from_start_all": boolean; "instance.exclude_from_start_all": boolean;
"instance.restart_after_crash": boolean;
"instance.crash_retry_attempts": number;
"instance.is_template": boolean; "instance.is_template": boolean;
"instance.template_id": number | null; "instance.template_id": number | null;
"instance.inherited_fields": Record<string, true | string[]>; "instance.inherited_fields": Record<string, true | string[]>;
@@ -633,6 +635,16 @@ export class InstanceConfig extends classes.Config<InstanceConfigFields> {
type: "boolean", type: "boolean",
initialValue: false, initialValue: false,
}, },
"instance.restart_after_crash": {
description: "Start this instance again if it crashes after running for at least 10 minutes",
type: "boolean",
initialValue: false,
},
"instance.crash_retry_attempts": {
description: "Restarts after a crash before giving up, 0 for no limit. Reset when the instance is started",
type: "number",
initialValue: 3,
},
"instance.is_template": { "instance.is_template": {
description: description:
"Mark this instance as a template other instances can inherit config from. " + "Mark this instance as a template other instances can inherit config from. " +
+76 -1
View File
@@ -95,7 +95,9 @@ describe("class Instance", function() {
instanceConfig.set("instance.name", "foo"); instanceConfig.set("instance.name", "foo");
src = addr({ instanceId: instanceConfig.get("instance.id") }); src = addr({ instanceId: instanceConfig.get("instance.id") });
connector = new MockConnector(src, dst); connector = new MockConnector(src, dst);
instance = new Instance({}, connector, "dir", instanceConfig, new MockServer()); instance = new Instance(
{ _crashRetryAttempts: new Map() }, connector, "dir", instanceConfig, new MockServer()
);
}); });
describe(".name", function() { describe(".name", function() {
@@ -308,6 +310,79 @@ describe("class Instance", function() {
}); });
}); });
describe("restart after crash", function() {
let errors;
let starts;
beforeEach(function() {
errors = [];
starts = [];
instance.logger = new MockLogger();
instance.logger.error = message => errors.push(message);
instance._saveStats = async () => {};
instance._host._connectInstance = async id => {
starts.push(id);
return { instance: { _startSave: async () => {} } };
};
instance.config.set("instance.restart_after_crash", true);
});
function crash(uptimeMinutes) {
instance.notifyStatus("running");
instance._startedAtMs = Date.now() - uptimeMinutes * 60e3;
instance.notifyExit();
}
it("should restart after running for 10 minutes until the attempts run out", function() {
for (let i = 0; i < 4; i++) {
crash(11);
}
assert.deepEqual(starts, [instance.id, instance.id, instance.id]);
assert.deepEqual(errors, ["Instance foo reached the limit of 3 crash restarts, not restarting it"]);
});
it("should restart without limit when attempts is 0", function() {
instance.config.set("instance.crash_retry_attempts", 0);
for (let i = 0; i < 5; i++) {
crash(11);
}
assert.equal(starts.length, 5);
assert.deepEqual(errors, []);
});
it("should not restart a crash within 10 minutes of starting", function() {
crash(9);
assert.deepEqual(starts, []);
assert.deepEqual(errors, ["Instance foo crashed within 10 minutes of starting, not restarting it"]);
});
it("should not restart when disabled", function() {
instance.config.set("instance.restart_after_crash", false);
crash(11);
assert.deepEqual(starts, []);
assert.deepEqual(errors, []);
});
it("should not restart after a stop or kill", async function() {
instance.server.kill = async () => {};
instance.notifyStatus("running");
instance._startedAtMs = 0;
await instance.handleInstanceKillRequest();
instance.notifyExit();
instance.notifyStatus("stopping");
instance.notifyExit();
assert.deepEqual(starts, []);
assert.deepEqual(errors, []);
});
it("should not restart while the host shuts down", function() {
instance._host._shuttingDown = true;
crash(11);
assert.deepEqual(starts, []);
});
it("should reset the attempts on a manual start", async function() {
instance.config.set("instance.crash_retry_attempts", 1);
instance._startSave = async () => {};
crash(11);
await instance.handleInstanceStartRequest(new lib.InstanceStartRequest());
crash(11);
assert.equal(starts.length, 2);
});
});
describe(".checkModPackVersion()", function() { describe(".checkModPackVersion()", function() {
function modPack(factorioVersion) { function modPack(factorioVersion) {
return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion }); return lib.ModPack.fromJSON({ name: "pack", factorio_version: factorioVersion });