Keep auto paused instances paused

Every RCON command makes an auto paused Factorio server run one tick, so
subspace_storage's 1 s inventory update and 5 s ping, plus a
statistics_exporter gather on every Prometheus scrape, kept a server with
nobody online running at about 1.3 ticks per second.

The instance now records the auto_pause server setting it was started
with and exposes autoPaused, true while that setting is on and no players
are online, along with an autoPauseChanged hook. subspace_storage skips
its timers while paused and sends a ping and inventory as soon as a player
joins. statistics_exporter gathers once after the game pauses and keeps
serving those values until it resumes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
bbassieandClaude Opus 5.5 committed 2026-10-28 23:40:30 +00:00
1 parent ebdbb1fe56
commit 11e067a933
11 files changed
+227 -19

No files matched your search

+2
View File
@@ -644,6 +644,8 @@ The settings in `data/server-settings.example.json` of the Factorio installation
Changes to the following properties will be applied live if the instance is running while it is changed: `afk_autokick_interval`, `allow_commands`, `autosave_interval`, `autosave_only_on_server`, `description`, `ignore_player_limit_for_returning_players`, `max_players`, `max_upload_slots`, `max_upload_in_kilobytes_per_second`, `name`, `only_admins_can_pause_the_game`, `game_password`, `require_user_verification`, `tags`, `visibility`. Changes to the following properties will be applied live if the instance is running while it is changed: `afk_autokick_interval`, `allow_commands`, `autosave_interval`, `autosave_only_on_server`, `description`, `ignore_player_limit_for_returning_players`, `max_players`, `max_upload_slots`, `max_upload_in_kilobytes_per_second`, `name`, `only_admins_can_pause_the_game`, `game_password`, `require_user_verification`, `tags`, `visibility`.
With `auto_pause` on, the plugins included with Clusterio stop their periodic commands while no players are online, so the game stays paused.
Defaults to a set of common settings filled in when the instance is created, such as `"tags": ["clusterio"]`, `"auto_pause": false` and a `name` and `description` based on the cluster and instance names. Defaults to a set of common settings filled in when the instance is created, such as `"tags": ["clusterio"]`, `"auto_pause": false` and a `name` and `description` based on the cluster and instance names.
+19
View File
@@ -505,6 +505,25 @@ Data out from Factorio does not have the same limits as data into Factorio, RCON
For `send_json` it's possible that payloads greater than 4kB are received after payloads that were sent at a later point in time. For `send_json` it's possible that payloads greater than 4kB are received after payloads that were sent at a later point in time.
For RCON, commands longer than 50 characters may end up being executed after shorter commands sent after it. For RCON, commands longer than 50 characters may end up being executed after shorter commands sent after it.
### Auto pause
Each RCON command makes a paused game run one tick, so commands sent on a timer keep a server with `auto_pause` on running while nobody is online.
`instance.autoPaused` is true while the instance was started with `auto_pause` on and no players are online, and the `autoPauseChanged` hook is invoked with the new value when it changes.
Skip periodic commands while it is true:
```js
instance.hooks.start.attach(plugin.name, async () => {
timer = setInterval(() => {
if (!instance.autoPaused) {
instance.sendRcon("/sc my_plugin.update()").catch(handleError);
}
}, 1000);
});
```
Commands sent in response to events from elsewhere in the cluster are fine, they cost one tick each.
A game paused with `game.tick_paused` does not run ticks for commands.
### Communicating over UDP ### Communicating over UDP
Instances can optionally exchange UDP packets with the Factorio server for low latency fire-and-forget communication, using the `--enable-lua-udp` option of Factorio. Instances can optionally exchange UDP packets with the Factorio server for low latency fire-and-forget communication, using the `--enable-lua-udp` option of Factorio.
+16
View File
@@ -23,6 +23,7 @@ export class InstanceHooks extends lib.AsyncHookCollection {
this.controllerConnectionEvent = this.newHook(); this.controllerConnectionEvent = this.newHook();
this.prepareControllerDisconnect = this.newHook(); this.prepareControllerDisconnect = this.newHook();
this.playerEvent = this.newHook(); this.playerEvent = this.newHook();
this.autoPauseChanged = this.newHook();
} }
/** /**
@@ -147,6 +148,19 @@ export class InstanceHooks extends lib.AsyncHookCollection {
* @param event - Information about the event. * @param event - Information about the event.
*/ */
readonly playerEvent: lib.AsyncHook<[event: lib.PlayerEvent]>; readonly playerEvent: lib.AsyncHook<[event: lib.PlayerEvent]>;
/**
* Called when Factorio auto pauses or unpauses the game
*
* Invoked when the last player leaves or the first player joins an
* instance started with the auto_pause server setting on. Every RCON
* command makes a paused game run one tick, so periodic commands
* should be skipped while paused. See `Instance.autoPaused`. It is
* not invoked on start, read `autoPaused` in the start hook instead.
*
* @param paused - true if the game is now paused.
*/
readonly autoPauseChanged: lib.AsyncHook<[paused: boolean]>;
} }
// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
@@ -160,6 +174,7 @@ export interface BaseInstancePlugin {
onControllerConnectionEvent?(event: "connect" | "drop" | "resume" | "close"): void; onControllerConnectionEvent?(event: "connect" | "drop" | "resume" | "close"): void;
onPrepareControllerDisconnect?(connection: Instance): Promise<void>; onPrepareControllerDisconnect?(connection: Instance): Promise<void>;
onPlayerEvent?(event: lib.PlayerEvent): Promise<void>; onPlayerEvent?(event: lib.PlayerEvent): Promise<void>;
onAutoPauseChanged?(paused: boolean): Promise<void>;
} }
/** /**
@@ -222,6 +237,7 @@ export class BaseInstancePlugin {
attach(instance.hooks.controllerConnectionEvent, this.onControllerConnectionEvent); attach(instance.hooks.controllerConnectionEvent, this.onControllerConnectionEvent);
attach(instance.hooks.prepareControllerDisconnect, this.onPrepareControllerDisconnect); attach(instance.hooks.prepareControllerDisconnect, this.onPrepareControllerDisconnect);
attach(instance.hooks.playerEvent, this.onPlayerEvent); attach(instance.hooks.playerEvent, this.onPlayerEvent);
attach(instance.hooks.autoPauseChanged, this.onAutoPauseChanged);
} }
static fromContext(context: InstancePluginContext) { static fromContext(context: InstancePluginContext) {
+21 -1
View File
@@ -190,6 +190,7 @@ export default class Instance extends lib.Link {
_loadedSave: string | null = null; _loadedSave: string | null = null;
_playerCheckInterval: ReturnType<typeof setInterval> | undefined; _playerCheckInterval: ReturnType<typeof setInterval> | undefined;
_hadPlayersOnline = false; _hadPlayersOnline = false;
_autoPause = false;
_playerAutosaveSlot = 1; _playerAutosaveSlot = 1;
_expectedUserUpdates: { action: string, name: string, reason: string }[] = []; _expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
_settleWaiters: (() => void)[] = []; _settleWaiters: (() => void)[] = [];
@@ -388,10 +389,19 @@ export default class Instance extends lib.Link {
} }
} }
/**
* True while Factorio has paused the game because auto_pause is on
* and no players are online.
*/
get autoPaused() {
return this._autoPause && this.playersOnline.size === 0;
}
_recordPlayerJoin(name: string) { _recordPlayerJoin(name: string) {
if (this.playersOnline.has(name)) { if (this.playersOnline.has(name)) {
return; return;
} }
const wasPaused = this.autoPaused;
this.playersOnline.add(name); this.playersOnline.add(name);
let stats = this.playerStats.get(name); let stats = this.playerStats.get(name);
@@ -411,6 +421,9 @@ export default class Instance extends lib.Link {
stats, stats,
}; };
this.sendTo("controller", new lib.InstancePlayerUpdateEvent("join", name, stats)); this.sendTo("controller", new lib.InstancePlayerUpdateEvent("join", name, stats));
if (wasPaused) {
this.hooks.autoPauseChanged.invoke(false);
}
this.hooks.playerEvent.invoke(event); this.hooks.playerEvent.invoke(event);
} }
@@ -432,6 +445,9 @@ export default class Instance extends lib.Link {
stats, stats,
}; };
this.sendTo("controller", new lib.InstancePlayerUpdateEvent("leave", name, stats, reason)); this.sendTo("controller", new lib.InstancePlayerUpdateEvent("leave", name, stats, reason));
if (this.autoPaused) {
this.hooks.autoPauseChanged.invoke(true);
}
this.hooks.playerEvent.invoke(event); this.hooks.playerEvent.invoke(event);
} }
@@ -628,6 +644,7 @@ end`.replace(/\r?\n/g, " ");
notifyExit() { notifyExit() {
this._loadedSave = null; this._loadedSave = null;
this._autoPause = false;
this.notifyStatus("stopped"); this.notifyStatus("stopped");
this.connector.emit("close"); this.connector.emit("close");
@@ -857,6 +874,7 @@ end`.replace(/\r?\n/g, " ");
this.server.writePath("server-settings.json"), this.server.writePath("server-settings.json"),
JSON.stringify(serverSettings, null, "\t") JSON.stringify(serverSettings, null, "\t")
); );
return serverSettings;
} }
/** /**
@@ -971,7 +989,9 @@ end`.replace(/\r?\n/g, " ");
*/ */
async prepare() { async prepare() {
this.logger.verbose("Writing server-settings.json"); this.logger.verbose("Writing server-settings.json");
await this.writeServerSettings(true); // auto_pause is only read on startup
const serverSettings = await this.writeServerSettings(true);
this._autoPause = serverSettings.auto_pause === true;
this._expectedUserUpdates = []; this._expectedUserUpdates = [];
if (this.config.get("factorio.sync_adminlist") !== "disabled") { if (this.config.get("factorio.sync_adminlist") !== "disabled") {
+7
View File
@@ -175,18 +175,25 @@ export async function gatherMetrics(instance: Instance, pluginName: string) {
export default async function(context: InstancePluginContext) { export default async function(context: InstancePluginContext) {
const { instance, plugin } = context; const { instance, plugin } = context;
// Stats of an auto paused game do not change, and gathering them runs a tick.
let gatheredWhilePaused = false;
instance.hooks.metrics.attach(plugin.name, async () => { instance.hooks.metrics.attach(plugin.name, async () => {
if (instance.status !== "running") { if (instance.status !== "running") {
return; return;
} }
if (instance.autoPaused && gatheredWhilePaused) {
return;
}
// Wait configured timeout for the metrics to be collected. It may // Wait configured timeout for the metrics to be collected. It may
// take a long time for the command to go through if the command // take a long time for the command to go through if the command
// stream is overloaded. Should the timeout be exceeded the // stream is overloaded. Should the timeout be exceeded the
// previous values for the metrics will end up being sent to controller. // previous values for the metrics will end up being sent to controller.
let timeoutMs = instance.config.get("statistics_exporter.command_timeout") * 1000; let timeoutMs = instance.config.get("statistics_exporter.command_timeout") * 1000;
const paused = instance.autoPaused;
await lib.timeout(gatherMetrics(instance, plugin.name), timeoutMs, undefined); await lib.timeout(gatherMetrics(instance, plugin.name), timeoutMs, undefined);
gatheredWhilePaused = paused;
}); });
} }
@@ -86,6 +86,19 @@ describe("statistics_exporter plugin", function() {
"7357", "player", "nauvis").get(), "7357", "player", "nauvis").get(),
2); 2);
}); });
it("should gather once while auto paused", async function() {
mockInstance.server.rconCommands.length = 0;
mockInstance.autoPaused = true;
try {
await mockInstance.hooks.metrics.invoke();
await mockInstance.hooks.metrics.invoke();
assert.equal(mockInstance.server.rconCommands.length, 1);
} finally {
mockInstance.autoPaused = false;
}
await mockInstance.hooks.metrics.invoke();
assert.equal(mockInstance.server.rconCommands.length, 2);
});
it("should pass on JSON parse errors", async function() { it("should pass on JSON parse errors", async function() {
let string = "An error occured\n"; let string = "An error occured\n";
mockInstance.server.rconCommandResults.set( mockInstance.server.rconCommandResults.set(
+38 -18
View File
@@ -87,14 +87,38 @@ export default async function(context: InstancePluginContext) {
cachedInventoryItems = event.items; cachedInventoryItems = event.items;
}); });
function sendPing() {
if (!host.connected) {
return; // Only ping if we are actually connected to the controller.
}
instance.sendRcon(
"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0", true, plugin.name
).catch(err => unexpectedError(err));
}
function sendInventory() {
if (instance.status !== "running") {
return;
}
const payloadItems = [
...cachedInventoryItems,
new Item("signal-unixtime", Math.floor(Date.now() / 1000), "normal"),
];
const payloadJson = lib.escapeString(JSON.stringify(payloadItems));
const task = instance.sendRcon(
`/sc __subspace_storage__ UpdateInvData("${payloadJson}")`, true, plugin.name
).catch(err => unexpectedError(err));
pendingTasks.add(task);
task.finally(() => { pendingTasks.delete(task); });
}
instance.hooks.start.attach(plugin.name, async () => { instance.hooks.start.attach(plugin.name, async () => {
// Each command runs a tick on an auto paused game.
pingId = setInterval(() => { pingId = setInterval(() => {
if (!host.connected) { if (!instance.autoPaused) {
return; // Only ping if we are actually connected to the controller. sendPing();
} }
instance.sendRcon(
"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0", true, plugin.name
).catch(err => unexpectedError(err));
}, 5000); }, 5000);
let items = await instance.sendTo("controller", new GetStorageRequest()); let items = await instance.sendTo("controller", new GetStorageRequest());
@@ -105,23 +129,19 @@ export default async function(context: InstancePluginContext) {
// Ensure a payload with updated time is sent every second // Ensure a payload with updated time is sent every second
timeUpdateId = setInterval(() => { timeUpdateId = setInterval(() => {
if (instance.status !== "running") { if (!instance.autoPaused) {
return; sendInventory();
} }
const payloadItems = [
...cachedInventoryItems,
new Item("signal-unixtime", Math.floor(Date.now() / 1000), "normal"),
];
const payloadJson = lib.escapeString(JSON.stringify(payloadItems));
const task = instance.sendRcon(
`/sc __subspace_storage__ UpdateInvData("${payloadJson}")`, true, plugin.name
).catch(err => unexpectedError(err));
pendingTasks.add(task);
task.finally(() => { pendingTasks.delete(task); });
}, 1000); }, 1000);
}); });
instance.hooks.autoPauseChanged.attach(plugin.name, async (paused: boolean) => {
if (!paused && instance.status === "running") {
sendPing();
sendInventory();
}
});
instance.hooks.stop.attach(plugin.name, async () => { instance.hooks.stop.attach(plugin.name, async () => {
clearInterval(pingId); clearInterval(pingId);
clearInterval(timeUpdateId); clearInterval(timeUpdateId);
+25
View File
@@ -129,5 +129,30 @@ describe("subspace_storage plugin", function() {
nodeMock.timers.reset(); nodeMock.timers.reset();
} }
}); });
it("should not send commands while auto paused and catch up on unpause", async function() {
nodeMock.timers.enable({ apis: ["setInterval", "Date"], now: 1000000 });
const { instance: startedInstance, host } = await mock.loadInstancePlugin(instanceEntrypoint, info);
host.connected = true;
startedInstance.sendTo = async () => [];
try {
await startedInstance.hooks.start.invoke();
startedInstance.server.rconCommands.length = 0;
startedInstance.autoPaused = true;
nodeMock.timers.tick(10000);
assert.deepEqual(startedInstance.server.rconCommands, []);
startedInstance.autoPaused = false;
await startedInstance.hooks.autoPauseChanged.invoke(false);
assert.deepEqual(startedInstance.server.rconCommands, [
"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0",
`/sc __subspace_storage__ UpdateInvData("${lib.escapeString(JSON.stringify(
[new Item("signal-unixtime", 1010)]
))}")`,
]);
} finally {
await startedInstance.hooks.stop.invoke();
nodeMock.timers.reset();
}
});
}); });
}); });
+36
View File
@@ -418,6 +418,42 @@ describe("class Instance", function() {
}); });
}); });
describe(".autoPaused", function() {
let changes;
beforeEach(function() {
changes = [];
instance.hooks.autoPauseChanged.attach("test", async paused => { changes.push(paused); });
});
it("should be false when auto_pause is off", function() {
assert.equal(instance.autoPaused, false);
instance._recordPlayerJoin("player");
instance._recordPlayerLeave("player", "quit");
assert.equal(instance.autoPaused, false);
assert.deepEqual(changes, []);
});
it("should follow players online when auto_pause is on", function() {
instance._autoPause = true;
assert.equal(instance.autoPaused, true);
instance._recordPlayerJoin("a");
instance._recordPlayerJoin("b");
assert.equal(instance.autoPaused, false);
instance._recordPlayerLeave("a", "quit");
assert.equal(instance.autoPaused, false);
instance._recordPlayerLeave("b", "quit");
assert.equal(instance.autoPaused, true);
assert.deepEqual(changes, [false, true]);
});
it("should not invoke the hook on exit", function() {
instance._autoPause = true;
instance._recordPlayerJoin("player");
instance._saveStats = async () => {};
instance.notifyExit();
assert.equal(instance.autoPaused, false);
assert.deepEqual(changes, [false]);
});
});
describe("._checkOnlinePlayers()", function() { describe("._checkOnlinePlayers()", function() {
it("should do nothing on empty server", async function() { it("should do nothing on empty server", async function() {
await instance._checkOnlinePlayers(); await instance._checkOnlinePlayers();
+49
View File
@@ -1332,6 +1332,55 @@ describe("Integration of Clusterio", function() {
}); });
}); });
describe("auto pause", function() {
requiresFactorio(this);
async function getTick() {
return Number(await getControl().sendTo(
{ instanceId: 44 }, new lib.InstanceSendRconRequest("/sc rcon.print(game.tick)")
));
}
async function sendPlayerEvent(type) {
const json = JSON.stringify({ type, name: "auto_pause_player", reason: "quit" });
await getControl().sendTo(
{ instanceId: 44 },
new lib.InstanceSendRconRequest(`/sc print("\\f$ipc:player_event?j${lib.escapeString(json)}")`),
);
}
let started = false;
before(async function() {
slowTest(this);
this.timeout(60000);
started = true;
await execCtl("instance stop test");
await execCtl("instance config set-prop test factorio.settings auto_pause true");
await execCtl("instance start test");
});
after(async function() {
if (!started) {
return;
}
this.timeout(60000);
await execCtl("instance stop test");
await execCtl("instance config set-prop test factorio.settings auto_pause false");
await execCtl("instance start test");
});
it("does not tick the game while no players are online", async function() {
slowTest(this);
const start = await getTick();
await wait(6000);
assert.equal(await getTick() - start, 1);
});
it("resumes plugin work when a player joins", async function() {
slowTest(this);
const start = await getTick();
await sendPlayerEvent("join");
await wait(6000);
const ticks = await getTick() - start;
await sendPlayerEvent("leave");
assert(ticks > 3, `game ran ${ticks} ticks`);
});
});
describe("instance stop", function() { describe("instance stop", function() {
it("stops the instance", async function() { it("stops the instance", async function() {
slowTest(this); slowTest(this);
+1
View File
@@ -129,6 +129,7 @@ export class MockInstance extends lib.Link {
this.name = "test"; this.name = "test";
this.id = 7357; this.id = 7357;
this.status = "running"; this.status = "running";
this.autoPaused = false;
this.hooks = new InstanceHooks(this.logger); this.hooks = new InstanceHooks(this.logger);
this.loadedPlugins = new Set(); this.loadedPlugins = new Set();
this.mockConfigEntries = new Map([ this.mockConfigEntries = new Map([