Keep auto paused instances paused (#1143)

* Keep auto paused instances paused

Every RCON command makes an auto paused Factorio server run one tick, so
subspace_storage's 1 s inventory update and 5 s ping, plus a
statistics_exporter gather on every Prometheus scrape, kept a server with
nobody online running at about 1.3 ticks per second.

The instance now records the auto_pause server setting it was started
with and exposes autoPaused, true while that setting is on and no players
are online, along with an autoPauseChanged hook. subspace_storage skips
its timers while paused and sends a ping and inventory as soon as a player
joins. statistics_exporter gathers once after the game pauses and keeps
serving those values until it resumes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Check connection and status in subspace_storage timers

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

docs/configuration.md

@@ -644,6 +644,8 @@ The settings in `data/server-settings.example.json` of the Factorio installation
 
 Changes to the following properties will be applied live if the instance is running while it is changed: `afk_autokick_interval`, `allow_commands`, `autosave_interval`, `autosave_only_on_server`, `description`, `ignore_player_limit_for_returning_players`, `max_players`, `max_upload_slots`, `max_upload_in_kilobytes_per_second`, `name`, `only_admins_can_pause_the_game`, `game_password`, `require_user_verification`, `tags`, `visibility`.
 
+With `auto_pause` on, the plugins included with Clusterio stop their periodic commands while no players are online, so the game stays paused.
+
 Defaults to a set of common settings filled in when the instance is created, such as `"tags": ["clusterio"]`, `"auto_pause": false` and a `name` and `description` based on the cluster and instance names.
 
 

docs/writing-plugins.md

@@ -505,6 +505,25 @@ Data out from Factorio does not have the same limits as data into Factorio, RCON
 For `send_json` it's possible that payloads greater than 4kB are received after payloads that were sent at a later point in time.
 For RCON, commands longer than 50 characters may end up being executed after shorter commands sent after it.
 
+### Auto pause
+
+Each RCON command makes a paused game run one tick, so commands sent on a timer keep a server with `auto_pause` on running while nobody is online.
+`instance.autoPaused` is true while the instance was started with `auto_pause` on and no players are online, and the `autoPauseChanged` hook is invoked with the new value when it changes.
+Skip periodic commands while it is true:
+
+```js
+instance.hooks.start.attach(plugin.name, async () => {
+    timer = setInterval(() => {
+        if (!instance.autoPaused) {
+            instance.sendRcon("/sc my_plugin.update()").catch(handleError);
+        }
+    }, 1000);
+});
+```
+
+Commands sent in response to events from elsewhere in the cluster are fine, they cost one tick each.
+A game paused with `game.tick_paused` does not run ticks for commands.
+
 ### Communicating over UDP
 
 Instances can optionally exchange UDP packets with the Factorio server for low latency fire-and-forget communication, using the `--enable-lua-udp` option of Factorio.

packages/host/src/BaseInstancePlugin.ts

@@ -23,6 +23,7 @@ export class InstanceHooks extends lib.AsyncHookCollection {
 		this.controllerConnectionEvent = this.newHook();
 		this.prepareControllerDisconnect = this.newHook();
 		this.playerEvent = this.newHook();
+		this.autoPauseChanged = this.newHook();
 	}
 
 	/**
@@ -147,6 +148,19 @@ export class InstanceHooks extends lib.AsyncHookCollection {
 	 * @param event - Information about the event.
 	 */
 	readonly playerEvent: lib.AsyncHook<[event: lib.PlayerEvent]>;
+
+	/**
+	 * Called when Factorio auto pauses or unpauses the game
+	 *
+	 * Invoked when the last player leaves or the first player joins an
+	 * instance started with the auto_pause server setting on.  Every RCON
+	 * command makes a paused game run one tick, so periodic commands
+	 * should be skipped while paused.  See `Instance.autoPaused`.  It is
+	 * not invoked on start, read `autoPaused` in the start hook instead.
+	 *
+	 * @param paused - true if the game is now paused.
+	 */
+	readonly autoPauseChanged: lib.AsyncHook<[paused: boolean]>;
 }
 
 // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
@@ -160,6 +174,7 @@ export interface BaseInstancePlugin {
 	onControllerConnectionEvent?(event: "connect" | "drop" | "resume" | "close"): void;
 	onPrepareControllerDisconnect?(connection: Instance): Promise<void>;
 	onPlayerEvent?(event: lib.PlayerEvent): Promise<void>;
+	onAutoPauseChanged?(paused: boolean): Promise<void>;
 }
 
 /**
@@ -222,6 +237,7 @@ export class BaseInstancePlugin {
 		attach(instance.hooks.controllerConnectionEvent, this.onControllerConnectionEvent);
 		attach(instance.hooks.prepareControllerDisconnect, this.onPrepareControllerDisconnect);
 		attach(instance.hooks.playerEvent, this.onPlayerEvent);
+		attach(instance.hooks.autoPauseChanged, this.onAutoPauseChanged);
 	}
 
 	static fromContext(context: InstancePluginContext) {

packages/host/src/Instance.ts

@@ -190,6 +190,7 @@ export default class Instance extends lib.Link {
 	_loadedSave: string | null = null;
 	_playerCheckInterval: ReturnType<typeof setInterval> | undefined;
 	_hadPlayersOnline = false;
+	_autoPause = false;
 	_playerAutosaveSlot = 1;
 	_expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
 	_settleWaiters: (() => void)[] = [];
@@ -388,10 +389,19 @@ export default class Instance extends lib.Link {
 		}
 	}
 
+	/**
+	 * True while Factorio has paused the game because auto_pause is on
+	 * and no players are online.
+	 */
+	get autoPaused() {
+		return this._autoPause && this.playersOnline.size === 0;
+	}
+
 	_recordPlayerJoin(name: string) {
 		if (this.playersOnline.has(name)) {
 			return;
 		}
+		const wasPaused = this.autoPaused;
 		this.playersOnline.add(name);
 
 		let stats = this.playerStats.get(name);
@@ -411,6 +421,9 @@ export default class Instance extends lib.Link {
 			stats,
 		};
 		this.sendTo("controller", new lib.InstancePlayerUpdateEvent("join", name, stats));
+		if (wasPaused) {
+			this.hooks.autoPauseChanged.invoke(false);
+		}
 		this.hooks.playerEvent.invoke(event);
 	}
 
@@ -432,6 +445,9 @@ export default class Instance extends lib.Link {
 			stats,
 		};
 		this.sendTo("controller", new lib.InstancePlayerUpdateEvent("leave", name, stats, reason));
+		if (this.autoPaused) {
+			this.hooks.autoPauseChanged.invoke(true);
+		}
 		this.hooks.playerEvent.invoke(event);
 	}
 
@@ -628,6 +644,7 @@ end`.replace(/\r?\n/g, " ");
 
 	notifyExit() {
 		this._loadedSave = null;
+		this._autoPause = false;
 		this.notifyStatus("stopped");
 		this.connector.emit("close");
 
@@ -857,6 +874,7 @@ end`.replace(/\r?\n/g, " ");
 			this.server.writePath("server-settings.json"),
 			JSON.stringify(serverSettings, null, "\t")
 		);
+		return serverSettings;
 	}
 
 	/**
@@ -971,7 +989,9 @@ end`.replace(/\r?\n/g, " ");
 	 */
 	async prepare() {
 		this.logger.verbose("Writing server-settings.json");
-		await this.writeServerSettings(true);
+		// auto_pause is only read on startup
+		const serverSettings = await this.writeServerSettings(true);
+		this._autoPause = serverSettings.auto_pause === true;
 		this._expectedUserUpdates = [];
 
 		if (this.config.get("factorio.sync_adminlist") !== "disabled") {

plugins/statistics_exporter/instance.ts

@@ -175,18 +175,25 @@ export async function gatherMetrics(instance: Instance, pluginName: string) {
 
 export default async function(context: InstancePluginContext) {
 	const { instance, plugin } = context;
+	// Stats of an auto paused game do not change, and gathering them runs a tick.
+	let gatheredWhilePaused = false;
 
 	instance.hooks.metrics.attach(plugin.name, async () => {
 		if (instance.status !== "running") {
 			return;
 		}
+		if (instance.autoPaused && gatheredWhilePaused) {
+			return;
+		}
 
 		// Wait configured timeout for the metrics to be collected.  It may
 		// take a long time for the command to go through if the command
 		// stream is overloaded.  Should the timeout be exceeded the
 		// previous values for the metrics will end up being sent to controller.
 		let timeoutMs = instance.config.get("statistics_exporter.command_timeout") * 1000;
+		const paused = instance.autoPaused;
 		await lib.timeout(gatherMetrics(instance, plugin.name), timeoutMs, undefined);
+		gatheredWhilePaused = paused;
 	});
 }
 

plugins/statistics_exporter/test/plugin.js

@@ -86,6 +86,19 @@ describe("statistics_exporter plugin", function() {
 					"7357", "player", "nauvis").get(),
 				2);
 			});
+			it("should gather once while auto paused", async function() {
+				mockInstance.server.rconCommands.length = 0;
+				mockInstance.autoPaused = true;
+				try {
+					await mockInstance.hooks.metrics.invoke();
+					await mockInstance.hooks.metrics.invoke();
+					assert.equal(mockInstance.server.rconCommands.length, 1);
+				} finally {
+					mockInstance.autoPaused = false;
+				}
+				await mockInstance.hooks.metrics.invoke();
+				assert.equal(mockInstance.server.rconCommands.length, 2);
+			});
 			it("should pass on JSON parse errors", async function() {
 				let string = "An error occured\n";
 				mockInstance.server.rconCommandResults.set(

plugins/subspace_storage/instance.ts

@@ -87,14 +87,32 @@ export default async function(context: InstancePluginContext) {
 		cachedInventoryItems = event.items;
 	});
 
+	function sendPing() {
+		instance.sendRcon(
+			"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0", true, plugin.name
+		).catch(err => unexpectedError(err));
+	}
+
+	function sendInventory() {
+		const payloadItems = [
+			...cachedInventoryItems,
+			new Item("signal-unixtime", Math.floor(Date.now() / 1000), "normal"),
+		];
+		const payloadJson = lib.escapeString(JSON.stringify(payloadItems));
+		const task = instance.sendRcon(
+			`/sc __subspace_storage__ UpdateInvData("${payloadJson}")`, true, plugin.name
+		).catch(err => unexpectedError(err));
+		pendingTasks.add(task);
+		task.finally(() => { pendingTasks.delete(task); });
+	}
+
 	instance.hooks.start.attach(plugin.name, async () => {
+		// Each command runs a tick on an auto paused game.
 		pingId = setInterval(() => {
-			if (!host.connected) {
-				return; // Only ping if we are actually connected to the controller.
+			// Only ping if we are actually connected to the controller.
+			if (host.connected && !instance.autoPaused) {
+				sendPing();
 			}
-			instance.sendRcon(
-				"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0", true, plugin.name
-			).catch(err => unexpectedError(err));
 		}, 5000);
 
 		let items = await instance.sendTo("controller", new GetStorageRequest());
@@ -105,23 +123,22 @@ export default async function(context: InstancePluginContext) {
 
 		// Ensure a payload with updated time is sent every second
 		timeUpdateId = setInterval(() => {
-			if (instance.status !== "running") {
-				return;
+			if (instance.status === "running" && !instance.autoPaused) {
+				sendInventory();
 			}
-
-			const payloadItems = [
-				...cachedInventoryItems,
-				new Item("signal-unixtime", Math.floor(Date.now() / 1000), "normal"),
-			];
-			const payloadJson = lib.escapeString(JSON.stringify(payloadItems));
-			const task = instance.sendRcon(
-				`/sc __subspace_storage__ UpdateInvData("${payloadJson}")`, true, plugin.name
-			).catch(err => unexpectedError(err));
-			pendingTasks.add(task);
-			task.finally(() => { pendingTasks.delete(task); });
 		}, 1000);
 	});
 
+	instance.hooks.autoPauseChanged.attach(plugin.name, async (paused: boolean) => {
+		if (paused || instance.status !== "running") {
+			return;
+		}
+		if (host.connected) {
+			sendPing();
+		}
+		sendInventory();
+	});
+
 	instance.hooks.stop.attach(plugin.name, async () => {
 		clearInterval(pingId);
 		clearInterval(timeUpdateId);

plugins/subspace_storage/test/plugin.js

@@ -129,5 +129,63 @@ describe("subspace_storage plugin", function() {
 				nodeMock.timers.reset();
 			}
 		});
+		it("should not send commands while auto paused and catch up on unpause", async function() {
+			nodeMock.timers.enable({ apis: ["setInterval", "Date"], now: 1000000 });
+			const { instance: startedInstance, host } = await mock.loadInstancePlugin(instanceEntrypoint, info);
+			host.connected = true;
+			startedInstance.sendTo = async () => [];
+			try {
+				await startedInstance.hooks.start.invoke();
+				startedInstance.server.rconCommands.length = 0;
+				startedInstance.autoPaused = true;
+				nodeMock.timers.tick(10000);
+				assert.deepEqual(startedInstance.server.rconCommands, []);
+
+				startedInstance.autoPaused = false;
+				await startedInstance.hooks.autoPauseChanged.invoke(false);
+				assert.deepEqual(startedInstance.server.rconCommands, [
+					"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0",
+					`/sc __subspace_storage__ UpdateInvData("${lib.escapeString(JSON.stringify(
+						[new Item("signal-unixtime", 1010)]
+					))}")`,
+				]);
+			} finally {
+				await startedInstance.hooks.stop.invoke();
+				nodeMock.timers.reset();
+			}
+		});
+		describe("timers and unpause", function() {
+			const ping = "/sc __subspace_storage__ global.ticksSinceMasterPinged = 0";
+			const isInventory = command => command.includes("UpdateInvData");
+			let startedInstance;
+			let host;
+			beforeEach(async function() {
+				nodeMock.timers.enable({ apis: ["setInterval", "Date"], now: 1000000 });
+				({ instance: startedInstance, host } = await mock.loadInstancePlugin(instanceEntrypoint, info));
+				host.connected = true;
+				startedInstance.sendTo = async () => [];
+				await startedInstance.hooks.start.invoke();
+				startedInstance.server.rconCommands.length = 0;
+			});
+			afterEach(async function() {
+				await startedInstance.hooks.stop.invoke();
+				nodeMock.timers.reset();
+			});
+
+			it("should not ping while the host is disconnected", async function() {
+				host.connected = false;
+				nodeMock.timers.tick(5000);
+				await startedInstance.hooks.autoPauseChanged.invoke(false);
+				const commands = startedInstance.server.rconCommands;
+				assert(!commands.includes(ping), "ping was sent");
+				assert.equal(commands.filter(isInventory).length, 6);
+			});
+			it("should not send inventory while not running", async function() {
+				startedInstance.status = "stopping";
+				nodeMock.timers.tick(5000);
+				await startedInstance.hooks.autoPauseChanged.invoke(false);
+				assert.deepEqual(startedInstance.server.rconCommands, [ping]);
+			});
+		});
 	});
 });

test/host/Instance.js

@@ -418,6 +418,42 @@ describe("class Instance", function() {
 		});
 	});
 
+	describe(".autoPaused", function() {
+		let changes;
+		beforeEach(function() {
+			changes = [];
+			instance.hooks.autoPauseChanged.attach("test", async paused => { changes.push(paused); });
+		});
+
+		it("should be false when auto_pause is off", function() {
+			assert.equal(instance.autoPaused, false);
+			instance._recordPlayerJoin("player");
+			instance._recordPlayerLeave("player", "quit");
+			assert.equal(instance.autoPaused, false);
+			assert.deepEqual(changes, []);
+		});
+		it("should follow players online when auto_pause is on", function() {
+			instance._autoPause = true;
+			assert.equal(instance.autoPaused, true);
+			instance._recordPlayerJoin("a");
+			instance._recordPlayerJoin("b");
+			assert.equal(instance.autoPaused, false);
+			instance._recordPlayerLeave("a", "quit");
+			assert.equal(instance.autoPaused, false);
+			instance._recordPlayerLeave("b", "quit");
+			assert.equal(instance.autoPaused, true);
+			assert.deepEqual(changes, [false, true]);
+		});
+		it("should not invoke the hook on exit", function() {
+			instance._autoPause = true;
+			instance._recordPlayerJoin("player");
+			instance._saveStats = async () => {};
+			instance.notifyExit();
+			assert.equal(instance.autoPaused, false);
+			assert.deepEqual(changes, [false]);
+		});
+	});
+
 	describe("._checkOnlinePlayers()", function() {
 		it("should do nothing on empty server", async function() {
 			await instance._checkOnlinePlayers();

test/integration/clusterio.js

@@ -1332,6 +1332,55 @@ describe("Integration of Clusterio", function() {
 			});
 		});
 
+		describe("auto pause", function() {
+			requiresFactorio(this);
+			async function getTick() {
+				return Number(await getControl().sendTo(
+					{ instanceId: 44 }, new lib.InstanceSendRconRequest("/sc rcon.print(game.tick)")
+				));
+			}
+			async function sendPlayerEvent(type) {
+				const json = JSON.stringify({ type, name: "auto_pause_player", reason: "quit" });
+				await getControl().sendTo(
+					{ instanceId: 44 },
+					new lib.InstanceSendRconRequest(`/sc print("\\f$ipc:player_event?j${lib.escapeString(json)}")`),
+				);
+			}
+			let started = false;
+			before(async function() {
+				slowTest(this);
+				this.timeout(60000);
+				started = true;
+				await execCtl("instance stop test");
+				await execCtl("instance config set-prop test factorio.settings auto_pause true");
+				await execCtl("instance start test");
+			});
+			after(async function() {
+				if (!started) {
+					return;
+				}
+				this.timeout(60000);
+				await execCtl("instance stop test");
+				await execCtl("instance config set-prop test factorio.settings auto_pause false");
+				await execCtl("instance start test");
+			});
+			it("does not tick the game while no players are online", async function() {
+				slowTest(this);
+				const start = await getTick();
+				await wait(6000);
+				assert.equal(await getTick() - start, 1);
+			});
+			it("resumes plugin work when a player joins", async function() {
+				slowTest(this);
+				const start = await getTick();
+				await sendPlayerEvent("join");
+				await wait(6000);
+				const ticks = await getTick() - start;
+				await sendPlayerEvent("leave");
+				assert(ticks > 3, `game ran ${ticks} ticks`);
+			});
+		});
+
 		describe("instance stop", function() {
 			it("stops the instance", async function() {
 				slowTest(this);

test/mock.js

@@ -129,6 +129,7 @@ export class MockInstance extends lib.Link {
 		this.name = "test";
 		this.id = 7357;
 		this.status = "running";
+		this.autoPaused = false;
 		this.hooks = new InstanceHooks(this.logger);
 		this.loadedPlugins = new Set();
 		this.mockConfigEntries = new Map([