mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-11 15:04:52 +09:00
Keep auto paused instances paused
Every RCON command makes an auto paused Factorio server run one tick, so subspace_storage's 1 s inventory update and 5 s ping, plus a statistics_exporter gather on every Prometheus scrape, kept a server with nobody online running at about 1.3 ticks per second. The instance now records the auto_pause server setting it was started with and exposes autoPaused, true while that setting is on and no players are online, along with an autoPauseChanged hook. subspace_storage skips its timers while paused and sends a ping and inventory as soon as a player joins. statistics_exporter gathers once after the game pauses and keeps serving those values until it resumes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ebdbb1fe56
commit
11e067a933
11 files changed
+227
-19
No files matched your search
@@ -644,6 +644,8 @@ The settings in `data/server-settings.example.json` of the Factorio installation
|
||||
|
||||
Changes to the following properties will be applied live if the instance is running while it is changed: `afk_autokick_interval`, `allow_commands`, `autosave_interval`, `autosave_only_on_server`, `description`, `ignore_player_limit_for_returning_players`, `max_players`, `max_upload_slots`, `max_upload_in_kilobytes_per_second`, `name`, `only_admins_can_pause_the_game`, `game_password`, `require_user_verification`, `tags`, `visibility`.
|
||||
|
||||
With `auto_pause` on, the plugins included with Clusterio stop their periodic commands while no players are online, so the game stays paused.
|
||||
|
||||
Defaults to a set of common settings filled in when the instance is created, such as `"tags": ["clusterio"]`, `"auto_pause": false` and a `name` and `description` based on the cluster and instance names.
|
||||
|
||||
|
||||
|
||||
@@ -505,6 +505,25 @@ Data out from Factorio does not have the same limits as data into Factorio, RCON
|
||||
For `send_json` it's possible that payloads greater than 4kB are received after payloads that were sent at a later point in time.
|
||||
For RCON, commands longer than 50 characters may end up being executed after shorter commands sent after it.
|
||||
|
||||
### Auto pause
|
||||
|
||||
Each RCON command makes a paused game run one tick, so commands sent on a timer keep a server with `auto_pause` on running while nobody is online.
|
||||
`instance.autoPaused` is true while the instance was started with `auto_pause` on and no players are online, and the `autoPauseChanged` hook is invoked with the new value when it changes.
|
||||
Skip periodic commands while it is true:
|
||||
|
||||
```js
|
||||
instance.hooks.start.attach(plugin.name, async () => {
|
||||
timer = setInterval(() => {
|
||||
if (!instance.autoPaused) {
|
||||
instance.sendRcon("/sc my_plugin.update()").catch(handleError);
|
||||
}
|
||||
}, 1000);
|
||||
});
|
||||
```
|
||||
|
||||
Commands sent in response to events from elsewhere in the cluster are fine, they cost one tick each.
|
||||
A game paused with `game.tick_paused` does not run ticks for commands.
|
||||
|
||||
### Communicating over UDP
|
||||
|
||||
Instances can optionally exchange UDP packets with the Factorio server for low latency fire-and-forget communication, using the `--enable-lua-udp` option of Factorio.
|
||||
|
||||
@@ -23,6 +23,7 @@ export class InstanceHooks extends lib.AsyncHookCollection {
|
||||
this.controllerConnectionEvent = this.newHook();
|
||||
this.prepareControllerDisconnect = this.newHook();
|
||||
this.playerEvent = this.newHook();
|
||||
this.autoPauseChanged = this.newHook();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -147,6 +148,19 @@ export class InstanceHooks extends lib.AsyncHookCollection {
|
||||
* @param event - Information about the event.
|
||||
*/
|
||||
readonly playerEvent: lib.AsyncHook<[event: lib.PlayerEvent]>;
|
||||
|
||||
/**
|
||||
* Called when Factorio auto pauses or unpauses the game
|
||||
*
|
||||
* Invoked when the last player leaves or the first player joins an
|
||||
* instance started with the auto_pause server setting on. Every RCON
|
||||
* command makes a paused game run one tick, so periodic commands
|
||||
* should be skipped while paused. See `Instance.autoPaused`. It is
|
||||
* not invoked on start, read `autoPaused` in the start hook instead.
|
||||
*
|
||||
* @param paused - true if the game is now paused.
|
||||
*/
|
||||
readonly autoPauseChanged: lib.AsyncHook<[paused: boolean]>;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging
|
||||
@@ -160,6 +174,7 @@ export interface BaseInstancePlugin {
|
||||
onControllerConnectionEvent?(event: "connect" | "drop" | "resume" | "close"): void;
|
||||
onPrepareControllerDisconnect?(connection: Instance): Promise<void>;
|
||||
onPlayerEvent?(event: lib.PlayerEvent): Promise<void>;
|
||||
onAutoPauseChanged?(paused: boolean): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -222,6 +237,7 @@ export class BaseInstancePlugin {
|
||||
attach(instance.hooks.controllerConnectionEvent, this.onControllerConnectionEvent);
|
||||
attach(instance.hooks.prepareControllerDisconnect, this.onPrepareControllerDisconnect);
|
||||
attach(instance.hooks.playerEvent, this.onPlayerEvent);
|
||||
attach(instance.hooks.autoPauseChanged, this.onAutoPauseChanged);
|
||||
}
|
||||
|
||||
static fromContext(context: InstancePluginContext) {
|
||||
|
||||
@@ -190,6 +190,7 @@ export default class Instance extends lib.Link {
|
||||
_loadedSave: string | null = null;
|
||||
_playerCheckInterval: ReturnType<typeof setInterval> | undefined;
|
||||
_hadPlayersOnline = false;
|
||||
_autoPause = false;
|
||||
_playerAutosaveSlot = 1;
|
||||
_expectedUserUpdates: { action: string, name: string, reason: string }[] = [];
|
||||
_settleWaiters: (() => void)[] = [];
|
||||
@@ -388,10 +389,19 @@ export default class Instance extends lib.Link {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True while Factorio has paused the game because auto_pause is on
|
||||
* and no players are online.
|
||||
*/
|
||||
get autoPaused() {
|
||||
return this._autoPause && this.playersOnline.size === 0;
|
||||
}
|
||||
|
||||
_recordPlayerJoin(name: string) {
|
||||
if (this.playersOnline.has(name)) {
|
||||
return;
|
||||
}
|
||||
const wasPaused = this.autoPaused;
|
||||
this.playersOnline.add(name);
|
||||
|
||||
let stats = this.playerStats.get(name);
|
||||
@@ -411,6 +421,9 @@ export default class Instance extends lib.Link {
|
||||
stats,
|
||||
};
|
||||
this.sendTo("controller", new lib.InstancePlayerUpdateEvent("join", name, stats));
|
||||
if (wasPaused) {
|
||||
this.hooks.autoPauseChanged.invoke(false);
|
||||
}
|
||||
this.hooks.playerEvent.invoke(event);
|
||||
}
|
||||
|
||||
@@ -432,6 +445,9 @@ export default class Instance extends lib.Link {
|
||||
stats,
|
||||
};
|
||||
this.sendTo("controller", new lib.InstancePlayerUpdateEvent("leave", name, stats, reason));
|
||||
if (this.autoPaused) {
|
||||
this.hooks.autoPauseChanged.invoke(true);
|
||||
}
|
||||
this.hooks.playerEvent.invoke(event);
|
||||
}
|
||||
|
||||
@@ -628,6 +644,7 @@ end`.replace(/\r?\n/g, " ");
|
||||
|
||||
notifyExit() {
|
||||
this._loadedSave = null;
|
||||
this._autoPause = false;
|
||||
this.notifyStatus("stopped");
|
||||
this.connector.emit("close");
|
||||
|
||||
@@ -857,6 +874,7 @@ end`.replace(/\r?\n/g, " ");
|
||||
this.server.writePath("server-settings.json"),
|
||||
JSON.stringify(serverSettings, null, "\t")
|
||||
);
|
||||
return serverSettings;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -971,7 +989,9 @@ end`.replace(/\r?\n/g, " ");
|
||||
*/
|
||||
async prepare() {
|
||||
this.logger.verbose("Writing server-settings.json");
|
||||
await this.writeServerSettings(true);
|
||||
// auto_pause is only read on startup
|
||||
const serverSettings = await this.writeServerSettings(true);
|
||||
this._autoPause = serverSettings.auto_pause === true;
|
||||
this._expectedUserUpdates = [];
|
||||
|
||||
if (this.config.get("factorio.sync_adminlist") !== "disabled") {
|
||||
|
||||
@@ -175,18 +175,25 @@ export async function gatherMetrics(instance: Instance, pluginName: string) {
|
||||
|
||||
export default async function(context: InstancePluginContext) {
|
||||
const { instance, plugin } = context;
|
||||
// Stats of an auto paused game do not change, and gathering them runs a tick.
|
||||
let gatheredWhilePaused = false;
|
||||
|
||||
instance.hooks.metrics.attach(plugin.name, async () => {
|
||||
if (instance.status !== "running") {
|
||||
return;
|
||||
}
|
||||
if (instance.autoPaused && gatheredWhilePaused) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Wait configured timeout for the metrics to be collected. It may
|
||||
// take a long time for the command to go through if the command
|
||||
// stream is overloaded. Should the timeout be exceeded the
|
||||
// previous values for the metrics will end up being sent to controller.
|
||||
let timeoutMs = instance.config.get("statistics_exporter.command_timeout") * 1000;
|
||||
const paused = instance.autoPaused;
|
||||
await lib.timeout(gatherMetrics(instance, plugin.name), timeoutMs, undefined);
|
||||
gatheredWhilePaused = paused;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -86,6 +86,19 @@ describe("statistics_exporter plugin", function() {
|
||||
"7357", "player", "nauvis").get(),
|
||||
2);
|
||||
});
|
||||
it("should gather once while auto paused", async function() {
|
||||
mockInstance.server.rconCommands.length = 0;
|
||||
mockInstance.autoPaused = true;
|
||||
try {
|
||||
await mockInstance.hooks.metrics.invoke();
|
||||
await mockInstance.hooks.metrics.invoke();
|
||||
assert.equal(mockInstance.server.rconCommands.length, 1);
|
||||
} finally {
|
||||
mockInstance.autoPaused = false;
|
||||
}
|
||||
await mockInstance.hooks.metrics.invoke();
|
||||
assert.equal(mockInstance.server.rconCommands.length, 2);
|
||||
});
|
||||
it("should pass on JSON parse errors", async function() {
|
||||
let string = "An error occured\n";
|
||||
mockInstance.server.rconCommandResults.set(
|
||||
|
||||
@@ -87,14 +87,38 @@ export default async function(context: InstancePluginContext) {
|
||||
cachedInventoryItems = event.items;
|
||||
});
|
||||
|
||||
function sendPing() {
|
||||
if (!host.connected) {
|
||||
return; // Only ping if we are actually connected to the controller.
|
||||
}
|
||||
instance.sendRcon(
|
||||
"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0", true, plugin.name
|
||||
).catch(err => unexpectedError(err));
|
||||
}
|
||||
|
||||
function sendInventory() {
|
||||
if (instance.status !== "running") {
|
||||
return;
|
||||
}
|
||||
|
||||
const payloadItems = [
|
||||
...cachedInventoryItems,
|
||||
new Item("signal-unixtime", Math.floor(Date.now() / 1000), "normal"),
|
||||
];
|
||||
const payloadJson = lib.escapeString(JSON.stringify(payloadItems));
|
||||
const task = instance.sendRcon(
|
||||
`/sc __subspace_storage__ UpdateInvData("${payloadJson}")`, true, plugin.name
|
||||
).catch(err => unexpectedError(err));
|
||||
pendingTasks.add(task);
|
||||
task.finally(() => { pendingTasks.delete(task); });
|
||||
}
|
||||
|
||||
instance.hooks.start.attach(plugin.name, async () => {
|
||||
// Each command runs a tick on an auto paused game.
|
||||
pingId = setInterval(() => {
|
||||
if (!host.connected) {
|
||||
return; // Only ping if we are actually connected to the controller.
|
||||
if (!instance.autoPaused) {
|
||||
sendPing();
|
||||
}
|
||||
instance.sendRcon(
|
||||
"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0", true, plugin.name
|
||||
).catch(err => unexpectedError(err));
|
||||
}, 5000);
|
||||
|
||||
let items = await instance.sendTo("controller", new GetStorageRequest());
|
||||
@@ -105,23 +129,19 @@ export default async function(context: InstancePluginContext) {
|
||||
|
||||
// Ensure a payload with updated time is sent every second
|
||||
timeUpdateId = setInterval(() => {
|
||||
if (instance.status !== "running") {
|
||||
return;
|
||||
if (!instance.autoPaused) {
|
||||
sendInventory();
|
||||
}
|
||||
|
||||
const payloadItems = [
|
||||
...cachedInventoryItems,
|
||||
new Item("signal-unixtime", Math.floor(Date.now() / 1000), "normal"),
|
||||
];
|
||||
const payloadJson = lib.escapeString(JSON.stringify(payloadItems));
|
||||
const task = instance.sendRcon(
|
||||
`/sc __subspace_storage__ UpdateInvData("${payloadJson}")`, true, plugin.name
|
||||
).catch(err => unexpectedError(err));
|
||||
pendingTasks.add(task);
|
||||
task.finally(() => { pendingTasks.delete(task); });
|
||||
}, 1000);
|
||||
});
|
||||
|
||||
instance.hooks.autoPauseChanged.attach(plugin.name, async (paused: boolean) => {
|
||||
if (!paused && instance.status === "running") {
|
||||
sendPing();
|
||||
sendInventory();
|
||||
}
|
||||
});
|
||||
|
||||
instance.hooks.stop.attach(plugin.name, async () => {
|
||||
clearInterval(pingId);
|
||||
clearInterval(timeUpdateId);
|
||||
|
||||
@@ -129,5 +129,30 @@ describe("subspace_storage plugin", function() {
|
||||
nodeMock.timers.reset();
|
||||
}
|
||||
});
|
||||
it("should not send commands while auto paused and catch up on unpause", async function() {
|
||||
nodeMock.timers.enable({ apis: ["setInterval", "Date"], now: 1000000 });
|
||||
const { instance: startedInstance, host } = await mock.loadInstancePlugin(instanceEntrypoint, info);
|
||||
host.connected = true;
|
||||
startedInstance.sendTo = async () => [];
|
||||
try {
|
||||
await startedInstance.hooks.start.invoke();
|
||||
startedInstance.server.rconCommands.length = 0;
|
||||
startedInstance.autoPaused = true;
|
||||
nodeMock.timers.tick(10000);
|
||||
assert.deepEqual(startedInstance.server.rconCommands, []);
|
||||
|
||||
startedInstance.autoPaused = false;
|
||||
await startedInstance.hooks.autoPauseChanged.invoke(false);
|
||||
assert.deepEqual(startedInstance.server.rconCommands, [
|
||||
"/sc __subspace_storage__ global.ticksSinceMasterPinged = 0",
|
||||
`/sc __subspace_storage__ UpdateInvData("${lib.escapeString(JSON.stringify(
|
||||
[new Item("signal-unixtime", 1010)]
|
||||
))}")`,
|
||||
]);
|
||||
} finally {
|
||||
await startedInstance.hooks.stop.invoke();
|
||||
nodeMock.timers.reset();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -418,6 +418,42 @@ describe("class Instance", function() {
|
||||
});
|
||||
});
|
||||
|
||||
describe(".autoPaused", function() {
|
||||
let changes;
|
||||
beforeEach(function() {
|
||||
changes = [];
|
||||
instance.hooks.autoPauseChanged.attach("test", async paused => { changes.push(paused); });
|
||||
});
|
||||
|
||||
it("should be false when auto_pause is off", function() {
|
||||
assert.equal(instance.autoPaused, false);
|
||||
instance._recordPlayerJoin("player");
|
||||
instance._recordPlayerLeave("player", "quit");
|
||||
assert.equal(instance.autoPaused, false);
|
||||
assert.deepEqual(changes, []);
|
||||
});
|
||||
it("should follow players online when auto_pause is on", function() {
|
||||
instance._autoPause = true;
|
||||
assert.equal(instance.autoPaused, true);
|
||||
instance._recordPlayerJoin("a");
|
||||
instance._recordPlayerJoin("b");
|
||||
assert.equal(instance.autoPaused, false);
|
||||
instance._recordPlayerLeave("a", "quit");
|
||||
assert.equal(instance.autoPaused, false);
|
||||
instance._recordPlayerLeave("b", "quit");
|
||||
assert.equal(instance.autoPaused, true);
|
||||
assert.deepEqual(changes, [false, true]);
|
||||
});
|
||||
it("should not invoke the hook on exit", function() {
|
||||
instance._autoPause = true;
|
||||
instance._recordPlayerJoin("player");
|
||||
instance._saveStats = async () => {};
|
||||
instance.notifyExit();
|
||||
assert.equal(instance.autoPaused, false);
|
||||
assert.deepEqual(changes, [false]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("._checkOnlinePlayers()", function() {
|
||||
it("should do nothing on empty server", async function() {
|
||||
await instance._checkOnlinePlayers();
|
||||
|
||||
@@ -1332,6 +1332,55 @@ describe("Integration of Clusterio", function() {
|
||||
});
|
||||
});
|
||||
|
||||
describe("auto pause", function() {
|
||||
requiresFactorio(this);
|
||||
async function getTick() {
|
||||
return Number(await getControl().sendTo(
|
||||
{ instanceId: 44 }, new lib.InstanceSendRconRequest("/sc rcon.print(game.tick)")
|
||||
));
|
||||
}
|
||||
async function sendPlayerEvent(type) {
|
||||
const json = JSON.stringify({ type, name: "auto_pause_player", reason: "quit" });
|
||||
await getControl().sendTo(
|
||||
{ instanceId: 44 },
|
||||
new lib.InstanceSendRconRequest(`/sc print("\\f$ipc:player_event?j${lib.escapeString(json)}")`),
|
||||
);
|
||||
}
|
||||
let started = false;
|
||||
before(async function() {
|
||||
slowTest(this);
|
||||
this.timeout(60000);
|
||||
started = true;
|
||||
await execCtl("instance stop test");
|
||||
await execCtl("instance config set-prop test factorio.settings auto_pause true");
|
||||
await execCtl("instance start test");
|
||||
});
|
||||
after(async function() {
|
||||
if (!started) {
|
||||
return;
|
||||
}
|
||||
this.timeout(60000);
|
||||
await execCtl("instance stop test");
|
||||
await execCtl("instance config set-prop test factorio.settings auto_pause false");
|
||||
await execCtl("instance start test");
|
||||
});
|
||||
it("does not tick the game while no players are online", async function() {
|
||||
slowTest(this);
|
||||
const start = await getTick();
|
||||
await wait(6000);
|
||||
assert.equal(await getTick() - start, 1);
|
||||
});
|
||||
it("resumes plugin work when a player joins", async function() {
|
||||
slowTest(this);
|
||||
const start = await getTick();
|
||||
await sendPlayerEvent("join");
|
||||
await wait(6000);
|
||||
const ticks = await getTick() - start;
|
||||
await sendPlayerEvent("leave");
|
||||
assert(ticks > 3, `game ran ${ticks} ticks`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("instance stop", function() {
|
||||
it("stops the instance", async function() {
|
||||
slowTest(this);
|
||||
|
||||
@@ -129,6 +129,7 @@ export class MockInstance extends lib.Link {
|
||||
this.name = "test";
|
||||
this.id = 7357;
|
||||
this.status = "running";
|
||||
this.autoPaused = false;
|
||||
this.hooks = new InstanceHooks(this.logger);
|
||||
this.loadedPlugins = new Set();
|
||||
this.mockConfigEntries = new Map([
|
||||
|
||||
Reference in new issue
Block a user