From 235a15b9cca3b5d38a915dfac665a11fcbe63d89 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:15:27 +0000 Subject: [PATCH] Limit unacknowledged messages buffered for a connection Every message sent over a link is kept in the send buffer until the other side acknowledges it with a heartbeat, so it can be resent if the session resumes. Nothing forced a peer to acknowledge, so a client that kept heartbeating without a sequence number while subscribed to a busy event made the controller buffer messages until it ran out of memory. Store buffered messages serialised, track their total size, and close the connection once it goes over maxSendBufferSize. The controller sets this to 64 MiB for the connections it accepts. A client that far behind could not resume in a reasonable time anyway. Co-Authored-By: Claude Opus 5.5 --- packages/controller/src/WsServerConnector.ts | 6 ++- packages/lib/src/link/connectors.ts | 38 ++++++++++++++++--- test/lib/link/connector.js | 39 ++++++++++++++++++-- 3 files changed, 74 insertions(+), 9 deletions(-) diff --git a/packages/controller/src/WsServerConnector.ts b/packages/controller/src/WsServerConnector.ts index 439a2056..a66f1f61 100644 --- a/packages/controller/src/WsServerConnector.ts +++ b/packages/controller/src/WsServerConnector.ts @@ -13,6 +13,9 @@ import { logger } from "@clusterio/lib"; * @extends module:lib.WebSocketBaseConnector * @alias module:controller/src/WsServerConnector */ +// Clients that leave this much unacknowledged are dropped rather than buffered for. +const maxSendBufferSize = 64 * 1024 * 1024; + export default class WsServerConnector extends lib.WebSocketBaseConnector { _timeoutId: ReturnType | null = null; @@ -23,6 +26,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector { public _heartbeatInterval: number, ) { super(new lib.Address(lib.Address.controller, 0), dst); + this.maxSendBufferSize = maxSendBufferSize; // The following states are used in the server connector // closed: Connection is closed @@ -110,7 +114,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector { `Connector | resuming session with ${this.dst}, resending ${this._sendBuffer.length} buffered messages` ); for (let message of this._sendBuffer) { - this._socket.send(JSON.stringify(message)); + this._socket.send(message.text); } this.emit("resume"); } diff --git a/packages/lib/src/link/connectors.ts b/packages/lib/src/link/connectors.ts index d14b89e3..c9890915 100644 --- a/packages/lib/src/link/connectors.ts +++ b/packages/lib/src/link/connectors.ts @@ -171,7 +171,11 @@ export abstract class WebSocketBaseConnector< _heartbeatId: ReturnType | null = null; _heartbeatInterval: number | null = null; _lastReceivedSeq = undefined; - _sendBuffer: (libData.MessageRoutable)[] = []; + /** Serialised messages sent but not yet acknowledged by the other side */ + _sendBuffer: { seq: number, text: string }[] = []; + _sendBufferSize = 0; + /** Connection is closed if the unacknowledged messages exceed this many characters */ + maxSendBufferSize = Infinity; _reset() { this._state = "closed"; @@ -182,6 +186,7 @@ export abstract class WebSocketBaseConnector< this._heartbeatInterval = null; this._lastReceivedSeq = undefined; this._sendBuffer.length = 0; + this._sendBufferSize = 0; super._reset(); } @@ -194,6 +199,7 @@ export abstract class WebSocketBaseConnector< this._heartbeatInterval = null; this._lastReceivedSeq = undefined; this._sendBuffer.length = 0; + this._sendBufferSize = 0; super._invalidate(); } @@ -219,6 +225,7 @@ export abstract class WebSocketBaseConnector< for (let index = 0; index < this._sendBuffer.length; index++) { if (this._sendBuffer[index].seq <= seq) { dropCount += 1; + this._sendBufferSize -= this._sendBuffer[index].text.length; } else { break; } @@ -326,9 +333,30 @@ export abstract class WebSocketBaseConnector< throw new libErrors.SessionLost("No session"); } - this._sendBuffer.push(message); + const text = JSON.stringify(message); + this._sendBuffer.push({ seq: message.seq, text }); + this._sendBufferSize += text.length; + if (this._sendBufferSize > this.maxSendBufferSize) { + this._sendBufferOverflow(); + return; + } if (this._state === "connected") { - this._sendInternal(message); + this._socket!.send(text); + } + } + + _sendBufferOverflow() { + logger.warn( + `Connector | closing connection to ${this.dst} after ${this._sendBuffer.length} messages ` + + "were left unacknowledged" + ); + // Resuming is no longer possible, free the memory right away. + this._sendBuffer.length = 0; + this._sendBufferSize = 0; + if (!this._closing) { + this.close(ConnectionClosed.PolicyViolation, "Send buffer limit exceeded").catch(err => { + logger.error(`Connector | error closing connection:\n${err.stack}`); + }); } } @@ -708,7 +736,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector