Merge pull request #1057 from bbassie/fix/bounded-send-buffer

Limit unacknowledged messages buffered for a connection
This commit is contained in:
Cooldude2606 authored and GitHub committed 2026-10-08 17:25:57 +01:00
commit 8a170eb2ac
7 files changed
+124 -11

No files matched your search

+9
View File
@@ -25,6 +25,7 @@ export interface ControllerConfigFields {
"controller.auth_secret": string | null;
"controller.heartbeat_interval": number;
"controller.session_timeout": number;
"controller.max_send_buffer_size": number;
"controller.metrics_timeout": number;
"controller.max_subscription_filters": number;
"controller.system_metrics_interval": number;
@@ -202,6 +203,14 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
initialValue: 60,
validator: validators.greaterThanZero,
},
"controller.max_send_buffer_size": {
title: "Max Send Buffer Size",
description:
"MiB of unacknowledged messages to keep for a connection before ending its session, 0 for no limit.",
type: "number",
initialValue: 64,
validator: validators.greaterThanEqualZero,
},
"controller.metrics_timeout": {
title: "Metrics Timeout",
description: "Timeout in seconds for metrics gathering from hosts.",
+34 -5
View File
@@ -171,7 +171,11 @@ export abstract class WebSocketBaseConnector<
_heartbeatId: ReturnType<typeof setInterval> | null = null;
_heartbeatInterval: number | null = null;
_lastReceivedSeq = undefined;
_sendBuffer: (libData.MessageRoutable)[] = [];
/** Serialised messages sent but not yet acknowledged by the other side */
_sendBuffer: { seq: number, text: string }[] = [];
_sendBufferSize = 0;
/** Session is ended if the unacknowledged messages exceed this many characters, 0 for no limit */
maxSendBufferSize = 0;
_reset() {
this._state = "closed";
@@ -182,6 +186,7 @@ export abstract class WebSocketBaseConnector<
this._heartbeatInterval = null;
this._lastReceivedSeq = undefined;
this._sendBuffer.length = 0;
this._sendBufferSize = 0;
super._reset();
}
@@ -194,6 +199,7 @@ export abstract class WebSocketBaseConnector<
this._heartbeatInterval = null;
this._lastReceivedSeq = undefined;
this._sendBuffer.length = 0;
this._sendBufferSize = 0;
super._invalidate();
}
@@ -219,6 +225,7 @@ export abstract class WebSocketBaseConnector<
for (let index = 0; index < this._sendBuffer.length; index++) {
if (this._sendBuffer[index].seq <= seq) {
dropCount += 1;
this._sendBufferSize -= this._sendBuffer[index].text.length;
} else {
break;
}
@@ -326,9 +333,31 @@ export abstract class WebSocketBaseConnector<
throw new libErrors.SessionLost("No session");
}
this._sendBuffer.push(message);
const text = JSON.stringify(message);
this._sendBuffer.push({ seq: message.seq, text });
this._sendBufferSize += text.length;
if (this.maxSendBufferSize && this._sendBufferSize > this.maxSendBufferSize) {
this._sendBufferOverflow();
return;
}
if (this._state === "connected") {
this._sendInternal(message);
this._socket!.send(text);
}
}
_sendBufferOverflow() {
logger.warn(
`Connector | closing connection to ${this.dst} after ${this._sendBuffer.length} messages ` +
"were left unacknowledged"
);
// Resuming is no longer possible, free the memory right away. The
// other side sees a normal drop and gets a new session when it reconnects.
this._sendBuffer.length = 0;
this._sendBufferSize = 0;
if (!this._closing) {
this.close(ConnectionClosed.TryAgainLater, "Send buffer limit exceeded").catch(err => {
logger.error(`Connector | error closing connection:\n${err.stack}`);
});
}
}
@@ -708,7 +737,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
this._heartbeatInterval = data.heartbeatInterval;
this.startHeartbeat();
for (let bufferedMessage of this._sendBuffer) {
this._sendInternal(bufferedMessage);
this._socket!.send(bufferedMessage.text);
}
this.emit("connect", data);
@@ -722,7 +751,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
`Connector | resuming existing session, resending ${this._sendBuffer.length} buffered messages`
);
for (let bufferedMessage of this._sendBuffer) {
this._sendInternal(bufferedMessage);
this._socket!.send(bufferedMessage.text);
}
this._startedResumingMs = null;
this.emit("resume");