mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-16 15:30:41 +09:00
Limit unacknowledged messages buffered for a connection
Every message sent over a link is kept in the send buffer until the other side acknowledges it with a heartbeat, so it can be resent if the session resumes. Nothing forced a peer to acknowledge, so a client that kept heartbeating without a sequence number while subscribed to a busy event made the controller buffer messages until it ran out of memory. Store buffered messages serialised, track their total size, and close the connection once it goes over maxSendBufferSize. The controller sets this to 64 MiB for the connections it accepts. A client that far behind could not resume in a reasonable time anyway. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
ce6bfe282f
commit
235a15b9cc
3 files changed
+74
-9
No files matched your search
@@ -13,6 +13,9 @@ import { logger } from "@clusterio/lib";
|
||||
* @extends module:lib.WebSocketBaseConnector
|
||||
* @alias module:controller/src/WsServerConnector
|
||||
*/
|
||||
// Clients that leave this much unacknowledged are dropped rather than buffered for.
|
||||
const maxSendBufferSize = 64 * 1024 * 1024;
|
||||
|
||||
export default class WsServerConnector extends lib.WebSocketBaseConnector {
|
||||
_timeoutId: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
@@ -23,6 +26,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
|
||||
public _heartbeatInterval: number,
|
||||
) {
|
||||
super(new lib.Address(lib.Address.controller, 0), dst);
|
||||
this.maxSendBufferSize = maxSendBufferSize;
|
||||
|
||||
// The following states are used in the server connector
|
||||
// closed: Connection is closed
|
||||
@@ -110,7 +114,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
|
||||
`Connector | resuming session with ${this.dst}, resending ${this._sendBuffer.length} buffered messages`
|
||||
);
|
||||
for (let message of this._sendBuffer) {
|
||||
this._socket.send(JSON.stringify(message));
|
||||
this._socket.send(message.text);
|
||||
}
|
||||
this.emit("resume");
|
||||
}
|
||||
|
||||
@@ -171,7 +171,11 @@ export abstract class WebSocketBaseConnector<
|
||||
_heartbeatId: ReturnType<typeof setInterval> | null = null;
|
||||
_heartbeatInterval: number | null = null;
|
||||
_lastReceivedSeq = undefined;
|
||||
_sendBuffer: (libData.MessageRoutable)[] = [];
|
||||
/** Serialised messages sent but not yet acknowledged by the other side */
|
||||
_sendBuffer: { seq: number, text: string }[] = [];
|
||||
_sendBufferSize = 0;
|
||||
/** Connection is closed if the unacknowledged messages exceed this many characters */
|
||||
maxSendBufferSize = Infinity;
|
||||
|
||||
_reset() {
|
||||
this._state = "closed";
|
||||
@@ -182,6 +186,7 @@ export abstract class WebSocketBaseConnector<
|
||||
this._heartbeatInterval = null;
|
||||
this._lastReceivedSeq = undefined;
|
||||
this._sendBuffer.length = 0;
|
||||
this._sendBufferSize = 0;
|
||||
super._reset();
|
||||
}
|
||||
|
||||
@@ -194,6 +199,7 @@ export abstract class WebSocketBaseConnector<
|
||||
this._heartbeatInterval = null;
|
||||
this._lastReceivedSeq = undefined;
|
||||
this._sendBuffer.length = 0;
|
||||
this._sendBufferSize = 0;
|
||||
super._invalidate();
|
||||
}
|
||||
|
||||
@@ -219,6 +225,7 @@ export abstract class WebSocketBaseConnector<
|
||||
for (let index = 0; index < this._sendBuffer.length; index++) {
|
||||
if (this._sendBuffer[index].seq <= seq) {
|
||||
dropCount += 1;
|
||||
this._sendBufferSize -= this._sendBuffer[index].text.length;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
@@ -326,9 +333,30 @@ export abstract class WebSocketBaseConnector<
|
||||
throw new libErrors.SessionLost("No session");
|
||||
}
|
||||
|
||||
this._sendBuffer.push(message);
|
||||
const text = JSON.stringify(message);
|
||||
this._sendBuffer.push({ seq: message.seq, text });
|
||||
this._sendBufferSize += text.length;
|
||||
if (this._sendBufferSize > this.maxSendBufferSize) {
|
||||
this._sendBufferOverflow();
|
||||
return;
|
||||
}
|
||||
if (this._state === "connected") {
|
||||
this._sendInternal(message);
|
||||
this._socket!.send(text);
|
||||
}
|
||||
}
|
||||
|
||||
_sendBufferOverflow() {
|
||||
logger.warn(
|
||||
`Connector | closing connection to ${this.dst} after ${this._sendBuffer.length} messages ` +
|
||||
"were left unacknowledged"
|
||||
);
|
||||
// Resuming is no longer possible, free the memory right away.
|
||||
this._sendBuffer.length = 0;
|
||||
this._sendBufferSize = 0;
|
||||
if (!this._closing) {
|
||||
this.close(ConnectionClosed.PolicyViolation, "Send buffer limit exceeded").catch(err => {
|
||||
logger.error(`Connector | error closing connection:\n${err.stack}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -708,7 +736,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
|
||||
this._heartbeatInterval = data.heartbeatInterval;
|
||||
this.startHeartbeat();
|
||||
for (let bufferedMessage of this._sendBuffer) {
|
||||
this._sendInternal(bufferedMessage);
|
||||
this._socket!.send(bufferedMessage.text);
|
||||
}
|
||||
this.emit("connect", data);
|
||||
|
||||
@@ -722,7 +750,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
|
||||
`Connector | resuming existing session, resending ${this._sendBuffer.length} buffered messages`
|
||||
);
|
||||
for (let bufferedMessage of this._sendBuffer) {
|
||||
this._sendInternal(bufferedMessage);
|
||||
this._socket!.send(bufferedMessage.text);
|
||||
}
|
||||
this._startedResumingMs = null;
|
||||
this.emit("resume");
|
||||
|
||||
@@ -13,10 +13,11 @@ describe("lib/link/connectors", function() {
|
||||
describe("._dropSendBufferSeq()", function() {
|
||||
beforeEach(function() {
|
||||
testConnector._sendBuffer = [
|
||||
{ seq: 1, type: "test", data: {} },
|
||||
{ seq: 2, type: "test", data: {} },
|
||||
{ seq: 4, type: "test", data: {} },
|
||||
{ seq: 1, text: "a" },
|
||||
{ seq: 2, text: "bb" },
|
||||
{ seq: 4, text: "cccc" },
|
||||
];
|
||||
testConnector._sendBufferSize = 7;
|
||||
});
|
||||
it("should do nothing if passed null", function() {
|
||||
testConnector._dropSendBufferSeq(null);
|
||||
@@ -25,6 +26,7 @@ describe("lib/link/connectors", function() {
|
||||
it("should drop up to sequence", function() {
|
||||
testConnector._dropSendBufferSeq(2);
|
||||
assert.equal(testConnector._sendBuffer.length, 1, "incorrect number of entries dropped");
|
||||
assert.equal(testConnector._sendBufferSize, 4);
|
||||
});
|
||||
it("should ignore holes", function() {
|
||||
testConnector._dropSendBufferSeq(3);
|
||||
@@ -61,6 +63,37 @@ describe("lib/link/connectors", function() {
|
||||
});
|
||||
});
|
||||
|
||||
describe("send buffer limit", function() {
|
||||
afterEach(function() {
|
||||
testConnector.maxSendBufferSize = Infinity;
|
||||
testConnector._closing = false;
|
||||
});
|
||||
it("should keep unacknowledged messages under the limit", function() {
|
||||
testConnector._state = "connected";
|
||||
testConnector._sendBuffer = [];
|
||||
testConnector._sendBufferSize = 0;
|
||||
testConnector._socket.closeCalled = false;
|
||||
testConnector.maxSendBufferSize = 1000;
|
||||
testConnector.send(new lib.MessageHeartbeat(1));
|
||||
assert.equal(testConnector._sendBuffer.length, 1);
|
||||
assert(!testConnector._socket.closeCalled, "Close was called on the socket");
|
||||
});
|
||||
it("should clear the buffer and close when the limit is exceeded", function() {
|
||||
testConnector._state = "connected";
|
||||
testConnector._sendBuffer = [];
|
||||
testConnector._sendBufferSize = 0;
|
||||
testConnector._socket.closeCalled = false;
|
||||
testConnector._socket.sentMessages = [];
|
||||
testConnector.maxSendBufferSize = 30;
|
||||
testConnector.send(new lib.MessageHeartbeat(1));
|
||||
testConnector.send(new lib.MessageHeartbeat(2));
|
||||
assert.equal(testConnector._sendBuffer.length, 0);
|
||||
assert.equal(testConnector._sendBufferSize, 0);
|
||||
assert(testConnector._socket.closeCalled, "Close was not called on the socket");
|
||||
assert.equal(testConnector._socket.sentMessages.length, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe(".close()", function() {
|
||||
it("should call close", function() {
|
||||
testConnector._socket.closeCalled = false;
|
||||
|
||||
Reference in new issue
Block a user