mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-17 07:30:41 +09:00
Merge pull request #1057 from bbassie/fix/bounded-send-buffer
Limit unacknowledged messages buffered for a connection
This commit is contained in:
7 files changed
+124
-11
No files matched your search
@@ -226,7 +226,8 @@ ${err.stack}`
|
||||
let sessionToken = jwt.sign({ aud: this.sessionAud, sid: sessionId }, this.controller.authSecret);
|
||||
let sessionTimeout = this.controller.config.get("controller.session_timeout");
|
||||
let heartbeatInterval = this.controller.config.get("controller.heartbeat_interval");
|
||||
let connector = new WsServerConnector(dst, sessionId, sessionTimeout, heartbeatInterval);
|
||||
let maxSendBufferSize = this.controller.config.get("controller.max_send_buffer_size") * 1024 * 1024;
|
||||
let connector = new WsServerConnector(dst, sessionId, sessionTimeout, heartbeatInterval, maxSendBufferSize);
|
||||
this.activeConnectors.set(sessionId, connector);
|
||||
connector.on("close", () => {
|
||||
this.activeConnectors.delete(sessionId);
|
||||
|
||||
@@ -21,8 +21,10 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
|
||||
private _sessionId: number,
|
||||
private _sessionTimeout: number,
|
||||
public _heartbeatInterval: number,
|
||||
maxSendBufferSize = 0,
|
||||
) {
|
||||
super(new lib.Address(lib.Address.controller, 0), dst);
|
||||
this.maxSendBufferSize = maxSendBufferSize;
|
||||
|
||||
// The following states are used in the server connector
|
||||
// closed: Connection is closed
|
||||
@@ -110,7 +112,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
|
||||
`Connector | resuming session with ${this.dst}, resending ${this._sendBuffer.length} buffered messages`
|
||||
);
|
||||
for (let message of this._sendBuffer) {
|
||||
this._socket.send(JSON.stringify(message));
|
||||
this._socket.send(message.text);
|
||||
}
|
||||
this.emit("resume");
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ export interface ControllerConfigFields {
|
||||
"controller.auth_secret": string | null;
|
||||
"controller.heartbeat_interval": number;
|
||||
"controller.session_timeout": number;
|
||||
"controller.max_send_buffer_size": number;
|
||||
"controller.metrics_timeout": number;
|
||||
"controller.max_subscription_filters": number;
|
||||
"controller.system_metrics_interval": number;
|
||||
@@ -202,6 +203,14 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
|
||||
initialValue: 60,
|
||||
validator: validators.greaterThanZero,
|
||||
},
|
||||
"controller.max_send_buffer_size": {
|
||||
title: "Max Send Buffer Size",
|
||||
description:
|
||||
"MiB of unacknowledged messages to keep for a connection before ending its session, 0 for no limit.",
|
||||
type: "number",
|
||||
initialValue: 64,
|
||||
validator: validators.greaterThanEqualZero,
|
||||
},
|
||||
"controller.metrics_timeout": {
|
||||
title: "Metrics Timeout",
|
||||
description: "Timeout in seconds for metrics gathering from hosts.",
|
||||
|
||||
@@ -171,7 +171,11 @@ export abstract class WebSocketBaseConnector<
|
||||
_heartbeatId: ReturnType<typeof setInterval> | null = null;
|
||||
_heartbeatInterval: number | null = null;
|
||||
_lastReceivedSeq = undefined;
|
||||
_sendBuffer: (libData.MessageRoutable)[] = [];
|
||||
/** Serialised messages sent but not yet acknowledged by the other side */
|
||||
_sendBuffer: { seq: number, text: string }[] = [];
|
||||
_sendBufferSize = 0;
|
||||
/** Session is ended if the unacknowledged messages exceed this many characters, 0 for no limit */
|
||||
maxSendBufferSize = 0;
|
||||
|
||||
_reset() {
|
||||
this._state = "closed";
|
||||
@@ -182,6 +186,7 @@ export abstract class WebSocketBaseConnector<
|
||||
this._heartbeatInterval = null;
|
||||
this._lastReceivedSeq = undefined;
|
||||
this._sendBuffer.length = 0;
|
||||
this._sendBufferSize = 0;
|
||||
super._reset();
|
||||
}
|
||||
|
||||
@@ -194,6 +199,7 @@ export abstract class WebSocketBaseConnector<
|
||||
this._heartbeatInterval = null;
|
||||
this._lastReceivedSeq = undefined;
|
||||
this._sendBuffer.length = 0;
|
||||
this._sendBufferSize = 0;
|
||||
super._invalidate();
|
||||
}
|
||||
|
||||
@@ -219,6 +225,7 @@ export abstract class WebSocketBaseConnector<
|
||||
for (let index = 0; index < this._sendBuffer.length; index++) {
|
||||
if (this._sendBuffer[index].seq <= seq) {
|
||||
dropCount += 1;
|
||||
this._sendBufferSize -= this._sendBuffer[index].text.length;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
@@ -326,9 +333,31 @@ export abstract class WebSocketBaseConnector<
|
||||
throw new libErrors.SessionLost("No session");
|
||||
}
|
||||
|
||||
this._sendBuffer.push(message);
|
||||
const text = JSON.stringify(message);
|
||||
this._sendBuffer.push({ seq: message.seq, text });
|
||||
this._sendBufferSize += text.length;
|
||||
if (this.maxSendBufferSize && this._sendBufferSize > this.maxSendBufferSize) {
|
||||
this._sendBufferOverflow();
|
||||
return;
|
||||
}
|
||||
if (this._state === "connected") {
|
||||
this._sendInternal(message);
|
||||
this._socket!.send(text);
|
||||
}
|
||||
}
|
||||
|
||||
_sendBufferOverflow() {
|
||||
logger.warn(
|
||||
`Connector | closing connection to ${this.dst} after ${this._sendBuffer.length} messages ` +
|
||||
"were left unacknowledged"
|
||||
);
|
||||
// Resuming is no longer possible, free the memory right away. The
|
||||
// other side sees a normal drop and gets a new session when it reconnects.
|
||||
this._sendBuffer.length = 0;
|
||||
this._sendBufferSize = 0;
|
||||
if (!this._closing) {
|
||||
this.close(ConnectionClosed.TryAgainLater, "Send buffer limit exceeded").catch(err => {
|
||||
logger.error(`Connector | error closing connection:\n${err.stack}`);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -708,7 +737,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
|
||||
this._heartbeatInterval = data.heartbeatInterval;
|
||||
this.startHeartbeat();
|
||||
for (let bufferedMessage of this._sendBuffer) {
|
||||
this._sendInternal(bufferedMessage);
|
||||
this._socket!.send(bufferedMessage.text);
|
||||
}
|
||||
this.emit("connect", data);
|
||||
|
||||
@@ -722,7 +751,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector<We
|
||||
`Connector | resuming existing session, resending ${this._sendBuffer.length} buffered messages`
|
||||
);
|
||||
for (let bufferedMessage of this._sendBuffer) {
|
||||
this._sendInternal(bufferedMessage);
|
||||
this._socket!.send(bufferedMessage.text);
|
||||
}
|
||||
this._startedResumingMs = null;
|
||||
this.emit("resume");
|
||||
|
||||
Reference in new issue
Block a user