diff --git a/docs/configuration.md b/docs/configuration.md index 818f0e48..5878e7a8 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -165,6 +165,17 @@ Must be greater than 0. Defaults to 60. +### controller.max_send_buffer_size + +Size in MiB of messages sent to a connection that it has not yet acknowledged before the controller ends its session. +The controller keeps these so they can be resent if the connection drops and resumes. +A client that stops acknowledging messages, or falls far behind, gets disconnected and starts a new session when it reconnects. +Changes apply to new sessions. +Set to 0 for no limit. + +Defaults to 64. + + ### controller.metrics_timeout Timeout in seconds before a call to gather metrics from a host times out. diff --git a/packages/controller/src/WsServer.ts b/packages/controller/src/WsServer.ts index efaf1061..db582ec1 100644 --- a/packages/controller/src/WsServer.ts +++ b/packages/controller/src/WsServer.ts @@ -226,7 +226,8 @@ ${err.stack}` let sessionToken = jwt.sign({ aud: this.sessionAud, sid: sessionId }, this.controller.authSecret); let sessionTimeout = this.controller.config.get("controller.session_timeout"); let heartbeatInterval = this.controller.config.get("controller.heartbeat_interval"); - let connector = new WsServerConnector(dst, sessionId, sessionTimeout, heartbeatInterval); + let maxSendBufferSize = this.controller.config.get("controller.max_send_buffer_size") * 1024 * 1024; + let connector = new WsServerConnector(dst, sessionId, sessionTimeout, heartbeatInterval, maxSendBufferSize); this.activeConnectors.set(sessionId, connector); connector.on("close", () => { this.activeConnectors.delete(sessionId); diff --git a/packages/controller/src/WsServerConnector.ts b/packages/controller/src/WsServerConnector.ts index a66f1f61..3b256d1a 100644 --- a/packages/controller/src/WsServerConnector.ts +++ b/packages/controller/src/WsServerConnector.ts @@ -13,9 +13,6 @@ import { logger } from "@clusterio/lib"; * @extends module:lib.WebSocketBaseConnector * @alias module:controller/src/WsServerConnector */ -// Clients that leave this much unacknowledged are dropped rather than buffered for. -const maxSendBufferSize = 64 * 1024 * 1024; - export default class WsServerConnector extends lib.WebSocketBaseConnector { _timeoutId: ReturnType | null = null; @@ -24,6 +21,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector { private _sessionId: number, private _sessionTimeout: number, public _heartbeatInterval: number, + maxSendBufferSize = 0, ) { super(new lib.Address(lib.Address.controller, 0), dst); this.maxSendBufferSize = maxSendBufferSize; diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 27042c44..83437105 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -25,6 +25,7 @@ export interface ControllerConfigFields { "controller.auth_secret": string | null; "controller.heartbeat_interval": number; "controller.session_timeout": number; + "controller.max_send_buffer_size": number; "controller.metrics_timeout": number; "controller.system_metrics_interval": number; "controller.proxy_stream_timeout": number; @@ -201,6 +202,14 @@ export class ControllerConfig extends classes.Config { initialValue: 60, validator: validators.greaterThanZero, }, + "controller.max_send_buffer_size": { + title: "Max Send Buffer Size", + description: + "MiB of unacknowledged messages to keep for a connection before ending its session, 0 for no limit.", + type: "number", + initialValue: 64, + validator: validators.greaterThanEqualZero, + }, "controller.metrics_timeout": { title: "Metrics Timeout", description: "Timeout in seconds for metrics gathering from hosts.", diff --git a/packages/lib/src/link/connectors.ts b/packages/lib/src/link/connectors.ts index c9890915..7bd62609 100644 --- a/packages/lib/src/link/connectors.ts +++ b/packages/lib/src/link/connectors.ts @@ -174,8 +174,8 @@ export abstract class WebSocketBaseConnector< /** Serialised messages sent but not yet acknowledged by the other side */ _sendBuffer: { seq: number, text: string }[] = []; _sendBufferSize = 0; - /** Connection is closed if the unacknowledged messages exceed this many characters */ - maxSendBufferSize = Infinity; + /** Session is ended if the unacknowledged messages exceed this many characters, 0 for no limit */ + maxSendBufferSize = 0; _reset() { this._state = "closed"; @@ -336,7 +336,7 @@ export abstract class WebSocketBaseConnector< const text = JSON.stringify(message); this._sendBuffer.push({ seq: message.seq, text }); this._sendBufferSize += text.length; - if (this._sendBufferSize > this.maxSendBufferSize) { + if (this.maxSendBufferSize && this._sendBufferSize > this.maxSendBufferSize) { this._sendBufferOverflow(); return; } @@ -350,11 +350,12 @@ export abstract class WebSocketBaseConnector< `Connector | closing connection to ${this.dst} after ${this._sendBuffer.length} messages ` + "were left unacknowledged" ); - // Resuming is no longer possible, free the memory right away. + // Resuming is no longer possible, free the memory right away. The + // other side sees a normal drop and gets a new session when it reconnects. this._sendBuffer.length = 0; this._sendBufferSize = 0; if (!this._closing) { - this.close(ConnectionClosed.PolicyViolation, "Send buffer limit exceeded").catch(err => { + this.close(ConnectionClosed.TryAgainLater, "Send buffer limit exceeded").catch(err => { logger.error(`Connector | error closing connection:\n${err.stack}`); }); } diff --git a/test/integration/link.js b/test/integration/link.js index 8bfdd211..01beb471 100644 --- a/test/integration/link.js +++ b/test/integration/link.js @@ -3,8 +3,9 @@ import events from "node:events"; import jwt from "jsonwebtoken"; import { - TestControlConnector, TestHostConnector, TestControl, url, controlToken, + TestControlConnector, TestHostConnector, TestControl, url, controlToken, getControl, } from "./index.js"; +import * as lib from "@clusterio/lib"; import { ConnectionClosed, ProtocolError, PolicyViolation, AuthenticationFailed } from "@clusterio/lib"; import { slowTest } from "../common.js"; @@ -92,6 +93,33 @@ describe("Integration of lib/link", function() { await events.once(controlConnector, "connect"); }); + it("should start a new session when the send buffer limit is exceeded", async function() { + await getControl().send(new lib.ControllerConfigSetFieldRequest("controller.max_send_buffer_size", "0.01")); + try { + await controlConnector.connect(); + } finally { + await getControl().send(new lib.ControllerConfigSetFieldRequest("controller.max_send_buffer_size", "64")); + } + + // Never acknowledge received messages + controlConnector._doHeartbeat = function() { + this._sendInternal(new lib.MessageHeartbeat()); + }; + let invalidated = false; + controlConnector.once("invalidate", () => { invalidated = true; }); + const connected = events.once(controlConnector, "invalidate").then( + () => events.once(controlConnector, "connect") + ); + for (let i = 0; i < 100; i++) { + if (invalidated) { + break; + } + await control.send(new lib.ControllerConfigGetRequest()).catch(() => {}); + } + assert(invalidated, "session was not invalidated"); + await connected; + }); + it("should properly close connector if close is called during reconnect wait", async function() { await controlConnector.connect(); controlConnector._socket.close(ConnectionClosed.Normal, "Test"); diff --git a/test/lib/link/connector.js b/test/lib/link/connector.js index e6f12316..a19f3696 100644 --- a/test/lib/link/connector.js +++ b/test/lib/link/connector.js @@ -65,7 +65,7 @@ describe("lib/link/connectors", function() { describe("send buffer limit", function() { afterEach(function() { - testConnector.maxSendBufferSize = Infinity; + testConnector.maxSendBufferSize = 0; testConnector._closing = false; }); it("should keep unacknowledged messages under the limit", function() {