Refactor connector logic

Refine the state flow used in the connector abstractions to better
handle close while connecting and resuming.  The new logic is to close
the connection immediately in these two states instead of continuing
trying to connect and then giving up some point later.

For the connectors the handshake state is split into connecting
and resuming, new is replaced by closed, and the closing state is turned
into a boolean flag as it's possible that closing is desired while
connecting, connected or resuming.  The connecting state is not present
in the server connector because there's nothing for the server to do in
that connector state.

The timeout logic is completely removed, once a connection attempt is
started for the client it will continue to attempt to connect, attempt
to resume dropped connections and attempt reconnects until either an
authentication error occurs or the client explicitly requests to close
the connection.

A new session timeout is added to control how long to attempt to resume
the connection until it's considered dead and invalidated.  This allows
stale connections to give up earlier, and swallow up less data while in
the dropped but not yet invalidated state.

For the client side sending messages while it's connecting has been
changed to throw an error instead of queuing up the message and
(possibly) sending it after the connection is established.  Code
shouldn't try to send messages without knowing if the connection it's
trying to send them over is active or not.
This commit is contained in:
Hornwitser committed 2021-08-30 20:20:49 +02:00
1 parent 3e617a81cf
commit de48d807f1
26 files changed
+2042 -821

No files matched your search

+9 -5
View File
@@ -2,6 +2,7 @@
const jwt = require("jsonwebtoken");
const WebSocket = require("ws");
const libErrors = require("@clusterio/lib/errors");
const { logger } = require("@clusterio/lib/logging");
const libPrometheus = require("@clusterio/lib/prometheus");
const libSchema = require("@clusterio/lib/schema");
@@ -64,12 +65,10 @@ class WsServer {
let disconnectTasks = [];
for (let controlConnection of this.controlConnections) {
controlConnection.connector.setTimeout(this.master.config.get("master.connector_shutdown_timeout"));
disconnectTasks.push(controlConnection.disconnect(1001, "Server Quit"));
}
for (let slaveConnection of this.slaveConnections.values()) {
slaveConnection.connector.setTimeout(this.master.config.get("master.connector_shutdown_timeout"));
disconnectTasks.push(slaveConnection.disconnect(1001, "Server Quit"));
}
@@ -254,8 +253,9 @@ ${err.stack}`
{ aud: this.sessionAud, sid: sessionId },
Buffer.from(this.master.config.get("master.auth_secret"), "base64"),
);
let sessionTimeout = this.master.config.get("master.session_timeout");
let heartbeatInterval = this.master.config.get("master.heartbeat_interval");
let connector = new WsServerConnector(socket, sessionId, heartbeatInterval);
let connector = new WsServerConnector(sessionId, sessionTimeout, heartbeatInterval);
this.activeConnectors.set(sessionId, connector);
connector.on("close", () => {
this.activeConnectors.delete(connector);
@@ -276,6 +276,8 @@ ${err.stack}`
if (this.slaveConnections.get(data.id) === connection) {
this.slaveConnections.delete(data.id);
this.master.slaveUpdated(this.master.slaves.get(data.id));
} else {
logger.warn("Unlisted SlaveConnection closed");
}
});
this.slaveConnections.set(data.id, connection);
@@ -286,9 +288,11 @@ ${err.stack}`
logger.verbose(`WsServer | registered control from ${req.socket.remoteAddress}`);
let connection = new ControlConnection(data, connector, this.master, user);
connector.on("close", () => {
let index = this.controlConnections.indexOf(this);
let index = this.controlConnections.indexOf(connection);
if (index !== -1) {
this.controlConnections.splice(index, 1);
} else {
logger.warn("Unlisted ControlConnection closed");
}
});
this.controlConnections.push(connection);
@@ -297,7 +301,7 @@ ${err.stack}`
};
}
connector.ready(sessionToken, additionalReadyData);
connector.ready(socket, sessionToken, additionalReadyData);
}
}
+54 -50
View File
@@ -12,28 +12,24 @@ const { logger } = require("@clusterio/lib/logging");
* @alias module:master/src/WsServerConnector
*/
class WsServerConnector extends libLink.WebSocketBaseConnector {
constructor(socket, sessionId, heartbeatInterval) {
super();
constructor(sessionId, sessionTimeout, heartbeatInterval) {
super(sessionTimeout);
this._socket = socket;
this._socket = null;
this._sessionId = sessionId;
this._heartbeatInterval = heartbeatInterval;
this._timeoutId = null;
// The following states are used in the server connector
// handshake: Waiting for client to (re)connect.
// closed: Connection is closed
// connected: Connection is online
// closing: Connection is in the process of being closed.
// closed: Connection has been closed
this._state = "handshake";
this._timeout = 15 * 60;
// resuming: Waiting for client to resume.
}
_check(expectedStates) {
if (!expectedStates.includes(this._state)) {
throw new Error(
`Expected state ${expectedStates} but state is ${this._state}`
);
}
_reset() {
clearTimeout(this._timeoutId);
this._timeoutId = null;
super._reset();
}
/**
@@ -41,17 +37,20 @@ class WsServerConnector extends libLink.WebSocketBaseConnector {
*
* Sends the ready message over the socket to initiate the session.
*
* @param {Object} socket - WebSocket connection to client.
* @param {string} sessionToken -
* the session token to send to the client.
* @param {Object=} additionalData -
* extra properties to send along the ready message.
*/
ready(sessionToken, additionalData) {
ready(socket, sessionToken, additionalData) {
this._socket = socket;
this._socket.send(JSON.stringify({
seq: null,
type: "ready",
data: {
session_token: sessionToken,
session_timeout: this._sessionTimeout,
heartbeat_interval: this._heartbeatInterval,
...additionalData,
},
@@ -77,7 +76,7 @@ class WsServerConnector extends libLink.WebSocketBaseConnector {
// stale connection. If this is the case the connector will have to
// wait for the previous socket to close down before it can continue
// from a new socket, or there will be overlapping events from both.
if (["connected", "closing"].includes(this._state)) {
if (this._state === "connected") {
this._socket.close(4003, "Session Hijacked");
// Correctly waiting for the connector to close and then
@@ -102,6 +101,7 @@ class WsServerConnector extends libLink.WebSocketBaseConnector {
type: "continue",
data: {
last_seq: this._lastReceivedSeq,
session_timeout: this._sessionTimeout,
heartbeat_interval: this._heartbeatInterval,
},
}));
@@ -115,63 +115,44 @@ class WsServerConnector extends libLink.WebSocketBaseConnector {
this.emit("resume");
}
setTimeout(timeout) {
if (this._timeoutId) {
clearTimeout(this._timeoutId);
}
this._timeoutId = setTimeout(() => { this._timedOut(); }, timeout * 1000);
this._timeout = timeout;
}
_timedOut() {
logger.verbose("SOCKET | Connection timed out");
this._timeoutId = null;
this._lastReceivedSeq = null;
this._sendBuffer.length = 0;
this._state = "closed";
logger.verbose("Connector | Connection timed out");
this._reset();
this.emit("close");
this.emit("invalidate");
}
_attachSocketHandlers() {
this.startHeartbeat();
this._socket.on("close", (code, reason) => {
logger.verbose(`SOCKET | Close (code: ${code}, reason: ${reason})`);
if (this._state === "closing") {
this._lastReceivedSeq = null;
this._sendBuffer.length = 0;
this._state = "closed";
logger.verbose(`Connector | Close (code: ${code}, reason: ${reason})`);
this.stopHeartbeat();
if (this._closing) {
this._reset();
this.emit("close");
if (this._timeoutId) {
clearTimeout(this._timeoutId);
this._timeoutId = null;
}
} else {
this._state = "handshake";
this._state = "resuming";
this.emit("drop");
this._timeoutId = setTimeout(() => { this._timedOut(); }, this._timeout * 1000);
this._timeoutId = setTimeout(() => { this._timedOut(); }, this._sessionTimeout * 1000);
}
this.stopHeartbeat();
});
this._socket.on("error", err => {
// It's assumed that close is always called by ws
logger.verbose("SOCKET | Error:", err);
logger.verbose("Connector | Error:", err);
});
this._socket.on("open", () => {
logger.verbose("SOCKET | Open");
logger.verbose("Connector | Open");
});
this._socket.on("ping", data => {
logger.verbose(`SOCKET | Ping (data: ${data}`);
logger.verbose(`Connector | Ping (data: ${data}`);
});
this._socket.on("pong", data => {
logger.verbose(`SOCKET | Pong (data: ${data}`);
logger.verbose(`Connector | Pong (data: ${data}`);
});
// Handle messages
@@ -208,11 +189,34 @@ class WsServerConnector extends libLink.WebSocketBaseConnector {
return;
}
this.stopHeartbeat();
this._state = "closing";
if (this._state === "resuming") {
this._reset();
this.emit("close");
return;
}
this._closing = true;
this._socket.close(code, reason);
await events.once(this, "close");
}
/**
* Notify the connection is expected to close
*
* Sets the internal flag to signal not to wait for the client to
* reconnect and resume the connection if the client closes it. This
* will close the connection if it's currently waiting for the client to
* resume it.
*/
setClosing() {
if (this._state === "resuming") {
this._reset();
this.emit("close");
return;
}
this._closing = true;
}
}
module.exports = WsServerConnector;