mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-18 23:33:12 +09:00
Refactor connector logic
Refine the state flow used in the connector abstractions to better handle close while connecting and resuming. The new logic is to close the connection immediately in these two states instead of continuing trying to connect and then giving up some point later. For the connectors the handshake state is split into connecting and resuming, new is replaced by closed, and the closing state is turned into a boolean flag as it's possible that closing is desired while connecting, connected or resuming. The connecting state is not present in the server connector because there's nothing for the server to do in that connector state. The timeout logic is completely removed, once a connection attempt is started for the client it will continue to attempt to connect, attempt to resume dropped connections and attempt reconnects until either an authentication error occurs or the client explicitly requests to close the connection. A new session timeout is added to control how long to attempt to resume the connection until it's considered dead and invalidated. This allows stale connections to give up earlier, and swallow up less data while in the dropped but not yet invalidated state. For the client side sending messages while it's connecting has been changed to throw an error instead of queuing up the message and (possibly) sending it after the connection is established. Code shouldn't try to send messages without knowing if the connection it's trying to send them over is active or not.
This commit is contained in:
1 parent
3e617a81cf
commit
de48d807f1
26 files changed
+2042
-821
No files matched your search
@@ -2,6 +2,7 @@
|
||||
const jwt = require("jsonwebtoken");
|
||||
const WebSocket = require("ws");
|
||||
|
||||
const libErrors = require("@clusterio/lib/errors");
|
||||
const { logger } = require("@clusterio/lib/logging");
|
||||
const libPrometheus = require("@clusterio/lib/prometheus");
|
||||
const libSchema = require("@clusterio/lib/schema");
|
||||
@@ -64,12 +65,10 @@ class WsServer {
|
||||
|
||||
let disconnectTasks = [];
|
||||
for (let controlConnection of this.controlConnections) {
|
||||
controlConnection.connector.setTimeout(this.master.config.get("master.connector_shutdown_timeout"));
|
||||
disconnectTasks.push(controlConnection.disconnect(1001, "Server Quit"));
|
||||
}
|
||||
|
||||
for (let slaveConnection of this.slaveConnections.values()) {
|
||||
slaveConnection.connector.setTimeout(this.master.config.get("master.connector_shutdown_timeout"));
|
||||
disconnectTasks.push(slaveConnection.disconnect(1001, "Server Quit"));
|
||||
}
|
||||
|
||||
@@ -254,8 +253,9 @@ ${err.stack}`
|
||||
{ aud: this.sessionAud, sid: sessionId },
|
||||
Buffer.from(this.master.config.get("master.auth_secret"), "base64"),
|
||||
);
|
||||
let sessionTimeout = this.master.config.get("master.session_timeout");
|
||||
let heartbeatInterval = this.master.config.get("master.heartbeat_interval");
|
||||
let connector = new WsServerConnector(socket, sessionId, heartbeatInterval);
|
||||
let connector = new WsServerConnector(sessionId, sessionTimeout, heartbeatInterval);
|
||||
this.activeConnectors.set(sessionId, connector);
|
||||
connector.on("close", () => {
|
||||
this.activeConnectors.delete(connector);
|
||||
@@ -276,6 +276,8 @@ ${err.stack}`
|
||||
if (this.slaveConnections.get(data.id) === connection) {
|
||||
this.slaveConnections.delete(data.id);
|
||||
this.master.slaveUpdated(this.master.slaves.get(data.id));
|
||||
} else {
|
||||
logger.warn("Unlisted SlaveConnection closed");
|
||||
}
|
||||
});
|
||||
this.slaveConnections.set(data.id, connection);
|
||||
@@ -286,9 +288,11 @@ ${err.stack}`
|
||||
logger.verbose(`WsServer | registered control from ${req.socket.remoteAddress}`);
|
||||
let connection = new ControlConnection(data, connector, this.master, user);
|
||||
connector.on("close", () => {
|
||||
let index = this.controlConnections.indexOf(this);
|
||||
let index = this.controlConnections.indexOf(connection);
|
||||
if (index !== -1) {
|
||||
this.controlConnections.splice(index, 1);
|
||||
} else {
|
||||
logger.warn("Unlisted ControlConnection closed");
|
||||
}
|
||||
});
|
||||
this.controlConnections.push(connection);
|
||||
@@ -297,7 +301,7 @@ ${err.stack}`
|
||||
};
|
||||
}
|
||||
|
||||
connector.ready(sessionToken, additionalReadyData);
|
||||
connector.ready(socket, sessionToken, additionalReadyData);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -12,28 +12,24 @@ const { logger } = require("@clusterio/lib/logging");
|
||||
* @alias module:master/src/WsServerConnector
|
||||
*/
|
||||
class WsServerConnector extends libLink.WebSocketBaseConnector {
|
||||
constructor(socket, sessionId, heartbeatInterval) {
|
||||
super();
|
||||
constructor(sessionId, sessionTimeout, heartbeatInterval) {
|
||||
super(sessionTimeout);
|
||||
|
||||
this._socket = socket;
|
||||
this._socket = null;
|
||||
this._sessionId = sessionId;
|
||||
this._heartbeatInterval = heartbeatInterval;
|
||||
this._timeoutId = null;
|
||||
|
||||
// The following states are used in the server connector
|
||||
// handshake: Waiting for client to (re)connect.
|
||||
// closed: Connection is closed
|
||||
// connected: Connection is online
|
||||
// closing: Connection is in the process of being closed.
|
||||
// closed: Connection has been closed
|
||||
this._state = "handshake";
|
||||
this._timeout = 15 * 60;
|
||||
// resuming: Waiting for client to resume.
|
||||
}
|
||||
|
||||
_check(expectedStates) {
|
||||
if (!expectedStates.includes(this._state)) {
|
||||
throw new Error(
|
||||
`Expected state ${expectedStates} but state is ${this._state}`
|
||||
);
|
||||
}
|
||||
_reset() {
|
||||
clearTimeout(this._timeoutId);
|
||||
this._timeoutId = null;
|
||||
super._reset();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -41,17 +37,20 @@ class WsServerConnector extends libLink.WebSocketBaseConnector {
|
||||
*
|
||||
* Sends the ready message over the socket to initiate the session.
|
||||
*
|
||||
* @param {Object} socket - WebSocket connection to client.
|
||||
* @param {string} sessionToken -
|
||||
* the session token to send to the client.
|
||||
* @param {Object=} additionalData -
|
||||
* extra properties to send along the ready message.
|
||||
*/
|
||||
ready(sessionToken, additionalData) {
|
||||
ready(socket, sessionToken, additionalData) {
|
||||
this._socket = socket;
|
||||
this._socket.send(JSON.stringify({
|
||||
seq: null,
|
||||
type: "ready",
|
||||
data: {
|
||||
session_token: sessionToken,
|
||||
session_timeout: this._sessionTimeout,
|
||||
heartbeat_interval: this._heartbeatInterval,
|
||||
...additionalData,
|
||||
},
|
||||
@@ -77,7 +76,7 @@ class WsServerConnector extends libLink.WebSocketBaseConnector {
|
||||
// stale connection. If this is the case the connector will have to
|
||||
// wait for the previous socket to close down before it can continue
|
||||
// from a new socket, or there will be overlapping events from both.
|
||||
if (["connected", "closing"].includes(this._state)) {
|
||||
if (this._state === "connected") {
|
||||
this._socket.close(4003, "Session Hijacked");
|
||||
|
||||
// Correctly waiting for the connector to close and then
|
||||
@@ -102,6 +101,7 @@ class WsServerConnector extends libLink.WebSocketBaseConnector {
|
||||
type: "continue",
|
||||
data: {
|
||||
last_seq: this._lastReceivedSeq,
|
||||
session_timeout: this._sessionTimeout,
|
||||
heartbeat_interval: this._heartbeatInterval,
|
||||
},
|
||||
}));
|
||||
@@ -115,63 +115,44 @@ class WsServerConnector extends libLink.WebSocketBaseConnector {
|
||||
this.emit("resume");
|
||||
}
|
||||
|
||||
setTimeout(timeout) {
|
||||
if (this._timeoutId) {
|
||||
clearTimeout(this._timeoutId);
|
||||
}
|
||||
|
||||
this._timeoutId = setTimeout(() => { this._timedOut(); }, timeout * 1000);
|
||||
this._timeout = timeout;
|
||||
}
|
||||
|
||||
_timedOut() {
|
||||
logger.verbose("SOCKET | Connection timed out");
|
||||
this._timeoutId = null;
|
||||
this._lastReceivedSeq = null;
|
||||
this._sendBuffer.length = 0;
|
||||
this._state = "closed";
|
||||
logger.verbose("Connector | Connection timed out");
|
||||
this._reset();
|
||||
this.emit("close");
|
||||
this.emit("invalidate");
|
||||
}
|
||||
|
||||
_attachSocketHandlers() {
|
||||
this.startHeartbeat();
|
||||
|
||||
this._socket.on("close", (code, reason) => {
|
||||
logger.verbose(`SOCKET | Close (code: ${code}, reason: ${reason})`);
|
||||
if (this._state === "closing") {
|
||||
this._lastReceivedSeq = null;
|
||||
this._sendBuffer.length = 0;
|
||||
this._state = "closed";
|
||||
logger.verbose(`Connector | Close (code: ${code}, reason: ${reason})`);
|
||||
this.stopHeartbeat();
|
||||
|
||||
if (this._closing) {
|
||||
this._reset();
|
||||
this.emit("close");
|
||||
|
||||
if (this._timeoutId) {
|
||||
clearTimeout(this._timeoutId);
|
||||
this._timeoutId = null;
|
||||
}
|
||||
|
||||
} else {
|
||||
this._state = "handshake";
|
||||
this._state = "resuming";
|
||||
this.emit("drop");
|
||||
this._timeoutId = setTimeout(() => { this._timedOut(); }, this._timeout * 1000);
|
||||
this._timeoutId = setTimeout(() => { this._timedOut(); }, this._sessionTimeout * 1000);
|
||||
}
|
||||
|
||||
this.stopHeartbeat();
|
||||
});
|
||||
|
||||
this._socket.on("error", err => {
|
||||
// It's assumed that close is always called by ws
|
||||
logger.verbose("SOCKET | Error:", err);
|
||||
logger.verbose("Connector | Error:", err);
|
||||
});
|
||||
|
||||
this._socket.on("open", () => {
|
||||
logger.verbose("SOCKET | Open");
|
||||
logger.verbose("Connector | Open");
|
||||
});
|
||||
this._socket.on("ping", data => {
|
||||
logger.verbose(`SOCKET | Ping (data: ${data}`);
|
||||
logger.verbose(`Connector | Ping (data: ${data}`);
|
||||
});
|
||||
this._socket.on("pong", data => {
|
||||
logger.verbose(`SOCKET | Pong (data: ${data}`);
|
||||
logger.verbose(`Connector | Pong (data: ${data}`);
|
||||
});
|
||||
|
||||
// Handle messages
|
||||
@@ -208,11 +189,34 @@ class WsServerConnector extends libLink.WebSocketBaseConnector {
|
||||
return;
|
||||
}
|
||||
|
||||
this.stopHeartbeat();
|
||||
this._state = "closing";
|
||||
if (this._state === "resuming") {
|
||||
this._reset();
|
||||
this.emit("close");
|
||||
return;
|
||||
}
|
||||
|
||||
this._closing = true;
|
||||
this._socket.close(code, reason);
|
||||
await events.once(this, "close");
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify the connection is expected to close
|
||||
*
|
||||
* Sets the internal flag to signal not to wait for the client to
|
||||
* reconnect and resume the connection if the client closes it. This
|
||||
* will close the connection if it's currently waiting for the client to
|
||||
* resume it.
|
||||
*/
|
||||
setClosing() {
|
||||
if (this._state === "resuming") {
|
||||
this._reset();
|
||||
this.emit("close");
|
||||
return;
|
||||
}
|
||||
|
||||
this._closing = true;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = WsServerConnector;
|
||||
Reference in new issue
Block a user