mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-16 07:24:53 +09:00
Rewrite reconnection logic
Implement proper session tracking and resume on reconnect for the WebSocket server.
This commit is contained in:
1 parent
026b7f1306
commit
d4e072f376
24 files changed
+1403
-319
No files matched your search
@@ -73,6 +73,13 @@ MasterGroup.define({
|
||||
return bytes.toString("base64");
|
||||
},
|
||||
});
|
||||
MasterGroup.define({
|
||||
name: "heartbeat_interval",
|
||||
title: "Heartbeat Interval",
|
||||
description: "Interval heartbeats are sent out on WebSocket connections",
|
||||
type: "number",
|
||||
initial_value: 60,
|
||||
});
|
||||
MasterGroup.finalize();
|
||||
|
||||
class MasterConfig extends classes.Config { }
|
||||
@@ -123,6 +130,13 @@ SlaveGroup.define({
|
||||
type: "string",
|
||||
initial_value: "localhost",
|
||||
});
|
||||
SlaveGroup.define({
|
||||
name: "reconnect_delay",
|
||||
title: "Reconnect Delay",
|
||||
description: "Maximum delay to wait before attempting to reconnect WebSocket",
|
||||
type: "number",
|
||||
initial_value: 5,
|
||||
});
|
||||
SlaveGroup.finalize();
|
||||
|
||||
class SlaveConfig extends classes.Config { }
|
||||
|
||||
@@ -14,6 +14,9 @@ class InvalidMessage extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
// Thrown when WebSocket authentication failed
|
||||
class AuthenticationFailed extends Error { }
|
||||
|
||||
// Errror class for known errors occuring during startup
|
||||
class StartupError extends Error { }
|
||||
|
||||
@@ -33,6 +36,7 @@ module.exports = {
|
||||
CommandError,
|
||||
RequestError,
|
||||
InvalidMessage,
|
||||
AuthenticationFailed,
|
||||
StartupError,
|
||||
EnvironmentError,
|
||||
PluginError,
|
||||
|
||||
+235
-50
@@ -3,19 +3,122 @@ const events = require("events");
|
||||
const WebSocket = require("ws");
|
||||
|
||||
const schema = require("lib/schema");
|
||||
const errors = require("lib/errors");
|
||||
|
||||
|
||||
class WebSocketBaseConnector extends events.EventEmitter {
|
||||
constructor() {
|
||||
super();
|
||||
|
||||
this._socket = null;
|
||||
this._seq = 1;
|
||||
this._lastHeartbeat = null;
|
||||
this._heartbeatId = null;
|
||||
this._heartbeatInterval = null;
|
||||
this._lastReceivedSeq = null;
|
||||
this._sendBuffer = [];
|
||||
}
|
||||
|
||||
_dropSendBufferSeq(seq) {
|
||||
if (seq === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
let dropCount = 0;
|
||||
for (let index = 0; index < this._sendBuffer.length; index++) {
|
||||
if (this._sendBuffer[index].seq <= seq) {
|
||||
dropCount++;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
this._sendBuffer.splice(0, dropCount);
|
||||
}
|
||||
|
||||
_doHeartbeat() {
|
||||
this._check(["connected"]);
|
||||
if (Date.now() - this._lastHeartbeat > 2000 * this._heartbeatInterval) {
|
||||
console.log("SOCKET | closing after heartbeat timed out");
|
||||
this._socket.close(1008, "Heartbeat timeout");
|
||||
clearInterval(this._heartbeatId);
|
||||
this._heartbeatId = null;
|
||||
|
||||
} else {
|
||||
this._socket.send(JSON.stringify({
|
||||
seq: null,
|
||||
type: "heartbeat",
|
||||
data: { seq: this._lastReceivedSeq },
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
_processHeartbeat(message) {
|
||||
if (!schema.heartbeat(message)) {
|
||||
console.log("SOCKET | closing after received invalid heartbeat:", message);
|
||||
this._socket.close(1002, "Invalid heartbeat");
|
||||
console.log(schema.heartbeat.errors);
|
||||
return;
|
||||
}
|
||||
|
||||
this._lastHeartbeat = Date.now();
|
||||
this._dropSendBufferSeq(message.data.seq);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start sending heartbeats
|
||||
*/
|
||||
startHeartbeat() {
|
||||
this._lastHeartbeat = Date.now();
|
||||
this._heartbeatId = setInterval(() => {
|
||||
this._doHeartbeat();
|
||||
}, this._heartbeatInterval * 1000);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop sending heartbeats
|
||||
*/
|
||||
stopHeartbeat() {
|
||||
if (this._heartbeatId) {
|
||||
clearInterval(this._heartbeatId);
|
||||
this._heartbeatId = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a message over the socket
|
||||
*
|
||||
* @returns the sequence number of the message sent
|
||||
*/
|
||||
send(type, data) {
|
||||
this._check(["handshake", "connected"]);
|
||||
let message = { seq: this._seq, type, data };
|
||||
this._sendBuffer.push(message);
|
||||
if (this._state === "connected") {
|
||||
this._socket.send(JSON.stringify(message));
|
||||
}
|
||||
return this._seq++;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connector for master server clients
|
||||
*/
|
||||
class WebSocketClientConnector extends events.EventEmitter {
|
||||
constructor(url) {
|
||||
class WebSocketClientConnector extends WebSocketBaseConnector {
|
||||
constructor(url, reconnectDelay) {
|
||||
super();
|
||||
|
||||
this._url = url;
|
||||
|
||||
this._seq = 1;
|
||||
// The following states are used in the client connector
|
||||
// new: Not connected
|
||||
// handshake: Attempting to (re)connect to server.
|
||||
// connected: Connection is online
|
||||
// closing: Connection is in the process of being closed.
|
||||
this._state = "new";
|
||||
this._reconnectId = null;
|
||||
this._sessionToken = null;
|
||||
this._connected = false;
|
||||
this._reconnectDelay = reconnectDelay;
|
||||
}
|
||||
|
||||
_check(expectedStates) {
|
||||
@@ -27,23 +130,38 @@ class WebSocketClientConnector extends events.EventEmitter {
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a message over the socket
|
||||
* Send a handshake message over the socket
|
||||
*
|
||||
* @returns the sequence number of the message sent
|
||||
* Used in the register function in order to send the register handshake
|
||||
* message over the WebSocket.
|
||||
*/
|
||||
send(type, data) {
|
||||
this._socket.send(JSON.stringify({ seq: this._seq, type, data }));
|
||||
return this._seq++;
|
||||
sendHandshake(type, data) {
|
||||
this._check(["handshake"]);
|
||||
this._socket.send(JSON.stringify({ seq: null, type, data }));
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the connection with the given reason.
|
||||
*
|
||||
* Sends a close message and disconnects the connector.
|
||||
* Sends a close frame and disconnects the connector.
|
||||
*/
|
||||
close(reason) {
|
||||
this.send('close', { reason });
|
||||
this._socket.close();
|
||||
close(code, reason) {
|
||||
if (this._state === "new") {
|
||||
return;
|
||||
}
|
||||
|
||||
if (this._reconnectId) {
|
||||
clearTimeout(this._reconnectId);
|
||||
this._reconnectId = null;
|
||||
}
|
||||
|
||||
this.stopHeartbeat();
|
||||
|
||||
if (["handshake", "connected"].includes(this._state)) {
|
||||
this._socket.close(code, reason);
|
||||
}
|
||||
|
||||
this._state = "closing";
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -53,6 +171,13 @@ class WebSocketClientConnector extends events.EventEmitter {
|
||||
this._check(["new"]);
|
||||
this._state = "handshake";
|
||||
|
||||
this._doConnect();
|
||||
|
||||
// Wait for the connection to be ready
|
||||
await events.once(this, "connected");
|
||||
}
|
||||
|
||||
async _doConnect() {
|
||||
let url = new URL(this._url);
|
||||
url.pathname += "api/socket";
|
||||
|
||||
@@ -66,27 +191,73 @@ class WebSocketClientConnector extends events.EventEmitter {
|
||||
});
|
||||
|
||||
this._attachSocketHandlers();
|
||||
}
|
||||
|
||||
// Wait for the connection to be ready
|
||||
await events.once(this, 'ready');
|
||||
/**
|
||||
* Attempt re-establishing the connecting again
|
||||
*/
|
||||
reconnect() {
|
||||
if (this._reconnectId !== null) {
|
||||
return;
|
||||
}
|
||||
|
||||
let delay = Math.random() * this._reconnectDelay;
|
||||
console.log(`SOCKET | waiting ${delay.toLocaleString("en", { maximumFractionDigits: 2 })} seconds for reconnect`)
|
||||
this._reconnectId = setTimeout(() => {
|
||||
this._reconnectId = null;
|
||||
this._doConnect();
|
||||
}, delay * 1000);
|
||||
}
|
||||
|
||||
_attachSocketHandlers() {
|
||||
this._socket.on("close", (code, reason) => {
|
||||
console.log(`SOCKET | Close (code: ${code}, reason: ${reason})`);
|
||||
// Authentication failed
|
||||
if (code === 4003) {
|
||||
this.emit("error", new errors.AuthenticationFailed("Authentication failed"));
|
||||
this._state = "closing";
|
||||
}
|
||||
|
||||
if (this._state === "closing") {
|
||||
this._lastReceivedSeq = null;
|
||||
this._sessionToken = null;
|
||||
this._sendBuffer.length = 0;
|
||||
this._state = "new";
|
||||
|
||||
if (this._connected) {
|
||||
this.emit("close");
|
||||
this._connected = false;
|
||||
}
|
||||
|
||||
} else {
|
||||
this._state = "handshake";
|
||||
this.reconnect();
|
||||
|
||||
if (this._connected) {
|
||||
this.emit("dropped");
|
||||
this._connected = false;
|
||||
}
|
||||
}
|
||||
|
||||
this.stopHeartbeat();
|
||||
});
|
||||
|
||||
this._socket.on("error", err => {
|
||||
// It's assumed that close is always called by ws
|
||||
console.error("SOCKET | Error:", err);
|
||||
this._socket.close();
|
||||
});
|
||||
|
||||
/*
|
||||
this._socket.on("unexpected-response", (req, res) => {
|
||||
let data = "";
|
||||
res.on('data', (chunk) => { data += chunk; });
|
||||
res.on('end', () => {
|
||||
console.error(`SOCKET | Unexpeced Response ${res.statusCode}`);
|
||||
console.error(data);
|
||||
this._socket.emit("error", new Error(`Unexpected Response ${res.statusCode}`));
|
||||
});
|
||||
});
|
||||
*/
|
||||
|
||||
this._socket.on("open", () => {
|
||||
console.log("SOCKET | Open");
|
||||
@@ -103,42 +274,73 @@ class WebSocketClientConnector extends events.EventEmitter {
|
||||
let message = JSON.parse(data);
|
||||
if (this._state === "handshake") {
|
||||
this._processHandshake(message);
|
||||
} else if (this._state === "ready") {
|
||||
this.emit("message", message);
|
||||
|
||||
} else if (["connected", "closing"].includes(this._state)) {
|
||||
if (message.seq !== null) {
|
||||
this._lastReceivedSeq = message.seq;
|
||||
}
|
||||
|
||||
if (message.type === "heartbeat") {
|
||||
this._processHeartbeat(message);
|
||||
|
||||
} else {
|
||||
this.emit("message", message);
|
||||
}
|
||||
|
||||
} else {
|
||||
throw new Error(`Received message in unexpected state ${this._state}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Immediatly close the connection
|
||||
*/
|
||||
disconnect() {
|
||||
this._socket.terminate();
|
||||
}
|
||||
|
||||
_processHandshake(message) {
|
||||
if (!schema.serverHandshake(message)) {
|
||||
console.log("SOCKET | closing after received invalid handshake:", message);
|
||||
this.close("Invalid handshake");
|
||||
this._socket.close(1002, "Invalid handshake");
|
||||
return;
|
||||
}
|
||||
|
||||
let { seq, type, data } = message;
|
||||
if (type == 'hello') {
|
||||
console.log(`SOCKET | received hello from master version ${data.version}`);
|
||||
this.register();
|
||||
if (this._sessionToken) {
|
||||
console.log("SOCKET | Attempting resume");
|
||||
this.sendHandshake("resume", {
|
||||
session_token: this._sessionToken,
|
||||
last_seq: this._lastReceivedSeq,
|
||||
});
|
||||
} else {
|
||||
this.register();
|
||||
}
|
||||
|
||||
} else if (type === 'ready') {
|
||||
console.log("SOCKET | received ready from master");
|
||||
this._state = "ready";
|
||||
this.emit('ready');
|
||||
this._state = "connected";
|
||||
this._sessionToken = data.session_token;
|
||||
this._heartbeatInterval = data.heartbeat_interval;
|
||||
this.startHeartbeat();
|
||||
this._connected = true;
|
||||
this.emit("connected");
|
||||
|
||||
} else if (type === 'close') {
|
||||
console.log(`SOCKET | received close from server: ${data.reason}`);
|
||||
this.emit('error', new Error(`server closed during handshake: ${data.reason}`));
|
||||
this._socket.terminate();
|
||||
} else if (type === "continue") {
|
||||
console.log("SOCKET | resuming existing session");
|
||||
this._state = "connected";
|
||||
this._heartbeatInterval = data.heartbeat_interval;
|
||||
this.startHeartbeat();
|
||||
this._dropSendBufferSeq(data.last_seq);
|
||||
for (let message of this._sendBuffer) {
|
||||
this._socket.send(JSON.stringify(message));
|
||||
}
|
||||
this._connected = true;
|
||||
this.emit("connected");
|
||||
|
||||
} else if (type === "invalidate") {
|
||||
this._heartbeatInterval = null;
|
||||
this._lastReceivedSeq = null;
|
||||
this._sessionToken = null;
|
||||
this._sendBuffer.length = 0;
|
||||
this.emit("invalidate");
|
||||
this.register();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -186,27 +388,10 @@ class VirtualConnector extends events.EventEmitter {
|
||||
this.other.emit('message', { seq: this._seq, type, data });
|
||||
return this._seq++;
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the connection with the given reason.
|
||||
*
|
||||
* Sends a close message to the other end of the connector with the
|
||||
* given reason. Doesn't actually disconnect as virtual connectors
|
||||
* doen't have a concept of disconnecting.
|
||||
*/
|
||||
close(reason) {
|
||||
this.send('close', { reason });
|
||||
}
|
||||
|
||||
/**
|
||||
* Does nothing
|
||||
*
|
||||
* Virtual connectors do not have a concept of disconnecting.
|
||||
*/
|
||||
disconnect() { }
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
WebSocketBaseConnector,
|
||||
WebSocketClientConnector,
|
||||
VirtualConnector,
|
||||
};
|
||||
+40
-8
@@ -32,7 +32,6 @@ class Link {
|
||||
this.processMessage(payload);
|
||||
} catch (err) {
|
||||
if (err instanceof errors.InvalidMessage) {
|
||||
this.connector.close(`Invalid message: ${err.message}`);
|
||||
console.error(`Invalid message on ${this.source}-${this.target} link: ${err.message}`)
|
||||
if (err.errors) {
|
||||
console.error(err.errors);
|
||||
@@ -43,10 +42,15 @@ class Link {
|
||||
}
|
||||
});
|
||||
|
||||
this.setHandler('close', message => {
|
||||
console.log(`SOCKET | Received close: ${message.data.reason}`);
|
||||
this.connector.disconnect();
|
||||
}, schema.close);
|
||||
connector.on("invalidate", () => {
|
||||
for (let waiterType of this._waiters.values()) {
|
||||
for (let waiter of waiterType) {
|
||||
waiter.reject(new Error("Session lost"));
|
||||
}
|
||||
}
|
||||
|
||||
this._waiters.clear();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -185,13 +189,41 @@ class Link {
|
||||
throw new Error(`No validator for ${type} on ${this.source}-${this.target}`);
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
let waiter = { data };
|
||||
waiter.promise = new Promise((resolve, reject) => {
|
||||
waiter.resolve = resolve;
|
||||
waiter.reject = reject;
|
||||
if (!this._waiters.has(type)) {
|
||||
this._waiters.set(type, [{ data, resolve }]);
|
||||
this._waiters.set(type, [waiter]);
|
||||
} else {
|
||||
this._waiters.get(type).push({ data, resolve });
|
||||
this._waiters.get(type).push(waiter);
|
||||
}
|
||||
});
|
||||
return waiter.promise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle ping requests
|
||||
*/
|
||||
async pingRequestHandler() { }
|
||||
|
||||
/**
|
||||
* Prepare connection for close
|
||||
*
|
||||
* Waits for all pending requests on the connection to resolve.
|
||||
* Sub-classes should implement this handler with code that prevents
|
||||
* additional requests to be sent out before calling the super class
|
||||
* method.
|
||||
*/
|
||||
async shutdownConnectionRequestHandler() {
|
||||
let promises = [];
|
||||
for (let waiterType of this._waiters.values()) {
|
||||
for (let waiter of waiterType) {
|
||||
promises.push(waiter.promise.then(() => {}, () => {}));
|
||||
}
|
||||
}
|
||||
|
||||
await Promise.all(promises);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -183,6 +183,18 @@ class Request {
|
||||
|
||||
let messages = {}
|
||||
|
||||
// Connection requests
|
||||
let wsLinks = ["master-control", "control-master", "master-slave", "slave-master"];
|
||||
messages.shutdownConnection = new Request({
|
||||
type: "shutdown_connection",
|
||||
links: wsLinks,
|
||||
});
|
||||
|
||||
messages.ping = new Request({
|
||||
type: "ping",
|
||||
links: wsLinks,
|
||||
});
|
||||
|
||||
// Management requests
|
||||
messages.listSlaves = new Request({
|
||||
type: 'list_slaves',
|
||||
|
||||
+58
-40
@@ -4,27 +4,9 @@ const ajv = new (require('ajv'))({
|
||||
extendRefs: "fail",
|
||||
strictDefaluts: true,
|
||||
strictKeywords: true,
|
||||
})
|
||||
});
|
||||
|
||||
|
||||
const closeSchema = {
|
||||
$schema: "http://json-schema.org/draft-07/schema#",
|
||||
title: "Close Message",
|
||||
description: "Format for a close messages on a link",
|
||||
properties: {
|
||||
"type": { const: "close" },
|
||||
"data": {
|
||||
additionalProperties: false,
|
||||
required: ["reason"],
|
||||
properties: {
|
||||
"reason": { type: "string" },
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
const close = ajv.compile(closeSchema);
|
||||
|
||||
/**
|
||||
* The base format for any message sent over the link
|
||||
*/
|
||||
@@ -36,28 +18,34 @@ const messageSchema = {
|
||||
additionalProperties: false,
|
||||
required: ["seq", "type", "data"],
|
||||
properties: {
|
||||
"seq": { type: "integer" },
|
||||
"seq": { type: ["integer", "null"] },
|
||||
"type": { type: "string" },
|
||||
"data": { type: "object" },
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
const message = ajv.compile(messageSchema);
|
||||
|
||||
const serverActions = [
|
||||
{},
|
||||
];
|
||||
|
||||
const actionResponse = {
|
||||
const heartbeat = ajv.compile({
|
||||
$schema: "http://json-schema.org/draft-07/schema#",
|
||||
title: "Heartbeat",
|
||||
description: "Heartbeat sent over link",
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["action", "id", "status", "message"],
|
||||
required: ["seq", "type", "data"],
|
||||
properties: {
|
||||
"action": { enum: ["TODO"] },
|
||||
"id": { type: "integer" },
|
||||
"status": { enum: ["started", "done", "error"] },
|
||||
"message": { type: "string" },
|
||||
"seq": { const: null },
|
||||
"type": { const: "heartbeat" },
|
||||
"data": {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
required: ["seq"],
|
||||
properties: {
|
||||
"seq": { type: ["integer", "null"] },
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
const serverHandshake = ajv.compile({
|
||||
$schema: "http://json-schema.org/draft-07/schema#",
|
||||
@@ -82,15 +70,33 @@ const serverHandshake = ajv.compile({
|
||||
"type": { const: "ready" },
|
||||
"data": {
|
||||
additionalProperties: false,
|
||||
/* XXX anything here?
|
||||
required: ["version"],
|
||||
required: ["session_token", "heartbeat_interval"],
|
||||
properties: {
|
||||
"version": { type: "string" },
|
||||
}*/
|
||||
"session_token": { type: "string" },
|
||||
"heartbeat_interval": { type: "number" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
closeSchema,
|
||||
{
|
||||
properties: {
|
||||
"type": { const: "continue" },
|
||||
"data": {
|
||||
additionalProperties: false,
|
||||
required: ["last_seq", "heartbeat_interval"],
|
||||
properties: {
|
||||
"last_seq": { type: ["integer", "null"] },
|
||||
"heartbeat_interval": { type: "number" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
properties: {
|
||||
"type": { const: "invalidate" },
|
||||
"data": { const: {} },
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
@@ -130,7 +136,19 @@ const clientHandshake = ajv.compile({
|
||||
},
|
||||
},
|
||||
},
|
||||
closeSchema,
|
||||
{
|
||||
properties: {
|
||||
"type": { const: "resume" },
|
||||
"data": {
|
||||
additionalProperties: false,
|
||||
required: ["session_token", "last_seq"],
|
||||
properties: {
|
||||
"session_token": { type: "string" },
|
||||
"last_seq": { type: ["integer", "null"] },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
@@ -139,8 +157,8 @@ module.exports = {
|
||||
compile: ajv.compile.bind(ajv),
|
||||
|
||||
message,
|
||||
close,
|
||||
heartbeat,
|
||||
|
||||
serverHandshake,
|
||||
clientHandshake,
|
||||
}
|
||||
};
|
||||
Reference in new issue
Block a user