Rewrite reconnection logic

Implement proper session tracking and resume on reconnect for the
WebSocket server.
This commit is contained in:
Hornwitser committed 2020-04-02 21:58:44 +02:00
1 parent 026b7f1306
commit d4e072f376
24 files changed
+1403 -319

No files matched your search

+14
View File
@@ -73,6 +73,13 @@ MasterGroup.define({
return bytes.toString("base64");
},
});
MasterGroup.define({
name: "heartbeat_interval",
title: "Heartbeat Interval",
description: "Interval heartbeats are sent out on WebSocket connections",
type: "number",
initial_value: 60,
});
MasterGroup.finalize();
class MasterConfig extends classes.Config { }
@@ -123,6 +130,13 @@ SlaveGroup.define({
type: "string",
initial_value: "localhost",
});
SlaveGroup.define({
name: "reconnect_delay",
title: "Reconnect Delay",
description: "Maximum delay to wait before attempting to reconnect WebSocket",
type: "number",
initial_value: 5,
});
SlaveGroup.finalize();
class SlaveConfig extends classes.Config { }
+4
View File
@@ -14,6 +14,9 @@ class InvalidMessage extends Error {
}
}
// Thrown when WebSocket authentication failed
class AuthenticationFailed extends Error { }
// Errror class for known errors occuring during startup
class StartupError extends Error { }
@@ -33,6 +36,7 @@ module.exports = {
CommandError,
RequestError,
InvalidMessage,
AuthenticationFailed,
StartupError,
EnvironmentError,
PluginError,
+235 -50
View File
@@ -3,19 +3,122 @@ const events = require("events");
const WebSocket = require("ws");
const schema = require("lib/schema");
const errors = require("lib/errors");
class WebSocketBaseConnector extends events.EventEmitter {
constructor() {
super();
this._socket = null;
this._seq = 1;
this._lastHeartbeat = null;
this._heartbeatId = null;
this._heartbeatInterval = null;
this._lastReceivedSeq = null;
this._sendBuffer = [];
}
_dropSendBufferSeq(seq) {
if (seq === null) {
return;
}
let dropCount = 0;
for (let index = 0; index < this._sendBuffer.length; index++) {
if (this._sendBuffer[index].seq <= seq) {
dropCount++;
} else {
break;
}
}
this._sendBuffer.splice(0, dropCount);
}
_doHeartbeat() {
this._check(["connected"]);
if (Date.now() - this._lastHeartbeat > 2000 * this._heartbeatInterval) {
console.log("SOCKET | closing after heartbeat timed out");
this._socket.close(1008, "Heartbeat timeout");
clearInterval(this._heartbeatId);
this._heartbeatId = null;
} else {
this._socket.send(JSON.stringify({
seq: null,
type: "heartbeat",
data: { seq: this._lastReceivedSeq },
}));
}
}
_processHeartbeat(message) {
if (!schema.heartbeat(message)) {
console.log("SOCKET | closing after received invalid heartbeat:", message);
this._socket.close(1002, "Invalid heartbeat");
console.log(schema.heartbeat.errors);
return;
}
this._lastHeartbeat = Date.now();
this._dropSendBufferSeq(message.data.seq);
}
/**
* Start sending heartbeats
*/
startHeartbeat() {
this._lastHeartbeat = Date.now();
this._heartbeatId = setInterval(() => {
this._doHeartbeat();
}, this._heartbeatInterval * 1000);
}
/**
* Stop sending heartbeats
*/
stopHeartbeat() {
if (this._heartbeatId) {
clearInterval(this._heartbeatId);
this._heartbeatId = null;
}
}
/**
* Send a message over the socket
*
* @returns the sequence number of the message sent
*/
send(type, data) {
this._check(["handshake", "connected"]);
let message = { seq: this._seq, type, data };
this._sendBuffer.push(message);
if (this._state === "connected") {
this._socket.send(JSON.stringify(message));
}
return this._seq++;
}
}
/**
* Connector for master server clients
*/
class WebSocketClientConnector extends events.EventEmitter {
constructor(url) {
class WebSocketClientConnector extends WebSocketBaseConnector {
constructor(url, reconnectDelay) {
super();
this._url = url;
this._seq = 1;
// The following states are used in the client connector
// new: Not connected
// handshake: Attempting to (re)connect to server.
// connected: Connection is online
// closing: Connection is in the process of being closed.
this._state = "new";
this._reconnectId = null;
this._sessionToken = null;
this._connected = false;
this._reconnectDelay = reconnectDelay;
}
_check(expectedStates) {
@@ -27,23 +130,38 @@ class WebSocketClientConnector extends events.EventEmitter {
}
/**
* Send a message over the socket
* Send a handshake message over the socket
*
* @returns the sequence number of the message sent
* Used in the register function in order to send the register handshake
* message over the WebSocket.
*/
send(type, data) {
this._socket.send(JSON.stringify({ seq: this._seq, type, data }));
return this._seq++;
sendHandshake(type, data) {
this._check(["handshake"]);
this._socket.send(JSON.stringify({ seq: null, type, data }));
}
/**
* Close the connection with the given reason.
*
* Sends a close message and disconnects the connector.
* Sends a close frame and disconnects the connector.
*/
close(reason) {
this.send('close', { reason });
this._socket.close();
close(code, reason) {
if (this._state === "new") {
return;
}
if (this._reconnectId) {
clearTimeout(this._reconnectId);
this._reconnectId = null;
}
this.stopHeartbeat();
if (["handshake", "connected"].includes(this._state)) {
this._socket.close(code, reason);
}
this._state = "closing";
}
/**
@@ -53,6 +171,13 @@ class WebSocketClientConnector extends events.EventEmitter {
this._check(["new"]);
this._state = "handshake";
this._doConnect();
// Wait for the connection to be ready
await events.once(this, "connected");
}
async _doConnect() {
let url = new URL(this._url);
url.pathname += "api/socket";
@@ -66,27 +191,73 @@ class WebSocketClientConnector extends events.EventEmitter {
});
this._attachSocketHandlers();
}
// Wait for the connection to be ready
await events.once(this, 'ready');
/**
* Attempt re-establishing the connecting again
*/
reconnect() {
if (this._reconnectId !== null) {
return;
}
let delay = Math.random() * this._reconnectDelay;
console.log(`SOCKET | waiting ${delay.toLocaleString("en", { maximumFractionDigits: 2 })} seconds for reconnect`)
this._reconnectId = setTimeout(() => {
this._reconnectId = null;
this._doConnect();
}, delay * 1000);
}
_attachSocketHandlers() {
this._socket.on("close", (code, reason) => {
console.log(`SOCKET | Close (code: ${code}, reason: ${reason})`);
// Authentication failed
if (code === 4003) {
this.emit("error", new errors.AuthenticationFailed("Authentication failed"));
this._state = "closing";
}
if (this._state === "closing") {
this._lastReceivedSeq = null;
this._sessionToken = null;
this._sendBuffer.length = 0;
this._state = "new";
if (this._connected) {
this.emit("close");
this._connected = false;
}
} else {
this._state = "handshake";
this.reconnect();
if (this._connected) {
this.emit("dropped");
this._connected = false;
}
}
this.stopHeartbeat();
});
this._socket.on("error", err => {
// It's assumed that close is always called by ws
console.error("SOCKET | Error:", err);
this._socket.close();
});
/*
this._socket.on("unexpected-response", (req, res) => {
let data = "";
res.on('data', (chunk) => { data += chunk; });
res.on('end', () => {
console.error(`SOCKET | Unexpeced Response ${res.statusCode}`);
console.error(data);
this._socket.emit("error", new Error(`Unexpected Response ${res.statusCode}`));
});
});
*/
this._socket.on("open", () => {
console.log("SOCKET | Open");
@@ -103,42 +274,73 @@ class WebSocketClientConnector extends events.EventEmitter {
let message = JSON.parse(data);
if (this._state === "handshake") {
this._processHandshake(message);
} else if (this._state === "ready") {
this.emit("message", message);
} else if (["connected", "closing"].includes(this._state)) {
if (message.seq !== null) {
this._lastReceivedSeq = message.seq;
}
if (message.type === "heartbeat") {
this._processHeartbeat(message);
} else {
this.emit("message", message);
}
} else {
throw new Error(`Received message in unexpected state ${this._state}`);
}
});
}
/**
* Immediatly close the connection
*/
disconnect() {
this._socket.terminate();
}
_processHandshake(message) {
if (!schema.serverHandshake(message)) {
console.log("SOCKET | closing after received invalid handshake:", message);
this.close("Invalid handshake");
this._socket.close(1002, "Invalid handshake");
return;
}
let { seq, type, data } = message;
if (type == 'hello') {
console.log(`SOCKET | received hello from master version ${data.version}`);
this.register();
if (this._sessionToken) {
console.log("SOCKET | Attempting resume");
this.sendHandshake("resume", {
session_token: this._sessionToken,
last_seq: this._lastReceivedSeq,
});
} else {
this.register();
}
} else if (type === 'ready') {
console.log("SOCKET | received ready from master");
this._state = "ready";
this.emit('ready');
this._state = "connected";
this._sessionToken = data.session_token;
this._heartbeatInterval = data.heartbeat_interval;
this.startHeartbeat();
this._connected = true;
this.emit("connected");
} else if (type === 'close') {
console.log(`SOCKET | received close from server: ${data.reason}`);
this.emit('error', new Error(`server closed during handshake: ${data.reason}`));
this._socket.terminate();
} else if (type === "continue") {
console.log("SOCKET | resuming existing session");
this._state = "connected";
this._heartbeatInterval = data.heartbeat_interval;
this.startHeartbeat();
this._dropSendBufferSeq(data.last_seq);
for (let message of this._sendBuffer) {
this._socket.send(JSON.stringify(message));
}
this._connected = true;
this.emit("connected");
} else if (type === "invalidate") {
this._heartbeatInterval = null;
this._lastReceivedSeq = null;
this._sessionToken = null;
this._sendBuffer.length = 0;
this.emit("invalidate");
this.register();
}
}
@@ -186,27 +388,10 @@ class VirtualConnector extends events.EventEmitter {
this.other.emit('message', { seq: this._seq, type, data });
return this._seq++;
}
/**
* Close the connection with the given reason.
*
* Sends a close message to the other end of the connector with the
* given reason. Doesn't actually disconnect as virtual connectors
* doen't have a concept of disconnecting.
*/
close(reason) {
this.send('close', { reason });
}
/**
* Does nothing
*
* Virtual connectors do not have a concept of disconnecting.
*/
disconnect() { }
}
module.exports = {
WebSocketBaseConnector,
WebSocketClientConnector,
VirtualConnector,
};
+40 -8
View File
@@ -32,7 +32,6 @@ class Link {
this.processMessage(payload);
} catch (err) {
if (err instanceof errors.InvalidMessage) {
this.connector.close(`Invalid message: ${err.message}`);
console.error(`Invalid message on ${this.source}-${this.target} link: ${err.message}`)
if (err.errors) {
console.error(err.errors);
@@ -43,10 +42,15 @@ class Link {
}
});
this.setHandler('close', message => {
console.log(`SOCKET | Received close: ${message.data.reason}`);
this.connector.disconnect();
}, schema.close);
connector.on("invalidate", () => {
for (let waiterType of this._waiters.values()) {
for (let waiter of waiterType) {
waiter.reject(new Error("Session lost"));
}
}
this._waiters.clear();
});
}
/**
@@ -185,13 +189,41 @@ class Link {
throw new Error(`No validator for ${type} on ${this.source}-${this.target}`);
}
return new Promise((resolve, reject) => {
let waiter = { data };
waiter.promise = new Promise((resolve, reject) => {
waiter.resolve = resolve;
waiter.reject = reject;
if (!this._waiters.has(type)) {
this._waiters.set(type, [{ data, resolve }]);
this._waiters.set(type, [waiter]);
} else {
this._waiters.get(type).push({ data, resolve });
this._waiters.get(type).push(waiter);
}
});
return waiter.promise;
}
/**
* Handle ping requests
*/
async pingRequestHandler() { }
/**
* Prepare connection for close
*
* Waits for all pending requests on the connection to resolve.
* Sub-classes should implement this handler with code that prevents
* additional requests to be sent out before calling the super class
* method.
*/
async shutdownConnectionRequestHandler() {
let promises = [];
for (let waiterType of this._waiters.values()) {
for (let waiter of waiterType) {
promises.push(waiter.promise.then(() => {}, () => {}));
}
}
await Promise.all(promises);
}
}
+12
View File
@@ -183,6 +183,18 @@ class Request {
let messages = {}
// Connection requests
let wsLinks = ["master-control", "control-master", "master-slave", "slave-master"];
messages.shutdownConnection = new Request({
type: "shutdown_connection",
links: wsLinks,
});
messages.ping = new Request({
type: "ping",
links: wsLinks,
});
// Management requests
messages.listSlaves = new Request({
type: 'list_slaves',
+58 -40
View File
@@ -4,27 +4,9 @@ const ajv = new (require('ajv'))({
extendRefs: "fail",
strictDefaluts: true,
strictKeywords: true,
})
});
const closeSchema = {
$schema: "http://json-schema.org/draft-07/schema#",
title: "Close Message",
description: "Format for a close messages on a link",
properties: {
"type": { const: "close" },
"data": {
additionalProperties: false,
required: ["reason"],
properties: {
"reason": { type: "string" },
}
},
},
}
const close = ajv.compile(closeSchema);
/**
* The base format for any message sent over the link
*/
@@ -36,28 +18,34 @@ const messageSchema = {
additionalProperties: false,
required: ["seq", "type", "data"],
properties: {
"seq": { type: "integer" },
"seq": { type: ["integer", "null"] },
"type": { type: "string" },
"data": { type: "object" },
},
}
};
const message = ajv.compile(messageSchema);
const serverActions = [
{},
];
const actionResponse = {
const heartbeat = ajv.compile({
$schema: "http://json-schema.org/draft-07/schema#",
title: "Heartbeat",
description: "Heartbeat sent over link",
type: "object",
additionalProperties: false,
required: ["action", "id", "status", "message"],
required: ["seq", "type", "data"],
properties: {
"action": { enum: ["TODO"] },
"id": { type: "integer" },
"status": { enum: ["started", "done", "error"] },
"message": { type: "string" },
"seq": { const: null },
"type": { const: "heartbeat" },
"data": {
type: "object",
additionalProperties: false,
required: ["seq"],
properties: {
"seq": { type: ["integer", "null"] },
},
},
},
};
});
const serverHandshake = ajv.compile({
$schema: "http://json-schema.org/draft-07/schema#",
@@ -82,15 +70,33 @@ const serverHandshake = ajv.compile({
"type": { const: "ready" },
"data": {
additionalProperties: false,
/* XXX anything here?
required: ["version"],
required: ["session_token", "heartbeat_interval"],
properties: {
"version": { type: "string" },
}*/
"session_token": { type: "string" },
"heartbeat_interval": { type: "number" },
},
},
},
},
closeSchema,
{
properties: {
"type": { const: "continue" },
"data": {
additionalProperties: false,
required: ["last_seq", "heartbeat_interval"],
properties: {
"last_seq": { type: ["integer", "null"] },
"heartbeat_interval": { type: "number" },
},
},
},
},
{
properties: {
"type": { const: "invalidate" },
"data": { const: {} },
},
},
],
});
@@ -130,7 +136,19 @@ const clientHandshake = ajv.compile({
},
},
},
closeSchema,
{
properties: {
"type": { const: "resume" },
"data": {
additionalProperties: false,
required: ["session_token", "last_seq"],
properties: {
"session_token": { type: "string" },
"last_seq": { type: ["integer", "null"] },
},
},
},
},
],
});
@@ -139,8 +157,8 @@ module.exports = {
compile: ajv.compile.bind(ajv),
message,
close,
heartbeat,
serverHandshake,
clientHandshake,
}
};