aperx/factorio-clusterio
1
0 Fork 0
mirror of https://github.com/clusterio/clusterio.git synced 2026-11-12 15:04:52 +09:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite reconnection logic

Browse Source 
Implement proper session tracking and resume on reconnect for the
WebSocket server.
This commit is contained in:
Hornwitser committed 2020-04-02 21:58:44 +02:00
1 parent 026b7f1306
commit d4e072f376
24 files changed
+1403 -319
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

CHANGELOG.md
+2
View File
@@ -29,6 +29,8 @@ Version 2.0.0
### Features
- Added export of pollution statitics.
- Reconnection logic that esures no data is dropped talking to the master
server provided the session can be resumed.
### Changes
client.js
+13 -13
View File
@@ -1,6 +1,7 @@
const fs = require('fs-extra');
const path = require('path');
const yargs = require("yargs");
const events = require("events");
const version = require("./package").version;
// internal libraries
@@ -402,21 +403,18 @@ class InstanceConnection extends link.Link {
class SlaveConnector extends link.WebSocketClientConnector {
constructor(slaveConfig) {
super(slaveConfig.get("slave.master_url"));
this.id = slaveConfig.get("slave.id");
this.name = slaveConfig.get("slave.name");
this._token = slaveConfig.get("slave.master_token");
super(slaveConfig.get("slave.master_url"), slaveConfig.get("slave.reconnect_delay"));
this.slaveConfig = slaveConfig;
}
register() {
console.log("SOCKET | registering slave");
this.send('register_slave', {
token: this._token,
agent: 'Clusterio Slave',
this.sendHandshake("register_slave", {
token: this.slaveConfig.get("slave.master_token"),
agent: "Clusterio Slave",
version,
id: this.id,
name: this.name,
id: this.slaveConfig.get("slave.id"),
name: this.slaveConfig.get("slave.name"),
});
}
}
@@ -649,11 +647,13 @@ class Slave extends link.Link {
await this.updateInstances();
}
async stop() {
async shutdown() {
await link.messages.shutdownConnection.send(this);
for (let instanceId of this.instanceConnections.keys()) {
await this.stopInstance(instanceId);
}
this.connector.close("shutdown");
this.connector.close(1001, "Slave Shutdown");
await events.once(this.connector, "close");
}
}
@@ -851,7 +851,7 @@ async function startClient() {
secondSigint = true;
console.log("Caught interrupt signal, shutting down");
slave.stop().catch(err => {
slave.shutdown().catch(err => {
console.error(`
+---------------------------------------------------------------+
| Unexpected error occured while stopping slave, please report |
clusterctl.js
+32 -10
View File
@@ -4,6 +4,7 @@ const yargs = require("yargs");
const version = require("./package").version;
const asTable = require("as-table").configure({delimiter: ' | '});
const chalk = require("chalk");
const events = require("events");
const link = require("lib/link");
const errors = require("lib/errors");
@@ -315,16 +316,16 @@ commands = new Map([...commands.map(command => [command.name, command])]);
* Connector for control connection to master server
*/
class ControlConnector extends link.WebSocketClientConnector {
constructor(url, token) {
super(url);
constructor(url, reconnectDelay, token) {
super(url, reconnectDelay);
this._token = token;
}
register() {
console.log("SOCKET | registering control");
this.send('register_control', {
this.sendHandshake("register_control", {
token: this._token,
agent: 'clusterctl',
agent: "clusterctl",
version: version,
});
}
@@ -348,6 +349,12 @@ class Control extends link.Link {
let { instance_id, output } = message.data;
console.log(formatOutputColored(output));
}
async shutdown() {
await link.messages.shutdownConnection.send(this);
this.connector.close(1001, "Control Quit")
await events.once(this.connector, "close");
}
}
async function findCredentials(controlConfig) {
@@ -411,6 +418,13 @@ ControlGroup.define({
type: "string",
optional: true,
});
ControlGroup.define({
name: "reconnect_delay",
title: "Reconnect Delay",
description: "Maximum delay to wait before attempting to reconnect WebSocket",
type: "number",
initial_value: 2,
});
ControlGroup.finalize();
class ControlConfig extends config.Config { }
@@ -471,10 +485,19 @@ async function startControl() {
}
let controlConnector = new ControlConnector(
controlConfig.get("control.master_url"), controlConfig.get("control.master_token")
controlConfig.get("control.master_url"),
controlConfig.get("control.reconnect_delay"),
controlConfig.get("control.master_token")
);
let control = new Control(controlConnector);
await controlConnector.connect();
try {
await controlConnector.connect();
} catch(err) {
if (err instanceof errors.AuthenticationFailed) {
throw new errors.StartupError(err.message);
}
throw err;
}
if (commands.has(commandName)) {
command = commands.get(commandName);
@@ -492,13 +515,12 @@ async function startControl() {
process.exitCode = 1;
} else {
controlConnector.close("error");
throw err;
}
}
controlConnector.close("quit");
return; // ??
} finally {
await control.shutdown();
}
}
//XXX control.close("done");
docs/assets/connection_flow.dot
+20
View File
@@ -0,0 +1,20 @@
digraph "Client Connection Flow" {
node [shape=box];
graph [newrank=true];
start [label="Start"];
wait_hello [label="Wait for hello"];
new_session [label="New session"];
wait_ready [label="Wait for ready"];
connected [label="Connected"];
abort [label="Abort"];
lost_connection [label="Connection lost or\nunexpectedly closed\nwhile connecting" style="dotted"];
lost_connection -> start [label="Wait random delay"];
start -> wait_hello [label="Connect to /api/socket"];
wait_hello -> new_session [label="Receive hello"];
new_session -> wait_ready [label="Send register_*"];
wait_ready -> connected [label="Receive ready"];
wait_ready -> abort [label="Connection closed with 4003"];
}
docs/assets/connection_flow.svg
+99
View File
@@ -0,0 +1,99 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.42.3 (0)
-->
<!-- Title: Client Connection Flow Pages: 1 -->
<svg width="371pt" height="496pt"
viewBox="0.00 0.00 371.00 496.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 492)">
<title>Client Connection Flow</title>
<polygon fill="white" stroke="transparent" points="-4,4 -4,-492 367,-492 367,4 -4,4"/>
<!-- start -->
<g id="node1" class="node">
<title>start</title>
<polygon fill="none" stroke="black" points="135,-384 81,-384 81,-348 135,-348 135,-384"/>
<text text-anchor="middle" x="108" y="-362.3" font-family="Times,serif" font-size="14.00">Start</text>
</g>
<!-- wait_hello -->
<g id="node2" class="node">
<title>wait_hello</title>
<polygon fill="none" stroke="black" points="164.5,-297 51.5,-297 51.5,-261 164.5,-261 164.5,-297"/>
<text text-anchor="middle" x="108" y="-275.3" font-family="Times,serif" font-size="14.00">Wait for hello</text>
</g>
<!-- start&#45;&gt;wait_hello -->
<g id="edge2" class="edge">
<title>start&#45;&gt;wait_hello</title>
<path fill="none" stroke="black" d="M108,-347.8C108,-336.16 108,-320.55 108,-307.24"/>
<polygon fill="black" stroke="black" points="111.5,-307.18 108,-297.18 104.5,-307.18 111.5,-307.18"/>
<text text-anchor="middle" x="187.5" y="-318.8" font-family="Times,serif" font-size="14.00">Connect to /api/socket</text>
</g>
<!-- new_session -->
<g id="node3" class="node">
<title>new_session</title>
<polygon fill="none" stroke="black" points="161,-210 55,-210 55,-174 161,-174 161,-210"/>
<text text-anchor="middle" x="108" y="-188.3" font-family="Times,serif" font-size="14.00">New session</text>
</g>
<!-- wait_hello&#45;&gt;new_session -->
<g id="edge3" class="edge">
<title>wait_hello&#45;&gt;new_session</title>
<path fill="none" stroke="black" d="M108,-260.8C108,-249.16 108,-233.55 108,-220.24"/>
<polygon fill="black" stroke="black" points="111.5,-220.18 108,-210.18 104.5,-220.18 111.5,-220.18"/>
<text text-anchor="middle" x="155.5" y="-231.8" font-family="Times,serif" font-size="14.00">Receive hello</text>
</g>
<!-- wait_ready -->
<g id="node4" class="node">
<title>wait_ready</title>
<polygon fill="none" stroke="black" points="167.5,-123 48.5,-123 48.5,-87 167.5,-87 167.5,-123"/>
<text text-anchor="middle" x="108" y="-101.3" font-family="Times,serif" font-size="14.00">Wait for ready</text>
</g>
<!-- new_session&#45;&gt;wait_ready -->
<g id="edge4" class="edge">
<title>new_session&#45;&gt;wait_ready</title>
<path fill="none" stroke="black" d="M108,-173.8C108,-162.16 108,-146.55 108,-133.24"/>
<polygon fill="black" stroke="black" points="111.5,-133.18 108,-123.18 104.5,-133.18 111.5,-133.18"/>
<text text-anchor="middle" x="164" y="-144.8" font-family="Times,serif" font-size="14.00">Send register_*</text>
</g>
<!-- connected -->
<g id="node5" class="node">
<title>connected</title>
<polygon fill="none" stroke="black" points="92,-36 0,-36 0,0 92,0 92,-36"/>
<text text-anchor="middle" x="46" y="-14.3" font-family="Times,serif" font-size="14.00">Connected</text>
</g>
<!-- wait_ready&#45;&gt;connected -->
<g id="edge5" class="edge">
<title>wait_ready&#45;&gt;connected</title>
<path fill="none" stroke="black" d="M64.78,-86.86C57.71,-82.13 51.29,-76.25 47,-69 43.04,-62.32 41.73,-54.22 41.69,-46.49"/>
<polygon fill="black" stroke="black" points="45.19,-46.55 42.31,-36.36 38.21,-46.13 45.19,-46.55"/>
<text text-anchor="middle" x="97.5" y="-57.8" font-family="Times,serif" font-size="14.00">Receive ready</text>
</g>
<!-- abort -->
<g id="node6" class="node">
<title>abort</title>
<polygon fill="none" stroke="black" points="198,-36 142,-36 142,0 198,0 198,-36"/>
<text text-anchor="middle" x="170" y="-14.3" font-family="Times,serif" font-size="14.00">Abort</text>
</g>
<!-- wait_ready&#45;&gt;abort -->
<g id="edge6" class="edge">
<title>wait_ready&#45;&gt;abort</title>
<path fill="none" stroke="black" d="M131.28,-86.66C137.25,-81.45 143.3,-75.4 148,-69 153.19,-61.93 157.5,-53.45 160.88,-45.5"/>
<polygon fill="black" stroke="black" points="164.14,-46.78 164.56,-36.19 157.63,-44.21 164.14,-46.78"/>
<text text-anchor="middle" x="259.5" y="-57.8" font-family="Times,serif" font-size="14.00">Connection closed with 4003</text>
</g>
<!-- lost_connection -->
<g id="node7" class="node">
<title>lost_connection</title>
<polygon fill="none" stroke="black" stroke-dasharray="1,5" points="188.5,-488 27.5,-488 27.5,-435 188.5,-435 188.5,-488"/>
<text text-anchor="middle" x="108" y="-472.8" font-family="Times,serif" font-size="14.00">Connection lost or</text>
<text text-anchor="middle" x="108" y="-457.8" font-family="Times,serif" font-size="14.00">unexpectedly closed</text>
<text text-anchor="middle" x="108" y="-442.8" font-family="Times,serif" font-size="14.00">while connecting</text>
</g>
<!-- lost_connection&#45;&gt;start -->
<g id="edge1" class="edge">
<title>lost_connection&#45;&gt;start</title>
<path fill="none" stroke="black" d="M108,-434.95C108,-422.36 108,-407.17 108,-394.35"/>
<polygon fill="black" stroke="black" points="111.5,-394.24 108,-384.24 104.5,-394.24 111.5,-394.24"/>
<text text-anchor="middle" x="175.5" y="-405.8" font-family="Times,serif" font-size="14.00">Wait random delay</text>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 5.3 KiB

docs/assets/generate.sh
Executable
+4
View File
@@ -0,0 +1,4 @@
for x in $(dirname $0)/*.dot; do
echo $x
dot $x -Tsvg -o ${x/.dot/.svg}
done
docs/assets/reconnection_flow.dot
+23
View File
@@ -0,0 +1,23 @@
digraph "Client Reconnection Flow" {
node [shape=box];
graph [newrank=true];
reconnect [label="Reconnect"];
wait_hello [label="Wait for hello"];
resume_session [label="Resume session"];
new_session [label="New session" style="dashed"];
wait_continue [label="Wait for continue"];
connected [label="Connected"];
lost_connection [label="Connection lost or\nunexpectedly closed\nwhile reconnecting" style="dotted"];
abort [label="Abort"];
{ rank="same"; wait_continue; new_session };
lost_connection -> reconnect [label="Wait random delay"];
reconnect -> wait_hello [label="Connect to /api/socket"];
wait_hello -> resume_session [label="Receive hello"];
resume_session -> wait_continue [label="Send resume"];
wait_continue -> abort [label="Connection closed with 4003"];
wait_continue -> new_session [label="Receive invalidate"];
wait_continue -> connected [label="Receive ready"];
}
docs/assets/reconnection_flow.svg
+112
View File
@@ -0,0 +1,112 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.42.3 (0)
-->
<!-- Title: Client Reconnection Flow Pages: 1 -->
<svg width="441pt" height="496pt"
viewBox="0.00 0.00 441.00 496.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 492)">
<title>Client Reconnection Flow</title>
<polygon fill="white" stroke="transparent" points="-4,4 -4,-492 437,-492 437,4 -4,4"/>
<!-- reconnect -->
<g id="node1" class="node">
<title>reconnect</title>
<polygon fill="none" stroke="black" points="153.5,-384 62.5,-384 62.5,-348 153.5,-348 153.5,-384"/>
<text text-anchor="middle" x="108" y="-362.3" font-family="Times,serif" font-size="14.00">Reconnect</text>
</g>
<!-- wait_hello -->
<g id="node2" class="node">
<title>wait_hello</title>
<polygon fill="none" stroke="black" points="164.5,-297 51.5,-297 51.5,-261 164.5,-261 164.5,-297"/>
<text text-anchor="middle" x="108" y="-275.3" font-family="Times,serif" font-size="14.00">Wait for hello</text>
</g>
<!-- reconnect&#45;&gt;wait_hello -->
<g id="edge2" class="edge">
<title>reconnect&#45;&gt;wait_hello</title>
<path fill="none" stroke="black" d="M108,-347.8C108,-336.16 108,-320.55 108,-307.24"/>
<polygon fill="black" stroke="black" points="111.5,-307.18 108,-297.18 104.5,-307.18 111.5,-307.18"/>
<text text-anchor="middle" x="187.5" y="-318.8" font-family="Times,serif" font-size="14.00">Connect to /api/socket</text>
</g>
<!-- resume_session -->
<g id="node3" class="node">
<title>resume_session</title>
<polygon fill="none" stroke="black" points="173.5,-210 42.5,-210 42.5,-174 173.5,-174 173.5,-210"/>
<text text-anchor="middle" x="108" y="-188.3" font-family="Times,serif" font-size="14.00">Resume session</text>
</g>
<!-- wait_hello&#45;&gt;resume_session -->
<g id="edge3" class="edge">
<title>wait_hello&#45;&gt;resume_session</title>
<path fill="none" stroke="black" d="M108,-260.8C108,-249.16 108,-233.55 108,-220.24"/>
<polygon fill="black" stroke="black" points="111.5,-220.18 108,-210.18 104.5,-220.18 111.5,-220.18"/>
<text text-anchor="middle" x="155.5" y="-231.8" font-family="Times,serif" font-size="14.00">Receive hello</text>
</g>
<!-- wait_continue -->
<g id="node5" class="node">
<title>wait_continue</title>
<polygon fill="none" stroke="black" points="178,-123 38,-123 38,-87 178,-87 178,-123"/>
<text text-anchor="middle" x="108" y="-101.3" font-family="Times,serif" font-size="14.00">Wait for continue</text>
</g>
<!-- resume_session&#45;&gt;wait_continue -->
<g id="edge4" class="edge">
<title>resume_session&#45;&gt;wait_continue</title>
<path fill="none" stroke="black" d="M108,-173.8C108,-162.16 108,-146.55 108,-133.24"/>
<polygon fill="black" stroke="black" points="111.5,-133.18 108,-123.18 104.5,-133.18 111.5,-133.18"/>
<text text-anchor="middle" x="155" y="-144.8" font-family="Times,serif" font-size="14.00">Send resume</text>
</g>
<!-- new_session -->
<g id="node4" class="node">
<title>new_session</title>
<polygon fill="none" stroke="black" stroke-dasharray="5,2" points="433,-123 327,-123 327,-87 433,-87 433,-123"/>
<text text-anchor="middle" x="380" y="-101.3" font-family="Times,serif" font-size="14.00">New session</text>
</g>
<!-- wait_continue&#45;&gt;new_session -->
<g id="edge6" class="edge">
<title>wait_continue&#45;&gt;new_session</title>
<path fill="none" stroke="black" d="M178.05,-105C220.94,-105 275.39,-105 316.94,-105"/>
<polygon fill="black" stroke="black" points="316.94,-108.5 326.94,-105 316.94,-101.5 316.94,-108.5"/>
<text text-anchor="middle" x="252.5" y="-111.8" font-family="Times,serif" font-size="14.00">Receive invalidate</text>
</g>
<!-- connected -->
<g id="node6" class="node">
<title>connected</title>
<polygon fill="none" stroke="black" points="92,-36 0,-36 0,0 92,0 92,-36"/>
<text text-anchor="middle" x="46" y="-14.3" font-family="Times,serif" font-size="14.00">Connected</text>
</g>
<!-- wait_continue&#45;&gt;connected -->
<g id="edge7" class="edge">
<title>wait_continue&#45;&gt;connected</title>
<path fill="none" stroke="black" d="M61.23,-86.99C53.95,-82.28 47.4,-76.37 43,-69 39.01,-62.33 38.18,-54.22 38.78,-46.49"/>
<polygon fill="black" stroke="black" points="42.27,-46.77 40.32,-36.36 35.35,-45.72 42.27,-46.77"/>
<text text-anchor="middle" x="93.5" y="-57.8" font-family="Times,serif" font-size="14.00">Receive ready</text>
</g>
<!-- abort -->
<g id="node8" class="node">
<title>abort</title>
<polygon fill="none" stroke="black" points="198,-36 142,-36 142,0 198,0 198,-36"/>
<text text-anchor="middle" x="170" y="-14.3" font-family="Times,serif" font-size="14.00">Abort</text>
</g>
<!-- wait_continue&#45;&gt;abort -->
<g id="edge5" class="edge">
<title>wait_continue&#45;&gt;abort</title>
<path fill="none" stroke="black" d="M131.28,-86.66C137.25,-81.45 143.3,-75.4 148,-69 153.19,-61.93 157.5,-53.45 160.88,-45.5"/>
<polygon fill="black" stroke="black" points="164.14,-46.78 164.56,-36.19 157.63,-44.21 164.14,-46.78"/>
<text text-anchor="middle" x="260.5" y="-57.8" font-family="Times,serif" font-size="14.00">Connection closed with 4003</text>
</g>
<!-- lost_connection -->
<g id="node7" class="node">
<title>lost_connection</title>
<polygon fill="none" stroke="black" stroke-dasharray="1,5" points="188.5,-488 27.5,-488 27.5,-435 188.5,-435 188.5,-488"/>
<text text-anchor="middle" x="108" y="-472.8" font-family="Times,serif" font-size="14.00">Connection lost or</text>
<text text-anchor="middle" x="108" y="-457.8" font-family="Times,serif" font-size="14.00">unexpectedly closed</text>
<text text-anchor="middle" x="108" y="-442.8" font-family="Times,serif" font-size="14.00">while reconnecting</text>
</g>
<!-- lost_connection&#45;&gt;reconnect -->
<g id="edge1" class="edge">
<title>lost_connection&#45;&gt;reconnect</title>
<path fill="none" stroke="black" d="M108,-434.95C108,-422.36 108,-407.17 108,-394.35"/>
<polygon fill="black" stroke="black" points="111.5,-394.24 108,-384.24 104.5,-394.24 111.5,-394.24"/>
<text text-anchor="middle" x="175.5" y="-405.8" font-family="Times,serif" font-size="14.00">Wait random delay</text>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 6.1 KiB

docs/assets/server_flow.dot
+21
View File
@@ -0,0 +1,21 @@
digraph "Server Connection Flow" {
node [shape=box];
graph [newrank=true];
new_connection [label="New connection"];
wait_handshake [label="Wait for handshake"];
validate_register [label="Validate register"];
validate_resume [label="Validate resume"];
connected [label="Connected"];
close [label="Close connection"];
new_connection -> wait_handshake [label="Send hello"];
wait_handshake -> validate_register [label="Receive register_*"];
wait_handshake -> validate_resume [label="Receive resume"];
wait_handshake -> close [label="Bad handshake\nTimeout"];
validate_register -> close [label="Bad token\nBad register"];
validate_register -> connected [label="Valid register:\nSend ready"];
validate_resume -> close [label="Bad token\nBad resume"];
validate_resume -> wait_handshake [label="Bad session:\nsend invalidate"];
validate_resume -> connected [label="Valid resume:\nSend continue"];
}
docs/assets/server_flow.svg
+118
View File
@@ -0,0 +1,118 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<!-- Generated by graphviz version 2.42.3 (0)
-->
<!-- Title: Server Connection Flow Pages: 1 -->
<svg width="603pt" height="335pt"
viewBox="0.00 0.00 603.00 335.00" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 331)">
<title>Server Connection Flow</title>
<polygon fill="white" stroke="transparent" points="-4,4 -4,-331 599,-331 599,4 -4,4"/>
<!-- new_connection -->
<g id="node1" class="node">
<title>new_connection</title>
<polygon fill="none" stroke="black" points="355.5,-327 224.5,-327 224.5,-291 355.5,-291 355.5,-327"/>
<text text-anchor="middle" x="290" y="-305.3" font-family="Times,serif" font-size="14.00">New connection</text>
</g>
<!-- wait_handshake -->
<g id="node2" class="node">
<title>wait_handshake</title>
<polygon fill="none" stroke="black" points="367.5,-240 212.5,-240 212.5,-204 367.5,-204 367.5,-240"/>
<text text-anchor="middle" x="290" y="-218.3" font-family="Times,serif" font-size="14.00">Wait for handshake</text>
</g>
<!-- new_connection&#45;&gt;wait_handshake -->
<g id="edge1" class="edge">
<title>new_connection&#45;&gt;wait_handshake</title>
<path fill="none" stroke="black" d="M290,-290.8C290,-279.16 290,-263.55 290,-250.24"/>
<polygon fill="black" stroke="black" points="293.5,-250.18 290,-240.18 286.5,-250.18 293.5,-250.18"/>
<text text-anchor="middle" x="328" y="-261.8" font-family="Times,serif" font-size="14.00">Send hello</text>
</g>
<!-- validate_register -->
<g id="node3" class="node">
<title>validate_register</title>
<polygon fill="none" stroke="black" points="136,-138 0,-138 0,-102 136,-102 136,-138"/>
<text text-anchor="middle" x="68" y="-116.3" font-family="Times,serif" font-size="14.00">Validate register</text>
</g>
<!-- wait_handshake&#45;&gt;validate_register -->
<g id="edge2" class="edge">
<title>wait_handshake&#45;&gt;validate_register</title>
<path fill="none" stroke="black" d="M212.45,-216.73C161.73,-212.32 101.57,-203.5 83,-186 72.82,-176.41 68.91,-161.41 67.62,-148.23"/>
<polygon fill="black" stroke="black" points="71.11,-147.99 67.07,-138.2 64.12,-148.37 71.11,-147.99"/>
<text text-anchor="middle" x="148.5" y="-167.3" font-family="Times,serif" font-size="14.00">Receive register_*</text>
</g>
<!-- validate_resume -->
<g id="node4" class="node">
<title>validate_resume</title>
<polygon fill="none" stroke="black" points="356,-138 224,-138 224,-102 356,-102 356,-138"/>
<text text-anchor="middle" x="290" y="-116.3" font-family="Times,serif" font-size="14.00">Validate resume</text>
</g>
<!-- wait_handshake&#45;&gt;validate_resume -->
<g id="edge3" class="edge">
<title>wait_handshake&#45;&gt;validate_resume</title>
<path fill="none" stroke="black" d="M241.15,-203.81C233.91,-199.14 227.42,-193.3 223,-186 216.09,-174.6 216.09,-167.4 223,-156 225.69,-151.55 229.16,-147.65 233.09,-144.22"/>
<polygon fill="black" stroke="black" points="235.23,-146.98 241.15,-138.19 231.04,-141.38 235.23,-146.98"/>
<text text-anchor="middle" x="280" y="-167.3" font-family="Times,serif" font-size="14.00">Receive resume</text>
</g>
<!-- close -->
<g id="node6" class="node">
<title>close</title>
<polygon fill="none" stroke="black" points="450.5,-36 313.5,-36 313.5,0 450.5,0 450.5,-36"/>
<text text-anchor="middle" x="382" y="-14.3" font-family="Times,serif" font-size="14.00">Close connection</text>
</g>
<!-- wait_handshake&#45;&gt;close -->
<g id="edge4" class="edge">
<title>wait_handshake&#45;&gt;close</title>
<path fill="none" stroke="black" d="M367.54,-213.17C403.99,-207.82 442.06,-199.24 454,-186 493.43,-142.29 497.55,-103.05 465,-54 461.92,-49.36 458.09,-45.33 453.78,-41.84"/>
<polygon fill="black" stroke="black" points="455.76,-38.95 445.55,-36.1 451.76,-44.69 455.76,-38.95"/>
<text text-anchor="middle" x="540.5" y="-123.8" font-family="Times,serif" font-size="14.00">Bad handshake</text>
<text text-anchor="middle" x="540.5" y="-108.8" font-family="Times,serif" font-size="14.00">Timeout</text>
</g>
<!-- connected -->
<g id="node5" class="node">
<title>connected</title>
<polygon fill="none" stroke="black" points="127,-36 35,-36 35,0 127,0 127,-36"/>
<text text-anchor="middle" x="81" y="-14.3" font-family="Times,serif" font-size="14.00">Connected</text>
</g>
<!-- validate_register&#45;&gt;connected -->
<g id="edge6" class="edge">
<title>validate_register&#45;&gt;connected</title>
<path fill="none" stroke="black" d="M34.78,-101.9C28.43,-97.02 22.64,-91.06 19,-84 12.9,-72.14 12.28,-65.51 19,-54 21.52,-49.69 24.78,-45.87 28.5,-42.49"/>
<polygon fill="black" stroke="black" points="30.9,-45.05 36.66,-36.16 26.61,-39.52 30.9,-45.05"/>
<text text-anchor="middle" x="70" y="-72.8" font-family="Times,serif" font-size="14.00">Valid register:</text>
<text text-anchor="middle" x="70" y="-57.8" font-family="Times,serif" font-size="14.00">Send ready</text>
</g>
<!-- validate_register&#45;&gt;close -->
<g id="edge5" class="edge">
<title>validate_register&#45;&gt;close</title>
<path fill="none" stroke="black" d="M136,-111.34C178.72,-105.47 229.03,-96.29 247,-84 260.03,-75.09 253.43,-63.55 266,-54 277.12,-45.55 290.27,-39.13 303.66,-34.26"/>
<polygon fill="black" stroke="black" points="305.06,-37.48 313.42,-30.97 302.83,-30.84 305.06,-37.48"/>
<text text-anchor="middle" x="311" y="-72.8" font-family="Times,serif" font-size="14.00">Bad token</text>
<text text-anchor="middle" x="311" y="-57.8" font-family="Times,serif" font-size="14.00">Bad register</text>
</g>
<!-- validate_resume&#45;&gt;wait_handshake -->
<g id="edge8" class="edge">
<title>validate_resume&#45;&gt;wait_handshake</title>
<path fill="none" stroke="black" d="M321.89,-138.33C327.98,-143.2 333.53,-149.09 337,-156 342.98,-167.92 342.98,-174.08 337,-186 335.05,-189.89 332.44,-193.45 329.44,-196.69"/>
<polygon fill="black" stroke="black" points="326.86,-194.32 321.89,-203.67 331.61,-199.46 326.86,-194.32"/>
<text text-anchor="middle" x="395.5" y="-174.8" font-family="Times,serif" font-size="14.00">Bad session:</text>
<text text-anchor="middle" x="395.5" y="-159.8" font-family="Times,serif" font-size="14.00">send invalidate</text>
</g>
<!-- validate_resume&#45;&gt;connected -->
<g id="edge9" class="edge">
<title>validate_resume&#45;&gt;connected</title>
<path fill="none" stroke="black" d="M223.7,-112.81C197.28,-107.96 167.61,-99.33 144,-84 129.91,-74.85 132.49,-66.26 121,-54 117.46,-50.23 113.56,-46.43 109.61,-42.79"/>
<polygon fill="black" stroke="black" points="111.86,-40.11 102.07,-36.06 107.2,-45.33 111.86,-40.11"/>
<text text-anchor="middle" x="195.5" y="-72.8" font-family="Times,serif" font-size="14.00">Valid resume:</text>
<text text-anchor="middle" x="195.5" y="-57.8" font-family="Times,serif" font-size="14.00">Send continue</text>
</g>
<!-- validate_resume&#45;&gt;close -->
<g id="edge7" class="edge">
<title>validate_resume&#45;&gt;close</title>
<path fill="none" stroke="black" d="M336.9,-101.98C345.46,-97.2 353.66,-91.27 360,-84 369.17,-73.47 374.56,-58.96 377.7,-46.32"/>
<polygon fill="black" stroke="black" points="381.18,-46.78 379.85,-36.27 374.33,-45.32 381.18,-46.78"/>
<text text-anchor="middle" x="418" y="-72.8" font-family="Times,serif" font-size="14.00">Bad token</text>
<text text-anchor="middle" x="418" y="-57.8" font-family="Times,serif" font-size="14.00">Bad resume</text>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 7.3 KiB

docs/socket.md
+214 -50
View File
@@ -1,4 +1,4 @@
Socket.io protocol
WebSocket protocol
==================
<sub>This document describes the protocol used between the master and
@@ -7,64 +7,211 @@ to be very useful for you.</sub>
The protocol is built on WebSocket and uses JSON encoded text messages
to communicate to and from the master server. These messages contain
the following three properties:
- seq - integer - The sequence number of the message
a JSON encoded object with the following three properties:
- seq - integer|null - The sequence number of the message
- type - string - The type of message
- data - object - Message data
For the handshake and heartbeats the seq property is null.
Handshake
---------
Upon connecting to the WebSocket endpoint the server will respond with a
`hello` message containing the master server version as the data. The
client is then expected to send a `register_slave` or `register_control`
message back depending on what kind of client it is. If the register
operation is successful the master will reply with a `ready` message
with an empty data payload.
client is then expected to send either `register_*` or a `resume`
message to set up the connection with the master. The exact flow is
described below.
**Master `hello` data**
### Client connection flow
For a client making a new connecting to the master server the first
course of action after obtaining a token is to connect with WebSocket to
the /api/socket endpoint and wait for the server hello. After the
hello, the client sends a register\_\* message corresponding to the type
of the client. If successful the server will reply with a ready message
and the connection has been established. See the flowchart below.
![Client connection flowchart](assets/connection_flow.svg)
If for any reason the connection is closed or lost during the connection
attempt the process is started over after a random delay, with the
exception of the connection being closed with a 4003 code which means
authentication failed and the client should stop trying to connect. The
wait for hello and wait for ready states should also have a timeout,
after which the connection is closed if the hello/reply wasn't received
in time.
### Client reconnection flow
If the connection is lost after the client has received the ready
message from the server it should attempt a reconnect instead. This is
very similar to a new connection, but instead of a register\_\* messsage
the client sends a resume message and the server replies with a continue
message if resuming the session was successful. This continue message
has sequence number of the last message the master received and the
client is expected to imediately send all messages from its send buffer
that occured after the sequence number given.
The server will respond with an invalidate message if resuming the
session is not possible. In this case the client should continue from
the new session state of the connection flow (see the previous section).
![Client reconnection flowchart](assets/reconnection_flow.svg)
Like the connection flow the process starts over after a delay if the
connection is closed or lost. The same exception for connection closed
with 4003 applies as with the connection flow. Additionally after an
invalidate message has been received and the connection flow fails, it
should start over from the connection flow.
### Server connection flow
From the master server side new connections and reconnection attempts
start out the same, a new connection has been made and the server sends
the hello messoge to the client. If a vaild register\_\* message is
received a new connection is set up for the client and the ready message
is sent in reply. If a valid resume message is received and the session
that's attempted to be resumed is active it'll signal that the session
is resumed with a continue message and the client upon receiving the
continue message should send the messages from its send buffer from
after the last\_seq given in the continue message.
If the session token used to resume is not valid it'll send an invalidate
message and expect the client to continue with a registr\_\* message. If an
invalid or unexpected message is received during the handshake the connection
is closed. See the flowchart below.
![Client connection flowchart](assets/server_flow.svg)
For authentication failure the connection is closed with code 4003,
which the connection client should interpret as connecting will never
succeed and stop trying to reconnect.
### Connection tracking and heartbeats
Once connected as indicated by the ready and continue messages from the
master server the client must start sending heartbeats at the interval
given in the ready/continue message, and storing messages sent over the
link in a sending side buffer. The heartbeats sent must have the seq of
the last received message on the link and acts as an ack for received
messages. The master server also sends heartbeats in the same manner
and the client should remove messages up to and including the sequence
number received in the heartbeat from its send buffer.
Handshake messages
------------------
### `hello`
Sent by the master server when the WebSocket connection has been opened.
- version - string - The version of the master, e.g. "2.0.0".
**Slave `register_slave` response data**
### `register_slave`
- agent - string - Human readable identifying the software connecting.
Slave handshake for establishing a new connection session.
- token - string - Authentication token.
- agent - string - Human readable string of the software connecting.
- version - string - The protocol version of the slave, e.g. "2.0.0".
- name - string - Name of the slave.
- id - integer - ID of the slave.
**Control `register_control repsonse data**
### `register_control`
- agent - string - Human readable identifying the software connecting.
Control handshake for enstablishing a new connection session.
- token - string - Authentication token.
- agent - string - Human readable string of the software connecting.
- version - string - The protocol version of the control, e.g. "2.0.0".
**Example handshake**
```js
// Server hello
({
type: 'hello',
data: {
version: "2.0.0",
}
})
### `resume`
// Client response
({
type: 'register_slave',
data: {
agent: 'Clusterio Slave',
version: "2.0.0",
name: "Foo",
id: 123,
}
})
Handshake for resuming an existing connection session where the
connection had dropped unexpectedly.
// Server repsonse
({
type: 'ready',
data: {}
})
- session_token - string - Session token to resume session with.
- last_seq - integer|null - Last message seq received by the client.
### `invalidate`
Sent by the master server in reply to a resume for a session that cannot
be resumed. The client should start a new session with a register\_\*
message in response.
### `continue`
Sent by the master server to signal resuming the session was successful.
- last_seq - integer|null - Last message seq received by the server.
- heartbeat_interval - number - Rate client should send heartbeats at.
### `ready`
Sent by the master server to signal the connection has been established
with a new session.
- session_token - string - Token to resume the session with.
- heartbeat_interval - number - Rate client should send heartbeats at.
### `heartbeat`
Sent by both the master server and clients at the interval given in the
continue/ready messages.
- last_seq - integer - Last message seq received over the link.
Example handshake
-----------------
Server hello after opening the WebSocket.
```json
{
"seq": null,
"type": "hello",
"data": {
"version": "2.0.0"
}
}
```
Slave registering a new connection session.
```json
{
"seq": null,
"type": "register_slave",
"data": {
"token": "xyzabc123",
"agent": "Clusterio Slave",
"version": "2.0.0",
"name": "Foo",
"id": 123
}
}
```
Master server reply signaling the session is established and ready.
```json
{
"seq": null,
"type": "ready",
"data": {
"session_token": "xyz.azy.bzy",
"heartbeat_interval": 45
}
}
```
@@ -73,11 +220,11 @@ Events
Events are messages send in one direction over the link, that the
receiver is expected to act upon, but not make any replies (for that
there's [Requests](#requests). By convention a message types ending in
`*_event` is an event. The data payload of an event consists solely of
there's [Requests](#requests). By convention message types ending in
`*_event` are events. The data payload of an event consists solely of
event specific properties.
See [lib/link/messages.js](lib/link/messages.js) for the recognized
See [lib/link/messages.js](/lib/link/messages.js) for the recognized
events and their contents.
@@ -87,30 +234,47 @@ Requests
Requests are messages which expects a response in return. They function
a lot like HTTP requests except that both parties of a connection can
inniate a request that is expected to be responded to. By convention
message types ending in `*_request` is a request, and is expected to be
message types ending in `*_request` are requests, and is expected to be
replied to with a corresponding `*_response` message. If an error
occured while processing the request an error response containing an
error property in the data payload is sent instead.
**`*_response` data**
See [lib/link/messages.js](/lib/link/messages.js) for the recognized
requests and their contents.
### `*_response`
- seq - integer - The seq number of the request message.
- additional properties may be present depending on the type of request.
**`*_resopnse` error data**
### `*_response` error
- seq - integer - The seq number of the request message.
- error - string - Human readable text message describing the error.
See [lib/link/messages.js](lib/link/messages.js) for the recognized
requests and their contents.
Graceful shutdown
-----------------
Closing the connection
----------------------
To ensure no data is lost when closing the link it's important to
send connection shutdown request and wait for it to complete before
closing the WebSocket. Either party of the link can send this request
and the party sending the request should be the one that initates the
close of the WebSocket.
The party that innitates the shut down of the connection should send a
close message containing the reason for closing before disconnecting.
### `connection_shutdown_request`
**`close` data**
Request graceful shutdown of the connection. The receiving party should
stop tasks that send events and requests over the link and wait for
pending requests it has over the link to complete, but it should not
stop responding to requests sent to it. Only after this process is done
should the response be sent.
- reason - string - Reason for closing the connection
### `connection_shutdown_response`
Signals the shutdown is complete and that the connection can be closed.
The party that initiatied the shutdown request may still send other
requests/events over the link to finish its part of the shutdown.
If the error response variant is received it should be logged if
possible and treated as a success.
lib/config/definitions.js
+14
View File
@@ -73,6 +73,13 @@ MasterGroup.define({
return bytes.toString("base64");
},
});
MasterGroup.define({
name: "heartbeat_interval",
title: "Heartbeat Interval",
description: "Interval heartbeats are sent out on WebSocket connections",
type: "number",
initial_value: 60,
});
MasterGroup.finalize();
class MasterConfig extends classes.Config { }
@@ -123,6 +130,13 @@ SlaveGroup.define({
type: "string",
initial_value: "localhost",
});
SlaveGroup.define({
name: "reconnect_delay",
title: "Reconnect Delay",
description: "Maximum delay to wait before attempting to reconnect WebSocket",
type: "number",
initial_value: 5,
});
SlaveGroup.finalize();
class SlaveConfig extends classes.Config { }
lib/errors.js
+4
View File
@@ -14,6 +14,9 @@ class InvalidMessage extends Error {
}
}
// Thrown when WebSocket authentication failed
class AuthenticationFailed extends Error { }
// Errror class for known errors occuring during startup
class StartupError extends Error { }
@@ -33,6 +36,7 @@ module.exports = {
CommandError,
RequestError,
InvalidMessage,
AuthenticationFailed,
StartupError,
EnvironmentError,
PluginError,
lib/link/connectors.js
+235 -50
View File
@@ -3,19 +3,122 @@ const events = require("events");
const WebSocket = require("ws");
const schema = require("lib/schema");
const errors = require("lib/errors");
class WebSocketBaseConnector extends events.EventEmitter {
constructor() {
super();
this._socket = null;
this._seq = 1;
this._lastHeartbeat = null;
this._heartbeatId = null;
this._heartbeatInterval = null;
this._lastReceivedSeq = null;
this._sendBuffer = [];
}
_dropSendBufferSeq(seq) {
if (seq === null) {
return;
}
let dropCount = 0;
for (let index = 0; index < this._sendBuffer.length; index++) {
if (this._sendBuffer[index].seq <= seq) {
dropCount++;
} else {
break;
}
}
this._sendBuffer.splice(0, dropCount);
}
_doHeartbeat() {
this._check(["connected"]);
if (Date.now() - this._lastHeartbeat > 2000 * this._heartbeatInterval) {
console.log("SOCKET | closing after heartbeat timed out");
this._socket.close(1008, "Heartbeat timeout");
clearInterval(this._heartbeatId);
this._heartbeatId = null;
} else {
this._socket.send(JSON.stringify({
seq: null,
type: "heartbeat",
data: { seq: this._lastReceivedSeq },
}));
}
}
_processHeartbeat(message) {
if (!schema.heartbeat(message)) {
console.log("SOCKET | closing after received invalid heartbeat:", message);
this._socket.close(1002, "Invalid heartbeat");
console.log(schema.heartbeat.errors);
return;
}
this._lastHeartbeat = Date.now();
this._dropSendBufferSeq(message.data.seq);
}
/**
* Start sending heartbeats
*/
startHeartbeat() {
this._lastHeartbeat = Date.now();
this._heartbeatId = setInterval(() => {
this._doHeartbeat();
}, this._heartbeatInterval * 1000);
}
/**
* Stop sending heartbeats
*/
stopHeartbeat() {
if (this._heartbeatId) {
clearInterval(this._heartbeatId);
this._heartbeatId = null;
}
}
/**
* Send a message over the socket
*
* @returns the sequence number of the message sent
*/
send(type, data) {
this._check(["handshake", "connected"]);
let message = { seq: this._seq, type, data };
this._sendBuffer.push(message);
if (this._state === "connected") {
this._socket.send(JSON.stringify(message));
}
return this._seq++;
}
}
/**
* Connector for master server clients
*/
class WebSocketClientConnector extends events.EventEmitter {
constructor(url) {
class WebSocketClientConnector extends WebSocketBaseConnector {
constructor(url, reconnectDelay) {
super();
this._url = url;
this._seq = 1;
// The following states are used in the client connector
// new: Not connected
// handshake: Attempting to (re)connect to server.
// connected: Connection is online
// closing: Connection is in the process of being closed.
this._state = "new";
this._reconnectId = null;
this._sessionToken = null;
this._connected = false;
this._reconnectDelay = reconnectDelay;
}
_check(expectedStates) {
@@ -27,23 +130,38 @@ class WebSocketClientConnector extends events.EventEmitter {
}
/**
* Send a message over the socket
* Send a handshake message over the socket
*
* @returns the sequence number of the message sent
* Used in the register function in order to send the register handshake
* message over the WebSocket.
*/
send(type, data) {
this._socket.send(JSON.stringify({ seq: this._seq, type, data }));
return this._seq++;
sendHandshake(type, data) {
this._check(["handshake"]);
this._socket.send(JSON.stringify({ seq: null, type, data }));
}
/**
* Close the connection with the given reason.
*
* Sends a close message and disconnects the connector.
* Sends a close frame and disconnects the connector.
*/
close(reason) {
this.send('close', { reason });
this._socket.close();
close(code, reason) {
if (this._state === "new") {
return;
}
if (this._reconnectId) {
clearTimeout(this._reconnectId);
this._reconnectId = null;
}
this.stopHeartbeat();
if (["handshake", "connected"].includes(this._state)) {
this._socket.close(code, reason);
}
this._state = "closing";
}
/**
@@ -53,6 +171,13 @@ class WebSocketClientConnector extends events.EventEmitter {
this._check(["new"]);
this._state = "handshake";
this._doConnect();
// Wait for the connection to be ready
await events.once(this, "connected");
}
async _doConnect() {
let url = new URL(this._url);
url.pathname += "api/socket";
@@ -66,27 +191,73 @@ class WebSocketClientConnector extends events.EventEmitter {
});
this._attachSocketHandlers();
}
// Wait for the connection to be ready
await events.once(this, 'ready');
/**
* Attempt re-establishing the connecting again
*/
reconnect() {
if (this._reconnectId !== null) {
return;
}
let delay = Math.random() * this._reconnectDelay;
console.log(`SOCKET | waiting ${delay.toLocaleString("en", { maximumFractionDigits: 2 })} seconds for reconnect`)
this._reconnectId = setTimeout(() => {
this._reconnectId = null;
this._doConnect();
}, delay * 1000);
}
_attachSocketHandlers() {
this._socket.on("close", (code, reason) => {
console.log(`SOCKET | Close (code: ${code}, reason: ${reason})`);
// Authentication failed
if (code === 4003) {
this.emit("error", new errors.AuthenticationFailed("Authentication failed"));
this._state = "closing";
}
if (this._state === "closing") {
this._lastReceivedSeq = null;
this._sessionToken = null;
this._sendBuffer.length = 0;
this._state = "new";
if (this._connected) {
this.emit("close");
this._connected = false;
}
} else {
this._state = "handshake";
this.reconnect();
if (this._connected) {
this.emit("dropped");
this._connected = false;
}
}
this.stopHeartbeat();
});
this._socket.on("error", err => {
// It's assumed that close is always called by ws
console.error("SOCKET | Error:", err);
this._socket.close();
});
/*
this._socket.on("unexpected-response", (req, res) => {
let data = "";
res.on('data', (chunk) => { data += chunk; });
res.on('end', () => {
console.error(`SOCKET | Unexpeced Response ${res.statusCode}`);
console.error(data);
this._socket.emit("error", new Error(`Unexpected Response ${res.statusCode}`));
});
});
*/
this._socket.on("open", () => {
console.log("SOCKET | Open");
@@ -103,42 +274,73 @@ class WebSocketClientConnector extends events.EventEmitter {
let message = JSON.parse(data);
if (this._state === "handshake") {
this._processHandshake(message);
} else if (this._state === "ready") {
this.emit("message", message);
} else if (["connected", "closing"].includes(this._state)) {
if (message.seq !== null) {
this._lastReceivedSeq = message.seq;
}
if (message.type === "heartbeat") {
this._processHeartbeat(message);
} else {
this.emit("message", message);
}
} else {
throw new Error(`Received message in unexpected state ${this._state}`);
}
});
}
/**
* Immediatly close the connection
*/
disconnect() {
this._socket.terminate();
}
_processHandshake(message) {
if (!schema.serverHandshake(message)) {
console.log("SOCKET | closing after received invalid handshake:", message);
this.close("Invalid handshake");
this._socket.close(1002, "Invalid handshake");
return;
}
let { seq, type, data } = message;
if (type == 'hello') {
console.log(`SOCKET | received hello from master version ${data.version}`);
this.register();
if (this._sessionToken) {
console.log("SOCKET | Attempting resume");
this.sendHandshake("resume", {
session_token: this._sessionToken,
last_seq: this._lastReceivedSeq,
});
} else {
this.register();
}
} else if (type === 'ready') {
console.log("SOCKET | received ready from master");
this._state = "ready";
this.emit('ready');
this._state = "connected";
this._sessionToken = data.session_token;
this._heartbeatInterval = data.heartbeat_interval;
this.startHeartbeat();
this._connected = true;
this.emit("connected");
} else if (type === 'close') {
console.log(`SOCKET | received close from server: ${data.reason}`);
this.emit('error', new Error(`server closed during handshake: ${data.reason}`));
this._socket.terminate();
} else if (type === "continue") {
console.log("SOCKET | resuming existing session");
this._state = "connected";
this._heartbeatInterval = data.heartbeat_interval;
this.startHeartbeat();
this._dropSendBufferSeq(data.last_seq);
for (let message of this._sendBuffer) {
this._socket.send(JSON.stringify(message));
}
this._connected = true;
this.emit("connected");
} else if (type === "invalidate") {
this._heartbeatInterval = null;
this._lastReceivedSeq = null;
this._sessionToken = null;
this._sendBuffer.length = 0;
this.emit("invalidate");
this.register();
}
}
@@ -186,27 +388,10 @@ class VirtualConnector extends events.EventEmitter {
this.other.emit('message', { seq: this._seq, type, data });
return this._seq++;
}
/**
* Close the connection with the given reason.
*
* Sends a close message to the other end of the connector with the
* given reason. Doesn't actually disconnect as virtual connectors
* doen't have a concept of disconnecting.
*/
close(reason) {
this.send('close', { reason });
}
/**
* Does nothing
*
* Virtual connectors do not have a concept of disconnecting.
*/
disconnect() { }
}
module.exports = {
WebSocketBaseConnector,
WebSocketClientConnector,
VirtualConnector,
};
lib/link/link.js
+40 -8
View File
@@ -32,7 +32,6 @@ class Link {
this.processMessage(payload);
} catch (err) {
if (err instanceof errors.InvalidMessage) {
this.connector.close(`Invalid message: ${err.message}`);
console.error(`Invalid message on ${this.source}-${this.target} link: ${err.message}`)
if (err.errors) {
console.error(err.errors);
@@ -43,10 +42,15 @@ class Link {
}
});
this.setHandler('close', message => {
console.log(`SOCKET | Received close: ${message.data.reason}`);
this.connector.disconnect();
}, schema.close);
connector.on("invalidate", () => {
for (let waiterType of this._waiters.values()) {
for (let waiter of waiterType) {
waiter.reject(new Error("Session lost"));
}
}
this._waiters.clear();
});
}
/**
@@ -185,13 +189,41 @@ class Link {
throw new Error(`No validator for ${type} on ${this.source}-${this.target}`);
}
return new Promise((resolve, reject) => {
let waiter = { data };
waiter.promise = new Promise((resolve, reject) => {
waiter.resolve = resolve;
waiter.reject = reject;
if (!this._waiters.has(type)) {
this._waiters.set(type, [{ data, resolve }]);
this._waiters.set(type, [waiter]);
} else {
this._waiters.get(type).push({ data, resolve });
this._waiters.get(type).push(waiter);
}
});
return waiter.promise;
}
/**
* Handle ping requests
*/
async pingRequestHandler() { }
/**
* Prepare connection for close
*
* Waits for all pending requests on the connection to resolve.
* Sub-classes should implement this handler with code that prevents
* additional requests to be sent out before calling the super class
* method.
*/
async shutdownConnectionRequestHandler() {
let promises = [];
for (let waiterType of this._waiters.values()) {
for (let waiter of waiterType) {
promises.push(waiter.promise.then(() => {}, () => {}));
}
}
await Promise.all(promises);
}
}
lib/link/messages.js
+12
View File
@@ -183,6 +183,18 @@ class Request {
let messages = {}
// Connection requests
let wsLinks = ["master-control", "control-master", "master-slave", "slave-master"];
messages.shutdownConnection = new Request({
type: "shutdown_connection",
links: wsLinks,
});
messages.ping = new Request({
type: "ping",
links: wsLinks,
});
// Management requests
messages.listSlaves = new Request({
type: 'list_slaves',
lib/schema.js
+58 -40
View File
@@ -4,27 +4,9 @@ const ajv = new (require('ajv'))({
extendRefs: "fail",
strictDefaluts: true,
strictKeywords: true,
})
});
const closeSchema = {
$schema: "http://json-schema.org/draft-07/schema#",
title: "Close Message",
description: "Format for a close messages on a link",
properties: {
"type": { const: "close" },
"data": {
additionalProperties: false,
required: ["reason"],
properties: {
"reason": { type: "string" },
}
},
},
}
const close = ajv.compile(closeSchema);
/**
* The base format for any message sent over the link
*/
@@ -36,28 +18,34 @@ const messageSchema = {
additionalProperties: false,
required: ["seq", "type", "data"],
properties: {
"seq": { type: "integer" },
"seq": { type: ["integer", "null"] },
"type": { type: "string" },
"data": { type: "object" },
},
}
};
const message = ajv.compile(messageSchema);
const serverActions = [
{},
];
const actionResponse = {
const heartbeat = ajv.compile({
$schema: "http://json-schema.org/draft-07/schema#",
title: "Heartbeat",
description: "Heartbeat sent over link",
type: "object",
additionalProperties: false,
required: ["action", "id", "status", "message"],
required: ["seq", "type", "data"],
properties: {
"action": { enum: ["TODO"] },
"id": { type: "integer" },
"status": { enum: ["started", "done", "error"] },
"message": { type: "string" },
"seq": { const: null },
"type": { const: "heartbeat" },
"data": {
type: "object",
additionalProperties: false,
required: ["seq"],
properties: {
"seq": { type: ["integer", "null"] },
},
},
},
};
});
const serverHandshake = ajv.compile({
$schema: "http://json-schema.org/draft-07/schema#",
@@ -82,15 +70,33 @@ const serverHandshake = ajv.compile({
"type": { const: "ready" },
"data": {
additionalProperties: false,
/* XXX anything here?
required: ["version"],
required: ["session_token", "heartbeat_interval"],
properties: {
"version": { type: "string" },
}*/
"session_token": { type: "string" },
"heartbeat_interval": { type: "number" },
},
},
},
},
closeSchema,
{
properties: {
"type": { const: "continue" },
"data": {
additionalProperties: false,
required: ["last_seq", "heartbeat_interval"],
properties: {
"last_seq": { type: ["integer", "null"] },
"heartbeat_interval": { type: "number" },
},
},
},
},
{
properties: {
"type": { const: "invalidate" },
"data": { const: {} },
},
},
],
});
@@ -130,7 +136,19 @@ const clientHandshake = ajv.compile({
},
},
},
closeSchema,
{
properties: {
"type": { const: "resume" },
"data": {
additionalProperties: false,
required: ["session_token", "last_seq"],
properties: {
"session_token": { type: "string" },
"last_seq": { type: ["integer", "null"] },
},
},
},
},
],
});
@@ -139,8 +157,8 @@ module.exports = {
compile: ajv.compile.bind(ajv),
message,
close,
heartbeat,
serverHandshake,
clientHandshake,
}
};
master.js
+252 -75
View File
@@ -222,11 +222,15 @@ async function shutdown() {
}
for (let controlConnection of controlConnections) {
controlConnection.connector.close("shutdown");
controlConnection.connector.close(1001, "Server Quit");
}
for (let slaveConnection of slaveConnections.values()) {
slaveConnection.connector.close("shutdown");
slaveConnection.connector.close(1001, "Server Quit");
}
for (let socket of pendingSockets) {
socket.close(1001, "Server Quit");
}
if (httpServer) {
@@ -537,6 +541,12 @@ class SlaveConnection extends BaseConnection {
}
}
}
async shutdown(code, reason) {
await link.messages.shutdownConnection.send(this);
this.connector.close(code, reason);
await events.once(this.connector, "close");
}
}
const wss = new WebSocket.Server({
@@ -559,110 +569,277 @@ function isInteger(value) {
/**
* Connector for master server connections
*/
class WebSocketServerConnector extends events.EventEmitter {
constructor(socket) {
class WebSocketServerConnector extends link.WebSocketBaseConnector {
constructor(socket, sessionId) {
super();
this._seq = 1;
this._socket = socket;
this._sessionId = sessionId;
this._socket.on('message', message => {
this.emit('message', JSON.parse(message));
});
this._socket.on('close', (code, reason) => {
this.emit('disconnect');
});
// The following states are used in the server connector
// handshake: Wating for client to (re)connect.
// connected: Connection is online
// closing: Connection is in the process of being closed.
// closed: Connection has been closed
this._state = "handshake";
}
_check(expectedStates) {
if (!expectedStates.includes(this._state)) {
throw new Error(
`Expected state ${expectedStates} but state is ${this._state}`
);
}
}
/**
* Send a message over the socket
* Send ready over the socket
*
* @returns the sequence number of the message sent
* Sends the ready message over the socket to initiate the session.
*/
send(type, data = {}) {
this._socket.send(JSON.stringify({ seq: this._seq, type, data }));
return this._seq++;
ready(sessionToken) {
this._heartbeatInterval = masterConfig.get("master.heartbeat_interval")
this._socket.send(JSON.stringify({
seq: null,
type: "ready",
data: {
session_token: sessionToken,
heartbeat_interval: this._heartbeatInterval,
},
}));
this._state = "connected";
this._attachSocketHandlers();
}
/**
* Continue connection with the given socket
*
* Terminates the current socket and contiunes the session over the
* socket given from the message sequence given.
*/
continue(socket, lastSeq) {
this._socket.terminate();
this._socket = socket;
this._heartbeatInterval = masterConfig.get("master.heartbeat_interval")
this._socket.send(JSON.stringify({
seq: null,
type: "continue",
data: {
last_seq: this._lastReceivedSeq,
heartbeat_interval: this._heartbeatInterval,
},
}));
this._state = "connected";
this._attachSocketHandlers();
this._dropSendBufferSeq(lastSeq);
for (let message of this._sendBuffer) {
this._socket.send(JSON.stringify(message));
}
}
_attachSocketHandlers() {
this.startHeartbeat();
this._socket.on("close", (code, reason) => {
console.log(`SOCKET | Close (code: ${code}, reason: ${reason})`);
if (this._state === "closing") {
this._lastReceivedSeq = null;
this._sendBuffer.length = 0;
this._state = "closed";
activeConnections.delete(this._sessionId);
if (this._connected) {
this.emit("close");
this._connected = false;
}
} else {
this._state = "handshake";
if (this._connected) {
this.emit("dropped");
this._connected = false;
}
}
this.stopHeartbeat();
});
this._socket.on("error", err => {
// It's assumed that close is always called by ws
console.error("SOCKET | Error:", err);
});
this._socket.on("open", () => {
console.log("SOCKET | Open");
});
this._socket.on("ping", data => {
console.log(`SOCKET | Ping (data: ${data}`);
});
this._socket.on("pong", data => {
console.log(`SOCKET | Pong (data: ${data}`);
});
// Handle messages
this._socket.on("message", data => {
let message = JSON.parse(data);
if (["connected", "closing"].includes(this._state)) {
if (message.seq !== null) {
this._lastReceivedSeq = message.seq;
}
if (message.type === "heartbeat") {
this._processHeartbeat(message);
} else {
this.emit("message", message);
}
} else {
throw new Error(`Received message in unexpected state ${this._state}`);
}
});
}
/**
* Close the connection with the given reason.
*
* Sends a close message and disconnects the connector.
* Sends a close frame and disconnects the connector.
*/
close(reason) {
this.send('close', { reason });
this._socket.close();
}
close(code, reason) {
if (this._state === "new") {
return;
}
/**
* Immediatly close the connection
*/
disconnect() {
this._socket.terminate();
this.stopHeartbeat();
if (["handshake", "connected"].includes(this._state)) {
this._socket.close(code, reason);
}
this._state = "closing";
}
}
wss.on('connection', function (socket, req) {
console.log(`SOCKET | new connection from ${socket.remoteAddress}`);
let pendingSockets = new Set();
let activeConnectors = new Map();
// Start connection handshake
let connector = new WebSocketServerConnector(socket);
connector.send('hello', { version });
wss.on("connection", function (socket, req) {
console.log(`SOCKET | new connection from ${req.socket.remoteAddress}`);
// Handle socket handshake
socket.once('message', (payload) => {
payload = JSON.parse(payload);
// Start connection handshake.
socket.send(JSON.stringify({ seq: null, type: "hello", data: { version }}));
// XXX prometheusWsUsageCounter.labels('message', "other").inc();
if (!schema.clientHandshake(payload)) {
console.log(`SOCKET | closing ${req.socket.remoteAddress} after receiving invalid handshake`, payload);
connector.close("Invalid handshake");
return;
}
function attachHandler() {
pendingSockets.add(socket);
let { seq, type, data } = payload;
if (["register_slave", "register_control"].includes(type)) {
authenticate.check(data.token).then((result) => {
if (result.ok) {
if (type === "register_slave") {
let connection = slaveConnections.get(data.id);
if (connection) {
console.log(`SOCKET | disconecting existing connection for slave ${data.id}`);
connection.connector.close("Registered from another connection");
}
let timeoutId = setTimeout(() => {
console.log(`SOCKET | closing ${req.socket.remoteAddress} after timing out on handshake`);
socket.close(1008, "Handshake timeout");
pendingSockets.delete(socket);
}, 30*1000);
console.log(`SOCKET | registered slave ${data.id} version ${data.version}`);
slaveConnections.set(data.id, new SlaveConnection(data, connector));
connector.send('ready');
} else { // type === "register_control"
console.log(`SOCKET | registered control from ${req.socket.remoteAddress}`);
controlConnections.push(new ControlConnection(data, connector));
connector.send('ready');
}
} else {
console.error(`SOCKET | authentication failed for ${req.socket.remoteAddress}`);
connector.close("Authentication failed");
}
}).catch((err) => {
socket.once("message", (message) => {
clearTimeout(timeoutId);
pendingSockets.delete(socket);
handleHandshake(
message, socket, req, attachHandler
).catch(err => {
console.error(`
+---------------------------------------------------------------------+
| Unexpected error occured while authenticating client, please report |
| it to https://github.com/clusterio/factorioClusterio/issues |
+---------------------------------------------------------------------+`
+----------------------------------------------------------------+
| Unexpected error occured in WebSocket handshake, please report |
| it to https://github.com/clusterio/factorioClusterio/issues |
+----------------------------------------------------------------+`
);
console.error(err);
connection.close("Unexpected error");
socket.close(1011, "Unexpected error");
});
});
}
} else if (type === "close") {
console.log(`SOCKET | received close from ${req.socket.remoteAddress}: ${data.reason}`);
connector.disconnect();
attachHandler();
});
let nextSessionId = 1;
// Unique string for the session token audience
let masterSession = `session-${Date.now()}`;
// Handle socket handshake
async function handleHandshake(message, socket, req, attachHandler) {
try {
message = JSON.parse(message);
} catch (err) {
console.log(`SOCKET | closing ${req.socket.remoteAddress} after receiving invalid JSON`);
socket.close(1001, "Invalid JSON");
return;
}
if (!schema.clientHandshake(message)) {
console.log(`SOCKET | closing ${req.socket.remoteAddress} after receiving invalid handshake:`, message);
socket.close(1001, "Bad handshake");
return;
}
let { seq, type, data } = message;
if (type === "resume") {
let connector;
try {
let payload = jwt.verify(
data.session_token,
masterConfig.get("master.auth_secret"),
{ audience: masterSession }
);
connector = activeConnectors.get(payload.sid);
if (!connector) {
throw new Error();
}
} catch(err) {
socket.send(JSON.stringify({ seq: null, type: "invalidate", data: {}}));
attachHandler();
return;
}
});
});
connector.continue(socket, data.last_seq);
return
}
let result = await authenticate.check(data.token);
if (!result.ok) {
console.error(`SOCKET | authentication failed for ${req.socket.remoteAddress}`);
socket.close(4003, "Authentication failed");
return;
}
let sessionId = nextSessionId++;
let sessionToken = jwt.sign({ aud: masterSession, sid: sessionId, }, masterConfig.get("master.auth_secret"));
let connector = new WebSocketServerConnector(socket, sessionId);
activeConnectors.set(sessionId, connector);
if (type === "register_slave") {
let connection = slaveConnections.get(data.id);
if (connection) {
console.log(`SOCKET | disconecting existing connection for slave ${data.id}`);
await connection.shutdown();
}
console.log(`SOCKET | registered slave ${data.id} version ${data.version}`);
slaveConnections.set(data.id, new SlaveConnection(data, connector));
} else if (type === "register_control") {
console.log(`SOCKET | registered control from ${req.socket.remoteAddress}`);
controlConnections.push(new ControlConnection(data, connector));
}
connector.ready(sessionToken);
}
// handle plugins on the master
var masterPlugins = new Map();
test/integration/index.js
+6 -2
View File
@@ -30,6 +30,8 @@ async function get(path) {
let masterProcess;
let slaveProcess;
let url = "https://localhost:4443/";
let token = jwt.sign({ id: "api" }, "TestSecretDoNotUse");
let instancesDir = path.join("test", "temp", "instances");
let databaseDir = path.join("test", "temp", "databse");
let masterConfigPath = path.join("test", "temp", "master-integration.json");
@@ -71,16 +73,16 @@ before(async function() {
await fs.remove(slaveConfigPath);
await fs.remove(controlConfigPath);
let token = jwt.sign({ id: "api" }, "TestSecretDoNotUse");
await exec(`node master --config ${masterConfigPath} config set master.database_directory ${databaseDir}`);
await exec(`node master --config ${masterConfigPath} config set master.auth_secret TestSecretDoNotUse`);
await exec(`node master --config ${masterConfigPath} config set master.http_port 8880`);
await exec(`node master --config ${masterConfigPath} config set master.https_port 4443`);
await exec(`node master --config ${masterConfigPath} config set master.heartbeat_interval 0.25`);
await exec(`node client --config ${slaveConfigPath} config set slave.id 4`);
await exec(`node client --config ${slaveConfigPath} config set slave.name slave`);
await exec(`node client --config ${slaveConfigPath} config set slave.instances_directory ${instancesDir}`);
await exec(`node client --config ${slaveConfigPath} config set slave.master_url "https://localhost:4443/"`);
await exec(`node client --config ${slaveConfigPath} config set slave.master_url "${url}"`);
await exec(`node client --config ${slaveConfigPath} config set slave.master_token "${token}"`);
await exec(`node clusterctl --config ${controlConfigPath} control-config set control.master_url "https://localhost:4443/"`);
@@ -111,6 +113,8 @@ module.exports = {
get,
exec,
url,
token,
instancesDir,
databaseDir,
masterConfigPath,
test/integration/link.js
+70
View File
@@ -0,0 +1,70 @@
const assert = require("assert").strict;
const events = require("events");
const link = require("lib/link");
const { get, exec, url, token } = require("./index");
class TestControl extends link.Link {
constructor(connector) {
super("control", "master", connector);
link.attachAllMessages(this);
}
async shutdownConnectionRequestHandler() { }
async instanceOutputEventHandler() { }
}
class TestControlConnector extends link.WebSocketClientConnector {
register() {
this.sendHandshake("register_control", { token: this.token, agent: "clusterctl", version: "test" });
}
}
describe("Integration of lib/link", function() {
let control;
let controlConnector;
it("should emitt an error if authentication failed", async function() {
controlConnector = new TestControlConnector(url, 0.2);
controlConnector.token = "gibberish";
control = new TestControl(controlConnector);
await assert.rejects(
controlConnector.connect(),
new Error("Authentication failed")
);
});
it("should connect with proper credentials", async function() {
controlConnector = new TestControlConnector(url, 0.2);
controlConnector.token = token;
control = new TestControl(controlConnector);
await controlConnector.connect();
});
it("should reconnect on connection lost", async function() {
controlConnector._socket.close(1008, "Test");
await events.once(controlConnector, "connected");
});
it("should reconnect on connection terminated", async function() {
controlConnector._socket.terminate();
await events.once(controlConnector, "connected");
});
it("should reconnect on connection heartbeat timeout", async function() {
controlConnector.stopHeartbeat();
await events.once(controlConnector, "connected");
});
it("should invalidate on reconnection with bad session token", async function() {
controlConnector._sessionToken = "blah";
controlConnector.stopHeartbeat();
await events.once(controlConnector, "invalidate");
await events.once(controlConnector, "connected");
});
after(async function() {
controlConnector.close(1001, "Test Quit");
});
});
test/lib/link/connector.js
+54 -33
View File
@@ -6,16 +6,35 @@ const link = require('lib/link');
describe("lib/link/connectors", function() {
describe("class WebSocketClientConnector", function() {
let testConnector = new link.WebSocketClientConnector('source', 'url');
testConnector._socket = new mock.MockSocket();
describe(".disconnect()", function() {
it("should call terminate on the socket", function() {
testConnector._socket.terminateCalled = false;
testConnector.disconnect();
assert(testConnector._socket.terminateCalled, "Close was not called");
describe("class WebSocketBaseConnector", function() {
let testConnector = new link.WebSocketBaseConnector();
describe("._dropSendBufferSeq()", function() {
beforeEach(function() {
testConnector._sendBuffer = [
{ seq: 1, type: "test", data: {} },
{ seq: 2, type: "test", data: {} },
{ seq: 4, type: "test", data: {} },
];
});
it("should do nothing if passed null", function() {
testConnector._dropSendBufferSeq(null);
assert(testConnector._sendBuffer.length === 3, "entries were dropped");
});
it("should drop up to sequence", function() {
testConnector._dropSendBufferSeq(2);
assert.equal(testConnector._sendBuffer.length, 1, "incorrect number of entries dropped");
});
it("should ignore holes", function() {
testConnector._dropSendBufferSeq(3);
assert.equal(testConnector._sendBuffer.length, 1, "incorrect number of entries dropped");
});
});
});
describe("class WebSocketClientConnector", function() {
let testConnector = new link.WebSocketClientConnector('url', 1);
testConnector._socket = new mock.MockSocket();
describe(".register()", function() {
it("is abstract", function() {
@@ -26,8 +45,21 @@ describe("lib/link/connectors", function() {
});
});
describe(".sendHandshake()", function() {
it("calls send on the socket", function() {
testConnector._state = "handshake";
testConnector._socket.sentMessages = [];
testConnector.sendHandshake('test', { test: true });
assert.deepEqual(
testConnector._socket.sentMessages.map(JSON.parse),
[{ seq: null, type: 'test', data: { test: true }}]
);
});
});
describe(".send()", function() {
it("calls send on the socket", function() {
testConnector._state = "connected";
testConnector._socket.sentMessages = [];
let seq = testConnector.send('test', { test: true });
assert.deepEqual(
@@ -38,43 +70,32 @@ describe("lib/link/connectors", function() {
});
describe(".close()", function() {
it("should send close", function() {
testConnector._socket.sentMessages = [];
let called = false;
it("should call close", function() {
testConnector._socket.closeCalled = false;
testConnector.close("test reason");
assert.deepEqual(
testConnector._socket.sentMessages.map(JSON.parse),
[{ seq: testConnector._seq - 1, type: 'close', data: { reason: "test reason" } }]
)
assert(testConnector._socket.closeCalled, "Close was not called on the socket");
});
});
describe(".processHandshake()", function() {
it("should close on invalid message", function() {
testConnector._socket.sentMessages = [];
testConnector._socket.closeCalled = false;
testConnector._processHandshake({ data: "invalid message" }),
assert.deepEqual(
testConnector._socket.sentMessages.map(JSON.parse),
[{seq: testConnector._seq - 1, type: 'close', data: {
reason: "Invalid handshake" }
}]
);
assert(testConnector._socket.closeCalled, "Close was not called on the socket");
});
it("should call register on hello", function() {
let called = false;
testConnector.register = () => { called = true; };
testConnector._processHandshake({ seq: 1, type: 'hello', data: { version: "test" }}),
testConnector._processHandshake({ seq: 1, type: 'hello', data: { version: "test" }});
assert(called, "register was not called");
});
it("should emit ready on ready", async function() {
let result = events.once(testConnector, 'ready');
testConnector._processHandshake({ seq: 1, type: 'ready', data: {}}),
it("should emit connected on ready", async function() {
let result = events.once(testConnector, 'connected');
testConnector._processHandshake(
{ seq: 1, type: 'ready', data: { session_token: "x", heartbeat_interval: 10 }}
);
await result;
});
it("should emit error on close", async function() {
let result = events.once(testConnector, 'ready');
testConnector._processHandshake({ seq: 1, type: 'close', data: { reason: "test" }}),
await assert.rejects(result, new Error("server closed during handshake: test"));
clearInterval(testConnector._heartbeatId);
});
});
@@ -97,9 +118,9 @@ describe("lib/link/connectors", function() {
testConnector._socket.events.get("message")("{}");
assert(called, "_processHandshake was not called");
});
it("should emit message on message in ready state", async function() {
it("should emit message on message in connected state", async function() {
testConnector._socket.sentMessages = [];
testConnector._state = "ready";
testConnector._state = "connected";
let result = events.once(testConnector, 'message');
testConnector._socket.events.get("message")('{"message":true}');
assert.deepEqual(await result, [{ message: true }]);
test/lib/link/link.js
-19
View File
@@ -27,25 +27,6 @@ describe("lib/link/link", function() {
}));
});
it("should close connector on invalid messages", function() {
testConnector.sentMessages = [];
testConnector.emit('message', { invalid: true });
assert.deepEqual(
testConnector.sentMessages,
[{
seq: testConnector._seq - 1,
type: 'close',
data: { reason: "Invalid message: Malformed" },
}]
);
});
it("should disconnect on close", function() {
testConnector.disconnectCalled = false;
testConnector.emit('message', { seq: 1, type: 'close', data: { reason: "test" }});
assert(testConnector.disconnectCalled, "disconnect was not called");
});
it("should emitt error processing message to connector", async function() {
testLink.setHandler('throws', () => { throw new Error("Handler error"); }, () => true);
let result = events.once(testConnector, 'message');
test/master.js
-10
View File
@@ -5,14 +5,4 @@ const master = require('../master');
describe('Master testing', function() {
describe("class WebSocketServerConnector", function() {
let testConnector = new master._WebSocketServerConnector(new mock.MockSocket());
describe(".disconnect()", function() {
it("should call disconnect on the socket", function() {
testConnector._socket.terminateCalled = false;
testConnector.disconnect();
assert(testConnector._socket.terminateCalled, "Terminate was not called");
});
});
});
});
test/mock.js
-9
View File
@@ -42,15 +42,6 @@ class MockConnector extends events.EventEmitter {
setImmediate(() => this.emit('send', message));
return this._seq++;
}
close(reason) {
this.send('close', { reason });
this.disconnect();
}
disconnect() {
this.disconnectCalled = true;
}
}
class MockServer {