Improve master shutdown

Close active connections and http servers during master shutdown and let
node exit on its own.  If this fails with an error set output streams to
blocking and report it then shutdown with process.exit.

Also add double SIGINT handling to immediately shutdown.
This commit is contained in:
Hornwitser committed 2019-12-20 02:37:27 +01:00
1 parent 8b9bb59357
commit 1f6574a89c
2 files changed
+74 -13

No files matched your search

+73 -13
View File
@@ -24,6 +24,7 @@ const crypto = require('crypto');
const base64url = require('base64url'); const base64url = require('base64url');
const moment = require("moment"); const moment = require("moment");
const request = require("request"); const request = require("request");
const setBlocking = require("set-blocking");
const events = require("events"); const events = require("events");
const version = require("./package").version; const version = require("./package").version;
@@ -49,6 +50,8 @@ const cookieParser = require('cookie-parser');
const bodyParser = require("body-parser"); const bodyParser = require("body-parser");
const fileUpload = require('express-fileupload'); const fileUpload = require('express-fileupload');
var app = express(); var app = express();
var httpServer;
var httpsServer;
app.use(cookieParser()); app.use(cookieParser());
app.use(bodyParser.json({ app.use(bodyParser.json({
@@ -176,9 +179,11 @@ async function saveMap(databaseDirectory, file, map) {
} }
// store slaves in a .json full of JSON data /**
* Innitiate shutdown of master server
*/
async function shutdown() { async function shutdown() {
console.log('Ctrl-C...'); console.log('Shutting down');
let exitStartTime = Date.now(); let exitStartTime = Date.now();
try { try {
await saveMap(config.databaseDirectory, "slaves.json", db.slaves); await saveMap(config.databaseDirectory, "slaves.json", db.slaves);
@@ -189,12 +194,37 @@ async function shutdown() {
await masterPlugin.onExit(); await masterPlugin.onExit();
console.log(`Plugin ${pluginName} exited in ${Date.now() - startTime}ms`); console.log(`Plugin ${pluginName} exited in ${Date.now() - startTime}ms`);
} }
console.log("Clusterio cleanly exited in "+(Date.now()-exitStartTime)+"ms");
process.exit(0); for (let controlConnection of controlConnections) {
} catch(e) { controlConnection.connector.close("shutdown");
console.log(e); }
console.log("Clusterio failed to exit cleanly. Time elapsed: "+(Date.now()-exitStartTime)+"ms");
process.exit(1) for (let slaveConnection of slaveConnections.values()) {
slaveConnection.connector.close("shutdown");
}
if (httpServer) {
console.log("Stopping HTTP server");
await new Promise(resolve => httpServer.close(resolve));
}
if (httpsServer) {
console.log("Stopping HTTPS server");
await new Promise(resolve => httpsServer.close(resolve));
}
console.log(`Clusterio cleanly exited in ${Date.now() - exitStartTime}ms`);
} catch(err) {
setBlocking(true);
console.error(`
+--------------------------------------------------------------------+
| Unexpected error occured while shutting down master, please report |
| it to https://github.com/clusterio/factorioClusterio/issues |
+--------------------------------------------------------------------+`
);
console.error(err);
process.exit(1);
} }
} }
@@ -468,6 +498,19 @@ class SocketIOServerConnector extends events.EventEmitter {
return this._seq++; return this._seq++;
} }
/**
* Close the connection with the given reason.
*
* Sends a close message and disconnects the connector.
*/
close(reason) {
this.send('close', { reason });
this.disconnect();
}
/**
* Immediatly close the connection
*/
disconnect() { disconnect() {
this._socket.disconnect(true); this._socket.disconnect(true);
} }
@@ -495,7 +538,7 @@ io.on('connection', function (socket) {
let connection = slaveConnections.get(data.id); let connection = slaveConnections.get(data.id);
if (connection) { if (connection) {
console.log(`SOCKET | disconecting existing connection for slave ${data.id}`); console.log(`SOCKET | disconecting existing connection for slave ${data.id}`);
connection.close("Registered from another connection"); connection.connector.close("Registered from another connection");
} }
console.log(`SOCKET | registered slave ${data.id} version ${data.version}`); console.log(`SOCKET | registered slave ${data.id} version ${data.version}`);
@@ -602,8 +645,25 @@ async function startServer() {
fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, { fs.writeFileSync("secret-api-token.txt", jwt.sign({ id: "api" }, config.masterAuthSecret, {
expiresIn: 86400*365 // expires in 1 year expiresIn: 86400*365 // expires in 1 year
})); }));
process.on('SIGINT', shutdown); // ctrl + c
process.on('SIGHUP', shutdown); // terminal closed let secondSigint = false
process.on('SIGINT', () => {
if (secondSigint) {
console.log("Caught second interrupt, terminating immediately");
process.exit(1);
}
secondSigint = true;
console.log("Caught interrupt signal, shutting down");
shutdown();
});
// terminal closed
process.on('SIGHUP', () => {
// No graceful cleanup, no warning out (stdout is likely closed.)
// Don't close the terminal with the clusterio master in it.
process.exit(1);
});
config.databaseDirectory = args.databaseDirectory || config.databaseDirectory || "./database"; config.databaseDirectory = args.databaseDirectory || config.databaseDirectory || "./database";
await fs.ensureDir(config.databaseDirectory); await fs.ensureDir(config.databaseDirectory);
@@ -635,7 +695,7 @@ async function startServer() {
// Only start listening for connections after all plugins have loaded // Only start listening for connections after all plugins have loaded
if (httpPort) { if (httpPort) {
let httpServer = require("http").Server(app); httpServer = require("http").Server(app);
await listen(httpServer, httpPort); await listen(httpServer, httpPort);
io.attach(httpServer); io.attach(httpServer);
console.log("Listening for HTTP on port %s...", httpServer.address().port); console.log("Listening for HTTP on port %s...", httpServer.address().port);
@@ -653,7 +713,7 @@ async function startServer() {
); );
} }
let httpsServer = require("https").createServer({ httpsServer = require("https").createServer({
key: privateKey, key: privateKey,
cert: certificate cert: certificate
}, app) }, app)
+1
View File
@@ -59,6 +59,7 @@
"rcon-client": "^4.2.0", "rcon-client": "^4.2.0",
"request": "^2.83.0", "request": "^2.83.0",
"sanitizer": "^0.1.3", "sanitizer": "^0.1.3",
"set-blocking": "^2.0.0",
"simple-git": "^1.96.0", "simple-git": "^1.96.0",
"single-line-log": "^1.1.2", "single-line-log": "^1.1.2",
"socket.io": "^2.2.0", "socket.io": "^2.2.0",