Add hang detection to FactorioServer.stop()

Attempt to detect if the Factorio server has hanged in
FactorioServer.stop() and send a second SIGTERM if this is the case.

lib/factorio/server.js

@@ -281,6 +281,17 @@ const outputHeuristics = [
 		}
 	},
 
+	// Message indicating the server is shutting down
+	{
+		filter: {
+			type: 'generic',
+			message: /^Quitting: /
+		},
+		action: function(output) {
+			this.emit('_quitting');
+		}
+	},
+
 	// Messages that might be tha cause of an unexpected shutdown.
 	{
 		filter: {
@@ -636,7 +647,33 @@ class FactorioServer extends events.EventEmitter {
 		// developers.  Rcon does not recognize /quit, stdin is not
 		// recognized, and there's no "send CTRL+C to process" on Windows.
 		this._server.kill();
-		await events.once(this._server, 'exit');
+
+		// There appears to be an race condition where sending SIGTERM
+		// immediatly before the RCON interface comes online causes the
+		// Factorio server to hang.  It's also possible the server doesn't
+		// stop if it's stuck with an infinite lua code loop.
+		if (process.platform !== "win32") {
+			let hanged = true;
+			function setAlive(output) { hanged = false; }
+			this.on('_quitting', setAlive);
+
+			let timeoutId = setTimeout(() => {
+				if (hanged) {
+					console.error("Factorio appears to have hanged, sending second SIGTERM");
+					this._server.kill();
+				}
+			}, 5000);
+
+			await events.once(this._server, 'exit');
+
+			clearTimeout(timeoutId);
+			this.off('_quitting', setAlive);
+
+		// On windows the process is terminated immediately, but to keep
+		// ordering wait until after the exit event here.
+		} else {
+			await events.once(this._server, 'exit');
+		}
 	}
 
 	/**

test/integration/server.js

@@ -1,4 +1,5 @@
 const assert = require("assert").strict;
+const events = require("events");
 const fs = require("fs-extra");
 const path = require("path");
 
@@ -165,5 +166,21 @@ describe("Integration of lib/factorio/server", function() {
 				assert(pass, "server did not output line from test scenario");
 			});
 		});
+
+		describe(".stop() hang detection", function() {
+			it("should detect factorio hanging on shutdown", async function() {
+				slowTest(this);
+				log(".start() for hang detection");
+
+				await server.start("test.zip");
+				if (!server._rconReady) {
+					await events.once(server, 'rcon-ready');
+				}
+				server.sendRcon("/c while true do end").catch(() => {});
+				await new Promise((resolve) => setTimeout(resolve, 300));
+				log(".stop() for hang detection");
+				await server.stop();
+			});
+		});
 	});
 });