Make the player proxy follow the Factorio handshake and route by version

The proxy now parses the full message header, so confirms and fragmented
ConnectionRequestReplyConfirm messages (which carry the mod list) are
forwarded intact, with the server request id patched into the first
fragment. It resends the connection request until the instance answers,
starts over when a client reconnects from the same socket (as Factorio
does after LuaPlayer::connect_to_server), answers pings from the server
browser and tells clients the same 508 byte max packet size Factorio
servers use.

Routing is decided by the new playerProxyRoute controller hook, which is
given the name and version of the player, then a dedicated port set with
instance.player_proxy_port, the instance the player was last on (taken
from the user's instance stats instead of a separate routing store), and
controller.player_proxy_default_instance_id.

The proxy is now off by default, as port 34197 is in the default
host.factorio_port_range. Leftovers of the per-instance proxy on hosts
are removed.

packages/controller/src/BaseControllerPlugin.ts

@@ -2,6 +2,7 @@ import type Controller from "./Controller.js";
 import type InstanceRecord from "./InstanceRecord.js";
 import type ControlConnection from "./ControlConnection.js";
 import type HostConnection from "./HostConnection.js";
+import type { PlayerProxyRouteRequest } from "./PlayerProxyServer.js";
 import * as lib from "@clusterio/lib";
 
 export type ControllerPluginContext = lib.PluginLoadContext<{
@@ -28,6 +29,7 @@ export class ControllerHooks extends lib.AsyncHookCollection {
 		this.modsUpdated = this.newHook();
 		this.rolesUpdated = this.newHook();
 		this.playerEvent = this.newHook();
+		this.playerProxyRoute = this.newHook();
 	}
 
 	/**
@@ -266,6 +268,24 @@ export class ControllerHooks extends lib.AsyncHookCollection {
 	 * @param event - Information about the event.
 	 */
 	readonly playerEvent: lib.AsyncHook<[instance: InstanceRecord, event: lib.PlayerEvent]>;
+
+	/**
+	 * Called to pick the instance a player connecting through the player proxy goes to
+	 *
+	 * Invoked when a Factorio client has sent its name to the player proxy
+	 * enabled with controller.player_proxy_port. Handlers return the ID of
+	 * the instance to send the player to, or undefined to leave the choice
+	 * to other plugins and the controller. If the instance is stopped the
+	 * proxy starts it, which can take longer than the client is willing to
+	 * wait.
+	 *
+	 * Factorio clients connect twice when joining a server they don't have a
+	 * key for, so a handler must give the same answer for both attempts.
+	 *
+	 * @param request - Name, version and connection details of the player.
+	 * @returns ID of instance to route the player to.
+	 */
+	readonly playerProxyRoute: lib.AsyncHook<[request: PlayerProxyRouteRequest], number | undefined>;
 }
 
 // eslint-disable-next-line @typescript-eslint/no-unsafe-declaration-merging

packages/controller/src/Controller.ts

@@ -32,8 +32,7 @@ import HostRecord from "./HostRecord.js";
 import { BaseControllerPlugin, ControllerHooks } from "./BaseControllerPlugin.js";
 import ControllerRouter from "./ControllerRouter.js";
 import InstanceManager from "./InstanceManager.js";
-import PlayerRoutingStore from "./PlayerRoutingStore.js";
-import PlayerProxyServer from "./PlayerProxyServer.js";
+import PlayerProxyServer, { type PlayerProxyRouteRequest } from "./PlayerProxyServer.js";
 
 const endpointDurationSummary = new Summary(
 	"clusterio_controller_http_endpoint_duration_seconds",
@@ -86,7 +85,6 @@ export default class Controller {
 	trustedProxies: BlockList;
 	debugEvents = new events.EventEmitter<ControllerDebugEvents>();
 	private _events = new events.EventEmitter<ControllerEvents>();
-	playerRoutingStore: PlayerRoutingStore;
 	playerProxyServer: PlayerProxyServer | null = null;
 
 	/** Event subscription controller */
@@ -290,10 +288,6 @@ export default class Controller {
 		this.clusterLogger = clusterLogger;
 		this.pluginInfos = pluginInfos;
 		this.config = config;
-		this.playerRoutingStore = new PlayerRoutingStore(
-			path.join(this.config.get("controller.database_directory"), "player-routing.json"),
-		);
-
 		this.app = express();
 		this.app.locals.controller = this;
 		this.app.locals.streams = new Map();
@@ -444,7 +438,6 @@ export default class Controller {
 		// Load plugins
 		await this.loadPlugins();
 
-		await this.playerRoutingStore.load();
 		const playerProxyPort = this.config.get("controller.player_proxy_port");
 		if (playerProxyPort !== null) {
 			this.playerProxyServer = new PlayerProxyServer(
@@ -816,7 +809,6 @@ export default class Controller {
 			this.users.records.save(),
 			this.hooks.save.invoke(),
 			this.notes.save(),
-			this.playerRoutingStore.save(),
 		]);
 	}
 
@@ -1051,87 +1043,65 @@ export default class Controller {
 		return hosts.length ? new lib.HostUpdatesEvent(hosts) : null;
 	}
 
-	resolvePlayerRoute(username: string): { hostId: number; instanceId: number; publicAddress: string; gamePort: number } | null {
-		const key = username.toLowerCase();
-		let route = this.playerRoutingStore.get(key);
-		let instance: Readonly<InstanceRecord> | undefined;
-
-		if (!route) {
-			const fallbackId = this.config.get("controller.player_proxy_default_instance_id");
-			if (fallbackId !== null) {
-				instance = this.instances.get(fallbackId);
-				if (instance) {
-					route = {
-						name: username,
-						instanceId: instance.id,
-						hostId: instance.config.get("instance.assigned_host") ?? -1,
-						gamePort: instance.gamePort ?? 0,
-						timestampMs: Date.now(),
-					};
-				}
-			} else {
-				instance = this._chooseFallbackInstance();
-				if (instance) {
-					route = {
-						name: username,
-						instanceId: instance.id,
-						hostId: instance.config.get("instance.assigned_host") ?? -1,
-						gamePort: instance.gamePort ?? 0,
-						timestampMs: Date.now(),
-					};
-				}
-			}
-		}
-
-		if (!route) {
-			return null;
-		}
-
-		instance = instance ?? this.instances.get(route.instanceId);
-		if (!instance || instance.isDeleted) {
-			this.playerRoutingStore.delete(key);
-			return null;
-		}
-
-		const hostId = instance.config.get("instance.assigned_host");
-		if (hostId === null || route.hostId !== hostId) {
-			this.playerRoutingStore.delete(key);
-			return null;
-		}
-
-		const host = this.hosts.get(hostId);
-		if (!host || !host.connected) {
-			return null;
-		}
-
-		const gamePort = instance.gamePort ?? route.gamePort;
-		const publicAddress = host.publicAddress || host.remoteAddress;
-		if (!gamePort || !publicAddress) {
-			return null;
-		}
-
-		return {
-			hostId,
-			instanceId: instance.id,
-			publicAddress,
-			gamePort,
+	/**
+	 * Pick the instance a player connecting through the player proxy goes to
+	 *
+	 * Plugins get the first say through the playerProxyRoute hook, then the
+	 * instance the proxy port is dedicated to, the instance the player was
+	 * last on, controller.player_proxy_default_instance_id, and finally any
+	 * running instance.
+	 *
+	 * @param request - Details about the connecting player.
+	 * @returns ID of the instance to route to, or undefined if there is none.
+	 */
+	async resolvePlayerRoute(request: PlayerProxyRouteRequest): Promise<number | undefined> {
+		const routable = (id: number | undefined) => {
+			const instance = id === undefined ? undefined : this.instances.get(id);
+			return Boolean(instance && !instance.isDeleted && instance.config.get("instance.assigned_host") !== null);
 		};
-	}
 
-	private _chooseFallbackInstance() {
-		for (const instance of this.instances.values()) {
-			if (instance.status === "running" && !instance.isDeleted) {
-				return instance;
+		for (const result of await this.hooks.playerProxyRoute.collect(request)) {
+			if (routable(result)) {
+				return result;
 			}
 		}
+
+		if (routable(request.portInstanceId)) {
+			return request.portInstanceId;
+		}
+
+		const lastInstanceId = this._lastInstanceOfPlayer(request.username);
+		if (routable(lastInstanceId)) {
+			return lastInstanceId;
+		}
+
+		const defaultId = this.config.get("controller.player_proxy_default_instance_id") ?? undefined;
+		if (routable(defaultId)) {
+			return defaultId;
+		}
+
 		for (const instance of this.instances.values()) {
-			if (!instance.isDeleted && instance.config.get("instance.assigned_host") !== null) {
-				return instance;
+			if (instance.status === "running") {
+				return instance.id;
 			}
 		}
 		return undefined;
 	}
 
+	private _lastInstanceOfPlayer(name: string) {
+		const user = this.users.getByName(name);
+		let lastId: number | undefined;
+		let lastJoinMs = -Infinity;
+		for (const [instanceId, stats] of user?.instanceStats ?? []) {
+			const joinMs = stats.lastJoinAt?.getTime() ?? -Infinity;
+			if (joinMs > lastJoinMs) {
+				lastJoinMs = joinMs;
+				lastId = instanceId;
+			}
+		}
+		return lastId;
+	}
+
 	/**
 	 * Removes all saves currently stored for the given instance, if any
 	 * @param instanceId - Id of Instance to clear saves for.

packages/controller/src/HostConnection.ts

@@ -72,7 +72,6 @@ export default class HostConnection extends BaseConnection {
 
 			this.info.connected = false;
 			this._controller.hosts.set(this.info);
-			this._controller.playerRoutingStore.dropHost(this.id);
 		});
 
 		this.handle(lib.GetFactorioCredentialsRequest, this.handleGetFactorioCredentialsRequest.bind(this));
@@ -82,7 +81,6 @@ export default class HostConnection extends BaseConnection {
 		this.handle(lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsUpdatesEvent.bind(this));
 		this.handle(lib.LogMessageEvent, this.handleLogMessageEvent.bind(this));
 		this.handle(lib.InstancePlayerUpdateEvent, this.handleInstancePlayerUpdateEvent.bind(this));
-		this.handle(lib.InstancePlayerRouteUpdateEvent, this.handleInstancePlayerRouteUpdateEvent.bind(this));
 		this.snoopEvent(lib.InstanceAdminlistUpdateEvent, this.snoopInstanceAdminlistUpdateEvent.bind(this));
 		this.snoopEvent(lib.InstanceBanlistUpdateEvent, this.snoopInstanceBanlistUpdateEvent.bind(this));
 		this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.snoopInstanceWhitelistUpdateEvent.bind(this));
@@ -413,30 +411,6 @@ export default class HostConnection extends BaseConnection {
 		});
 	}
 
-	async handleInstancePlayerRouteUpdateEvent(event: lib.InstancePlayerRouteUpdateEvent, src: lib.Address) {
-		const instanceId = src.id;
-		if (event.instanceId !== instanceId) {
-			logger.warn(
-				`Received route update for instance ${event.instanceId} from ${instanceId}, ignoring mismatched id`
-			);
-			return;
-		}
-		if (event.hostId !== this.id) {
-			logger.warn(
-				`Received route update for host ${event.hostId} from ${this.id}, ignoring mismatched host`
-			);
-			return;
-		}
-
-		this._controller.playerRoutingStore.update(
-			event.name,
-			instanceId,
-			this.id,
-			event.gamePort,
-			event.timestampMs,
-		);
-	}
-
 	async snoopInstanceAdminlistUpdateEvent(event: lib.InstanceAdminlistUpdateEvent, src: lib.Address) {
 		if (src.type !== lib.Address.instance) {
 			return;

packages/controller/src/PlayerRoutingStore.ts

@@ -1,75 +0,0 @@
-import fs from "node:fs/promises";
-
-import * as lib from "@clusterio/lib";
-
-export type StoredRoute = {
-	name: string;
-	instanceId: number;
-	hostId: number;
-	gamePort: number;
-	timestampMs: number;
-};
-
-export default class PlayerRoutingStore {
-	private _routes = new Map<string, StoredRoute>();
-	private _dirty = false;
-
-	constructor(private _filePath: string) {}
-
-	get(name: string): StoredRoute | undefined {
-		return this._routes.get(name.toLowerCase());
-	}
-
-	update(name: string, instanceId: number, hostId: number, gamePort: number, timestampMs: number) {
-		const key = name.toLowerCase();
-		this._routes.set(key, { name, instanceId, hostId, gamePort, timestampMs });
-		this._dirty = true;
-	}
-
-	delete(name: string) {
-		const key = name.toLowerCase();
-		if (this._routes.delete(key)) {
-			this._dirty = true;
-		}
-	}
-
-	dropHost(hostId: number) {
-		let removed = false;
-		for (const [key, route] of this._routes) {
-			if (route.hostId === hostId) {
-				this._routes.delete(key);
-				removed = true;
-			}
-		}
-		if (removed) {
-			this._dirty = true;
-		}
-	}
-
-	async load(): Promise<void> {
-		try {
-			const routes = JSON.parse(await fs.readFile(this._filePath, { encoding: "utf8" })) as StoredRoute[];
-			this._routes.clear();
-			for (const route of routes) {
-				if (!route || !route.name) {
-					continue;
-				}
-				this._routes.set(route.name.toLowerCase(), route);
-			}
-			this._dirty = false;
-		} catch (err: any) {
-			if (err.code !== "ENOENT") {
-				throw err;
-			}
-		}
-	}
-
-	async save(): Promise<void> {
-		if (!this._dirty) {
-			return;
-		}
-		const payload = JSON.stringify([...this._routes.values()], null, "\t");
-		await lib.safeOutputFile(this._filePath, payload);
-		this._dirty = false;
-	}
-}

packages/controller/src/proxy_protocol.ts

@@ -0,0 +1,302 @@
+/**
+ * Encoding and decoding of the parts of the Factorio network protocol the
+ * player proxy needs to look at: the message header and the connection
+ * handshake messages.
+ *
+ * Based on the Factorio 2.x wire format. Every multi-byte value is little
+ * endian, "space optimized" values are a single byte unless that byte is 0xff,
+ * in which case the full width value follows it.
+ *
+ * @module
+ */
+
+export const MessageType = {
+	Ping: 0,
+	PingReply: 1,
+	ConnectionRequest: 2,
+	ConnectionRequestReply: 3,
+	ConnectionRequestReplyConfirm: 4,
+	ConnectionAcceptOrDeny: 5,
+} as const;
+
+const TYPE_MASK = 0x1f;
+const FRAGMENTED_FLAG = 0x40;
+const LAST_FRAGMENT_FLAG = 0x80;
+const CONFIRM_FLAG = 0x8000;
+const MESSAGE_ID_MASK = 0x7fff;
+
+/** Message types that carry a message id even when not fragmented */
+const RELIABLE_TYPES: ReadonlySet<number> = new Set([
+	MessageType.ConnectionRequest,
+	MessageType.ConnectionRequestReplyConfirm,
+]);
+
+/**
+ * Max packet size Factorio servers tell clients to use, this is the largest
+ * datagram every host on the Internet is required to be able to receive.
+ */
+export const DEFAULT_MAX_PACKET_SIZE = 508;
+
+export interface ApplicationVersion {
+	major: number;
+	minor: number;
+	sub: number;
+	build: number;
+}
+
+export function formatVersion(version: ApplicationVersion) {
+	return `${version.major}.${version.minor}.${version.sub}`;
+}
+
+export function versionsEqual(a: ApplicationVersion, b: ApplicationVersion) {
+	return a.major === b.major && a.minor === b.minor && a.sub === b.sub && a.build === b.build;
+}
+
+export interface MessageHeader {
+	type: number;
+	fragmented: boolean;
+	lastFragment: boolean;
+	messageId?: number;
+	fragmentNumber?: number;
+	confirms: number[];
+	/** Offset of the payload of the message */
+	payloadOffset: number;
+}
+
+export interface ConnectionRequest {
+	header: MessageHeader;
+	version: ApplicationVersion;
+	clientRequestId: number;
+}
+
+export interface ConnectionRequestReply {
+	version: ApplicationVersion;
+	clientRequestId: number;
+	serverRequestId: number;
+	maxPacketSize: number;
+}
+
+export interface ConnectionRequestReplyConfirm {
+	clientRequestId: number;
+	serverRequestId: number;
+	/** Offset of serverRequestId in the packet */
+	serverRequestIdOffset: number;
+	username: string;
+}
+
+export class BufferReader {
+	constructor(
+		private _buffer: Buffer,
+		public offset = 0,
+	) { }
+
+	private _take(length: number) {
+		const start = this.offset;
+		if (start + length > this._buffer.length) {
+			throw new Error("Unexpected end of message");
+		}
+		this.offset += length;
+		return start;
+	}
+
+	readUInt8() {
+		return this._buffer.readUInt8(this._take(1));
+	}
+
+	readUInt16LE() {
+		return this._buffer.readUInt16LE(this._take(2));
+	}
+
+	readUInt32LE() {
+		return this._buffer.readUInt32LE(this._take(4));
+	}
+
+	/** Read a value stored as one byte or 0xff followed by a uint16 */
+	readSpaceOptimizedUInt16() {
+		const value = this.readUInt8();
+		return value === 0xff ? this.readUInt16LE() : value;
+	}
+
+	/** Read a value stored as one byte or 0xff followed by a uint32 */
+	readSpaceOptimizedUInt32() {
+		const value = this.readUInt8();
+		return value === 0xff ? this.readUInt32LE() : value;
+	}
+
+	readString() {
+		const length = this.readSpaceOptimizedUInt32();
+		const start = this._take(length);
+		return this._buffer.toString("utf8", start, start + length);
+	}
+
+	readVersion(): ApplicationVersion {
+		return {
+			major: this.readSpaceOptimizedUInt16(),
+			minor: this.readSpaceOptimizedUInt16(),
+			sub: this.readSpaceOptimizedUInt16(),
+			build: this.readUInt32LE(),
+		};
+	}
+}
+
+export class BufferWriter {
+	private _parts: number[] = [];
+
+	writeUInt8(value: number) {
+		this._parts.push(value & 0xff);
+	}
+
+	writeUInt16LE(value: number) {
+		this.writeUInt8(value);
+		this.writeUInt8(value >>> 8);
+	}
+
+	writeUInt32LE(value: number) {
+		this.writeUInt16LE(value);
+		this.writeUInt16LE(value >>> 16);
+	}
+
+	writeSpaceOptimizedUInt16(value: number) {
+		if (value >= 0xff) {
+			this.writeUInt8(0xff);
+			this.writeUInt16LE(value);
+		} else {
+			this.writeUInt8(value);
+		}
+	}
+
+	writeVersion(version: ApplicationVersion) {
+		this.writeSpaceOptimizedUInt16(version.major);
+		this.writeSpaceOptimizedUInt16(version.minor);
+		this.writeSpaceOptimizedUInt16(version.sub);
+		this.writeUInt32LE(version.build);
+	}
+
+	toBuffer() {
+		return Buffer.from(this._parts);
+	}
+}
+
+export function messageType(buffer: Buffer) {
+	if (!buffer.length) {
+		return undefined;
+	}
+	return buffer[0] & TYPE_MASK;
+}
+
+export function parseHeader(buffer: Buffer): MessageHeader {
+	const reader = new BufferReader(buffer);
+	const typeAndFlags = reader.readUInt8();
+	const type = typeAndFlags & TYPE_MASK;
+	const fragmented = (typeAndFlags & FRAGMENTED_FLAG) !== 0;
+	const lastFragment = (typeAndFlags & LAST_FRAGMENT_FLAG) !== 0;
+
+	let messageId: number | undefined;
+	let hasConfirm = false;
+	if (RELIABLE_TYPES.has(type) || fragmented) {
+		const messageIdAndFlags = reader.readUInt16LE();
+		messageId = messageIdAndFlags & MESSAGE_ID_MASK;
+		hasConfirm = (messageIdAndFlags & CONFIRM_FLAG) !== 0;
+	}
+
+	let fragmentNumber: number | undefined;
+	if (fragmented) {
+		fragmentNumber = reader.readSpaceOptimizedUInt16();
+	}
+
+	const confirms: number[] = [];
+	if (hasConfirm) {
+		const count = reader.readUInt8();
+		for (let i = 0; i < count; i++) {
+			confirms.push(reader.readUInt32LE());
+		}
+	}
+
+	return { type, fragmented, lastFragment, messageId, fragmentNumber, confirms, payloadOffset: reader.offset };
+}
+
+export function parseConnectionRequest(buffer: Buffer): ConnectionRequest {
+	const header = parseHeader(buffer);
+	if (header.type !== MessageType.ConnectionRequest) {
+		throw new Error(`Expected ConnectionRequest, got message type ${header.type}`);
+	}
+	const reader = new BufferReader(buffer, header.payloadOffset);
+	const version = reader.readVersion();
+	const clientRequestId = reader.readUInt32LE();
+	return { header, version, clientRequestId };
+}
+
+export function parseConnectionRequestReply(buffer: Buffer): ConnectionRequestReply {
+	const header = parseHeader(buffer);
+	if (header.type !== MessageType.ConnectionRequestReply) {
+		throw new Error(`Expected ConnectionRequestReply, got message type ${header.type}`);
+	}
+	const reader = new BufferReader(buffer, header.payloadOffset);
+	return {
+		version: reader.readVersion(),
+		clientRequestId: reader.readUInt32LE(),
+		serverRequestId: reader.readUInt32LE(),
+		maxPacketSize: reader.readUInt16LE(),
+	};
+}
+
+/**
+ * Parse the start of a ConnectionRequestReplyConfirm
+ *
+ * The message contains the mod list and startup settings of the client and
+ * is usually fragmented, but the fields read here are always in the first
+ * fragment.
+ *
+ * @param buffer - Unfragmented message or first fragment of it.
+ * @returns the parsed fields.
+ */
+export function parseConnectionRequestReplyConfirm(buffer: Buffer): ConnectionRequestReplyConfirm {
+	const header = parseHeader(buffer);
+	if (header.type !== MessageType.ConnectionRequestReplyConfirm) {
+		throw new Error(`Expected ConnectionRequestReplyConfirm, got message type ${header.type}`);
+	}
+	if (header.fragmented && header.fragmentNumber !== 0) {
+		throw new Error(`Expected first fragment, got fragment ${header.fragmentNumber}`);
+	}
+	const reader = new BufferReader(buffer, header.payloadOffset);
+	const clientRequestId = reader.readUInt32LE();
+	const serverRequestIdOffset = reader.offset;
+	const serverRequestId = reader.readUInt32LE();
+	reader.readUInt32LE(); // instanceID
+	const username = reader.readString();
+	return { clientRequestId, serverRequestId, serverRequestIdOffset, username };
+}
+
+export function encodeConnectionRequest(version: ApplicationVersion, clientRequestId: number, messageId = 0) {
+	const writer = new BufferWriter();
+	writer.writeUInt8(MessageType.ConnectionRequest);
+	writer.writeUInt16LE(messageId & MESSAGE_ID_MASK);
+	writer.writeVersion(version);
+	writer.writeUInt32LE(clientRequestId);
+	return writer.toBuffer();
+}
+
+export function encodeConnectionRequestReply(reply: ConnectionRequestReply) {
+	const writer = new BufferWriter();
+	writer.writeUInt8(MessageType.ConnectionRequestReply);
+	writer.writeVersion(reply.version);
+	writer.writeUInt32LE(reply.clientRequestId);
+	writer.writeUInt32LE(reply.serverRequestId);
+	writer.writeUInt16LE(reply.maxPacketSize);
+	return writer.toBuffer();
+}
+
+/**
+ * Turn a Ping into the PingReply a server would send back
+ *
+ * @param ping - Ping message received.
+ * @returns PingReply message, or undefined if ping was malformed.
+ */
+export function pingReplyFor(ping: Buffer) {
+	if (ping.length < 3) {
+		return undefined;
+	}
+	const reply = Buffer.from(ping.subarray(0, 3));
+	reply[0] = MessageType.PingReply;
+	return reply;
+}