Add Factorio version management and download functionality

Co-authored-by: danielv <danielv@danielv.no>
Merge pull request #778 from Cooldude2606/refactor/test-exec-ctl
2026-11-12 23:04:52 +09:00 · 2025-07-21 18:47:19 +00:00 · 2025-07-10 13:09:27 +02:00 · 2025-07-10 13:02:38 +02:00 · 2025-07-06 23:17:52 +01:00 · 2025-07-06 22:58:39 +01:00
52 changed files with 961 additions and 612 deletions
@@ -130,8 +130,16 @@ async function handleBootstrapCommand(
 	controllerConfig: lib.ControllerConfig
 ): Promise<void> {
 	let subCommand = args._[1];
-	let userManager = new UserManager(controllerConfig);
-	await userManager.load(path.join(controllerConfig.get("controller.database_directory"), "users.json"));
+
+	const databaseDirectory = controllerConfig.get("controller.database_directory");
+	const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		path.join(databaseDirectory, "roles.json"),
+		lib.Role.fromJSON.bind(lib.Role),
+	).bootstrap());
+
+	const userManager = new UserManager(controllerConfig, roles);
+	await userManager.load(path.join(databaseDirectory, "users.json"));
+
 	if (subCommand === "create-admin") {
 		if (!args.name) {
 			logger.error("name cannot be blank");
@@ -144,7 +152,7 @@ async function handleBootstrapCommand(
 			admin = userManager.createUser(args.name);
 		}

-		let adminRole = lib.ensureDefaultAdminRole(userManager.roles);
+		let adminRole = lib.ensureDefaultAdminRole(roles);
 		admin.roleIds.add(adminRole.id);
 		admin.isAdmin = true;
 		await userManager.save(path.join(controllerConfig.get("controller.database_directory"), "users.json"));
@@ -313,7 +313,7 @@ export default class ControlConnection extends BaseConnection {
 		let stream = await routes.createProxyStream(this._controller.app);
 		stream.filename = name;

-		let ready = new Promise((resolve, reject) => {
+		let ready = new Promise<void>((resolve, reject) => {
 			stream.events.on("source", resolve);
 			stream.events.on("timeout", () => reject(
 				new lib.RequestError("Timed out establishing stream from host")
@@ -671,22 +671,21 @@ export default class ControlConnection extends BaseConnection {
 	}

 	async handleRoleListRequest() {
-		return [...this._controller.userManager.roles.values()];
+		return [...this._controller.roles.values()];
 	}

 	async handleRoleCreateRequest(request: lib.RoleCreateRequest) {
-		let lastId = Math.max.apply(null, [...this._controller.userManager.roles.keys()]);
+		let lastId = Math.max.apply(null, [...this._controller.roles.keys()]);

 		// Start at 5 to leave space for future default roles
 		let id = Math.max(5, lastId+1);
-		this._controller.userManager.roles.set(id, lib.Role.fromJSON({ id, ...request }));
-		this._controller.userManager.dirty = true;
+		this._controller.roles.set(lib.Role.fromJSON({ id, ...request }));
 		return id;
 	}

 	async handleRoleUpdateRequest(request: lib.RoleUpdateRequest) {
 		let { id, name, description, permissions } = request;
-		let role = this._controller.userManager.roles.get(id);
+		let role = this._controller.roles.getMutable(id);
 		if (!role) {
 			throw new lib.RequestError(`Role with ID ${id} does not exist`);
 		}
@@ -694,27 +693,29 @@ export default class ControlConnection extends BaseConnection {
 		role.name = name;
 		role.description = description;
 		role.permissions = new Set(permissions);
-		this._controller.rolePermissionsUpdated(role);
+		this._controller.roles.set(role);
 	}

 	async handleRoleGrantDefaultPermissionsRequest(request: lib.RoleGrantDefaultPermissionsRequest) {
-		let role = this._controller.userManager.roles.get(request.id);
+		let role = this._controller.roles.get(request.id);
 		if (!role) {
 			throw new lib.RequestError(`Role with ID ${request.id} does not exist`);
 		}

 		role.grantDefaultPermissions();
-		this._controller.rolePermissionsUpdated(role);
+		this._controller.roles.set(role);
 	}

 	async handleRoleDeleteRequest(request: lib.RoleDeleteRequest) {
 		let id = request.id;

-		if (!this._controller.userManager.roles.delete(id)) {
+		const role = this._controller.roles.get(id);
+		if (!role) {
 			throw new lib.RequestError(`Role with ID ${id} does not exist`);
 		}
-		this._controller.userManager.dirty = true;
+		this._controller.roles.delete(role);

+		this._controller.userManager.dirty = true;
 		for (let user of this._controller.userManager.users.values()) {
 			if (user.roleIds.delete(id)) {
 				this._controller.userPermissionsUpdated(user);
@@ -766,7 +767,7 @@ export default class ControlConnection extends BaseConnection {
 		}

 		for (let roleId of request.roles) {
-			if (!this._controller.userManager.roles.has(roleId)) {
+			if (!this._controller.roles.has(roleId)) {
 				throw new lib.RequestError(`Role with ID ${roleId} does not exist`);
 			}
 		}
@@ -6,7 +6,7 @@ import type { Static } from "@sinclair/typebox";
 import finalhandler from "finalhandler";

 import compression from "compression";
-import events, { EventEmitter } from "events";
+import events from "events";
 import fs from "fs-extra";
 import http from "http";
 import https from "https";
@@ -42,6 +42,14 @@ const logSizeGauge = new Gauge(

 type InstanceId = { instanceId: number };

+type ControllerDebugEvents = {
+	"message": [ { direction: "in" | "out", content: string } ],
+};
+
+type ControllerEvents = {
+	"stop": [],
+}
+
 /**
 * Manages all controller related operations
 * @alias module:controller/src/Controller
@@ -66,8 +74,8 @@ export default class Controller {
 	wsServer: WsServer;
 	router = new ControllerRouter(this);
 	trustedProxies: BlockList;
-	debugEvents: events.EventEmitter = new events.EventEmitter();
-	private _events: events.EventEmitter = new events.EventEmitter();
+	debugEvents = new events.EventEmitter<ControllerDebugEvents>();
+	private _events = new events.EventEmitter<ControllerEvents>();

 	/** Event subscription controller */
 	subscriptions = new lib.SubscriptionController();
@@ -130,17 +138,28 @@ export default class Controller {
 			lib.ModPack.fromJSON.bind(lib.ModPack)
 		).bootstrap());

-		if (modPacks.size === 0) {
-			modPacks.setMany(lib.ModPack.defaultModPacks);
-		}
+		const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "roles.json"),
+			lib.Role.fromJSON.bind(lib.Role),
+			this.migrateRoles,
+		).bootstrap());

-		const userManager = new UserManager(config);
+		const userManager = new UserManager(config, roles);
 		await userManager.load(path.join(databaseDirectory, "users.json"));

 		let modsDirectory = config.get("controller.mods_directory");
 		await fs.ensureDir(modsDirectory);
 		const modStore = await lib.ModStore.fromDirectory(modsDirectory);

+		// Add default mod packs
+		if (modPacks.size === 0) {
+			modPacks.setMany(lib.ModPack.defaultModPacks);
+		}
+
+		// Add default roles
+		lib.ensureDefaultAdminRole(roles);
+		lib.ensureDefaultPlayerRole(roles);
+
 		return [
 			systems,
 			hosts,
@@ -148,6 +167,7 @@ export default class Controller {
 			saves,
 			modPacks,
 			modStore,
+			roles,
 			userManager,
 		] as const;
 	}
@@ -178,8 +198,10 @@ export default class Controller {
 		public modPacks = new lib.SubscribableDatastore<lib.ModPack>(),
 		/** Mods stored on the controller */
 		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
+		/** Mapping of mod pack id to mod pack */
+		public roles = new lib.SubscribableDatastore<lib.Role>(),
 		/** User and roles manager for the cluster */
-		public userManager = new UserManager(config),
+		public userManager = new UserManager(config, roles),
 	) {
 		this.clusterLogger = clusterLogger;
 		this.pluginInfos = pluginInfos;
@@ -206,6 +228,7 @@ export default class Controller {
 		this.subscriptions.handle(lib.ModPackUpdatesEvent, this.handleModPackSubscription.bind(this));
 		this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this));
 		this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this));
+		this.subscriptions.handle(lib.RoleUpdatesEvent, this.handleRoleSubscription.bind(this));

 		// Handle updates for datastores
 		this.systems.on("update", this.systemsUpdated.bind(this));
@@ -213,6 +236,7 @@ export default class Controller {
 		this.instances.on("update", this.instanceDetailsUpdated.bind(this));
 		this.saves.on("update", this.savesUpdated.bind(this));
 		this.modPacks.on("update", this.modPacksUpdated.bind(this));
+		this.roles.on("update", this.rolesUpdated.bind(this));
 	}

 	async start(args: ControllerArgs) {
@@ -589,6 +613,7 @@ export default class Controller {
 			this.instances.save(),
 			this.saves.save(),
 			this.modPacks.save(),
+			this.roles.save(),
 		]);

 		if (this.userManager.dirty) {
@@ -649,6 +674,20 @@ export default class Controller {
 		return instance;
 	}

+	static migrateRoles(rawJson: unknown[]): Static<typeof lib.Role.jsonSchema>[] {
+		const serialized = rawJson as Static<typeof lib.Role.jsonSchema>[];
+		return serialized.map(json => {
+			json.permissions = json.permissions.map(permission => {
+				// migrate: core.instance.save.list.subscribe was renamed in alpha 17
+				if (permission === "core.instance.save.list.subscribe") {
+					return "core.instance.save.subscribe";
+				}
+				return permission;
+			});
+			return json;
+		});
+	}
+
 	static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
 		let manifest = {};
 		try {
@@ -1052,27 +1091,34 @@ export default class Controller {
 		}
 	}

-	/**
-	 * Notify connected control clients with the given role that the
-	 * permissions may have changed.
-	 * @param role - Role permisions updated for.
-	 */
-	rolePermissionsUpdated(role: lib.Role) {
-		for (let controlConnection of this.wsServer.controlConnections.values()) {
-			if (controlConnection.user.roles.has(role)) {
-				controlConnection.send(
-					new lib.AccountUpdateEvent(
-						[...controlConnection.user.roles].map(r => ({
-							name: r.name,
-							id: r.id,
-							permissions: [...r.permissions],
-						}))
-					)
-				);
+	rolesUpdated(roles: lib.Role[]) {
+		this.subscriptions.broadcast(new lib.RoleUpdatesEvent(roles));
+		// lib.invokeHook(this.plugins, "onRolesUpdated", roles); // This doesn't exist at the moment
+		// Notify connected control clients with the given role that the permissions may have changed.
+		for (const role of roles) {
+			for (let controlConnection of this.wsServer.controlConnections.values()) {
+				if (controlConnection.user.roles.has(role)) {
+					controlConnection.send(
+						new lib.AccountUpdateEvent(
+							[...controlConnection.user.roles].map(r => ({
+								name: r.name,
+								id: r.id,
+								permissions: [...r.permissions],
+							}))
+						)
+					);
+				}
 			}
 		}
 	}

+	async handleRoleSubscription(request: lib.SubscriptionRequest) {
+		const roles = [...this.roles.values()].filter(
+			role => role.updatedAtMs > request.lastRequestTimeMs,
+		);
+		return roles.length ? new lib.RoleUpdatesEvent(roles) : null;
+	}
+
 	async loadPlugins() {
 		for (let pluginInfo of this.pluginInfos) {
 			try {
@@ -1,10 +1,13 @@
 import { Static, Type } from "@sinclair/typebox";
-import { IControllerUser, PermissionError, PlayerStats, Role, User, permissions } from "@clusterio/lib";
-import type UserManager from "./UserManager";
+
+import {
+	IControllerUser, PermissionError, PlayerStats,
+	Role, SubscribableDatastore, User, permissions,
+} from "@clusterio/lib";

 export default class ControllerUser extends User implements IControllerUser {
 	constructor(
-		public userManager: UserManager,
+		private _controllerRoles: SubscribableDatastore<Role>,
 		/** Unix time in seconds the user token must be issued after to be valid.  */
 		public tokenValidAfter = 0,
 		...args: ConstructorParameters<typeof User>
@@ -17,7 +20,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		token_valid_after: Type.Optional(Type.Number()),
 	});

-	static fromJSON(json: Static<typeof this.jsonSchema>, userManager: UserManager) {
+	static fromJSON(json: Static<typeof this.jsonSchema>, controllerRoles: SubscribableDatastore<Role>) {
 		const roleIds = new Set<number>(json.roles ?? []);
 		const instanceStats = new Map(
 			(json.instance_stats ?? []).map(
@@ -26,7 +29,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		);
 		const playerStats = User._calculatePlayerStats(instanceStats);
 		return new this(
-			userManager,
+			controllerRoles,
 			json.token_valid_after,
 			json.name,
 			roleIds,
@@ -57,7 +60,7 @@ export default class ControllerUser extends User implements IControllerUser {

 	get roles(): ReadonlySet<Readonly<Role>> {
 		return new Set([...this.roleIds].map(
-			id => this.userManager.roles.get(id)
+			id => this._controllerRoles.get(id)
 		).filter(
 			(r): r is Role => Boolean(r)
 		));
@@ -79,7 +82,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		}

 		for (let roleId of this.roleIds) {
-			let role = this.userManager.roles.get(roleId);
+			let role = this._controllerRoles.get(roleId);
 			if (!role) {
 				continue;
 			}
@@ -10,7 +10,6 @@ import ControllerUser from "./ControllerUser";
 * @alias module:controller/src/UserManager
 */
 export default class UserManager {
-	roles: Map<lib.Role["id"], lib.Role> = new Map();
 	users: Map<ControllerUser["id"], ControllerUser> = new Map();
 	dirty = false;

@@ -20,7 +19,8 @@ export default class UserManager {
 	onlineUsers = new Set<ControllerUser>();

 	constructor(
-		private _config: lib.ControllerConfig
+		private _config: lib.ControllerConfig,
+		private _roles: lib.SubscribableDatastore<lib.Role>,
 	) {
 	}

@@ -31,14 +31,15 @@ export default class UserManager {
 	async load(filePath: string): Promise<void> {
 		try {
 			let content = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-			for (let serializedRole of content.roles) {
-				let role = lib.Role.fromJSON(serializedRole);
-				this.roles.set(role.id, role);
+			if ("roles" in content) { // Split file format after 2.0.0-alpha.22
+				this._roles.setMany(content.roles.map((serializedRole: any) => lib.Role.fromJSON(serializedRole)));
+				await this._roles.save(); // Force a save to prevent data loss
+				content = content.users;
 			}

 			let duplicates = 0;
-			for (let serializedUser of content.users) {
-				let user = ControllerUser.fromJSON(serializedUser, this);
+			for (const serializedUser of content) {
+				const user = ControllerUser.fromJSON(serializedUser, this._roles);
 				const existingUser = this.users.get(user.id);
 				if (existingUser) {
 					// Required migration to all lowercase ids in alpha 19
@@ -65,30 +66,14 @@ export default class UserManager {
 			if (err.code !== "ENOENT") {
 				throw err;
 			}
-
-			// Create default roles if loading failed
-			lib.ensureDefaultAdminRole(this.roles);
-			lib.ensureDefaultPlayerRole(this.roles);
 		}
 	}

 	async save(filePath: string): Promise<void> {
-		let serializedRoles = [];
-		for (let role of this.roles.values()) {
-			serializedRoles.push(role.toJSON());
-		}
-
-		let serializedUsers = [];
-		for (let user of this.users.values()) {
-			serializedUsers.push(user.toJSON(true));
-		}
-
-		let serialized = {
-			users: serializedUsers,
-			roles: serializedRoles,
-		};
 		this.dirty = false;
-		await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t"));
+		await lib.safeOutputFile(filePath,
+			JSON.stringify([...this.users.values()].map(user => user.toJSON(true)), null, "\t")
+		);
 	}

 	/**
@@ -103,11 +88,11 @@ export default class UserManager {

 		let roles = new Set<number>();
 		let defaultRoleId = this._config.get("controller.default_role_id");
-		if (defaultRoleId !== null && this.roles.has(defaultRoleId)) {
+		if (defaultRoleId !== null && this._roles.has(defaultRoleId)) {
 			roles.add(defaultRoleId);
 		}

-		let user = new ControllerUser(this, 0, name, roles);
+		let user = new ControllerUser(this._roles, 0, name, roles);
 		this.users.set(user.id, user);
 		this.dirty = true;
 		return user;
@@ -1,6 +1,7 @@
 import type WebSocket from "ws";

 import events from "events";
+import assert from "assert/strict";

 import * as lib from "@clusterio/lib";
 const { logger } = lib;
@@ -113,8 +114,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {

 	_timedOut() {
 		logger.verbose("Connector | Connection timed out");
-		this._reset();
-		this.emit("close");
+		this._close();
 	}

 	_attachSocketHandlers() {
@@ -130,8 +130,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
 			this._socket = null;

 			if (this._closing) {
-				this._reset();
-				this.emit("close");
+				this._close();

 			} else {
 				this._state = "resuming";
@@ -187,8 +186,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
 		}

 		if (this._state === "resuming") {
-			this._reset();
-			this.emit("close");
+			this._close();
 			return;
 		}

@@ -223,8 +221,7 @@ export default class WsServerConnector extends lib.WebSocketBaseConnector {
 	 */
 	setClosing() {
 		if (this._state === "resuming") {
-			this._reset();
-			this.emit("close");
+			this._close();
 			return;
 		}

@@ -266,6 +266,12 @@ async function uploadExport(req: Request, res: Response) {
 	res.sendStatus(200);
 }

+type ProxyStreamEvents = {
+	"error": [ err: any ],
+	"close": [],
+	"timeout": [],
+	"source": [],
+}

 export interface ProxyStream {
 	id: string;
@@ -274,7 +280,7 @@ export interface ProxyStream {
 	size?: string;
 	mime?: string;
 	filename: string | null;
-	events: events.EventEmitter;
+	events: events.EventEmitter<ProxyStreamEvents>;
 	timeout: ReturnType<typeof setTimeout>;
 }

@@ -689,7 +689,7 @@ async function inquirerMissingArgs(args) {
 	}

 	if (["standalone", "controller", "host"].includes(answers.mode)) {
-		answers = inquirer.prompt([
+		answers = await inquirer.prompt([
 			{
 				type: "confirm",
 				name: "remoteNpm",
@@ -143,7 +143,19 @@ interface CtlArguments {
 	pluginList: string,
 }

-async function startControl() {
+interface InitializeParameters {
+	args: CtlArguments;
+	shouldRun: boolean;
+	ctlPlugins?: Map<string, BaseCtlPlugin>;
+	rootCommands?: lib.CommandTree;
+	controlConfig?: lib.ControlConfig;
+}
+
+export async function initialize(
+	argv: string | string[],
+	ctlPlugins?: Map<string, BaseCtlPlugin>,
+	noLoggerTransport?: boolean,
+): Promise<InitializeParameters> {
 	yargs
 		.scriptName("clusterioctl")
 		.usage("$0 <command> [options]")
@@ -175,38 +187,41 @@ async function startControl() {

 	// Parse the args first to get the configured plugin list.
 	// eslint-disable-next-line node/no-sync
-	let args = yargs.parseSync() as CtlArguments;
+	let args = yargs.parseSync(argv) as CtlArguments;

-	// Log stream for the ctl session.
-	logger.add(
-		new ConsoleTransport({
-			errorLevels: Object.keys(levels),
-			level: args.logLevel,
-			format: new lib.TerminalFormat(),
-		})
-	);
-	lib.handleUnhandledErrors();
-
-	logger.verbose(`Loading available plugins from ${args.pluginList}`);
-	let pluginList = await lib.loadPluginList(args.pluginList);
-
-	// If the command is plugin management we don't try to load plugins
-	if (args._[0] === "plugin") {
-		await lib.handlePluginCommand(args, pluginList, args.pluginList);
-		return;
+	// Log stream for the ctl session. Skipped in testing.
+	if (!noLoggerTransport) {
+		logger.add(
+			new ConsoleTransport({
+				errorLevels: Object.keys(levels),
+				level: args.logLevel,
+				format: new lib.TerminalFormat(),
+			})
+		);
+		lib.handleUnhandledErrors();
 	}

-	logger.verbose("Loading Plugins");
-	let ctlPlugins = await loadPlugins(pluginList);
+	// Discover and load plugins. This check exists to allow tests to inject plugins.
+	if (!ctlPlugins || args._[0] === "plugin") {
+		logger.verbose(`Loading available plugins from ${args.pluginList}`);
+		const pluginList = await lib.loadPluginList(args.pluginList);

-	// Add all cluster management commands including ones from plugins
-	let rootCommands = await commands.registerCommands(ctlPlugins, yargs);
+		// If the command is plugin management we don't try to load plugins
+		if (args._[0] === "plugin") {
+			await lib.handlePluginCommand(args, pluginList, args.pluginList);
+			return { args, shouldRun: false };
+		}

-	// Reparse after commands have been added with help and strict checking.
+		logger.verbose("Loading Plugins");
+		ctlPlugins = await loadPlugins(pluginList);
+	}
+
+	// Add all commands including from plugins and reparse with help and strict checking.
+	const rootCommands = await commands.registerCommands(ctlPlugins, yargs);
 	args = yargs
 		.help()
 		.strict()
-		.parse() as CtlArguments
+		.parse(argv) as CtlArguments
 	;

 	let controlConfig;
@@ -233,18 +248,34 @@ async function startControl() {
 	// Handle the control-config command before trying to connect.
 	if (args._[0] === "control-config") {
 		await lib.handleConfigCommand(args, controlConfig);
-		return;
+		return { args, controlConfig, ctlPlugins, rootCommands, shouldRun: false };
 	}

-	// Determine which command is being executed.
+	return { args, controlConfig, ctlPlugins, rootCommands, shouldRun: true };
+}
+
+export function selectTargetCommand(args: CtlArguments, rootCommands: lib.CommandTree): lib.Command {
 	let commandPath = [...args._] as string[];
 	let targetCommand: lib.CommandTree | lib.Command = rootCommands;
 	while (commandPath.length && targetCommand instanceof lib.CommandTree) {
 		targetCommand = targetCommand.get(commandPath.shift()!)!;
 	}
 	assert(targetCommand instanceof lib.Command);
+	return targetCommand;
+}
+
+async function startControl() {
+	const {
+		args,
+		shouldRun,
+		ctlPlugins,
+		rootCommands,
+		controlConfig,
+	} = await initialize(process.argv.slice(2));
+	if (!shouldRun || !ctlPlugins || !rootCommands || !controlConfig) {
+		return;
+	}

-	// The remaining commands require connecting to the controller.
 	if (!controlConfig.get("control.controller_url") || !controlConfig.get("control.controller_token")) {
 		logger.error("Missing URL and/or token to connect with.  See README.md for setting up access.");
 		process.exitCode = 1;
@@ -263,6 +294,7 @@ async function startControl() {
 		tlsCa,
 		controlConfig.get("control.controller_token")!,
 	);
+
 	let control = new Control(controlConnector, controlConfig, tlsCa, ctlPlugins);
 	try {
 		await controlConnector.connect();
@@ -284,6 +316,7 @@ async function startControl() {
 	});

 	try {
+		const targetCommand = selectTargetCommand(args, rootCommands);
 		await targetCommand.run(args, control);

 	} catch (err) {
@@ -197,28 +197,24 @@ controllerConfigCommands.add(new lib.Command({
 		editorSpawn.on("data", (data) => {
 			process.stdout.pipe(data);
 		});
-		let doneEmitter = new events.EventEmitter();
-		editorSpawn.on("exit", async (exit) => {
-			const data = await fs.readFile(tmpFile, "utf8");
-			const final = await configToKeyVal(data);
-			for (let index in final) {
-				if (index in final) {
-					try {
-						await control.send(new lib.ControllerConfigSetFieldRequest(
-							index,
-							final[index],
-						));
-					} catch (err) {
-						// eslint-disable-next-line
-						print(`Attempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
-						print(err);
-						// If the string is empty, it's better to just print "" instead of nothing
-					}
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.send(new lib.ControllerConfigSetFieldRequest(
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`Attempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					print(err);
+					// If the string is empty, it's better to just print "" instead of nothing
 				}
 			}
-			doneEmitter.emit("dot_on_done");
-		});
-		await events.once(doneEmitter, "dot_on_done");
+		}
 		try {
 			await fs.unlink(tmpFile);
 		} catch (err) {
@@ -487,33 +483,29 @@ hostConfigCommands.add(new lib.Command({
 		editorSpawn.on("data", (data) => {
 			process.stdout.pipe(data);
 		});
-		let doneEmitter = new events.EventEmitter();
-		editorSpawn.on("exit", async (exit) => {
-			const data = await fs.readFile(tmpFile, "utf8");
-			const final = await configToKeyVal(data);
-			for (let index in final) {
-				if (index in final) {
-					try {
-						await control.sendTo({ hostId }, new lib.HostConfigSetFieldRequest(
-							index,
-							final[index],
-						));
-					} catch (err) {
-						// eslint-disable-next-line
-						print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
-						// If the string is empty, it's better to just print "" instead of nothing
-						print("This message shouldn't normally appear; if the below message does not indicate it");
-						print("was a user mistake, please report it to the clustorio devs.");
-						// added this because it could be a missed entry in disallowedList
-						// i've added all the vanilla clustorio configs, but there may be
-						// some modded ones that need to be added.
-						print(err);
-					}
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.sendTo({ hostId }, new lib.HostConfigSetFieldRequest(
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					// If the string is empty, it's better to just print "" instead of nothing
+					print("This message shouldn't normally appear; if the below message does not indicate it");
+					print("was a user mistake, please report it to the clustorio devs.");
+					// added this because it could be a missed entry in disallowedList
+					// i've added all the vanilla clustorio configs, but there may be
+					// some modded ones that need to be added.
+					print(err);
 				}
 			}
-			doneEmitter.emit("dot_on_done");
-		});
-		await events.once(doneEmitter, "dot_on_done");
+		}
 		try {
 			await fs.unlink(tmpFile);
 		} catch (err) {
@@ -732,34 +724,30 @@ instanceConfigCommands.add(new lib.Command({
 		editorSpawn.on("data", (data) => {
 			process.stdout.pipe(data);
 		});
-		let doneEmitter = new events.EventEmitter();
-		editorSpawn.on("exit", async (exit) => {
-			const data = await fs.readFile(tmpFile, "utf8");
-			const final = await configToKeyVal(data);
-			for (let index in final) {
-				if (index in final) {
-					try {
-						await control.send(new lib.InstanceConfigSetFieldRequest(
-							instanceId,
-							index,
-							final[index],
-						));
-					} catch (err) {
-						// eslint-disable-next-line
-						print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
-						// If the string is empty, it's better to just print "" instead of nothing
-						print("This message shouldn't normally appear; if the below message does not indicate it");
-						print("was a user mistake, please report it to the clustorio devs.");
-						// added this because it could be a missed entry in disallowedList
-						// i've added all the vanilla clustorio configs, but there may be
-						// some modded ones that need to be added.
-						print(err);
-					}
+		await events.once(editorSpawn, "exit");
+		const data = await fs.readFile(tmpFile, "utf8");
+		const final = await configToKeyVal(data);
+		for (let index in final) {
+			if (index in final) {
+				try {
+					await control.send(new lib.InstanceConfigSetFieldRequest(
+						instanceId,
+						index,
+						final[index],
+					));
+				} catch (err) {
+					// eslint-disable-next-line
+					print(`\n\n\nAttempt to set ${index} to ${final[index] || String(null)} failed; set back to previous value.`);
+					// If the string is empty, it's better to just print "" instead of nothing
+					print("This message shouldn't normally appear; if the below message does not indicate it");
+					print("was a user mistake, please report it to the clustorio devs.");
+					// added this because it could be a missed entry in disallowedList
+					// i've added all the vanilla clustorio configs, but there may be
+					// some modded ones that need to be added.
+					print(err);
 				}
 			}
-			doneEmitter.emit("dot_on_done");
-		});
-		await events.once(doneEmitter, "dot_on_done");
+		}
 		try {
 			await fs.unlink(tmpFile);
 		} catch (err) {
@@ -6,6 +6,7 @@ import setBlocking from "set-blocking";
 import phin from "phin";
 import stream from "stream";
 import util from "util";
+import child_process from "child_process";

 // internal libraries
 import * as lib from "@clusterio/lib";
@@ -384,6 +385,8 @@ export default class Host extends lib.Link {
 		this.handle(lib.HostMetricsRequest, this.handleHostMetricsRequest.bind(this));
 		this.handle(lib.SyncUserListsEvent, this.handleSyncUserListsEvent.bind(this));
 		this.handle(lib.SystemInfoRequest, this.handleSystemInfoRequest.bind(this));
+		this.handle(lib.HostFactorioVersionsRequest, this.handleHostFactorioVersionsRequest.bind(this));
+		this.handle(lib.HostDownloadFactorioRequest, this.handleHostDownloadFactorioRequest.bind(this));
 		this.handle(lib.PluginListRequest, this.handlePluginListRequest.bind(this));
 		this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this));
 		this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this));
@@ -859,6 +862,160 @@ export default class Host extends lib.Link {
 		return { results };
 	}

+	async handleHostFactorioVersionsRequest() {
+		const factorioDir = this.config.get("host.factorio_directory");
+		const versions: string[] = [];
+
+		try {
+			// Check if factorioDir is a simple installation
+			const simpleVersion = await this.getFactorioVersion(path.join(factorioDir, "data", "changelog.txt"));
+			if (simpleVersion !== null) {
+				versions.push(simpleVersion);
+			} else {
+				// Check for multiple versions in subdirectories
+				const entries = await fs.readdir(factorioDir, { withFileTypes: true });
+				for (const entry of entries) {
+					if (!entry.isDirectory()) {
+						continue;
+					}
+
+					const version = await this.getFactorioVersion(path.join(factorioDir, entry.name, "data", "changelog.txt"));
+					if (version !== null) {
+						versions.push(version);
+					}
+				}
+			}
+		} catch (error: any) {
+			if (error.code !== "ENOENT") {
+				logger.warn(`Error reading Factorio directory: ${error.message}`);
+			}
+		}
+
+		// Sort versions in descending order (newest first)
+		versions.sort((a, b) => {
+			const aParts = a.split(".").map(s => parseInt(s, 10));
+			const bParts = b.split(".").map(s => parseInt(s, 10));
+
+			for (let i = 0; i < Math.max(aParts.length, bParts.length); i++) {
+				const aPart = aParts[i] || 0;
+				const bPart = bParts[i] || 0;
+				if (aPart !== bPart) {
+					return bPart - aPart;
+				}
+			}
+			return 0;
+		});
+
+		return new lib.HostFactorioVersionsRequest.Response(versions);
+	}
+
+	async handleHostDownloadFactorioRequest() {
+		try {
+			const factorioDir = this.config.get("host.factorio_directory");
+			
+			// Fetch the latest version info
+			const res = await fetch("https://factorio.com/get-download/stable/headless/linux64");
+			const location = res.headers.get("location");
+			if (!location) {
+				throw new Error("No redirect location found in response headers");
+			}
+			const url = new URL(location);
+			const filename = path.posix.basename(url.pathname);
+			const match = filename.match(/(?<=factorio_headless_x64_|factorio-headless_linux_)\d+\.\d+\.\d+(?=\.tar\.xz)/);
+			
+			if (!match || !match.length) {
+				throw new Error(`Unable to extract version from filename: ${filename}`);
+			}
+			
+			const version = match[0];
+			const tmpDir = "temp/create-temp/";
+			const archivePath = path.join(tmpDir, filename);
+			const tmpArchivePath = `${archivePath}.tmp`;
+			const versionDir = path.join(factorioDir, version);
+			const tmpFactorioDir = path.join(tmpDir, version);
+
+			// Check if version already exists
+			if (await fs.pathExists(versionDir)) {
+				return new lib.HostDownloadFactorioRequest.Response(true, version);
+			}
+
+			await fs.ensureDir(tmpDir);
+
+			// Download the factorio archive
+			const downloadRes = await fetch(url.href);
+			if (!downloadRes.ok) {
+				throw new Error(`Failed to download Factorio: ${downloadRes.statusText}`);
+			}
+
+			logger.info(`Downloading Factorio ${version}. This may take a while.`);
+			const writeStream = fs.createWriteStream(tmpArchivePath);
+			if (downloadRes.body) {
+				const readableStream = stream.Readable.fromWeb(downloadRes.body);
+				readableStream.pipe(writeStream);
+				await finished(writeStream);
+			} else {
+				throw new Error("No response body received from download");
+			}
+
+			await fs.rename(tmpArchivePath, archivePath);
+
+			// Extract the archive
+			await fs.ensureDir(tmpFactorioDir);
+			await this.execFile("tar", [
+				"xf", archivePath, "-C", tmpFactorioDir, "--strip-components", "1",
+			]);
+
+			// Clean up archive and move to final location
+			await fs.unlink(archivePath);
+			await fs.rename(tmpFactorioDir, versionDir);
+
+			logger.info(`Successfully downloaded and installed Factorio ${version}`);
+			return new lib.HostDownloadFactorioRequest.Response(true, version);
+
+		} catch (error: any) {
+			logger.error(`Failed to download Factorio: ${error.message}`);
+			return new lib.HostDownloadFactorioRequest.Response(false, undefined, error.message);
+		}
+	}
+
+	private async getFactorioVersion(changelogPath: string): Promise<string | null> {
+		try {
+			const changelog = await fs.readFile(changelogPath, "utf-8");
+			for (const line of changelog.split(/[\r\n]+/)) {
+				const index = line.indexOf(":");
+				if (index !== -1) {
+					const name = line.slice(0, index).trim();
+					if (name.toLowerCase() === "version") {
+						return line.slice(index + 1).trim();
+					}
+				}
+			}
+			return null;
+		} catch (error: any) {
+			if (error.code === "ENOENT") {
+				return null;
+			}
+			throw error;
+		}
+	}
+
+	private async execFile(cmd: string, args: string[]): Promise<{ stdout: string; stderr: string }> {
+		return new Promise((resolve, reject) => {
+			child_process.execFile(
+				cmd,
+				args,
+				{ shell: true },
+				(err, stdout, stderr) => {
+					if (err) {
+						reject(err);
+					} else {
+						resolve({ stdout, stderr });
+					}
+				}
+			);
+		});
+	}
+
 	async fallbackInstanceSaveDetailsListRequest(
 		request: lib.InstanceSaveDetailsListRequest,
 		src: lib.Address,
@@ -454,6 +454,32 @@ export interface FactorioServerOptions {
 	shutdownTimeoutMs?: number,
 }

+type FactorioServerEvents = {
+	// Public Events
+	"stdout": [ rawLine: Buffer ],
+	"stderr": [ rawLine: Buffer ],
+	"output": [ parsed: lib.ParsedFactorioOutput, line: string ]
+
+	"error": [ err: any ],
+	"rcon-ready": [],
+	"game-ready": [],
+	"exit": [],
+
+	"autosave-start": [ name: string ],
+	"autosave-finished": [ name: string ],
+	"save-finished": [],
+
+	"whitelist-change": [ added: string[], removed: string[] ],
+
+	// Private Events
+	"_autosave": [ name: string ]
+	"_saving": [],
+	"_saved": [],
+
+	// IPS events
+	[ipcEvent: `ipc-${string}`]: [ event: any ],
+};
+
 /**
 * Factorio Server interface
 *
@@ -473,7 +499,7 @@ export interface FactorioServerOptions {
 * - exit - invoked when the sterver has exited
 * @extends events.EventEmitter
 */
-export class FactorioServer extends events.EventEmitter {
+export class FactorioServer extends events.EventEmitter<FactorioServerEvents> {
 	/** Path to executable to invoke when starting the server */
 	executablePath?: string;
 	/** UDP port used for hosting the Factorio game server on */
@@ -1212,7 +1238,7 @@ export class FactorioServer extends events.EventEmitter {
 		// complete before the RCON connection can be used
 		if (!this._rconReady) {
 			// Not using events.once here to avoid throwing on error events.
-			await new Promise(resolve => this.once("rcon-ready", resolve));
+			await new Promise<void>(resolve => this.once("rcon-ready", resolve));
 		}

 		// The Factorio server may have decided to get ahead of us and
@@ -15,4 +15,3 @@ export * from "./src/subscriptions";

 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as RateLimiter } from "./src/RateLimiter";
-export { default as TypedEventEmitter } from "./src/TypedEventEmitter";
@@ -35,4 +35,3 @@ export * from "./src/rce_ops";
 export { default as ExponentialBackoff } from "./src/ExponentialBackoff";
 export { default as ModStore } from "./src/ModStore";
 export { default as RateLimiter } from "./src/RateLimiter";
-export { default as TypedEventEmitter } from "./src/TypedEventEmitter";
@@ -1,16 +1,16 @@
 import fs from "fs-extra";
 import path from "path";
+import events from "events";
 import { Static } from "@sinclair/typebox";
 import { logger } from "./logging";
-import { integerModVersion, ModInfo, ModPack, ModRecord } from "./data";
-import TypedEventEmitter from "./TypedEventEmitter";
+import { ModInfo, ModPack } from "./data";
 import { safeOutputFile } from "./file_ops";
 import { Writable } from "stream";
 import { ModPortalReleaseSchema, ModPortalDetailsSchema } from "./data/messages_mod";

 export interface ModStoreEvents {
 	/** A stored mod was created, updated or deleted */
-	change: (mod: ModInfo) => void;
+	"change": [ mod: ModInfo ];
 }

 interface ModsInfoResponse {
@@ -37,7 +37,7 @@ interface ModPortalResponse {
 	results: Static<typeof ModPortalDetailsSchema>[];
 }

-export default class ModStore extends TypedEventEmitter<keyof ModStoreEvents, ModStoreEvents> {
+export default class ModStore extends events.EventEmitter<ModStoreEvents> {
 	constructor(
 		public modsDirectory: string,
 		public files: Map<string, ModInfo>,
@@ -1,27 +0,0 @@
-import EventEmitter from "events";
-
-/**
- * Type checked version of EventEmitter
- *
- * @example
- * ```ts
- * interface Events {
- *     "foo": (a: number, b: number) => void,
- *     "bar": (spam: string) => void,
- * }
- *
- * class Emitter extends TypedEventEmitter<keyof Events, Events> {}
- *
- * const e = new Emitter();
- * e.emit("foo", 1, 2) // type checked arguments
- * ```
- */
-export default abstract class TypedEventEmitter<
-	Keys extends string,
-	Events extends Record<Keys, (...args: [...any]) => void>,
-> extends EventEmitter {
-	declare ["on"]: <E extends Keys>(event: E, listener: Events[E]) => this;
-	declare ["off"]: <E extends Keys>(event: E, listener: Events[E]) => this;
-	declare ["once"]: <E extends Keys>(event: E, listener: Events[E]) => this;
-	declare ["emit"]: <E extends Keys>(event: E, ...args: Parameters<Events[E]>) => boolean;
-}
@@ -1,8 +1,8 @@
 // Configuration classes
 import { Type, Static } from "@sinclair/typebox";
 import * as fs from "fs-extra";
+import events from "events";

-import TypedEventEmitter from "../TypedEventEmitter";
 import isDeepStrictEqual from "../is_deep_strict_equal";
 import { basicType } from "../helpers";
 import * as libSchema from "../schema";
@@ -152,7 +152,7 @@ function defaultValue(def: FieldDefinition) {
 }

 export type ConfigEvents = {
-	fieldChanged: (name: string, curr: FieldValue, prev: FieldValue) => void;
+	fieldChanged: [ name: string, curr: FieldValue, prev: FieldValue ],
 }

 export enum ConfigAccess {
@@ -178,10 +178,7 @@ export enum ConfigAccess {
 */
 export class Config<
 	Fields extends { [Field in keyof Fields]: FieldValue },
-> extends TypedEventEmitter<
-	keyof ConfigEvents,
-	ConfigEvents
-> {
+> extends events.EventEmitter<ConfigEvents> {
 	/**
 	 * Mapping of config name to field meta data
 	 * Note that mutating this object will lead to unexpected behaviour.
@@ -10,6 +10,8 @@ export default class Role {
 		public name: string,
 		public description: string,
 		public permissions: Set<string>,
+		public updatedAtMs = 0,
+		public isDeleted = false,
 	) { }

 	static jsonSchema = Type.Object({
@@ -17,22 +19,18 @@ export default class Role {
 		name: Type.String(),
 		description: Type.String(),
 		permissions: Type.Array(Type.String()),
+		updated_at_ms: Type.Optional(Type.Number()),
+		is_deleted: Type.Optional(Type.Boolean()),
 	});

-	static remapPermission(permission: string) {
-		// migrate: core.instance.save.list.subscribe was renamed in alpha 17
-		if (permission === "core.instance.save.list.subscribe") {
-			return "core.instance.save.subscribe";
-		}
-		return permission;
-	}
-
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
 		return new this(
 			json.id,
 			json.name,
 			json.description,
-			new Set(json.permissions.map(this.remapPermission)),
+			new Set(json.permissions),
+			json.updated_at_ms,
+			json.is_deleted,
 		);
 	}

@@ -42,6 +40,8 @@ export default class Role {
 			name: this.name,
 			description: this.description,
 			permissions: [...this.permissions],
+			updated_at_ms: this.updatedAtMs,
+			is_deleted: this.isDeleted,
 		};
 	}

@@ -21,5 +21,6 @@ export * from "./messages_host";
 export * from "./messages_instance";
 export * from "./messages_mod";
 export * from "./messages_user";
+export * from "./messages_role";
 export * from "./messages_plugin";
 export * from "./version";
@@ -288,3 +288,51 @@ export class HostRevokeTokensRequest {
 		return new this(json.hostId);
 	}
 }
+
+export class HostFactorioVersionsRequest {
+	declare ["constructor"]: typeof HostFactorioVersionsRequest;
+	static type = "request" as const;
+	static src = ["control", "controller"] as const;
+	static dst = "host" as const;
+	static permission = "core.host.list" as const;
+
+	static Response = class Response {
+		constructor(
+			public versions: string[],
+		) { }
+
+		static jsonSchema = Type.Object({
+			"versions": Type.Array(Type.String()),
+		});
+
+		static fromJSON(json: Static<typeof this.jsonSchema>) {
+			return new this(json.versions);
+		}
+	};
+}
+
+export class HostDownloadFactorioRequest {
+	declare ["constructor"]: typeof HostDownloadFactorioRequest;
+	static type = "request" as const;
+	static src = ["control", "controller"] as const;
+	static dst = "host" as const;
+	static permission = "core.host.update" as const;
+
+	static Response = class Response {
+		constructor(
+			public success: boolean,
+			public version?: string,
+			public error?: string,
+		) { }
+
+		static jsonSchema = Type.Object({
+			"success": Type.Boolean(),
+			"version": Type.Optional(Type.String()),
+			"error": Type.Optional(Type.String()),
+		});
+
+		static fromJSON(json: Static<typeof this.jsonSchema>) {
+			return new this(json.success, json.version, json.error);
+		}
+	};
+}
@@ -0,0 +1,134 @@
+import { Type, Static } from "@sinclair/typebox";
+import { JsonNumber, jsonArray } from "./composites";
+import Permission from "./Permission";
+import Role from "./Role";
+
+export class PermissionListRequest {
+	declare ["constructor"]: typeof PermissionListRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.permission.list" as const;
+	static Response = jsonArray(Permission);
+}
+
+export class RoleListRequest {
+	declare ["constructor"]: typeof RoleListRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.list" as const;
+	static Response = jsonArray(Role);
+}
+
+export class RoleUpdatesEvent {
+	declare ["constructor"]: typeof RoleUpdatesEvent;
+	static type = "event" as const;
+	static src = "controller" as const;
+	static dst = "control" as const;
+	static permission = "core.role.subscribe" as const;
+
+	constructor(
+		public updates: Role[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		"updates": Type.Array(Role.jsonSchema),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.updates.map(update => Role.fromJSON(update)));
+	}
+}
+
+export class RoleCreateRequest {
+	declare ["constructor"]: typeof RoleCreateRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.create" as const;
+
+	constructor(
+		public name: string,
+		public description: string,
+		public permissions: string[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		name: Type.String(),
+		description: Type.String(),
+		permissions: Type.Array(Type.String()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.name, json.description, json.permissions);
+	}
+
+	static Response = JsonNumber;
+}
+
+export class RoleUpdateRequest {
+	declare ["constructor"]: typeof RoleUpdateRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.update" as const;
+
+	constructor(
+		public id: number,
+		public name: string,
+		public description: string,
+		public permissions: string[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		id: Type.Integer(),
+		name: Type.String(),
+		description: Type.String(),
+		permissions: Type.Array(Type.String()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.id, json.name, json.description, json.permissions);
+	}
+}
+
+export class RoleGrantDefaultPermissionsRequest {
+	declare ["constructor"]: typeof RoleGrantDefaultPermissionsRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.update" as const;
+
+	constructor(
+		public id: number,
+	) { }
+
+	static jsonSchema = Type.Object({
+		id: Type.Integer(),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.id);
+	}
+}
+
+export class RoleDeleteRequest {
+	declare ["constructor"]: typeof RoleDeleteRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.delete" as const;
+
+	constructor(
+		public id: number,
+	) { }
+
+	static jsonSchema = Type.Object({
+		id: Type.Integer(),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.id);
+	}
+}
@@ -1,120 +1,8 @@
 import { Type, Static } from "@sinclair/typebox";
-import Permission from "./Permission";
-import Role from "./Role";
 import User, { IControllerUser } from "./User";
-import { JsonNumber, StringEnum, jsonArray, plainJson } from "./composites";
+import { StringEnum, jsonArray, plainJson } from "./composites";
 import { MessageRequest } from "./messages_core";

-export class PermissionListRequest {
-	declare ["constructor"]: typeof PermissionListRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.permission.list" as const;
-	static Response = jsonArray(Permission);
-}
-
-export class RoleListRequest {
-	declare ["constructor"]: typeof RoleListRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.list" as const;
-	static Response = jsonArray(Role);
-}
-
-export class RoleCreateRequest {
-	declare ["constructor"]: typeof RoleCreateRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.create" as const;
-
-	constructor(
-		public name: string,
-		public description: string,
-		public permissions: string[],
-	) { }
-
-	static jsonSchema = Type.Object({
-		name: Type.String(),
-		description: Type.String(),
-		permissions: Type.Array(Type.String()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.name, json.description, json.permissions);
-	}
-
-	static Response = JsonNumber;
-}
-
-export class RoleUpdateRequest {
-	declare ["constructor"]: typeof RoleUpdateRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.update" as const;
-
-	constructor(
-		public id: number,
-		public name: string,
-		public description: string,
-		public permissions: string[],
-	) { }
-
-	static jsonSchema = Type.Object({
-		id: Type.Integer(),
-		name: Type.String(),
-		description: Type.String(),
-		permissions: Type.Array(Type.String()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.id, json.name, json.description, json.permissions);
-	}
-}
-
-export class RoleGrantDefaultPermissionsRequest {
-	declare ["constructor"]: typeof RoleGrantDefaultPermissionsRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.update" as const;
-
-	constructor(
-		public id: number,
-	) { }
-
-	static jsonSchema = Type.Object({
-		id: Type.Integer(),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.id);
-	}
-}
-
-export class RoleDeleteRequest {
-	declare ["constructor"]: typeof RoleDeleteRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.delete" as const;
-
-	constructor(
-		public id: number,
-	) { }
-
-	static jsonSchema = Type.Object({
-		id: Type.Integer(),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.id);
-	}
-}
-
 export class UserGetRequest {
 	declare ["constructor"]: typeof UserGetRequest;
 	static type = "request" as const;
@@ -1,11 +1,11 @@
+import path from "path";
 import fs from "fs-extra";
+import { EventEmitter } from "events";

 import { safeOutputFile } from "./file_ops";
 import { SubscribableValue } from "./subscriptions";
 import { logger } from "./logging";
-import { EventEmitter } from "stream";
 import { ControllerConfig } from "./config";
-import path from "path";

 type DatastoreKey = string | number;
 type DatastoreValue = string | number | boolean | object;
@@ -30,7 +30,7 @@ export class MemoryDatastoreProvider<
 	V extends DatastoreValue,
 > extends DatastoreProvider<K, V> {
 	constructor(
-		private value: Map<K, V> = new Map()
+		private value = new Map<K, V>()
 	) {
 		super();
 	}
@@ -39,34 +39,33 @@ export class MemoryDatastoreProvider<
 		this.value = new Map(data);
 	}

-	async load() {
+	async load(): Promise<Map<K, V>> {
 		return new Map(this.value);
 	}
 }

 // Store all data within a json object file and validate the result against a schema
 export class JsonDatastoreProvider<
-	K extends DatastoreKey,
 	V extends DatastoreValue,
 	J, // Intermediate type returned from migrations
-> extends DatastoreProvider<K, V> {
+> extends DatastoreProvider<string, V> {
 	constructor(
 		private filePath: string,
-		private fromJson: (json: J) => V = v => v as any,
-		private migrations: (rawJson: Record<DatastoreKey, unknown>) => Record<DatastoreKey, J> = v => v as any,
+		private fromJson: (json: J) => V = v => v as any as V,
+		private migrations: (rawJson: Record<string, unknown>) => Record<string, J> = v => v as Record<string, J>,
 		private finalise: (obj: V) => V = v => v,
 	) {
 		super();
 	}

 	// Save the data to the json file
-	async save(data: Map<K, V>) {
+	async save(data: Map<string, V>) {
 		logger.verbose(`Saving ${this.filePath}`);
 		await safeOutputFile(this.filePath, JSON.stringify(Object.fromEntries(data), null, "\t"));
 	}

 	// Load the data from the json file
-	async load() {
+	async load(): Promise<Map<string, V>> {
 		// Read the raw json from the file
 		let rawJson;
 		try {
@@ -89,27 +88,26 @@ export class JsonDatastoreProvider<

 // Store all data within a json array file where each element has a unique id
 export class JsonIdDatastoreProvider<
-	K extends DatastoreKey,
-	V extends DatastoreValue & { id: K },
+	V extends DatastoreValue & { id: DatastoreKey },
 	J, // Intermediate type returned from migrations
-> extends DatastoreProvider<K, V> {
+> extends DatastoreProvider<V["id"], V> {
 	constructor(
 		private filePath: string,
-		private fromJson: (json: J) => V = v => v as any,
-		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as any,
+		private fromJson: (json: J) => V = v => v as any as V,
+		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as Array<J>,
 		private finalise: (obj: V) => V = v => v,
 	) {
 		super();
 	}

 	// Save the data to the json file
-	async save(data: Map<K, V>) {
+	async save(data: Map<V["id"], V>) {
 		logger.verbose(`Saving ${this.filePath}`);
 		await safeOutputFile(this.filePath, JSON.stringify([...data.values()], null, "\t"));
 	}

 	// Load the data from the json file
-	async load() {
+	async load(): Promise<Map<V["id"], V>> {
 		// Read the raw json from the file
 		let rawJson;
 		try {
@@ -137,7 +135,10 @@ export class JsonIdDatastoreProvider<
 export abstract class Datastore<
 	K extends DatastoreKey,
 	V extends DatastoreValue,
-> extends EventEmitter {
+	U = V,
+> extends EventEmitter<{
+	"update": [ updates: U[] ],
+}> {
 	private dirty = false;

 	constructor(
@@ -167,7 +168,7 @@ export abstract class Datastore<
 	}

 	// Set the dirty flag and call update handlers
-	protected touch(updates: any[]) {
+	protected touch(updates: U[]) {
 		if (updates.length) {
 			this.dirty = true;
 			this.emit("update", updates);
@@ -208,13 +209,17 @@ export abstract class Datastore<
 	valuesMutable() {
 		return this.data.values();
 	}
+
+	keys() {
+		return this.data.keys();
+	}
 }

 // General key value mapping which can be saved and loaded with the file system
 export class KeyValueDatastore<
 	K extends DatastoreKey,
 	V extends DatastoreValue,
-> extends Datastore<K, V> {
+> extends Datastore<K, V, [key: K, value: V, isDeleted?: boolean]> {
 	// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
 	set(key: K, value: V) {
 		this.data.set(key, value);
@@ -232,8 +237,10 @@ export class KeyValueDatastore<
 	// Delete a value in the datastore
 	delete(key: K) {
 		const value = this.data.get(key);
-		this.data.delete(key);
-		this.touch([[key, value!, true]]);
+		if (value !== undefined) {
+			this.data.delete(key);
+			this.touch([[key, value, true]]);
+		}
 	}

 	// Delete many values at once from an array of keys
@@ -1,8 +1,8 @@
 // Connection adapters for Links
 import { strict as assert } from "assert";
 import events from "events";
-import WebSocket from "./WebSocket";

+import WebSocket from "./WebSocket";
 import * as libData from "../data";
 import * as libErrors from "../errors";
 import { logger } from "../logging";
@@ -36,12 +36,21 @@ export enum ConnectionClosed {
 	RecoveryMode = 4001,
 };

+type BaseConnectorEvents = {
+	"message": [ message: libData.Message ],
+	"error": [ err: any ],
+	"invalidate": [],
+	"reset": [],
+};
+
 /**
 * Base connector for links
 *
 * @extends events.EventEmitter
 */
-export abstract class BaseConnector extends events.EventEmitter {
+export abstract class BaseConnector<
+	E extends Record<string, any[]> | [never] = [never], // Should be "extends EventMap = DefaultEventMap"
+> extends events.EventEmitter<E | BaseConnectorEvents> {
 	protected _seq = 1;

 	constructor(
@@ -53,10 +62,12 @@ export abstract class BaseConnector extends events.EventEmitter {

 	_reset() {
 		this._seq = 1;
+		this.emit("reset");
 	}

 	_invalidate() {
 		this._seq = 1;
+		this.emit("invalidate");
 	}

 	protected abstract send(message: libData.MessageRoutable): void;
@@ -131,6 +142,15 @@ export abstract class BaseConnector extends events.EventEmitter {

 type ConnectorState = "closed" | "connecting" | "connected" | "resuming";

+type WebSocketBaseConnectorEvents = {
+	"disconnectReady": [],
+	"disconnectPrepare": [],
+	"connect": [ data: any ],
+	"resume": [],
+	"close": [],
+	"drop": [],
+};
+
 export type WebSocketClusterio = WebSocket.WebSocket & {
 	clusterio_ignore_dump?: boolean;
 };
@@ -140,7 +160,9 @@ export type WebSocketClusterio = WebSocket.WebSocket & {
 *
 * @extends module:lib.BaseConnector
 */
-export abstract class WebSocketBaseConnector extends BaseConnector {
+export abstract class WebSocketBaseConnector<
+	E extends Record<string, any[]> | [never] = [never], // Should be "extends EventMap = DefaultEventMap"
+> extends BaseConnector<E | WebSocketBaseConnectorEvents> {
 	// One of closed, connecting (client only), connected and resuming.
 	_state: ConnectorState = "closed";
 	_closing = false;
@@ -166,7 +188,7 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 	_invalidate() {
 		this._state = "connecting";
 		assert(this._closing === false);
-		assert(this._socket);
+		// We do not care about the socket state
 		this._lastHeartbeatMs = null;
 		assert(this._heartbeatId === null);
 		this._heartbeatInterval = null;
@@ -175,6 +197,11 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 		super._invalidate();
 	}

+	_close() {
+		this._reset();
+		this.emit("close");
+	}
+
 	_check(...expectedStates: ConnectorState[]) {
 		if (!expectedStates.includes(this._state)) {
 			throw new Error(
@@ -385,12 +412,16 @@ export abstract class WebSocketBaseConnector extends BaseConnector {
 	}
 }

+type WebSocketClientConnectorEvents = {
+	"hello": [ data: any ],
+};
+
 /**
 * Connector for controller clients
 *
 * @extends module:lib.WebSocketBaseConnector
 */
-export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
+export abstract class WebSocketClientConnector extends WebSocketBaseConnector<WebSocketClientConnectorEvents> {
 	_reconnectId?: ReturnType<typeof setTimeout>;
 	_sessionToken: string | null = null;
 	_sessionTimeout: number | null = null;
@@ -418,8 +449,8 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 		this._sessionToken = null;
 		this._sessionTimeout = null;
 		this._startedResumingMs = null;
-		super._reset();
 		this.src = undefined as any;
+		super._reset();
 	}

 	_invalidate() {
@@ -427,8 +458,8 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 		this._sessionToken = null;
 		this._sessionTimeout = null;
 		this._startedResumingMs = null;
-		super._invalidate();
 		this.src = undefined as any;
+		super._invalidate();
 	}

 	/**
@@ -463,8 +494,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 			await events.once(this, "close");

 		} else {
-			this._reset();
-			this.emit("close");
+			this._close();
 		}
 	}

@@ -489,8 +519,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 			this._socket!.close(ConnectionClosed.NormalClosure, "Connector closing");

 		} else {
-			this._reset();
-			this.emit("close");
+			this._close();
 		}
 	}

@@ -546,13 +575,10 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 			&& this._startedResumingMs! + this._sessionTimeout! * 1000 < Date.now() + delayMs
 		) {
 			logger.error("Connector | Session timed out trying to resume");
-			this._reset();
 			if (this._closing) {
-				this._reset();
-				this.emit("close");
+				this._close();
 			} else {
-				this._state = "connecting";
-				this.emit("invalidate");
+				this._invalidate();
 			}
 		}
 		logger.verbose(
@@ -604,8 +630,7 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 			} else {
 				// eslint-disable-next-line no-lonely-if
 				if (this._closing) {
-					this._reset();
-					this.emit("close");
+					this._close();

 				} else {
 					this.reconnect();
@@ -711,11 +736,9 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector {
 		} else if (type === "invalidate") {
 			logger.warn("Connector | session invalidated by controller");
 			if (this._closing) {
-				this._reset();
-				this.emit("close");
+				this._close();
 			} else {
 				this._invalidate();
-				this.emit("invalidate");
 				this.register();
 			}
 		}
@@ -113,7 +113,7 @@ export class Link {
 		// Process messages received by the connector
 		connector.on("message", payload => {
 			try {
-				this._processMessage(payload);
+				this._processMessage(payload as MessageRoutable);
 			} catch (err) {
 				if (err instanceof libErrors.InvalidMessage) {
 					logger.error(err.message);
@@ -128,10 +128,10 @@ export class Link {
 			}
 		});

+		// The events below exist only on websocket connectors, we can't early return because of test mocking
+		const webSocketConnector = this.connector as WebSocketBaseConnector;
 		connector.on("disconnectPrepare", () => {
 			this.prepareDisconnect().finally(() => {
-				// Only WebSocket connectors issue disconnection events.
-				const webSocketConnector = this.connector as WebSocketBaseConnector;
 				if (webSocketConnector.hasSession) {
 					webSocketConnector.sendDisconnectReady();
 				}
@@ -4,6 +4,7 @@ import * as host from "../data/messages_host";
 import * as instance from "../data/messages_instance";
 import * as mod from "../data/messages_mod";
 import * as user from "../data/messages_user";
+import * as role from "../data/messages_role";
 import * as plugin from "../data/messages_plugin";
 import * as subscriptions from "../subscriptions";
 import type { RequestClass, EventClass } from "./link";
@@ -51,6 +52,8 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	host.PrepareControllerDisconnectRequest,
 	host.SyncUserListsEvent,
 	host.HostRevokeTokensRequest,
+	host.HostFactorioVersionsRequest,
+	host.HostDownloadFactorioRequest,

 	instance.InstanceDetailsGetRequest,
 	instance.InstanceDetailsListRequest,
@@ -107,12 +110,14 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	mod.ModUpdatesEvent,
 	mod.ModPortalDownloadRequest,

-	user.PermissionListRequest,
-	user.RoleListRequest,
-	user.RoleCreateRequest,
-	user.RoleUpdateRequest,
-	user.RoleGrantDefaultPermissionsRequest,
-	user.RoleDeleteRequest,
+	role.PermissionListRequest,
+	role.RoleListRequest,
+	role.RoleUpdatesEvent,
+	role.RoleCreateRequest,
+	role.RoleUpdateRequest,
+	role.RoleGrantDefaultPermissionsRequest,
+	role.RoleDeleteRequest,
+
 	user.UserGetRequest,
 	user.UserListRequest,
 	user.UserCreateRequest,
@@ -7,6 +7,7 @@
 * @module lib/users
 */
 import { Permission, Role } from "./data";
+import { SubscribableDatastore } from "./datastore";


 export const permissions = new Map<string, Permission>();
@@ -66,7 +67,7 @@ export function definePermission({
 * @param roles - role storage to modify.
 * @returns the default admin role
 */
-export function ensureDefaultAdminRole(roles: Map<number, Role>) {
+export function ensureDefaultAdminRole(roles: SubscribableDatastore<Role>) {
 	let admin = roles.get(0);
 	if (!admin) {
 		admin = Role.fromJSON({
@@ -75,9 +76,9 @@ export function ensureDefaultAdminRole(roles: Map<number, Role>) {
 			description: "Cluster wide administrator.",
 			permissions: [],
 		});
-		roles.set(0, admin);
 	}
 	admin.permissions.add("core.admin");
+	roles.set(admin);
 	return admin;
 }

@@ -90,7 +91,7 @@ export function ensureDefaultAdminRole(roles: Map<number, Role>) {
 *
 * @param roles - role storage to modify.
 */
-export function ensureDefaultPlayerRole(roles: Map<number, Role>) {
+export function ensureDefaultPlayerRole(roles: SubscribableDatastore<Role>) {
 	let player = roles.get(1);
 	if (!player) {
 		player = Role.fromJSON({
@@ -99,9 +100,9 @@ export function ensureDefaultPlayerRole(roles: Map<number, Role>) {
 			description: "Default player role.",
 			permissions: [],
 		});
-		roles.set(1, player);
 	}
 	player.grantDefaultPermissions();
+	roles.set(player);
 }


@@ -431,6 +432,12 @@ definePermission({
 	description: "Get the full list of roles and their permissions in the cluster.",
 	grantByDefault: true,
 });
+definePermission({
+	name: "core.role.subscribe",
+	title: "Subscribe to role updates",
+	description: "Subscribe to be notified on updates on the details of roles.",
+	grantByDefault: true,
+});
 definePermission({
 	name: "core.role.create",
 	title: "Create role",
@@ -222,10 +222,12 @@ export class EventSubscriber<
 		public control: Link,
 	) {
 		control.handle(Event, this._handleEvent.bind(this));
-		control.connector.on("connect", () => {
+		// The events below exist only on websocket connectors, we can't early return because of test mocking
+		const webSocketConnector = control.connector as WebSocketBaseConnector;
+		webSocketConnector.on("connect", () => {
 			this.handleConnectionEvent("connect");
 		});
-		control.connector.on("close", () => {
+		webSocketConnector.on("close", () => {
 			this.handleConnectionEvent("close");
 		});
 	}
@@ -6,6 +6,7 @@
 		"./src/api.ts",
 		"./src/config/*.ts",
 		"./src/data/*.ts",
+		"./src/datastore.ts",
 		"./src/errors.ts",
 		"./src/factorio/exchange_string.ts",
 		"./src/helpers.ts",
@@ -15,6 +15,7 @@ import { useHost } from "../model/host";
 import PageHeader from "./PageHeader";
 import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
+import FactorioVersionManager from "./FactorioVersionManager";
 import {
 	MetricCpuRatio, MetricCpuUsed, MetricMemoryRatio, MetricMemoryUsed,
 	MetricDiskUsed, MetricDiskRatio,
@@ -151,6 +152,7 @@ export default function HostViewPage() {
 			<Title level={5} style={{ marginTop: 16 }}>Instances</Title>
 			<InstanceList instances={hostInstances} size="small" hideAssignedHost />
 		</>}
+		<FactorioVersionManager hostId={hostId} system={system} connected={host.connected} />
 		{account.hasPermission("core.log.follow") && <>
 			<Flex justify="space-between" align="baseline">
 				<Title level={5} style={{ marginTop: 16 }}>Console</Title>
@@ -1,21 +1,10 @@
-import React, { useContext, useEffect, useState } from "react";
 import { Select } from "antd";

-import * as lib from "@clusterio/lib";
-
-import ControlContext from "./ControlContext";
-import { notifyErrorHandler } from "../util/notify";
 import { InputComponentProps } from "../BaseWebPlugin";
+import { useRoles } from "../model/roles";

 export default function InputRole(props: InputComponentProps) {
-	let control = useContext(ControlContext);
-	let [roles, setRoles] = useState<lib.Role[]>([]);
-
-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(newRoles);
-		}).catch(notifyErrorHandler("Error fetching role list"));
-	}, []);
+	const [roles] = useRoles();

 	return <Select
 		showSearch
@@ -5,6 +5,7 @@ import DeleteOutlined from "@ant-design/icons/DeleteOutlined";

 import * as lib from "@clusterio/lib";

+import { useRole } from "../model/roles";
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import PageHeader from "./PageHeader";
@@ -12,30 +13,6 @@ import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
 import { notifyErrorHandler } from "../util/notify";

-
-function useRole(id: number): [Readonly<lib.Role> | undefined, boolean, () => void] {
-	let control = useContext(ControlContext);
-	let [role, setRole] = useState<lib.Role>();
-	let [synced, setSynced] = useState(false);
-
-	function updateRole() {
-		// XXX optimize by requesting only the role in question
-		control.send(new lib.RoleListRequest()).then(roles => {
-			let match = roles.find(u => u.id === id);
-			if (match) {
-				setRole(match);
-			}
-			setSynced(true);
-		});
-	}
-
-	useEffect(() => {
-		updateRole();
-	}, [id]);
-
-	return [role, synced, updateRole];
-}
-
 export default function RoleViewPage() {
 	let params = useParams();
 	let roleId = Number(params.id);
@@ -44,7 +21,7 @@ export default function RoleViewPage() {

 	let account = useAccount();
 	let control = useContext(ControlContext);
-	const [role, synced, updateRole] = useRole(roleId);
+	const [role, synced] = useRole(roleId);
 	let [edited, setEdited] = useState(false);


@@ -63,6 +40,7 @@ export default function RoleViewPage() {
 		</PageLayout>;
 	}

+	// TODO: Update displayed perms when role perms change and have not been edited locally
 	let initialValues = {
 		name: role["name"],
 		description: role["description"],
@@ -88,7 +66,6 @@ export default function RoleViewPage() {
 					new lib.RoleUpdateRequest(roleId, values.name || "", values.description || "", newPermissions)
 				).then(() => {
 					setEdited(false);
-					updateRole();
 				}).catch(notifyErrorHandler("Error applying changes"));
 			}}
 		>
@@ -4,6 +4,7 @@ import { Button, Form, Input, Modal, Table } from "antd";

 import * as lib from "@clusterio/lib";

+import { useRoles } from "../model/roles";
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import { notifyErrorHandler } from "../util/notify";
@@ -60,16 +61,8 @@ function CreateRoleButton() {

 export default function RolesPage() {
 	let account = useAccount();
-	let control = useContext(ControlContext);
 	let navigate = useNavigate();
-
-	let [roles, setRoles] = useState<lib.Role[]>([]);
-
-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(newRoles);
-		}).catch(notifyErrorHandler("Error fetching role list"));
-	}, []);
+	const [roles] = useRoles();

 	return <PageLayout nav={[{ name: "Roles" }]}>
 		<PageHeader
@@ -88,7 +81,7 @@ export default function RolesPage() {
 					dataIndex: "description",
 				},
 			]}
-			dataSource={roles}
+			dataSource={[...roles.values()]}
 			pagination={false}
 			rowKey={role => role.id}
 			onRow={(role, rowIndex) => ({
@@ -8,6 +8,7 @@ import DeleteOutlined from "@ant-design/icons/DeleteOutlined";

 import * as lib from "@clusterio/lib";

+import { useRoles } from "../model/roles";
 import { useAccount } from "../model/account";
 import { useInstances } from "../model/instance";
 import ControlContext from "./ControlContext";
@@ -35,21 +36,13 @@ export default function UserViewPage() {
 	let control = useContext(ControlContext);
 	let [instances] = useInstances();
 	const [user, synced] = useUser(userName);
-	let [roles, setRoles] = useState<Map<number, lib.Role>>(new Map());
+	let [roles] = useRoles();
 	let [form] = Form.useForm();
 	let [rolesDirty, setRolesDirty] = useState<boolean>(false);
 	let [banReasonDirty, setBanReasonDirty] = useState<boolean>(false);
 	let [applyingRoles, setApplyingRoles] = useState<boolean>(false);
 	let [rolesError, setRolesError] = useState<string|undefined>();

-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(new Map(newRoles.map(role => [role.id, role])));
-		}).catch(() => {
-			setRoles(new Map());
-		});
-	}, []);
-
 	useEffect(() => {
 		if (
 			account.hasPermission("core.user.set_banned")
@@ -76,6 +69,7 @@ export default function UserViewPage() {
 		</PageLayout>;
 	}

+	// TODO: Update displayed roles when user roles change and have not been edited locally
 	let roleSelector = <Form.Item
 		noStyle
 		name="roles"
@@ -13,6 +13,7 @@ import {

 import * as lib from "@clusterio/lib";

+import { useRoles } from "../model/roles";
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import { notifyErrorHandler } from "../util/notify";
@@ -373,21 +374,11 @@ function BulkUserActionButton() {

 export default function UsersPage() {
 	let account = useAccount();
-	let control = useContext(ControlContext);
 	let navigate = useNavigate();
 	let [users] = useUsers();
-
-	const [roles, setRoles] = useState(new Map());
+	const [roles] = useRoles();
 	const searchInput = useRef<InputRef>(null);

-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(new Map(newRoles.map(role => [role.id, role])));
-		}).catch(() => {
-			// ignore
-		});
-	}, []);
-
 	return <PageLayout nav={[{ name: "Users" }]}>
 		<PageHeader
 			title="Users"
@@ -0,0 +1,23 @@
+import { useCallback, useContext, useSyncExternalStore } from "react";
+import ControlContext from "../components/ControlContext";
+
+import * as lib from "@clusterio/lib";
+
+export function useRolesOfUser(user?: lib.User) {
+	const [roles, synced] = useRoles();
+	return [
+		user !== undefined ? new Map([...user.roleIds].map(roleId => [roleId, roles.get(roleId)])) : new Map(),
+		synced,
+	] as const;
+}
+
+export function useRole(id?: number) {
+	const [roles, synced] = useRoles();
+	return [id !== undefined ? roles.get(id) : undefined, synced] as const;
+}
+
+export function useRoles() {
+	const control = useContext(ControlContext);
+	const subscribe = useCallback((callback: () => void) => control.roles.subscribe(callback), [control]);
+	return useSyncExternalStore(subscribe, () => control.roles.getSnapshot());
+}
@@ -1,7 +1,11 @@
 import { useEffect, useState } from "react";
 import { EventEmitter } from "events";

-class Permission extends EventEmitter {
+type PermissionEvents = {
+	"change": [ status: string ]
+};
+
+class Permission extends EventEmitter<PermissionEvents> {
 	status!: { state: string };

 	constructor(name: PermissionName) {
@@ -67,6 +67,7 @@ export class Control extends lib.Link {
 	modPacks = new lib.EventSubscriber(lib.ModPackUpdatesEvent, this);
 	mods = new lib.EventSubscriber(lib.ModUpdatesEvent, this);
 	users = new lib.EventSubscriber(lib.UserUpdatesEvent, this);
+	roles = new lib.EventSubscriber(lib.RoleUpdatesEvent, this);

 	declare connector: ControlConnector;

@@ -26,7 +26,6 @@ describe("global_chat plugin", function() {
 		let instancePlugin;

 		before(async function() {
-			lib.Link.register(ChatEvent);
 			instancePlugin = new instance.InstancePlugin(info, new mock.MockInstance(), new mock.MockHost());
 			await instancePlugin.init();
 		});
@@ -1,7 +1,7 @@
 "use strict";
 const assert = require("assert").strict;
 const { Controller, HostInfo, InstanceInfo } = require("@clusterio/controller");
-const { ControllerConfig, Address, RequestError, InstanceConfig, HostConfig, SystemInfo } = require("@clusterio/lib");
+const { ControllerConfig, Address, RequestError, InstanceConfig, SystemInfo, Role } = require("@clusterio/lib");
 const { EventEmitter } = require("stream");

 class MockEvent {}
@@ -314,5 +314,33 @@ describe("controller/src/Controller", function() {
 				]);
 			});
 		});
+		describe("Role", function() {
+			it("migrates renamed permissions", function() { // Alpha 17
+				const result = Controller.migrateRoles([
+					{ permissions: ["foo", "bar", "core.instance.save.list.subscribe"] },
+					{ permissions: ["bar", "core.instance.save.list.subscribe"] },
+					{ permissions: ["core.instance.save.subscribe"] },
+				]);
+				assert.deepEqual(result, [
+					{ permissions: ["foo", "bar", "core.instance.save.subscribe"] },
+					{ permissions: ["bar", "core.instance.save.subscribe"] },
+					{ permissions: ["core.instance.save.subscribe"] },
+				]);
+			});
+			it("does nothing for upto date data", function() {
+				const role = new Role(
+					0, "n", "d", new Set("p1", "p2"), 0, false
+				);
+				const jsonString = JSON.stringify(role);
+				const roleJson = JSON.parse(jsonString);
+				const roleJsonCopy = JSON.parse(jsonString);
+				const result = Controller.migrateRoles([
+					roleJson, roleJson, roleJson,
+				]);
+				assert.deepEqual(result, [
+					roleJsonCopy, roleJsonCopy, roleJsonCopy,
+				]);
+			});
+		});
 	});
 });
@@ -5,20 +5,17 @@ const lib = require("@clusterio/lib");

 describe("controller/src/ControllerUser", function() {
 	describe("class ControllerUser", function() {
-		class MockUserManager {
-			roles = new Map([
-				[1, new lib.Role(1, "a", "a role", new Set(["core.admin"]))],
-				[2, new lib.Role(2, "b", "b role", new Set(["user-test"]))],
-			]);
-		}
-		const userManager = new MockUserManager();
+		const roles = new Map([
+			[1, new lib.Role(1, "a", "a role", new Set(["core.admin"]))],
+			[2, new lib.Role(2, "b", "b role", new Set(["user-test"]))],
+		]);

 		it("should round trip serialize", function() {
 			function test_roundtrip(serialized) {
-				let user = ControllerUser.fromJSON(serialized, userManager);
+				let user = ControllerUser.fromJSON(serialized, roles);
 				let user_serialized = user.toJSON(true);
 				assert.deepEqual(user_serialized, serialized);
-				let user_deserialized = ControllerUser.fromJSON(user_serialized, userManager);
+				let user_deserialized = ControllerUser.fromJSON(user_serialized, roles);
 				assert.deepEqual(user_deserialized, user);
 			}

@@ -31,9 +28,9 @@ describe("controller/src/ControllerUser", function() {
 		describe(".checkPermission()", function() {
 			it("should correctly resolve permissions", function() {
 				lib.definePermission({ name: "user-test", title: "Test", description: "User Test" });
-				let a = ControllerUser.fromJSON({ name: "admin", roles: [1] }, userManager);
-				let b = ControllerUser.fromJSON({ name: "user", roles: [2] }, userManager);
-				let c = ControllerUser.fromJSON({ name: "null", roles: [] }, userManager);
+				let a = ControllerUser.fromJSON({ name: "admin", roles: [1] }, roles);
+				let b = ControllerUser.fromJSON({ name: "user", roles: [2] }, roles);
+				let c = ControllerUser.fromJSON({ name: "null", roles: [] }, roles);

 				a.checkPermission("core.control.connect");
 				assert.throws(() => b.checkPermission("core.control.connect"), new Error("Permission denied"));
@@ -13,7 +13,8 @@ const { wait } = lib;
 const testStrings = require("../lib/factorio/test_strings");
 const {
 	TestControl, TestControlConnector, url, controlToken, slowTest,
-	exec, execCtl, sendRcon, getControl, spawnNode, instancesDir, factorioDir,
+	exec, execCtl, execCtlProcess, execHost, sendRcon, getControl,
+	spawnNode, instancesDir, factorioDir,
 } = require("./index");


@@ -39,11 +40,11 @@ async function startAltHost() {
 	await fs.remove(path.join("temp", "test", "alt-instances"));
 	await fs.remove(path.join("temp", "test", "alt-mods"));
 	await execCtl(`host create-config --id 5 --name alt-host --generate-token --output ${config}`);
-	await exec(`node ../../packages/host --config ${config} config set host.tls_ca ../../test/file/tls/cert.pem`);
-	await exec(`node ../../packages/host --config ${config} config set host.instances_directory alt-instances`);
+	await execHost(`--config ${config} config set host.tls_ca ../../test/file/tls/cert.pem`);
+	await execHost(`--config ${config} config set host.instances_directory alt-instances`);
 	const dir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir);
-	await exec(`node ../../packages/host --config ${config} config set host.factorio_directory ${dir}`);
-	await exec(`node ../../packages/host --config ${config} config set host.mods_directory alt-mods`);
+	await execHost(`--config ${config} config set host.factorio_directory ${dir}`);
+	await execHost(`--config ${config} config set host.mods_directory alt-mods`);
 	return await spawnAltHost(config);
 }

@@ -259,9 +260,9 @@ describe("Integration of Clusterio", function() {

 		describe("controller config set", function() {
 			it("sets given config option", async function() {
-				await execCtl('controller config set controller.name "Test Cluster"');
+				await execCtl("controller config set controller.name Test-Cluster");
 				let result = await getControl().send(new lib.ControllerConfigGetRequest());
-				assert.equal(result["controller.name"], "Test Cluster");
+				assert.equal(result["controller.name"], "Test-Cluster");
 			});
 			it("should not allow setting auth_secret", async function() {
 				await assert.rejects(execCtl("controller config set controller.auth_secret root"));
@@ -277,9 +278,9 @@ describe("Integration of Clusterio", function() {
 		describe("controller plugin update", function() {
 			it("runs", async function() {
 				// In dev plugins have no npm package, so best we can do is get an error from the controller
-				assert.rejects(
+				await assert.rejects(
 					execCtl("controller plugin update foo"),
-					"Plugin foo is not installed on this machine"
+					/Plugin foo is not installed on this machine/
 				);
 			});
 			// Update always fails, we can not test restart option
@@ -289,9 +290,9 @@ describe("Integration of Clusterio", function() {
 			it("runs", async function() {
 				// Default is to disallow updates, changing this value would require a restart
 				// Additionally, it can not be changed via ctl, so best we can do is get an error from the controller
-				assert.rejects(
+				await assert.rejects(
 					execCtl("controller plugin install foo"),
-					"Plugin installs are disabled on this machine"
+					/Plugin installs are disabled on this machine/
 				);
 			});
 			// Install always fails, we can not test restart option
@@ -303,9 +304,9 @@ describe("Integration of Clusterio", function() {
 			});
 			it("accepts --restart", async function() {
 				// We cannot restart the controller, so we check for controller error instead
-				assert.rejects(
+				await assert.rejects(
 					execCtl("controller update --restart"),
-					"Cannot restart, controller does not have a process monitor to restart it."
+					/Cannot restart, controller does not have a process monitor to restart it./
 				);
 			});
 		});
@@ -313,9 +314,9 @@ describe("Integration of Clusterio", function() {
 		describe("controller restart", function() {
 			it("runs", async function() {
 				// We cannot restart the controller, so we check for controller error instead
-				assert.rejects(
+				await assert.rejects(
 					execCtl("controller restart"),
-					"Cannot restart, controller does not have a process monitor to restart it."
+					/Cannot restart, controller does not have a process monitor to restart it./
 				);
 			});
 		});
@@ -329,7 +330,7 @@ describe("Integration of Clusterio", function() {
 		describe("host config", function() {
 			it("changes host config", async function() {
 				await execCtl("host config set 4 host.name My-Host");
-				const result = await execCtl("host config list 4");
+				const result = await execCtlProcess("host config list 4");
 				assert(/My-Host/.test(result.stdout), "New host name not in config output");
 			});
 			it("should not allow setting host.controller_token", async function() {
@@ -359,9 +360,9 @@ describe("Integration of Clusterio", function() {
 		describe("host plugin update", function() {
 			it("runs", async function() {
 				// In dev plugins have no npm package, so best we can do is get an error from the host
-				assert.rejects(
+				await assert.rejects(
 					execCtl("host plugin update 4 foo"),
-					"Plugin foo is not installed on this machine"
+					/Plugin foo is not installed on this machine/
 				);
 			});
 			// Update always fails, we can not test restart option
@@ -371,9 +372,9 @@ describe("Integration of Clusterio", function() {
 			it("runs", async function() {
 				// Default is to disallow updates, changing this value would require a restart
 				// Additionally, it can not be changed via ctl, so best we can do is get an error from the host
-				assert.rejects(
+				await assert.rejects(
 					execCtl("host plugin install 4 foo"),
-					"Plugin installs are disabled on this machine"
+					/Plugin installs are disabled on this machine/
 				);
 			});
 			// Install always fails, we can not test restart option
@@ -385,9 +386,9 @@ describe("Integration of Clusterio", function() {
 			});
 			it("accepts --restart", async function() {
 				// We cannot restart the host, so we check for host error instead
-				assert.rejects(
+				await assert.rejects(
 					execCtl("host update 4 --restart"),
-					"Cannot restart, host does not have a process monitor to restart it."
+					/Cannot restart, host does not have a process monitor to restart it./
 				);
 			});
 		});
@@ -395,9 +396,9 @@ describe("Integration of Clusterio", function() {
 		describe("host restart", function() {
 			it("runs", async function() {
 				// We cannot restart the host, so we check for host error instead
-				assert.rejects(
+				await assert.rejects(
 					execCtl("host restart 4"),
-					"Cannot restart, host does not have a process monitor to restart it."
+					/Cannot restart, host does not have a process monitor to restart it./
 				);
 			});
 		});
@@ -416,9 +417,12 @@ describe("Integration of Clusterio", function() {
 				slowTest(this);
 				let sawDisconnected;
 				await runWithAltHost(async () => {
-					getControl().hostUpdates = [];
+					const control = getControl();
+					control.hostUpdates = [];
+					await wait(50); // Allow for host to update instances
 					await execCtl("host revoke-token 5");
-					for (let update of getControl().hostUpdates) {
+					await wait(50); // Allow controller to notify control
+					for (let update of control.hostUpdates) {
 						if (update.name !== "alt-host") {
 							continue;
 						}
@@ -448,10 +452,8 @@ describe("Integration of Clusterio", function() {
 				assert.equal(instances.get(44).status, "unassigned", "incorrect instance status");

 				// Make sure the following tests does not fail due to not having internet
-				let value = JSON.stringify({ lan: true, public: false }).replace(
-					/"/g, process.platform === "win32" ? '""' : '\\"'
-				);
-				await execCtl(`instance config set-prop test factorio.settings visibility "${value}"`);
+				const value = jsonArg({ lan: true, public: false });
+				await execCtlProcess(`instance config set-prop test factorio.settings visibility ${value}`);
 				await execCtl("instance config set-prop test factorio.settings require_user_verification false");
 			});
 			it("can clone an instance", async function() {
@@ -466,7 +468,7 @@ describe("Integration of Clusterio", function() {
 			});
 			it("errors when cloning an invalid instance", async function() {
 				slowTest(this);
-				assert.rejects(
+				await assert.rejects(
 					execCtl("instance create testClone --id 440 --from 441"),
 					new lib.RequestError("Instance with ID 441 does not exist")
 				);
@@ -500,7 +502,7 @@ describe("Integration of Clusterio", function() {
 		describe("instance save list", function() {
 			it("lists the created save", async function() {
 				slowTest(this);
-				let result = await execCtl("instance save list test");
+				let result = await execCtlProcess("instance save list test");
 				assert(/world\.zip/.test(result.stdout), "world.zip not present in list save output");
 			});
 		});
@@ -615,7 +617,7 @@ describe("Integration of Clusterio", function() {
 				assert(r1.log.some(info => /\[COMMAND\]/.test(info.message)), "Command was not sent");
 				await execCtl("instance config set test factorio.enable_script_commands false");

-				assert.rejects(
+				await assert.rejects(
 					execCtl("instance send-rcon test /c"),
 					new Error(
 						"Attempted to use script command while disabled. " +
@@ -698,8 +700,8 @@ describe("Integration of Clusterio", function() {
 				];

 				for (let [prop, value] of testConfigs) {
-					let args = `test factorio.settings ${prop} ${jsonArg(value)}`;
-					await execCtl(`instance config set-prop ${args}`);
+					const args = `test factorio.settings ${prop} ${jsonArg(value)}`;
+					await execCtlProcess(`instance config set-prop ${args}`);
 				}

 				// Do this afterwards to leave enough to time for the
@@ -909,6 +911,7 @@ describe("Integration of Clusterio", function() {
 				await execCtl("instance config set 44 research_sync.load_plugin false");
 				await execCtl("instance config set 44 statistics_exporter.load_plugin false");
 				await execCtl("instance config set 44 subspace_storage.load_plugin false");
+				await execCtl("instance config set 44 inventory_sync.load_plugin false");

 				await execCtl("instance start 44");
 				const instance = (await getInstances()).get(44);
@@ -1188,7 +1191,7 @@ describe("Integration of Clusterio", function() {
 			});
 			it("should allow setting all fields", async function() {
 				const color = { r: 1, g: 0, b: 1, a: 0 };
-				await execCtl(
+				await execCtlProcess(
 					"mod-pack create full-pack 0.17.59 " +
 					"--description Description " +
 					"--mods empty_mod:1.0.0 " +
@@ -1220,7 +1223,7 @@ describe("Integration of Clusterio", function() {
 		describe("mod-pack list", function() {
 			let result = null;
 			it("runs", async function() {
-				result = await execCtl("mod-pack list");
+				result = await execCtlProcess("mod-pack list");
 			});
 			it("contains defaults", function() {
 				assert(result !== null, "Failed to return a value");
@@ -1249,7 +1252,7 @@ describe("Integration of Clusterio", function() {
 				reference.settings["runtime-global"].set("MyDouble", { value: 12.25 });
 				reference.settings["runtime-per-user"].set("MyString", { value: "a-string" });
 				await execCtl(`mod-pack import ${reference.toModPackString()}`);
-				const result = await execCtl("mod-pack export imported-pack");
+				const result = await execCtlProcess("mod-pack export imported-pack");
 				const roundtrip = lib.ModPack.fromModPackString(result.stdout.trim());
 				roundtrip.id = reference.id;
 				assert.deepEqual(roundtrip, reference);
@@ -1308,7 +1311,7 @@ describe("Integration of Clusterio", function() {

 		describe("mod show", function() {
 			it("gives details of a mod", async function() {
-				let result = await execCtl("mod show empty_mod 1.0.0");
+				let result = await execCtlProcess("mod show empty_mod 1.0.0");
 				let hash = await lib.hashFile(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip"));
 				let stat = await fs.stat(path.join("temp", "test", "mods", "empty_mod_1.0.0.zip"));
 				assert.equal(
@@ -1333,14 +1336,14 @@ describe("Integration of Clusterio", function() {

 		describe("mod list", function() {
 			it("shows the list of mods", async function() {
-				let result = await execCtl("mod list");
+				let result = await execCtlProcess("mod list");
 				assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the list");
 			});
 		});

 		describe("mod search", function() {
 			it("searches the list of mods", async function() {
-				let result = await execCtl("mod search 1.1 name:empty_mod");
+				let result = await execCtlProcess("mod search 1.1 name:empty_mod");
 				assert(result.stdout.indexOf("empty_mod") !== -1, "empty_mod is not in the result");
 			});
 		});
@@ -1404,6 +1407,7 @@ describe("Integration of Clusterio", function() {
 				await execCtl(`role create temp ${args}`);
 				let roles = await getControl().send(new lib.RoleListRequest());
 				let tempRole = roles.find(role => role.name === "temp");
+				if (tempRole) { tempRole.updatedAtMs = 0; }
 				assert.deepEqual(
 					tempRole,
 					new lib.Role(5, "temp", "A temp role", new Set(["core.control.connect"]))
@@ -1414,9 +1418,10 @@ describe("Integration of Clusterio", function() {
 		describe("role edit", function() {
 			it("should modify the given role", async function() {
 				let args = "--name new --description \"A new role\" --set-perms";
-				await execCtl(`role edit temp ${args}`);
+				await execCtlProcess(`role edit temp ${args}`);
 				let roles = await getControl().send(new lib.RoleListRequest());
 				let newRole = roles.find(role => role.name === "new");
+				if (newRole) { newRole.updatedAtMs = 0; }
 				assert.deepEqual(newRole, new lib.Role(5, "new", "A new role", new Set()));
 			});
 			it("should add permissions with --add-perms", async function() {
@@ -1495,7 +1500,7 @@ describe("Integration of Clusterio", function() {
 		describe("user set-roles", function() {
 			it("should set the roles on the user", async function() {
 				getControl().userUpdates = [];
-				await execCtl('user set-roles temp "Cluster Admin"');
+				await execCtlProcess('user set-roles temp "Cluster Admin"');
 				let users = await getControl().send(new lib.UserListRequest());
 				let tempUser = users.find(user => user.id === "temp");
 				assert.deepEqual(tempUser.roleIds, new Set([0]));
@@ -1503,7 +1508,7 @@ describe("Integration of Clusterio", function() {
 			});

 			it("should restrict actions based on roles", async function() {
-				await execCtl('user set-roles temp "Player"');
+				await execCtl("user set-roles temp Player");
 				let tempControl;
 				try {
 					let tlsCa = await fs.readFile("test/file/tls/cert.pem");
@@ -1648,7 +1653,7 @@ describe("Integration of Clusterio", function() {
 				);
 			});
 			it("should reject multiple provided options", async function() {
-				assert.rejects(execCtl("user import --bans --admins import-admins.json"));
+				await assert.rejects(execCtl("user import --bans --admins import-admins.json"));
 			});
 		});

@@ -1657,7 +1662,6 @@ describe("Integration of Clusterio", function() {
 				await execCtl("user create export_control");
 			});
 			it("should export users", async function() {
-				slowTest(this);
 				await execCtl("user set-admin export_user --create");
 				await execCtl("user set-whitelisted export_user");
 				await execCtl("user export --users user-export.json");
@@ -1669,7 +1673,6 @@ describe("Integration of Clusterio", function() {
 				});
 			});
 			it("should export bans", async function() {
-				slowTest(this);
 				await execCtl("user set-banned export_ban --create");
 				await execCtl("user set-banned export_ban_reason --create --reason banned");
 				await execCtl("user export --bans ban-export.json");
@@ -1779,7 +1782,7 @@ describe("Integration of Clusterio", function() {
 				assert(data.indexOf("restore_control") >= 0, "restore_control is missing");
 			});
 			it("should reject multiple provided options", async function() {
-				assert.rejects(execCtl("user restore --bans --admins restore-admins.json"));
+				await assert.rejects(execCtl("user restore --bans --admins restore-admins.json"));
 			});
 		});
 	});
@@ -11,6 +11,8 @@ const events = require("events");
 const lib = require("@clusterio/lib");
 const { LineSplitter, ConsoleTransport, logger } = lib;

+const { selectTargetCommand, initialize: initializeCtl } = require("@clusterio/ctl");
+
 // Make sure permissions from plugins are loaded
 require("../../plugins/global_chat/dist/node/index");
 require("../../plugins/player_auth/dist/node/index");
@@ -84,6 +86,10 @@ class TestControl extends lib.Link {
 	async handleUserUpdatesEvent(event) {
 		this.userUpdates.push(...event.updates);
 	}
+
+	async setLogSubscriptions(args) {
+		// Do nothing
+	}
 }

 class TestControlConnector extends lib.WebSocketClientConnector {
@@ -185,11 +191,27 @@ async function execHost(...args) {
 	return await exec(...args);
 }

-async function execCtl(...args) {
+async function execCtlProcess(...args) {
 	args[0] = `node --enable-source-maps ../../packages/ctl ${args[0]}`;
 	return await exec(...args);
 }

+async function execCtl(command) {
+	process.chdir("temp/test");
+	try {
+		const initArgs = await initializeCtl(command, control.plugins, true);
+		if (!initArgs.shouldRun) {
+			return;
+		}
+
+		control.config = initArgs.controlConfig;
+		const targetCommand = selectTargetCommand(initArgs.args, initArgs.rootCommands);
+		await targetCommand.run(initArgs.args, control);
+	} finally {
+		process.chdir("../..");
+	}
+}
+
 async function sendRcon(instanceId, command) {
 	return await control.sendTo({ instanceId }, new lib.InstanceSendRconRequest(command));
 }
@@ -291,33 +313,38 @@ before(async function() {
 	await execController("config set controller.tls_private_key ../../test/file/tls/key.pem");

 	console.log("Setting Controller Plugins");
-	await execCtl("plugin add ../../plugins/global_chat");
-	await execCtl("plugin add ../../plugins/research_sync");
-	await execCtl("plugin add ../../plugins/statistics_exporter");
-	await execCtl("plugin add ../../plugins/subspace_storage");
-	await execCtl("plugin add ../../plugins/player_auth");
+	await execCtlProcess("plugin add ../../plugins/global_chat");
+	await execCtlProcess("plugin add ../../plugins/research_sync");
+	await execCtlProcess("plugin add ../../plugins/statistics_exporter");
+	await execCtlProcess("plugin add ../../plugins/subspace_storage");
+	await execCtlProcess("plugin add ../../plugins/inventory_sync");
+	await execCtlProcess("plugin add ../../plugins/player_auth");

 	console.log("Bootstrapping");
 	await execController("bootstrap create-admin test");
 	await execController("bootstrap create-ctl-config test");
-	await execCtl("control-config set control.tls_ca ../../test/file/tls/cert.pem");
+	await execCtlProcess("control-config set control.tls_ca ../../test/file/tls/cert.pem");

 	controllerProcess = await spawnNode("controller:", "../../packages/controller run", /Started controller/);

-	await execCtl("host create-config --id 4 --name host --generate-token");
-
 	const relativeFactorioDir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir);
+	await execCtlProcess("host create-config --id 4 --name host --generate-token");
 	await execHost(`config set host.factorio_directory ${relativeFactorioDir}`);
 	await execHost("config set host.tls_ca ../../test/file/tls/cert.pem");

 	hostProcess = await spawnNode("host:", "../../packages/host run", /Started host/);

-	let tlsCa = await fs.readFile("test/file/tls/cert.pem");
-	let controlConnector = new TestControlConnector(url, 2, tlsCa);
+	const tlsCa = await fs.readFile("test/file/tls/cert.pem");
+	const controlConnector = new TestControlConnector(url, 2, tlsCa);
 	controlConnector.token = controlToken;
 	control = new TestControl(controlConnector);
 	await controlConnector.connect();

+	const initArgs = await initializeCtl(`--plugin-list ${pluginListPath} control-config list`, undefined, true);
+	control.config = initArgs.controlConfig;
+	control.plugins = initArgs.ctlPlugins;
+	control.tlsCa = tlsCa;
+
 	const testPack = lib.ModPack.fromJSON({});
 	testPack.id = 12;
 	testPack.name = "subspace_storage-pack";
@@ -358,10 +385,12 @@ module.exports = {
 	TestControl,
 	TestControlConnector,
 	TestHostConnector,
+	execCtlProcess,
 	slowTest,
 	get,
 	exec,
 	execCtl,
+	execHost,
 	sendRcon,
 	getControl,
 	spawn,
@@ -3,7 +3,7 @@ const assert = require("assert").strict;
 const lib = require("@clusterio/lib");

 const {
-	slowTest, exec, execCtl, sendRcon, getControl,
+	slowTest, exec, execCtl, execCtlProcess, sendRcon, getControl,
 } = require("./index");

 const instId = 48;
@@ -31,12 +31,12 @@ describe("Clusterio Instance", function() {
 		// Create a new instance
 		await execCtl(`instance create ${instName} --id ${instId}`);
 		await execCtl(`instance config set ${instName} factorio.enable_whitelist true`);
-		await execCtl(`instance config set-prop ${instName} factorio.settings visibility "${visibility}"`);
+		await execCtlProcess(`instance config set-prop ${instName} factorio.settings visibility "${visibility}"`);
 		await execCtl(`instance config set-prop ${instName} factorio.settings require_user_verification false`);
 		await execCtl(`instance assign ${instName} 4`);

 		for (const plugin of [
-			"global_chat", "research_sync", "statistics_exporter", "subspace_storage", "player_auth",
+			"global_chat", "research_sync", "statistics_exporter", "subspace_storage", "player_auth", "inventory_sync",
 		]) {
 			await execCtl(`instance config set ${instName} ${plugin}.load_plugin false`);
 		}
@@ -44,7 +44,7 @@ describe("Clusterio Instance", function() {
 		// Create a new alt instance
 		await execCtl(`instance create ${instAltName} --id ${instAltId}`);
 		await execCtl(`instance config set ${instAltName} factorio.enable_whitelist true`);
-		await execCtl(`instance config set-prop ${instAltName} factorio.settings visibility "${visibility}"`);
+		await execCtlProcess(`instance config set-prop ${instAltName} factorio.settings visibility "${visibility}"`);
 		await execCtl(`instance config set-prop ${instAltName} factorio.settings require_user_verification false`);
 		await execCtl(`instance assign ${instAltName} 4`);
 		await execCtl(`instance start ${instAltName}`);
@@ -75,12 +75,12 @@ describe("Clusterio Instance", function() {
 				if (savePatchingEnabled) {
 					// IPC expects save patching to be enabled
 					it("should do nothing for an unknown type", async function() {
-						assert.rejects(getUser("DoesNotExist"), "User should not exist");
+						await assert.rejects(getUser("DoesNotExist"), "User should not exist");
 						await sendRcon(instId,
 							`/sc ${requireApi} api.send_json("player_event",` +
 							"{ type='invalid type', name='DoesNotExist' })"
 						);
-						assert.rejects(getUser("DoesNotExist"), "User was created");
+						await assert.rejects(getUser("DoesNotExist"), "User was created");
 					});
 					it("should respond to a player join and leave event", async function() {
 						await sendRcon(instId,
@@ -201,32 +201,16 @@ describe("lib/config/classes", function() {
 			const filepath = path.join("temp", "test", "config_test.json");
 			it("should throw for file and json errors", async function() {
 				await fs.writeFile(filepath, "abc");
-				assert.rejects(
-					TestConfig.fromFile("local", filepath),
-					new Error(
-						`${filepath}: SyntaxError: JSON.parse: unexpected character at line 1 column 1 of the JSON data`
-					)
-				);
-				assert.rejects(
-					TestConfig.fromFile("local", `${filepath}.notExist`),
-				);
+				await assert.rejects(TestConfig.fromFile("local", filepath));
+				await assert.rejects(TestConfig.fromFile("local", `${filepath}.notExist`));
 			});
 			it("should throw on incorrect input passed", async function() {
 				await fs.writeFile(filepath, "null");
-				await assert.rejects(
-					TestConfig.fromFile("local", filepath),
-					new Error("Invalid config")
-				);
+				await assert.rejects(TestConfig.fromFile("local", filepath));
 				await fs.writeFile(filepath, "undefined");
-				assert.rejects(
-					TestConfig.fromFile("local", filepath),
-					new Error("Invalid config")
-				);
+				await assert.rejects(TestConfig.fromFile("local", filepath));
 				await fs.writeFile(filepath, "[]");
-				assert.rejects(
-					TestConfig.fromFile("local", filepath),
-					new Error("Invalid config")
-				);
+				await assert.rejects(TestConfig.fromFile("local", filepath));
 			});
 			it("should load defaults for missing fields", async function() {
 				await fs.writeJSON(filepath, {
@@ -507,10 +507,9 @@ describe("lib/datastore", function() {
 					["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => k !== "foo"))]
 				);
 			});
-			it("does not break when the key does not exist", function() {
+			it("does nothing when the key does not exist", function() {
 				datastore.delete("test");
-				assert.equal(eventsRaised.length, 1);
-				assert.deepEqual(eventsRaised[0], [["test", undefined, true]]);
+				assert.equal(eventsRaised.length, 0);
 			});
 		});

@@ -22,9 +22,9 @@ describe("rce_ops", function() {
 			await lib.handlePluginUpdate("foo", [{ npmPackage: "foo" }]);
 		});
 		it("rejects when plugin not installed", async function() {
-			assert.rejects(
+			await assert.rejects(
 				lib.handlePluginUpdate("foo", []),
-				"Plugin foo is not installed on this machine"
+				/Plugin foo is not installed on this machine/
 			);
 		});
 	});
@@ -50,16 +50,16 @@ describe("rce_ops", function() {
 		});
 		it("rejects when name too long", async function() {
 			const pluginName = "a".repeat(215);
-			assert.rejects(
+			await assert.rejects(
 				lib.handlePluginInstall(pluginName),
-				`Invalid plugin name: ${pluginName}`
+				{ message: `Invalid plugin name: ${pluginName}` }
 			);
 		});
 		it("rejects when invalid symbol present", async function() {
 			const pluginName = "?";
-			assert.rejects(
+			await assert.rejects(
 				lib.handlePluginInstall(pluginName),
-				`Invalid plugin name: ${pluginName}`
+				{ message: `Invalid plugin name: ${pluginName}` },
 			);
 		});
 		it("rejects unregistered packages", async function() {
@@ -69,7 +69,7 @@ describe("rce_ops", function() {
 				return { ok: false };
 			};

-			assert.rejects(lib.handlePluginInstall("foo"), "Unknown plugin: foo");
+			await assert.rejects(lib.handlePluginInstall("foo"), /Unknown plugin: foo/);
 			assert.equal(calledWith, "https://registry.npmjs.com/foo");
 		});
 	});
@@ -240,20 +240,20 @@ describe("lib/subscriptions", function() {
 				subscriptions.handle(RegisteredEvent);
 			});

-			it("should not accept subscriptions to unregistered events", function() {
+			it("should not accept subscriptions to unregistered events", async function() {
 				const request = new lib.SubscriptionRequest(RegisteredEvent.name, true);
 				request.eventName = UnregisteredEvent.name;
-				assert.rejects(
+				await assert.rejects(
 					subscriptions.handleRequest(getLink(0), request, addr({ controlId: 0 }), addr("controller")),
-					new Error(`Unregistered Event class ${UnregisteredEvent.name}`)
+					new Error(`Event ${UnregisteredEvent.name} is not a registered event`)
 				);
 			});
-			it("should not accept subscriptions to events not handled by the class", function() {
+			it("should not accept subscriptions to events not handled by the class", async function() {
 				const request = new lib.SubscriptionRequest(RegisteredEvent.name, true);
 				request.eventName = StringPermissionEvent.name;
-				assert.rejects(
+				await assert.rejects(
 					subscriptions.handleRequest(getLink(0), request, addr({ controlId: 0 }), addr("controller")),
-					new Error(`Event ${StringPermissionEvent.eventName} is not a registered as subscribable`)
+					new Error(`Event ${StringPermissionEvent.name} is not a registered as subscribable`)
 				);
 			});
 			it("should accept a subscription", async function() {
@@ -28,9 +28,9 @@ describe("messages/controller", function() {
 		});
 		it("rejects if updates are disabled", async function() {
 			controller.config.set("controller.allow_remote_updates", false);
-			assert.rejects(
+			await assert.rejects(
 				controlConnection.handleControllerUpdateRequest(new lib.ControllerUpdateRequest()),
-				"Remote updates are disabled on this machine"
+				/Remote updates are disabled on this machine/
 			);
 		});
 	});
@@ -24,9 +24,9 @@ describe("messages/controller", function() {
 		});
 		it("rejects if updates are disabled", async function() {
 			host.config.set("host.allow_remote_updates", false);
-			assert.rejects(
+			await assert.rejects(
 				host.handleHostUpdateRequest(new lib.HostUpdateRequest()),
-				"Remote updates are disabled on this machine"
+				/Remote updates are disabled on this machine/
 			);
 		});
 	});
@@ -115,9 +115,9 @@ describe("messages/plugin", function() {
 		});
 		it("rejects if updates are disabled on the controller", async function() {
 			controller.config.set("controller.allow_plugin_updates", false);
-			assert.rejects(
+			await assert.rejects(
 				controlConnection.handlePluginUpdateRequest(new lib.PluginUpdateRequest("foo")),
-				"Plugin updates are disabled on this machine"
+				/Plugin updates are disabled on this machine/
 			);
 		});
 		it("runs on a host", async function() {
@@ -127,9 +127,9 @@ describe("messages/plugin", function() {
 		});
 		it("rejects if updates are disabled on the host", async function() {
 			host.config.set("host.allow_plugin_updates", false);
-			assert.rejects(
+			await assert.rejects(
 				host.handlePluginUpdateRequest(new lib.PluginUpdateRequest("foo")),
-				"Plugin updates are disabled on this machine"
+				/Plugin updates are disabled on this machine/
 			);
 		});
 	});
@@ -154,22 +154,22 @@ describe("messages/plugin", function() {
 			controller.config.set("controller.allow_plugin_install", true);
 			await controlConnection.handlePluginInstallRequest(new lib.PluginInstallRequest("foo"));
 		});
-		it("rejects if updates are disabled on the controller", async function() {
+		it("rejects if installs are disabled on the controller", async function() {
 			controller.config.set("controller.allow_plugin_install", false);
-			assert.rejects(
+			await assert.rejects(
 				controlConnection.handlePluginInstallRequest(new lib.PluginInstallRequest("foo")),
-				"Plugin updates are disabled on this machine"
+				/Plugin installs are disabled on this machine/
 			);
 		});
 		it("runs on a host", async function() {
 			host.config.set("host.allow_plugin_install", true);
 			await host.handlePluginInstallRequest(new lib.PluginInstallRequest("foo"));
 		});
-		it("rejects if updates are disabled on the host", async function() {
+		it("rejects if installs are disabled on the host", async function() {
 			host.config.set("host.allow_plugin_install", false);
-			assert.rejects(
+			await assert.rejects(
 				host.handlePluginInstallRequest(new lib.PluginInstallRequest("foo")),
-				"Plugin updates are disabled on this machine"
+				/Plugin installs are disabled on this machine/
 			);
 		});
 	});
@@ -173,17 +173,20 @@ class MockController {
 			},
 		};

-		this.userManager = new UserManager(this.config);
-		this.userManager.roles = new Map([
+		const roleProvider = new lib.MemoryDatastoreProvider(new Map([
 			[0, lib.Role.fromJSON({ id: 0, name: "Admin", description: "admin", permissions: ["core.admin"] })],
 			[1, lib.Role.fromJSON({ id: 1, name: "Player", description: "player", permissions: [] })],
-		]);
-		this.userManager.roles.get(1).grantDefaultPermissions();
+		]));

+		this.roles = new lib.SubscribableDatastore(roleProvider, roleProvider.value);
+		this.roles.get(1).grantDefaultPermissions();
+
+		this.userManager = new UserManager(this.config, this.roles);
 		this.userManager.users = new Map([
-			["test", ControllerUser.fromJSON({ name: "test", roles: [0, 1] }, this.userManager)],
-			["player", ControllerUser.fromJSON({ name: "player", roles: [1] }, this.userManager)],
+			["test", ControllerUser.fromJSON({ name: "test", roles: [0, 1] }, this.roles)],
+			["player", ControllerUser.fromJSON({ name: "player", roles: [1] }, this.roles)],
 		]);
+
 		this.instances = new lib.KeyValueDatastore();
 		this.hosts = new lib.KeyValueDatastore();
 		this.handles = new Map();
@@ -215,7 +218,6 @@ class MockController {
 }

 async function createControllerPlugin(ControllerPluginClass, info) {
-	lib.registerPluginMessages([info]);
 	let controller = new MockController();
 	let metrics = {};
 	let logger = new MockLogger();
Author	SHA1	Message	Date
Cursor Agentanddanielv	f6424c5411	Add Factorio version management and download functionality Co-authored-by: danielv <danielv@danielv.no>	2025-07-21 18:47:19 +00:00
Daniel	02d14bd98d	Merge pull request #778 from Cooldude2606/refactor/test-exec-ctl Refactor execCtl and Ctl to speed up unit testing	2025-07-10 13:09:27 +02:00
Daniel	952ae21958	Merge pull request #777 from Cooldude2606/refactor/user-datastore Refactor roles to use datastore and subscriptions	2025-07-10 13:02:38 +02:00
Cooldude2606	50d7d28054	Fix expected error messages for assert rejects	2025-07-06 23:17:52 +01:00
Cooldude2606	2896ec1c15	Add initializeCtl and control reuse in tests	2025-07-06 22:58:39 +01:00
Cooldude2606	6b81e6383d	Fix usage of assert.rejects	2025-07-06 22:53:39 +01:00
Cooldude2606	5ed7163ed8	Add missing await statement Fixes #774	2025-06-14 23:05:14 +01:00
Cooldude2606	df1edec623	Add role subscriptions and data model	2025-06-14 00:28:03 +01:00
Cooldude2606	a5bf8d92d9	Separate roles to their own file	2025-06-13 22:24:56 +01:00
Cooldude2606	af95ca587b	Fix datastore types	2025-06-13 22:24:44 +01:00
Cooldude2606	331fc7d947	Add types to event emitters and clean up connections (#772 ) * Event types for controller * Clean up ctl event emitters * Event types for server * Event types for lib * Remove TypedEventEmitter * Event types for webui * Fix compile issues * Small cleanup of connectors	2025-06-13 22:20:50 +01:00