Merge pull request #777 from Cooldude2606/refactor/user-datastore

Refactor roles to use datastore and subscriptions

packages/controller/controller.ts

@@ -130,8 +130,16 @@ async function handleBootstrapCommand(
 	controllerConfig: lib.ControllerConfig
 ): Promise<void> {
 	let subCommand = args._[1];
-	let userManager = new UserManager(controllerConfig);
-	await userManager.load(path.join(controllerConfig.get("controller.database_directory"), "users.json"));
+
+	const databaseDirectory = controllerConfig.get("controller.database_directory");
+	const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		path.join(databaseDirectory, "roles.json"),
+		lib.Role.fromJSON.bind(lib.Role),
+	).bootstrap());
+
+	const userManager = new UserManager(controllerConfig, roles);
+	await userManager.load(path.join(databaseDirectory, "users.json"));
+
 	if (subCommand === "create-admin") {
 		if (!args.name) {
 			logger.error("name cannot be blank");
@@ -144,7 +152,7 @@ async function handleBootstrapCommand(
 			admin = userManager.createUser(args.name);
 		}
 
-		let adminRole = lib.ensureDefaultAdminRole(userManager.roles);
+		let adminRole = lib.ensureDefaultAdminRole(roles);
 		admin.roleIds.add(adminRole.id);
 		admin.isAdmin = true;
 		await userManager.save(path.join(controllerConfig.get("controller.database_directory"), "users.json"));

packages/controller/src/ControlConnection.ts

@@ -671,22 +671,21 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleRoleListRequest() {
-		return [...this._controller.userManager.roles.values()];
+		return [...this._controller.roles.values()];
 	}
 
 	async handleRoleCreateRequest(request: lib.RoleCreateRequest) {
-		let lastId = Math.max.apply(null, [...this._controller.userManager.roles.keys()]);
+		let lastId = Math.max.apply(null, [...this._controller.roles.keys()]);
 
 		// Start at 5 to leave space for future default roles
 		let id = Math.max(5, lastId+1);
-		this._controller.userManager.roles.set(id, lib.Role.fromJSON({ id, ...request }));
-		this._controller.userManager.dirty = true;
+		this._controller.roles.set(lib.Role.fromJSON({ id, ...request }));
 		return id;
 	}
 
 	async handleRoleUpdateRequest(request: lib.RoleUpdateRequest) {
 		let { id, name, description, permissions } = request;
-		let role = this._controller.userManager.roles.get(id);
+		let role = this._controller.roles.getMutable(id);
 		if (!role) {
 			throw new lib.RequestError(`Role with ID ${id} does not exist`);
 		}
@@ -694,27 +693,29 @@ export default class ControlConnection extends BaseConnection {
 		role.name = name;
 		role.description = description;
 		role.permissions = new Set(permissions);
-		this._controller.rolePermissionsUpdated(role);
+		this._controller.roles.set(role);
 	}
 
 	async handleRoleGrantDefaultPermissionsRequest(request: lib.RoleGrantDefaultPermissionsRequest) {
-		let role = this._controller.userManager.roles.get(request.id);
+		let role = this._controller.roles.get(request.id);
 		if (!role) {
 			throw new lib.RequestError(`Role with ID ${request.id} does not exist`);
 		}
 
 		role.grantDefaultPermissions();
-		this._controller.rolePermissionsUpdated(role);
+		this._controller.roles.set(role);
 	}
 
 	async handleRoleDeleteRequest(request: lib.RoleDeleteRequest) {
 		let id = request.id;
 
-		if (!this._controller.userManager.roles.delete(id)) {
+		const role = this._controller.roles.get(id);
+		if (!role) {
 			throw new lib.RequestError(`Role with ID ${id} does not exist`);
 		}
-		this._controller.userManager.dirty = true;
+		this._controller.roles.delete(role);
 
+		this._controller.userManager.dirty = true;
 		for (let user of this._controller.userManager.users.values()) {
 			if (user.roleIds.delete(id)) {
 				this._controller.userPermissionsUpdated(user);
@@ -766,7 +767,7 @@ export default class ControlConnection extends BaseConnection {
 		}
 
 		for (let roleId of request.roles) {
-			if (!this._controller.userManager.roles.has(roleId)) {
+			if (!this._controller.roles.has(roleId)) {
 				throw new lib.RequestError(`Role with ID ${roleId} does not exist`);
 			}
 		}

packages/controller/src/Controller.ts

@@ -138,17 +138,28 @@ export default class Controller {
 			lib.ModPack.fromJSON.bind(lib.ModPack)
 		).bootstrap());
 
-		if (modPacks.size === 0) {
-			modPacks.setMany(lib.ModPack.defaultModPacks);
-		}
+		const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "roles.json"),
+			lib.Role.fromJSON.bind(lib.Role),
+			this.migrateRoles,
+		).bootstrap());
 
-		const userManager = new UserManager(config);
+		const userManager = new UserManager(config, roles);
 		await userManager.load(path.join(databaseDirectory, "users.json"));
 
 		let modsDirectory = config.get("controller.mods_directory");
 		await fs.ensureDir(modsDirectory);
 		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
 
+		// Add default mod packs
+		if (modPacks.size === 0) {
+			modPacks.setMany(lib.ModPack.defaultModPacks);
+		}
+
+		// Add default roles
+		lib.ensureDefaultAdminRole(roles);
+		lib.ensureDefaultPlayerRole(roles);
+
 		return [
 			systems,
 			hosts,
@@ -156,6 +167,7 @@ export default class Controller {
 			saves,
 			modPacks,
 			modStore,
+			roles,
 			userManager,
 		] as const;
 	}
@@ -186,8 +198,10 @@ export default class Controller {
 		public modPacks = new lib.SubscribableDatastore<lib.ModPack>(),
 		/** Mods stored on the controller */
 		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
+		/** Mapping of mod pack id to mod pack */
+		public roles = new lib.SubscribableDatastore<lib.Role>(),
 		/** User and roles manager for the cluster */
-		public userManager = new UserManager(config),
+		public userManager = new UserManager(config, roles),
 	) {
 		this.clusterLogger = clusterLogger;
 		this.pluginInfos = pluginInfos;
@@ -214,6 +228,7 @@ export default class Controller {
 		this.subscriptions.handle(lib.ModPackUpdatesEvent, this.handleModPackSubscription.bind(this));
 		this.subscriptions.handle(lib.ModUpdatesEvent, this.handleModSubscription.bind(this));
 		this.subscriptions.handle(lib.UserUpdatesEvent, this.handleUserSubscription.bind(this));
+		this.subscriptions.handle(lib.RoleUpdatesEvent, this.handleRoleSubscription.bind(this));
 
 		// Handle updates for datastores
 		this.systems.on("update", this.systemsUpdated.bind(this));
@@ -221,6 +236,7 @@ export default class Controller {
 		this.instances.on("update", this.instanceDetailsUpdated.bind(this));
 		this.saves.on("update", this.savesUpdated.bind(this));
 		this.modPacks.on("update", this.modPacksUpdated.bind(this));
+		this.roles.on("update", this.rolesUpdated.bind(this));
 	}
 
 	async start(args: ControllerArgs) {
@@ -597,6 +613,7 @@ export default class Controller {
 			this.instances.save(),
 			this.saves.save(),
 			this.modPacks.save(),
+			this.roles.save(),
 		]);
 
 		if (this.userManager.dirty) {
@@ -657,6 +674,20 @@ export default class Controller {
 		return instance;
 	}
 
+	static migrateRoles(rawJson: unknown[]): Static<typeof lib.Role.jsonSchema>[] {
+		const serialized = rawJson as Static<typeof lib.Role.jsonSchema>[];
+		return serialized.map(json => {
+			json.permissions = json.permissions.map(permission => {
+				// migrate: core.instance.save.list.subscribe was renamed in alpha 17
+				if (permission === "core.instance.save.list.subscribe") {
+					return "core.instance.save.subscribe";
+				}
+				return permission;
+			});
+			return json;
+		});
+	}
+
 	static async loadJsonObject(filePath: string, throwOnMissing: boolean = false): Promise<any> {
 		let manifest = {};
 		try {
@@ -1060,27 +1091,34 @@ export default class Controller {
 		}
 	}
 
-	/**
-	 * Notify connected control clients with the given role that the
-	 * permissions may have changed.
-	 * @param role - Role permisions updated for.
-	 */
-	rolePermissionsUpdated(role: lib.Role) {
-		for (let controlConnection of this.wsServer.controlConnections.values()) {
-			if (controlConnection.user.roles.has(role)) {
-				controlConnection.send(
-					new lib.AccountUpdateEvent(
-						[...controlConnection.user.roles].map(r => ({
-							name: r.name,
-							id: r.id,
-							permissions: [...r.permissions],
-						}))
-					)
-				);
+	rolesUpdated(roles: lib.Role[]) {
+		this.subscriptions.broadcast(new lib.RoleUpdatesEvent(roles));
+		// lib.invokeHook(this.plugins, "onRolesUpdated", roles); // This doesn't exist at the moment
+		// Notify connected control clients with the given role that the permissions may have changed.
+		for (const role of roles) {
+			for (let controlConnection of this.wsServer.controlConnections.values()) {
+				if (controlConnection.user.roles.has(role)) {
+					controlConnection.send(
+						new lib.AccountUpdateEvent(
+							[...controlConnection.user.roles].map(r => ({
+								name: r.name,
+								id: r.id,
+								permissions: [...r.permissions],
+							}))
+						)
+					);
+				}
 			}
 		}
 	}
 
+	async handleRoleSubscription(request: lib.SubscriptionRequest) {
+		const roles = [...this.roles.values()].filter(
+			role => role.updatedAtMs > request.lastRequestTimeMs,
+		);
+		return roles.length ? new lib.RoleUpdatesEvent(roles) : null;
+	}
+
 	async loadPlugins() {
 		for (let pluginInfo of this.pluginInfos) {
 			try {

packages/controller/src/ControllerUser.ts

@@ -1,10 +1,13 @@
 import { Static, Type } from "@sinclair/typebox";
-import { IControllerUser, PermissionError, PlayerStats, Role, User, permissions } from "@clusterio/lib";
-import type UserManager from "./UserManager";
+
+import {
+	IControllerUser, PermissionError, PlayerStats,
+	Role, SubscribableDatastore, User, permissions,
+} from "@clusterio/lib";
 
 export default class ControllerUser extends User implements IControllerUser {
 	constructor(
-		public userManager: UserManager,
+		private _controllerRoles: SubscribableDatastore<Role>,
 		/** Unix time in seconds the user token must be issued after to be valid.  */
 		public tokenValidAfter = 0,
 		...args: ConstructorParameters<typeof User>
@@ -17,7 +20,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		token_valid_after: Type.Optional(Type.Number()),
 	});
 
-	static fromJSON(json: Static<typeof this.jsonSchema>, userManager: UserManager) {
+	static fromJSON(json: Static<typeof this.jsonSchema>, controllerRoles: SubscribableDatastore<Role>) {
 		const roleIds = new Set<number>(json.roles ?? []);
 		const instanceStats = new Map(
 			(json.instance_stats ?? []).map(
@@ -26,7 +29,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		);
 		const playerStats = User._calculatePlayerStats(instanceStats);
 		return new this(
-			userManager,
+			controllerRoles,
 			json.token_valid_after,
 			json.name,
 			roleIds,
@@ -57,7 +60,7 @@ export default class ControllerUser extends User implements IControllerUser {
 
 	get roles(): ReadonlySet<Readonly<Role>> {
 		return new Set([...this.roleIds].map(
-			id => this.userManager.roles.get(id)
+			id => this._controllerRoles.get(id)
 		).filter(
 			(r): r is Role => Boolean(r)
 		));
@@ -79,7 +82,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		}
 
 		for (let roleId of this.roleIds) {
-			let role = this.userManager.roles.get(roleId);
+			let role = this._controllerRoles.get(roleId);
 			if (!role) {
 				continue;
 			}

packages/controller/src/UserManager.ts

@@ -10,7 +10,6 @@ import ControllerUser from "./ControllerUser";
  * @alias module:controller/src/UserManager
  */
 export default class UserManager {
-	roles: Map<lib.Role["id"], lib.Role> = new Map();
 	users: Map<ControllerUser["id"], ControllerUser> = new Map();
 	dirty = false;
 
@@ -20,7 +19,8 @@ export default class UserManager {
 	onlineUsers = new Set<ControllerUser>();
 
 	constructor(
-		private _config: lib.ControllerConfig
+		private _config: lib.ControllerConfig,
+		private _roles: lib.SubscribableDatastore<lib.Role>,
 	) {
 	}
 
@@ -31,14 +31,15 @@ export default class UserManager {
 	async load(filePath: string): Promise<void> {
 		try {
 			let content = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-			for (let serializedRole of content.roles) {
-				let role = lib.Role.fromJSON(serializedRole);
-				this.roles.set(role.id, role);
+			if ("roles" in content) { // Split file format after 2.0.0-alpha.22
+				this._roles.setMany(content.roles.map((serializedRole: any) => lib.Role.fromJSON(serializedRole)));
+				await this._roles.save(); // Force a save to prevent data loss
+				content = content.users;
 			}
 
 			let duplicates = 0;
-			for (let serializedUser of content.users) {
-				let user = ControllerUser.fromJSON(serializedUser, this);
+			for (const serializedUser of content) {
+				const user = ControllerUser.fromJSON(serializedUser, this._roles);
 				const existingUser = this.users.get(user.id);
 				if (existingUser) {
 					// Required migration to all lowercase ids in alpha 19
@@ -65,30 +66,14 @@ export default class UserManager {
 			if (err.code !== "ENOENT") {
 				throw err;
 			}
-
-			// Create default roles if loading failed
-			lib.ensureDefaultAdminRole(this.roles);
-			lib.ensureDefaultPlayerRole(this.roles);
 		}
 	}
 
 	async save(filePath: string): Promise<void> {
-		let serializedRoles = [];
-		for (let role of this.roles.values()) {
-			serializedRoles.push(role.toJSON());
-		}
-
-		let serializedUsers = [];
-		for (let user of this.users.values()) {
-			serializedUsers.push(user.toJSON(true));
-		}
-
-		let serialized = {
-			users: serializedUsers,
-			roles: serializedRoles,
-		};
 		this.dirty = false;
-		await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t"));
+		await lib.safeOutputFile(filePath,
+			JSON.stringify([...this.users.values()].map(user => user.toJSON(true)), null, "\t")
+		);
 	}
 
 	/**
@@ -103,11 +88,11 @@ export default class UserManager {
 
 		let roles = new Set<number>();
 		let defaultRoleId = this._config.get("controller.default_role_id");
-		if (defaultRoleId !== null && this.roles.has(defaultRoleId)) {
+		if (defaultRoleId !== null && this._roles.has(defaultRoleId)) {
 			roles.add(defaultRoleId);
 		}
 
-		let user = new ControllerUser(this, 0, name, roles);
+		let user = new ControllerUser(this._roles, 0, name, roles);
 		this.users.set(user.id, user);
 		this.dirty = true;
 		return user;

packages/lib/src/data/Role.ts

@@ -10,6 +10,8 @@ export default class Role {
 		public name: string,
 		public description: string,
 		public permissions: Set<string>,
+		public updatedAtMs = 0,
+		public isDeleted = false,
 	) { }
 
 	static jsonSchema = Type.Object({
@@ -17,22 +19,18 @@ export default class Role {
 		name: Type.String(),
 		description: Type.String(),
 		permissions: Type.Array(Type.String()),
+		updated_at_ms: Type.Optional(Type.Number()),
+		is_deleted: Type.Optional(Type.Boolean()),
 	});
 
-	static remapPermission(permission: string) {
-		// migrate: core.instance.save.list.subscribe was renamed in alpha 17
-		if (permission === "core.instance.save.list.subscribe") {
-			return "core.instance.save.subscribe";
-		}
-		return permission;
-	}
-
 	static fromJSON(json: Static<typeof this.jsonSchema>) {
 		return new this(
 			json.id,
 			json.name,
 			json.description,
-			new Set(json.permissions.map(this.remapPermission)),
+			new Set(json.permissions),
+			json.updated_at_ms,
+			json.is_deleted,
 		);
 	}
 
@@ -42,6 +40,8 @@ export default class Role {
 			name: this.name,
 			description: this.description,
 			permissions: [...this.permissions],
+			updated_at_ms: this.updatedAtMs,
+			is_deleted: this.isDeleted,
 		};
 	}
 

packages/lib/src/data/index.ts

@@ -21,5 +21,6 @@ export * from "./messages_host";
 export * from "./messages_instance";
 export * from "./messages_mod";
 export * from "./messages_user";
+export * from "./messages_role";
 export * from "./messages_plugin";
 export * from "./version";

packages/lib/src/data/messages_role.ts

@@ -0,0 +1,134 @@
+import { Type, Static } from "@sinclair/typebox";
+import { JsonNumber, jsonArray } from "./composites";
+import Permission from "./Permission";
+import Role from "./Role";
+
+export class PermissionListRequest {
+	declare ["constructor"]: typeof PermissionListRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.permission.list" as const;
+	static Response = jsonArray(Permission);
+}
+
+export class RoleListRequest {
+	declare ["constructor"]: typeof RoleListRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.list" as const;
+	static Response = jsonArray(Role);
+}
+
+export class RoleUpdatesEvent {
+	declare ["constructor"]: typeof RoleUpdatesEvent;
+	static type = "event" as const;
+	static src = "controller" as const;
+	static dst = "control" as const;
+	static permission = "core.role.subscribe" as const;
+
+	constructor(
+		public updates: Role[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		"updates": Type.Array(Role.jsonSchema),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.updates.map(update => Role.fromJSON(update)));
+	}
+}
+
+export class RoleCreateRequest {
+	declare ["constructor"]: typeof RoleCreateRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.create" as const;
+
+	constructor(
+		public name: string,
+		public description: string,
+		public permissions: string[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		name: Type.String(),
+		description: Type.String(),
+		permissions: Type.Array(Type.String()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.name, json.description, json.permissions);
+	}
+
+	static Response = JsonNumber;
+}
+
+export class RoleUpdateRequest {
+	declare ["constructor"]: typeof RoleUpdateRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.update" as const;
+
+	constructor(
+		public id: number,
+		public name: string,
+		public description: string,
+		public permissions: string[],
+	) { }
+
+	static jsonSchema = Type.Object({
+		id: Type.Integer(),
+		name: Type.String(),
+		description: Type.String(),
+		permissions: Type.Array(Type.String()),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.id, json.name, json.description, json.permissions);
+	}
+}
+
+export class RoleGrantDefaultPermissionsRequest {
+	declare ["constructor"]: typeof RoleGrantDefaultPermissionsRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.update" as const;
+
+	constructor(
+		public id: number,
+	) { }
+
+	static jsonSchema = Type.Object({
+		id: Type.Integer(),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.id);
+	}
+}
+
+export class RoleDeleteRequest {
+	declare ["constructor"]: typeof RoleDeleteRequest;
+	static type = "request" as const;
+	static src = "control" as const;
+	static dst = "controller" as const;
+	static permission = "core.role.delete" as const;
+
+	constructor(
+		public id: number,
+	) { }
+
+	static jsonSchema = Type.Object({
+		id: Type.Integer(),
+	});
+
+	static fromJSON(json: Static<typeof this.jsonSchema>) {
+		return new this(json.id);
+	}
+}

packages/lib/src/data/messages_user.ts

@@ -1,120 +1,8 @@
 import { Type, Static } from "@sinclair/typebox";
-import Permission from "./Permission";
-import Role from "./Role";
 import User, { IControllerUser } from "./User";
-import { JsonNumber, StringEnum, jsonArray, plainJson } from "./composites";
+import { StringEnum, jsonArray, plainJson } from "./composites";
 import { MessageRequest } from "./messages_core";
 
-export class PermissionListRequest {
-	declare ["constructor"]: typeof PermissionListRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.permission.list" as const;
-	static Response = jsonArray(Permission);
-}
-
-export class RoleListRequest {
-	declare ["constructor"]: typeof RoleListRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.list" as const;
-	static Response = jsonArray(Role);
-}
-
-export class RoleCreateRequest {
-	declare ["constructor"]: typeof RoleCreateRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.create" as const;
-
-	constructor(
-		public name: string,
-		public description: string,
-		public permissions: string[],
-	) { }
-
-	static jsonSchema = Type.Object({
-		name: Type.String(),
-		description: Type.String(),
-		permissions: Type.Array(Type.String()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.name, json.description, json.permissions);
-	}
-
-	static Response = JsonNumber;
-}
-
-export class RoleUpdateRequest {
-	declare ["constructor"]: typeof RoleUpdateRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.update" as const;
-
-	constructor(
-		public id: number,
-		public name: string,
-		public description: string,
-		public permissions: string[],
-	) { }
-
-	static jsonSchema = Type.Object({
-		id: Type.Integer(),
-		name: Type.String(),
-		description: Type.String(),
-		permissions: Type.Array(Type.String()),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.id, json.name, json.description, json.permissions);
-	}
-}
-
-export class RoleGrantDefaultPermissionsRequest {
-	declare ["constructor"]: typeof RoleGrantDefaultPermissionsRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.update" as const;
-
-	constructor(
-		public id: number,
-	) { }
-
-	static jsonSchema = Type.Object({
-		id: Type.Integer(),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.id);
-	}
-}
-
-export class RoleDeleteRequest {
-	declare ["constructor"]: typeof RoleDeleteRequest;
-	static type = "request" as const;
-	static src = "control" as const;
-	static dst = "controller" as const;
-	static permission = "core.role.delete" as const;
-
-	constructor(
-		public id: number,
-	) { }
-
-	static jsonSchema = Type.Object({
-		id: Type.Integer(),
-	});
-
-	static fromJSON(json: Static<typeof this.jsonSchema>) {
-		return new this(json.id);
-	}
-}
-
 export class UserGetRequest {
 	declare ["constructor"]: typeof UserGetRequest;
 	static type = "request" as const;

packages/lib/src/datastore.ts

@@ -30,7 +30,7 @@ export class MemoryDatastoreProvider<
 	V extends DatastoreValue,
 > extends DatastoreProvider<K, V> {
 	constructor(
-		private value: Map<K, V> = new Map()
+		private value = new Map<K, V>()
 	) {
 		super();
 	}
@@ -39,34 +39,33 @@ export class MemoryDatastoreProvider<
 		this.value = new Map(data);
 	}
 
-	async load() {
+	async load(): Promise<Map<K, V>> {
 		return new Map(this.value);
 	}
 }
 
 // Store all data within a json object file and validate the result against a schema
 export class JsonDatastoreProvider<
-	K extends DatastoreKey,
 	V extends DatastoreValue,
 	J, // Intermediate type returned from migrations
-> extends DatastoreProvider<K, V> {
+> extends DatastoreProvider<string, V> {
 	constructor(
 		private filePath: string,
-		private fromJson: (json: J) => V = v => v as any,
-		private migrations: (rawJson: Record<DatastoreKey, unknown>) => Record<DatastoreKey, J> = v => v as any,
+		private fromJson: (json: J) => V = v => v as any as V,
+		private migrations: (rawJson: Record<string, unknown>) => Record<string, J> = v => v as Record<string, J>,
 		private finalise: (obj: V) => V = v => v,
 	) {
 		super();
 	}
 
 	// Save the data to the json file
-	async save(data: Map<K, V>) {
+	async save(data: Map<string, V>) {
 		logger.verbose(`Saving ${this.filePath}`);
 		await safeOutputFile(this.filePath, JSON.stringify(Object.fromEntries(data), null, "\t"));
 	}
 
 	// Load the data from the json file
-	async load() {
+	async load(): Promise<Map<string, V>> {
 		// Read the raw json from the file
 		let rawJson;
 		try {
@@ -89,27 +88,26 @@ export class JsonDatastoreProvider<
 
 // Store all data within a json array file where each element has a unique id
 export class JsonIdDatastoreProvider<
-	K extends DatastoreKey,
-	V extends DatastoreValue & { id: K },
+	V extends DatastoreValue & { id: DatastoreKey },
 	J, // Intermediate type returned from migrations
-> extends DatastoreProvider<K, V> {
+> extends DatastoreProvider<V["id"], V> {
 	constructor(
 		private filePath: string,
-		private fromJson: (json: J) => V = v => v as any,
-		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as any,
+		private fromJson: (json: J) => V = v => v as any as V,
+		private migrations: (rawJson: Array<unknown>) => Array<J> = v => v as Array<J>,
 		private finalise: (obj: V) => V = v => v,
 	) {
 		super();
 	}
 
 	// Save the data to the json file
-	async save(data: Map<K, V>) {
+	async save(data: Map<V["id"], V>) {
 		logger.verbose(`Saving ${this.filePath}`);
 		await safeOutputFile(this.filePath, JSON.stringify([...data.values()], null, "\t"));
 	}
 
 	// Load the data from the json file
-	async load() {
+	async load(): Promise<Map<V["id"], V>> {
 		// Read the raw json from the file
 		let rawJson;
 		try {
@@ -211,13 +209,17 @@ export abstract class Datastore<
 	valuesMutable() {
 		return this.data.values();
 	}
+
+	keys() {
+		return this.data.keys();
+	}
 }
 
 // General key value mapping which can be saved and loaded with the file system
 export class KeyValueDatastore<
 	K extends DatastoreKey,
 	V extends DatastoreValue,
-> extends Datastore<K, V, [K, V, isDeleted?: boolean]> {
+> extends Datastore<K, V, [key: K, value: V, isDeleted?: boolean]> {
 	// Set the value in the datastore, be careful of race conditions if you await any functions before calling set
 	set(key: K, value: V) {
 		this.data.set(key, value);
@@ -235,8 +237,10 @@ export class KeyValueDatastore<
 	// Delete a value in the datastore
 	delete(key: K) {
 		const value = this.data.get(key);
-		this.data.delete(key);
-		this.touch([[key, value!, true]]);
+		if (value !== undefined) {
+			this.data.delete(key);
+			this.touch([[key, value, true]]);
+		}
 	}
 
 	// Delete many values at once from an array of keys

packages/lib/src/link/messages.ts

@@ -4,6 +4,7 @@ import * as host from "../data/messages_host";
 import * as instance from "../data/messages_instance";
 import * as mod from "../data/messages_mod";
 import * as user from "../data/messages_user";
+import * as role from "../data/messages_role";
 import * as plugin from "../data/messages_plugin";
 import * as subscriptions from "../subscriptions";
 import type { RequestClass, EventClass } from "./link";
@@ -107,12 +108,14 @@ export const dataClasses: (RequestClass<unknown, unknown> | EventClass<unknown>)
 	mod.ModUpdatesEvent,
 	mod.ModPortalDownloadRequest,
 
-	user.PermissionListRequest,
-	user.RoleListRequest,
-	user.RoleCreateRequest,
-	user.RoleUpdateRequest,
-	user.RoleGrantDefaultPermissionsRequest,
-	user.RoleDeleteRequest,
+	role.PermissionListRequest,
+	role.RoleListRequest,
+	role.RoleUpdatesEvent,
+	role.RoleCreateRequest,
+	role.RoleUpdateRequest,
+	role.RoleGrantDefaultPermissionsRequest,
+	role.RoleDeleteRequest,
+
 	user.UserGetRequest,
 	user.UserListRequest,
 	user.UserCreateRequest,

packages/lib/src/permissions.ts

@@ -7,6 +7,7 @@
  * @module lib/users
  */
 import { Permission, Role } from "./data";
+import { SubscribableDatastore } from "./datastore";
 
 
 export const permissions = new Map<string, Permission>();
@@ -66,7 +67,7 @@ export function definePermission({
  * @param roles - role storage to modify.
  * @returns the default admin role
  */
-export function ensureDefaultAdminRole(roles: Map<number, Role>) {
+export function ensureDefaultAdminRole(roles: SubscribableDatastore<Role>) {
 	let admin = roles.get(0);
 	if (!admin) {
 		admin = Role.fromJSON({
@@ -75,9 +76,9 @@ export function ensureDefaultAdminRole(roles: Map<number, Role>) {
 			description: "Cluster wide administrator.",
 			permissions: [],
 		});
-		roles.set(0, admin);
 	}
 	admin.permissions.add("core.admin");
+	roles.set(admin);
 	return admin;
 }
 
@@ -90,7 +91,7 @@ export function ensureDefaultAdminRole(roles: Map<number, Role>) {
  *
  * @param roles - role storage to modify.
  */
-export function ensureDefaultPlayerRole(roles: Map<number, Role>) {
+export function ensureDefaultPlayerRole(roles: SubscribableDatastore<Role>) {
 	let player = roles.get(1);
 	if (!player) {
 		player = Role.fromJSON({
@@ -99,9 +100,9 @@ export function ensureDefaultPlayerRole(roles: Map<number, Role>) {
 			description: "Default player role.",
 			permissions: [],
 		});
-		roles.set(1, player);
 	}
 	player.grantDefaultPermissions();
+	roles.set(player);
 }
 
 
@@ -431,6 +432,12 @@ definePermission({
 	description: "Get the full list of roles and their permissions in the cluster.",
 	grantByDefault: true,
 });
+definePermission({
+	name: "core.role.subscribe",
+	title: "Subscribe to role updates",
+	description: "Subscribe to be notified on updates on the details of roles.",
+	grantByDefault: true,
+});
 definePermission({
 	name: "core.role.create",
 	title: "Create role",

packages/lib/tsconfig.browser.json

@@ -6,6 +6,7 @@
 		"./src/api.ts",
 		"./src/config/*.ts",
 		"./src/data/*.ts",
+		"./src/datastore.ts",
 		"./src/errors.ts",
 		"./src/factorio/exchange_string.ts",
 		"./src/helpers.ts",

packages/web_ui/src/components/InputRole.tsx

@@ -1,21 +1,10 @@
-import React, { useContext, useEffect, useState } from "react";
 import { Select } from "antd";
 
-import * as lib from "@clusterio/lib";
-
-import ControlContext from "./ControlContext";
-import { notifyErrorHandler } from "../util/notify";
 import { InputComponentProps } from "../BaseWebPlugin";
+import { useRoles } from "../model/roles";
 
 export default function InputRole(props: InputComponentProps) {
-	let control = useContext(ControlContext);
-	let [roles, setRoles] = useState<lib.Role[]>([]);
-
-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(newRoles);
-		}).catch(notifyErrorHandler("Error fetching role list"));
-	}, []);
+	const [roles] = useRoles();
 
 	return <Select
 		showSearch

packages/web_ui/src/components/RoleViewPage.tsx

@@ -5,6 +5,7 @@ import DeleteOutlined from "@ant-design/icons/DeleteOutlined";
 
 import * as lib from "@clusterio/lib";
 
+import { useRole } from "../model/roles";
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import PageHeader from "./PageHeader";
@@ -12,30 +13,6 @@ import PageLayout from "./PageLayout";
 import PluginExtra from "./PluginExtra";
 import { notifyErrorHandler } from "../util/notify";
 
-
-function useRole(id: number): [Readonly<lib.Role> | undefined, boolean, () => void] {
-	let control = useContext(ControlContext);
-	let [role, setRole] = useState<lib.Role>();
-	let [synced, setSynced] = useState(false);
-
-	function updateRole() {
-		// XXX optimize by requesting only the role in question
-		control.send(new lib.RoleListRequest()).then(roles => {
-			let match = roles.find(u => u.id === id);
-			if (match) {
-				setRole(match);
-			}
-			setSynced(true);
-		});
-	}
-
-	useEffect(() => {
-		updateRole();
-	}, [id]);
-
-	return [role, synced, updateRole];
-}
-
 export default function RoleViewPage() {
 	let params = useParams();
 	let roleId = Number(params.id);
@@ -44,7 +21,7 @@ export default function RoleViewPage() {
 
 	let account = useAccount();
 	let control = useContext(ControlContext);
-	const [role, synced, updateRole] = useRole(roleId);
+	const [role, synced] = useRole(roleId);
 	let [edited, setEdited] = useState(false);
 
 
@@ -63,6 +40,7 @@ export default function RoleViewPage() {
 		</PageLayout>;
 	}
 
+	// TODO: Update displayed perms when role perms change and have not been edited locally
 	let initialValues = {
 		name: role["name"],
 		description: role["description"],
@@ -88,7 +66,6 @@ export default function RoleViewPage() {
 					new lib.RoleUpdateRequest(roleId, values.name || "", values.description || "", newPermissions)
 				).then(() => {
 					setEdited(false);
-					updateRole();
 				}).catch(notifyErrorHandler("Error applying changes"));
 			}}
 		>

packages/web_ui/src/components/RolesPage.tsx

@@ -4,6 +4,7 @@ import { Button, Form, Input, Modal, Table } from "antd";
 
 import * as lib from "@clusterio/lib";
 
+import { useRoles } from "../model/roles";
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import { notifyErrorHandler } from "../util/notify";
@@ -60,16 +61,8 @@ function CreateRoleButton() {
 
 export default function RolesPage() {
 	let account = useAccount();
-	let control = useContext(ControlContext);
 	let navigate = useNavigate();
-
-	let [roles, setRoles] = useState<lib.Role[]>([]);
-
-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(newRoles);
-		}).catch(notifyErrorHandler("Error fetching role list"));
-	}, []);
+	const [roles] = useRoles();
 
 	return <PageLayout nav={[{ name: "Roles" }]}>
 		<PageHeader
@@ -88,7 +81,7 @@ export default function RolesPage() {
 					dataIndex: "description",
 				},
 			]}
-			dataSource={roles}
+			dataSource={[...roles.values()]}
 			pagination={false}
 			rowKey={role => role.id}
 			onRow={(role, rowIndex) => ({

packages/web_ui/src/components/UserViewPage.tsx

@@ -8,6 +8,7 @@ import DeleteOutlined from "@ant-design/icons/DeleteOutlined";
 
 import * as lib from "@clusterio/lib";
 
+import { useRoles } from "../model/roles";
 import { useAccount } from "../model/account";
 import { useInstances } from "../model/instance";
 import ControlContext from "./ControlContext";
@@ -35,21 +36,13 @@ export default function UserViewPage() {
 	let control = useContext(ControlContext);
 	let [instances] = useInstances();
 	const [user, synced] = useUser(userName);
-	let [roles, setRoles] = useState<Map<number, lib.Role>>(new Map());
+	let [roles] = useRoles();
 	let [form] = Form.useForm();
 	let [rolesDirty, setRolesDirty] = useState<boolean>(false);
 	let [banReasonDirty, setBanReasonDirty] = useState<boolean>(false);
 	let [applyingRoles, setApplyingRoles] = useState<boolean>(false);
 	let [rolesError, setRolesError] = useState<string|undefined>();
 
-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(new Map(newRoles.map(role => [role.id, role])));
-		}).catch(() => {
-			setRoles(new Map());
-		});
-	}, []);
-
 	useEffect(() => {
 		if (
 			account.hasPermission("core.user.set_banned")
@@ -76,6 +69,7 @@ export default function UserViewPage() {
 		</PageLayout>;
 	}
 
+	// TODO: Update displayed roles when user roles change and have not been edited locally
 	let roleSelector = <Form.Item
 		noStyle
 		name="roles"

packages/web_ui/src/components/UsersPage.tsx

@@ -13,6 +13,7 @@ import {
 
 import * as lib from "@clusterio/lib";
 
+import { useRoles } from "../model/roles";
 import { useAccount } from "../model/account";
 import ControlContext from "./ControlContext";
 import { notifyErrorHandler } from "../util/notify";
@@ -373,21 +374,11 @@ function BulkUserActionButton() {
 
 export default function UsersPage() {
 	let account = useAccount();
-	let control = useContext(ControlContext);
 	let navigate = useNavigate();
 	let [users] = useUsers();
-
-	const [roles, setRoles] = useState(new Map());
+	const [roles] = useRoles();
 	const searchInput = useRef<InputRef>(null);
 
-	useEffect(() => {
-		control.send(new lib.RoleListRequest()).then(newRoles => {
-			setRoles(new Map(newRoles.map(role => [role.id, role])));
-		}).catch(() => {
-			// ignore
-		});
-	}, []);
-
 	return <PageLayout nav={[{ name: "Users" }]}>
 		<PageHeader
 			title="Users"

packages/web_ui/src/model/roles.ts

@@ -0,0 +1,23 @@
+import { useCallback, useContext, useSyncExternalStore } from "react";
+import ControlContext from "../components/ControlContext";
+
+import * as lib from "@clusterio/lib";
+
+export function useRolesOfUser(user?: lib.User) {
+	const [roles, synced] = useRoles();
+	return [
+		user !== undefined ? new Map([...user.roleIds].map(roleId => [roleId, roles.get(roleId)])) : new Map(),
+		synced,
+	] as const;
+}
+
+export function useRole(id?: number) {
+	const [roles, synced] = useRoles();
+	return [id !== undefined ? roles.get(id) : undefined, synced] as const;
+}
+
+export function useRoles() {
+	const control = useContext(ControlContext);
+	const subscribe = useCallback((callback: () => void) => control.roles.subscribe(callback), [control]);
+	return useSyncExternalStore(subscribe, () => control.roles.getSnapshot());
+}

packages/web_ui/src/util/websocket.ts

@@ -67,6 +67,7 @@ export class Control extends lib.Link {
 	modPacks = new lib.EventSubscriber(lib.ModPackUpdatesEvent, this);
 	mods = new lib.EventSubscriber(lib.ModUpdatesEvent, this);
 	users = new lib.EventSubscriber(lib.UserUpdatesEvent, this);
+	roles = new lib.EventSubscriber(lib.RoleUpdatesEvent, this);
 
 	declare connector: ControlConnector;
 

test/controller/Controller.js

@@ -1,7 +1,7 @@
 "use strict";
 const assert = require("assert").strict;
 const { Controller, HostInfo, InstanceInfo } = require("@clusterio/controller");
-const { ControllerConfig, Address, RequestError, InstanceConfig, HostConfig, SystemInfo } = require("@clusterio/lib");
+const { ControllerConfig, Address, RequestError, InstanceConfig, SystemInfo, Role } = require("@clusterio/lib");
 const { EventEmitter } = require("stream");
 
 class MockEvent {}
@@ -314,5 +314,33 @@ describe("controller/src/Controller", function() {
 				]);
 			});
 		});
+		describe("Role", function() {
+			it("migrates renamed permissions", function() { // Alpha 17
+				const result = Controller.migrateRoles([
+					{ permissions: ["foo", "bar", "core.instance.save.list.subscribe"] },
+					{ permissions: ["bar", "core.instance.save.list.subscribe"] },
+					{ permissions: ["core.instance.save.subscribe"] },
+				]);
+				assert.deepEqual(result, [
+					{ permissions: ["foo", "bar", "core.instance.save.subscribe"] },
+					{ permissions: ["bar", "core.instance.save.subscribe"] },
+					{ permissions: ["core.instance.save.subscribe"] },
+				]);
+			});
+			it("does nothing for upto date data", function() {
+				const role = new Role(
+					0, "n", "d", new Set("p1", "p2"), 0, false
+				);
+				const jsonString = JSON.stringify(role);
+				const roleJson = JSON.parse(jsonString);
+				const roleJsonCopy = JSON.parse(jsonString);
+				const result = Controller.migrateRoles([
+					roleJson, roleJson, roleJson,
+				]);
+				assert.deepEqual(result, [
+					roleJsonCopy, roleJsonCopy, roleJsonCopy,
+				]);
+			});
+		});
 	});
 });

test/controller/ControllerUser.js

@@ -5,20 +5,17 @@ const lib = require("@clusterio/lib");
 
 describe("controller/src/ControllerUser", function() {
 	describe("class ControllerUser", function() {
-		class MockUserManager {
-			roles = new Map([
-				[1, new lib.Role(1, "a", "a role", new Set(["core.admin"]))],
-				[2, new lib.Role(2, "b", "b role", new Set(["user-test"]))],
-			]);
-		}
-		const userManager = new MockUserManager();
+		const roles = new Map([
+			[1, new lib.Role(1, "a", "a role", new Set(["core.admin"]))],
+			[2, new lib.Role(2, "b", "b role", new Set(["user-test"]))],
+		]);
 
 		it("should round trip serialize", function() {
 			function test_roundtrip(serialized) {
-				let user = ControllerUser.fromJSON(serialized, userManager);
+				let user = ControllerUser.fromJSON(serialized, roles);
 				let user_serialized = user.toJSON(true);
 				assert.deepEqual(user_serialized, serialized);
-				let user_deserialized = ControllerUser.fromJSON(user_serialized, userManager);
+				let user_deserialized = ControllerUser.fromJSON(user_serialized, roles);
 				assert.deepEqual(user_deserialized, user);
 			}
 
@@ -31,9 +28,9 @@ describe("controller/src/ControllerUser", function() {
 		describe(".checkPermission()", function() {
 			it("should correctly resolve permissions", function() {
 				lib.definePermission({ name: "user-test", title: "Test", description: "User Test" });
-				let a = ControllerUser.fromJSON({ name: "admin", roles: [1] }, userManager);
-				let b = ControllerUser.fromJSON({ name: "user", roles: [2] }, userManager);
-				let c = ControllerUser.fromJSON({ name: "null", roles: [] }, userManager);
+				let a = ControllerUser.fromJSON({ name: "admin", roles: [1] }, roles);
+				let b = ControllerUser.fromJSON({ name: "user", roles: [2] }, roles);
+				let c = ControllerUser.fromJSON({ name: "null", roles: [] }, roles);
 
 				a.checkPermission("core.control.connect");
 				assert.throws(() => b.checkPermission("core.control.connect"), new Error("Permission denied"));

test/integration/clusterio.js

@@ -1404,6 +1404,7 @@ describe("Integration of Clusterio", function() {
 				await execCtl(`role create temp ${args}`);
 				let roles = await getControl().send(new lib.RoleListRequest());
 				let tempRole = roles.find(role => role.name === "temp");
+				if (tempRole) { tempRole.updatedAtMs = 0; }
 				assert.deepEqual(
 					tempRole,
 					new lib.Role(5, "temp", "A temp role", new Set(["core.control.connect"]))
@@ -1417,6 +1418,7 @@ describe("Integration of Clusterio", function() {
 				await execCtl(`role edit temp ${args}`);
 				let roles = await getControl().send(new lib.RoleListRequest());
 				let newRole = roles.find(role => role.name === "new");
+				if (newRole) { newRole.updatedAtMs = 0; }
 				assert.deepEqual(newRole, new lib.Role(5, "new", "A new role", new Set()));
 			});
 			it("should add permissions with --add-perms", async function() {

test/lib/datastore.js

@@ -507,10 +507,9 @@ describe("lib/datastore", function() {
 					["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => k !== "foo"))]
 				);
 			});
-			it("does not break when the key does not exist", function() {
+			it("does nothing when the key does not exist", function() {
 				datastore.delete("test");
-				assert.equal(eventsRaised.length, 1);
-				assert.deepEqual(eventsRaised[0], [["test", undefined, true]]);
+				assert.equal(eventsRaised.length, 0);
 			});
 		});
 

test/mock.js

@@ -173,17 +173,20 @@ class MockController {
 			},
 		};
 
-		this.userManager = new UserManager(this.config);
-		this.userManager.roles = new Map([
+		const roleProvider = new lib.MemoryDatastoreProvider(new Map([
 			[0, lib.Role.fromJSON({ id: 0, name: "Admin", description: "admin", permissions: ["core.admin"] })],
 			[1, lib.Role.fromJSON({ id: 1, name: "Player", description: "player", permissions: [] })],
-		]);
-		this.userManager.roles.get(1).grantDefaultPermissions();
+		]));
 
+		this.roles = new lib.SubscribableDatastore(roleProvider, roleProvider.value);
+		this.roles.get(1).grantDefaultPermissions();
+
+		this.userManager = new UserManager(this.config, this.roles);
 		this.userManager.users = new Map([
-			["test", ControllerUser.fromJSON({ name: "test", roles: [0, 1] }, this.userManager)],
-			["player", ControllerUser.fromJSON({ name: "player", roles: [1] }, this.userManager)],
+			["test", ControllerUser.fromJSON({ name: "test", roles: [0, 1] }, this.roles)],
+			["player", ControllerUser.fromJSON({ name: "player", roles: [1] }, this.roles)],
 		]);
+
 		this.instances = new lib.KeyValueDatastore();
 		this.hosts = new lib.KeyValueDatastore();
 		this.handles = new Map();