Separate roles to their own file

packages/controller/controller.ts

@@ -130,8 +130,16 @@ async function handleBootstrapCommand(
 	controllerConfig: lib.ControllerConfig
 ): Promise<void> {
 	let subCommand = args._[1];
-	let userManager = new UserManager(controllerConfig);
-	await userManager.load(path.join(controllerConfig.get("controller.database_directory"), "users.json"));
+
+	const databaseDirectory = controllerConfig.get("controller.database_directory");
+	const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+		path.join(databaseDirectory, "roles.json"),
+		lib.Role.fromJSON.bind(lib.Role),
+	).bootstrap());
+
+	const userManager = new UserManager(controllerConfig, roles);
+	await userManager.load(path.join(databaseDirectory, "users.json"));
+
 	if (subCommand === "create-admin") {
 		if (!args.name) {
 			logger.error("name cannot be blank");
@@ -144,7 +152,7 @@ async function handleBootstrapCommand(
 			admin = userManager.createUser(args.name);
 		}
 
-		let adminRole = lib.ensureDefaultAdminRole(userManager.roles);
+		let adminRole = lib.ensureDefaultAdminRole(roles);
 		admin.roleIds.add(adminRole.id);
 		admin.isAdmin = true;
 		await userManager.save(path.join(controllerConfig.get("controller.database_directory"), "users.json"));

packages/controller/src/ControlConnection.ts

@@ -671,22 +671,21 @@ export default class ControlConnection extends BaseConnection {
 	}
 
 	async handleRoleListRequest() {
-		return [...this._controller.userManager.roles.values()];
+		return [...this._controller.roles.values()];
 	}
 
 	async handleRoleCreateRequest(request: lib.RoleCreateRequest) {
-		let lastId = Math.max.apply(null, [...this._controller.userManager.roles.keys()]);
+		let lastId = Math.max.apply(null, [...this._controller.roles.keys()]);
 
 		// Start at 5 to leave space for future default roles
 		let id = Math.max(5, lastId+1);
-		this._controller.userManager.roles.set(id, lib.Role.fromJSON({ id, ...request }));
-		this._controller.userManager.dirty = true;
+		this._controller.roles.set(lib.Role.fromJSON({ id, ...request }));
 		return id;
 	}
 
 	async handleRoleUpdateRequest(request: lib.RoleUpdateRequest) {
 		let { id, name, description, permissions } = request;
-		let role = this._controller.userManager.roles.get(id);
+		let role = this._controller.roles.getMutable(id);
 		if (!role) {
 			throw new lib.RequestError(`Role with ID ${id} does not exist`);
 		}
@@ -694,27 +693,29 @@ export default class ControlConnection extends BaseConnection {
 		role.name = name;
 		role.description = description;
 		role.permissions = new Set(permissions);
-		this._controller.rolePermissionsUpdated(role);
+		this._controller.roles.set(role);
 	}
 
 	async handleRoleGrantDefaultPermissionsRequest(request: lib.RoleGrantDefaultPermissionsRequest) {
-		let role = this._controller.userManager.roles.get(request.id);
+		let role = this._controller.roles.get(request.id);
 		if (!role) {
 			throw new lib.RequestError(`Role with ID ${request.id} does not exist`);
 		}
 
 		role.grantDefaultPermissions();
-		this._controller.rolePermissionsUpdated(role);
+		this._controller.roles.set(role);
 	}
 
 	async handleRoleDeleteRequest(request: lib.RoleDeleteRequest) {
 		let id = request.id;
 
-		if (!this._controller.userManager.roles.delete(id)) {
+		const role = this._controller.roles.get(id);
+		if (!role) {
 			throw new lib.RequestError(`Role with ID ${id} does not exist`);
 		}
-		this._controller.userManager.dirty = true;
+		this._controller.roles.delete(role);
 
+		this._controller.userManager.dirty = true;
 		for (let user of this._controller.userManager.users.values()) {
 			if (user.roleIds.delete(id)) {
 				this._controller.userPermissionsUpdated(user);
@@ -766,7 +767,7 @@ export default class ControlConnection extends BaseConnection {
 		}
 
 		for (let roleId of request.roles) {
-			if (!this._controller.userManager.roles.has(roleId)) {
+			if (!this._controller.roles.has(roleId)) {
 				throw new lib.RequestError(`Role with ID ${roleId} does not exist`);
 			}
 		}

packages/controller/src/Controller.ts

@@ -138,17 +138,27 @@ export default class Controller {
 			lib.ModPack.fromJSON.bind(lib.ModPack)
 		).bootstrap());
 
-		if (modPacks.size === 0) {
-			modPacks.setMany(lib.ModPack.defaultModPacks);
-		}
+		const roles = new lib.SubscribableDatastore(...await new lib.JsonIdDatastoreProvider(
+			path.join(databaseDirectory, "roles.json"),
+			lib.Role.fromJSON.bind(lib.Role),
+		).bootstrap());
 
-		const userManager = new UserManager(config);
+		const userManager = new UserManager(config, roles);
 		await userManager.load(path.join(databaseDirectory, "users.json"));
 
 		let modsDirectory = config.get("controller.mods_directory");
 		await fs.ensureDir(modsDirectory);
 		const modStore = await lib.ModStore.fromDirectory(modsDirectory);
 
+		// Add default mod packs
+		if (modPacks.size === 0) {
+			modPacks.setMany(lib.ModPack.defaultModPacks);
+		}
+
+		// Add default roles
+		lib.ensureDefaultAdminRole(roles);
+		lib.ensureDefaultPlayerRole(roles);
+
 		return [
 			systems,
 			hosts,
@@ -156,6 +166,7 @@ export default class Controller {
 			saves,
 			modPacks,
 			modStore,
+			roles,
 			userManager,
 		] as const;
 	}
@@ -186,8 +197,10 @@ export default class Controller {
 		public modPacks = new lib.SubscribableDatastore<lib.ModPack>(),
 		/** Mods stored on the controller */
 		public modStore = new lib.ModStore(config.get("controller.mods_directory"), new Map()),
+		/** Mapping of mod pack id to mod pack */
+		public roles = new lib.SubscribableDatastore<lib.Role>(),
 		/** User and roles manager for the cluster */
-		public userManager = new UserManager(config),
+		public userManager = new UserManager(config, roles),
 	) {
 		this.clusterLogger = clusterLogger;
 		this.pluginInfos = pluginInfos;
@@ -221,6 +234,7 @@ export default class Controller {
 		this.instances.on("update", this.instanceDetailsUpdated.bind(this));
 		this.saves.on("update", this.savesUpdated.bind(this));
 		this.modPacks.on("update", this.modPacksUpdated.bind(this));
+		this.roles.on("update", this.rolesUpdated.bind(this));
 	}
 
 	async start(args: ControllerArgs) {
@@ -597,6 +611,7 @@ export default class Controller {
 			this.instances.save(),
 			this.saves.save(),
 			this.modPacks.save(),
+			this.roles.save(),
 		]);
 
 		if (this.userManager.dirty) {
@@ -1063,20 +1078,22 @@ export default class Controller {
 	/**
 	 * Notify connected control clients with the given role that the
 	 * permissions may have changed.
-	 * @param role - Role permisions updated for.
+	 * @param roles - Roles updated.
 	 */
-	rolePermissionsUpdated(role: lib.Role) {
-		for (let controlConnection of this.wsServer.controlConnections.values()) {
-			if (controlConnection.user.roles.has(role)) {
-				controlConnection.send(
-					new lib.AccountUpdateEvent(
-						[...controlConnection.user.roles].map(r => ({
-							name: r.name,
-							id: r.id,
-							permissions: [...r.permissions],
-						}))
-					)
-				);
+	rolesUpdated(roles: lib.Role[]) {
+		for (const role of roles) {
+			for (let controlConnection of this.wsServer.controlConnections.values()) {
+				if (controlConnection.user.roles.has(role)) {
+					controlConnection.send(
+						new lib.AccountUpdateEvent(
+							[...controlConnection.user.roles].map(r => ({
+								name: r.name,
+								id: r.id,
+								permissions: [...r.permissions],
+							}))
+						)
+					);
+				}
 			}
 		}
 	}

packages/controller/src/ControllerUser.ts

@@ -1,10 +1,13 @@
 import { Static, Type } from "@sinclair/typebox";
-import { IControllerUser, PermissionError, PlayerStats, Role, User, permissions } from "@clusterio/lib";
-import type UserManager from "./UserManager";
+
+import {
+	IControllerUser, PermissionError, PlayerStats,
+	Role, SubscribableDatastore, User, permissions,
+} from "@clusterio/lib";
 
 export default class ControllerUser extends User implements IControllerUser {
 	constructor(
-		public userManager: UserManager,
+		private _controllerRoles: SubscribableDatastore<Role>,
 		/** Unix time in seconds the user token must be issued after to be valid.  */
 		public tokenValidAfter = 0,
 		...args: ConstructorParameters<typeof User>
@@ -17,7 +20,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		token_valid_after: Type.Optional(Type.Number()),
 	});
 
-	static fromJSON(json: Static<typeof this.jsonSchema>, userManager: UserManager) {
+	static fromJSON(json: Static<typeof this.jsonSchema>, controllerRoles: SubscribableDatastore<Role>) {
 		const roleIds = new Set<number>(json.roles ?? []);
 		const instanceStats = new Map(
 			(json.instance_stats ?? []).map(
@@ -26,7 +29,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		);
 		const playerStats = User._calculatePlayerStats(instanceStats);
 		return new this(
-			userManager,
+			controllerRoles,
 			json.token_valid_after,
 			json.name,
 			roleIds,
@@ -57,7 +60,7 @@ export default class ControllerUser extends User implements IControllerUser {
 
 	get roles(): ReadonlySet<Readonly<Role>> {
 		return new Set([...this.roleIds].map(
-			id => this.userManager.roles.get(id)
+			id => this._controllerRoles.get(id)
 		).filter(
 			(r): r is Role => Boolean(r)
 		));
@@ -79,7 +82,7 @@ export default class ControllerUser extends User implements IControllerUser {
 		}
 
 		for (let roleId of this.roleIds) {
-			let role = this.userManager.roles.get(roleId);
+			let role = this._controllerRoles.get(roleId);
 			if (!role) {
 				continue;
 			}

packages/controller/src/UserManager.ts

@@ -10,7 +10,6 @@ import ControllerUser from "./ControllerUser";
  * @alias module:controller/src/UserManager
  */
 export default class UserManager {
-	roles: Map<lib.Role["id"], lib.Role> = new Map();
 	users: Map<ControllerUser["id"], ControllerUser> = new Map();
 	dirty = false;
 
@@ -20,7 +19,8 @@ export default class UserManager {
 	onlineUsers = new Set<ControllerUser>();
 
 	constructor(
-		private _config: lib.ControllerConfig
+		private _config: lib.ControllerConfig,
+		private _roles: lib.SubscribableDatastore<lib.Role>,
 	) {
 	}
 
@@ -31,14 +31,15 @@ export default class UserManager {
 	async load(filePath: string): Promise<void> {
 		try {
 			let content = JSON.parse(await fs.readFile(filePath, { encoding: "utf8" }));
-			for (let serializedRole of content.roles) {
-				let role = lib.Role.fromJSON(serializedRole);
-				this.roles.set(role.id, role);
+			if ("roles" in content) { // Split file format after 2.0.0-alpha.22
+				this._roles.setMany(content.roles.map((serializedRole: any) => lib.Role.fromJSON(serializedRole)));
+				await this._roles.save(); // Force a save to prevent data loss
+				content = content.users;
 			}
 
 			let duplicates = 0;
-			for (let serializedUser of content.users) {
-				let user = ControllerUser.fromJSON(serializedUser, this);
+			for (const serializedUser of content) {
+				const user = ControllerUser.fromJSON(serializedUser, this._roles);
 				const existingUser = this.users.get(user.id);
 				if (existingUser) {
 					// Required migration to all lowercase ids in alpha 19
@@ -65,30 +66,14 @@ export default class UserManager {
 			if (err.code !== "ENOENT") {
 				throw err;
 			}
-
-			// Create default roles if loading failed
-			lib.ensureDefaultAdminRole(this.roles);
-			lib.ensureDefaultPlayerRole(this.roles);
 		}
 	}
 
 	async save(filePath: string): Promise<void> {
-		let serializedRoles = [];
-		for (let role of this.roles.values()) {
-			serializedRoles.push(role.toJSON());
-		}
-
-		let serializedUsers = [];
-		for (let user of this.users.values()) {
-			serializedUsers.push(user.toJSON(true));
-		}
-
-		let serialized = {
-			users: serializedUsers,
-			roles: serializedRoles,
-		};
 		this.dirty = false;
-		await lib.safeOutputFile(filePath, JSON.stringify(serialized, null, "\t"));
+		await lib.safeOutputFile(filePath,
+			JSON.stringify([...this.users.values()].map(user => user.toJSON(true)), null, "\t")
+		);
 	}
 
 	/**
@@ -103,11 +88,11 @@ export default class UserManager {
 
 		let roles = new Set<number>();
 		let defaultRoleId = this._config.get("controller.default_role_id");
-		if (defaultRoleId !== null && this.roles.has(defaultRoleId)) {
+		if (defaultRoleId !== null && this._roles.has(defaultRoleId)) {
 			roles.add(defaultRoleId);
 		}
 
-		let user = new ControllerUser(this, 0, name, roles);
+		let user = new ControllerUser(this._roles, 0, name, roles);
 		this.users.set(user.id, user);
 		this.dirty = true;
 		return user;

packages/lib/src/data/Role.ts

@@ -10,6 +10,8 @@ export default class Role {
 		public name: string,
 		public description: string,
 		public permissions: Set<string>,
+		public updatedAtMs = 0,
+		public isDeleted = false,
 	) { }
 
 	static jsonSchema = Type.Object({
@@ -17,6 +19,8 @@ export default class Role {
 		name: Type.String(),
 		description: Type.String(),
 		permissions: Type.Array(Type.String()),
+		updated_at_ms: Type.Optional(Type.Number()),
+		is_deleted: Type.Optional(Type.Boolean()),
 	});
 
 	static remapPermission(permission: string) {
@@ -33,6 +37,8 @@ export default class Role {
 			json.name,
 			json.description,
 			new Set(json.permissions.map(this.remapPermission)),
+			json.updated_at_ms,
+			json.is_deleted,
 		);
 	}
 
@@ -42,6 +48,8 @@ export default class Role {
 			name: this.name,
 			description: this.description,
 			permissions: [...this.permissions],
+			updated_at_ms: this.updatedAtMs,
+			is_deleted: this.isDeleted,
 		};
 	}
 

packages/lib/src/datastore.ts

@@ -209,6 +209,10 @@ export abstract class Datastore<
 	valuesMutable() {
 		return this.data.values();
 	}
+
+	keys() {
+		return this.data.keys();
+	}
 }
 
 // General key value mapping which can be saved and loaded with the file system
@@ -233,8 +237,10 @@ export class KeyValueDatastore<
 	// Delete a value in the datastore
 	delete(key: K) {
 		const value = this.data.get(key);
-		this.data.delete(key);
-		this.touch([[key, value!, true]]);
+		if (value !== undefined) {
+			this.data.delete(key);
+			this.touch([[key, value, true]]);
+		}
 	}
 
 	// Delete many values at once from an array of keys

packages/lib/src/permissions.ts

@@ -7,6 +7,7 @@
  * @module lib/users
  */
 import { Permission, Role } from "./data";
+import { SubscribableDatastore } from "./datastore";
 
 
 export const permissions = new Map<string, Permission>();
@@ -66,7 +67,7 @@ export function definePermission({
  * @param roles - role storage to modify.
  * @returns the default admin role
  */
-export function ensureDefaultAdminRole(roles: Map<number, Role>) {
+export function ensureDefaultAdminRole(roles: SubscribableDatastore<Role>) {
 	let admin = roles.get(0);
 	if (!admin) {
 		admin = Role.fromJSON({
@@ -75,9 +76,9 @@ export function ensureDefaultAdminRole(roles: Map<number, Role>) {
 			description: "Cluster wide administrator.",
 			permissions: [],
 		});
-		roles.set(0, admin);
 	}
 	admin.permissions.add("core.admin");
+	roles.set(admin);
 	return admin;
 }
 
@@ -90,7 +91,7 @@ export function ensureDefaultAdminRole(roles: Map<number, Role>) {
  *
  * @param roles - role storage to modify.
  */
-export function ensureDefaultPlayerRole(roles: Map<number, Role>) {
+export function ensureDefaultPlayerRole(roles: SubscribableDatastore<Role>) {
 	let player = roles.get(1);
 	if (!player) {
 		player = Role.fromJSON({
@@ -99,9 +100,9 @@ export function ensureDefaultPlayerRole(roles: Map<number, Role>) {
 			description: "Default player role.",
 			permissions: [],
 		});
-		roles.set(1, player);
 	}
 	player.grantDefaultPermissions();
+	roles.set(player);
 }
 
 

packages/lib/tsconfig.browser.json

@@ -6,6 +6,7 @@
 		"./src/api.ts",
 		"./src/config/*.ts",
 		"./src/data/*.ts",
+		"./src/datastore.ts",
 		"./src/errors.ts",
 		"./src/factorio/exchange_string.ts",
 		"./src/helpers.ts",

test/controller/ControllerUser.js

@@ -5,20 +5,17 @@ const lib = require("@clusterio/lib");
 
 describe("controller/src/ControllerUser", function() {
 	describe("class ControllerUser", function() {
-		class MockUserManager {
-			roles = new Map([
-				[1, new lib.Role(1, "a", "a role", new Set(["core.admin"]))],
-				[2, new lib.Role(2, "b", "b role", new Set(["user-test"]))],
-			]);
-		}
-		const userManager = new MockUserManager();
+		const roles = new Map([
+			[1, new lib.Role(1, "a", "a role", new Set(["core.admin"]))],
+			[2, new lib.Role(2, "b", "b role", new Set(["user-test"]))],
+		]);
 
 		it("should round trip serialize", function() {
 			function test_roundtrip(serialized) {
-				let user = ControllerUser.fromJSON(serialized, userManager);
+				let user = ControllerUser.fromJSON(serialized, roles);
 				let user_serialized = user.toJSON(true);
 				assert.deepEqual(user_serialized, serialized);
-				let user_deserialized = ControllerUser.fromJSON(user_serialized, userManager);
+				let user_deserialized = ControllerUser.fromJSON(user_serialized, roles);
 				assert.deepEqual(user_deserialized, user);
 			}
 
@@ -31,9 +28,9 @@ describe("controller/src/ControllerUser", function() {
 		describe(".checkPermission()", function() {
 			it("should correctly resolve permissions", function() {
 				lib.definePermission({ name: "user-test", title: "Test", description: "User Test" });
-				let a = ControllerUser.fromJSON({ name: "admin", roles: [1] }, userManager);
-				let b = ControllerUser.fromJSON({ name: "user", roles: [2] }, userManager);
-				let c = ControllerUser.fromJSON({ name: "null", roles: [] }, userManager);
+				let a = ControllerUser.fromJSON({ name: "admin", roles: [1] }, roles);
+				let b = ControllerUser.fromJSON({ name: "user", roles: [2] }, roles);
+				let c = ControllerUser.fromJSON({ name: "null", roles: [] }, roles);
 
 				a.checkPermission("core.control.connect");
 				assert.throws(() => b.checkPermission("core.control.connect"), new Error("Permission denied"));

test/integration/clusterio.js

@@ -1404,6 +1404,7 @@ describe("Integration of Clusterio", function() {
 				await execCtl(`role create temp ${args}`);
 				let roles = await getControl().send(new lib.RoleListRequest());
 				let tempRole = roles.find(role => role.name === "temp");
+				if (tempRole) { tempRole.updatedAtMs = 0; }
 				assert.deepEqual(
 					tempRole,
 					new lib.Role(5, "temp", "A temp role", new Set(["core.control.connect"]))
@@ -1417,6 +1418,7 @@ describe("Integration of Clusterio", function() {
 				await execCtl(`role edit temp ${args}`);
 				let roles = await getControl().send(new lib.RoleListRequest());
 				let newRole = roles.find(role => role.name === "new");
+				if (newRole) { newRole.updatedAtMs = 0; }
 				assert.deepEqual(newRole, new lib.Role(5, "new", "A new role", new Set()));
 			});
 			it("should add permissions with --add-perms", async function() {

test/lib/datastore.js

@@ -507,10 +507,9 @@ describe("lib/datastore", function() {
 					["save", new Map([...datastoreProvider.value.entries()].filter((v, k) => k !== "foo"))]
 				);
 			});
-			it("does not break when the key does not exist", function() {
+			it("does nothing when the key does not exist", function() {
 				datastore.delete("test");
-				assert.equal(eventsRaised.length, 1);
-				assert.deepEqual(eventsRaised[0], [["test", undefined, true]]);
+				assert.equal(eventsRaised.length, 0);
 			});
 		});
 

test/mock.js

@@ -173,17 +173,20 @@ class MockController {
 			},
 		};
 
-		this.userManager = new UserManager(this.config);
-		this.userManager.roles = new Map([
+		const roleProvider = new lib.MemoryDatastoreProvider(new Map([
 			[0, lib.Role.fromJSON({ id: 0, name: "Admin", description: "admin", permissions: ["core.admin"] })],
 			[1, lib.Role.fromJSON({ id: 1, name: "Player", description: "player", permissions: [] })],
-		]);
-		this.userManager.roles.get(1).grantDefaultPermissions();
+		]));
 
+		this.roles = new lib.SubscribableDatastore(roleProvider, roleProvider.value);
+		this.roles.get(1).grantDefaultPermissions();
+
+		this.userManager = new UserManager(this.config, this.roles);
 		this.userManager.users = new Map([
-			["test", ControllerUser.fromJSON({ name: "test", roles: [0, 1] }, this.userManager)],
-			["player", ControllerUser.fromJSON({ name: "player", roles: [1] }, this.userManager)],
+			["test", ControllerUser.fromJSON({ name: "test", roles: [0, 1] }, this.roles)],
+			["player", ControllerUser.fromJSON({ name: "player", roles: [1] }, this.roles)],
 		]);
+
 		this.instances = new lib.KeyValueDatastore();
 		this.hosts = new lib.KeyValueDatastore();
 		this.handles = new Map();