From 6989fbdd444ec7e7f01ee732b39c233228f7535a Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Fri, 9 May 2025 22:44:56 +0100 Subject: [PATCH 1/7] Move defaultAccess to static property --- packages/lib/src/config/classes.ts | 6 ++++-- test/lib/config/classes.js | 6 ++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/lib/src/config/classes.ts b/packages/lib/src/config/classes.ts index 310312ba..ab0b056d 100644 --- a/packages/lib/src/config/classes.ts +++ b/packages/lib/src/config/classes.ts @@ -198,6 +198,9 @@ export class Config< return config; } + /** Defines the default access locations for the config class */ + static defaultAccess: ConfigLocation[] = ["controller", "host", "control"]; + fields: Fields; _unknownFields: Record = {}; @@ -217,7 +220,6 @@ export class Config< constructor( public location: ConfigLocation, fields?: Static, - public defaultAccess: ConfigLocation[] = ["controller", "host", "control"], ) { if (typeof location !== "string") { throw new Error("location must be a string"); @@ -262,7 +264,7 @@ export class Config< || ( (!def.credential || mode & ConfigAccess.write) && (!def.readonly || mode & ConfigAccess.read) - && !(def.access ?? this.defaultAccess).includes(location) + && !(def.access ?? this.constructor.defaultAccess).includes(location) ) ) { if (error) { diff --git a/test/lib/config/classes.js b/test/lib/config/classes.js index b9ac399b..abe35efe 100644 --- a/test/lib/config/classes.js +++ b/test/lib/config/classes.js @@ -7,6 +7,8 @@ const CA = lib.ConfigAccess; describe("lib/config/classes", function() { describe("Config", function() { class TestConfig extends lib.Config { + static defaultAccess = ["local", "remote"]; + static migrations(config) { if (config.hasOwnProperty("test.migration")) { config["alpha.foo"] = config["test.migration"]; @@ -38,10 +40,6 @@ describe("lib/config/classes", function() { "test.cred": { credential: ["local"] }, "test.rdo": { readonly: ["local"] }, }; - - constructor(location, fields) { - super(location, fields, ["local", "remote"]); - } } describe("constructor", function() { From c650f422763f568204af3e7c231d91de79adbeb5 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sat, 10 May 2025 00:07:43 +0100 Subject: [PATCH 2/7] Enable source maps for test setup --- test/integration/index.js | 44 ++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/test/integration/index.js b/test/integration/index.js index 8810144f..5d4ee86a 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -175,6 +175,16 @@ async function exec(command, options = {}) { return await util.promisify(child_process.exec)(command, options); } +async function execController(...args) { + args[0] = `node --enable-source-maps ../../packages/controller ${args[0]}`; + return await exec(...args); +} + +async function execHost(...args) { + args[0] = `node --enable-source-maps ../../packages/host ${args[0]}`; + return await exec(...args); +} + async function execCtl(...args) { args[0] = `node --enable-source-maps ../../packages/ctl ${args[0]}`; return await exec(...args); @@ -265,33 +275,33 @@ before(async function() { }); console.log("Setting Controller Config"); - await exec("node ../../packages/controller config set controller.auth_secret TestSecretDoNotUse"); - await exec("node ../../packages/controller config set controller.http_port 8880"); - await exec("node ../../packages/controller config set controller.https_port 4443"); - await exec("node ../../packages/controller config set controller.heartbeat_interval 0.25"); - await exec("node ../../packages/controller config set controller.session_timeout 2"); - await exec("node ../../packages/controller config set controller.tls_certificate ../../test/file/tls/cert.pem"); - await exec("node ../../packages/controller config set controller.tls_private_key ../../test/file/tls/key.pem"); + await execController("config set controller.auth_secret TestSecretDoNotUse"); + await execController("config set controller.http_port 8880"); + await execController("config set controller.https_port 4443"); + await execController("config set controller.heartbeat_interval 0.25"); + await execController("config set controller.session_timeout 2"); + await execController("config set controller.tls_certificate ../../test/file/tls/cert.pem"); + await execController("config set controller.tls_private_key ../../test/file/tls/key.pem"); console.log("Setting Controller Plugins"); - await exec("node ../../packages/ctl plugin add ../../plugins/global_chat"); - await exec("node ../../packages/ctl plugin add ../../plugins/research_sync"); - await exec("node ../../packages/ctl plugin add ../../plugins/statistics_exporter"); - await exec("node ../../packages/ctl plugin add ../../plugins/subspace_storage"); - await exec("node ../../packages/ctl plugin add ../../plugins/player_auth"); + await execCtl("plugin add ../../plugins/global_chat"); + await execCtl("plugin add ../../plugins/research_sync"); + await execCtl("plugin add ../../plugins/statistics_exporter"); + await execCtl("plugin add ../../plugins/subspace_storage"); + await execCtl("plugin add ../../plugins/player_auth"); console.log("Bootstrapping"); - await exec("node ../../packages/controller bootstrap create-admin test"); - await exec("node ../../packages/controller bootstrap create-ctl-config test"); - await exec("node ../../packages/ctl control-config set control.tls_ca ../../test/file/tls/cert.pem"); + await execController("bootstrap create-admin test"); + await execController("bootstrap create-ctl-config test"); + await execCtl("control-config set control.tls_ca ../../test/file/tls/cert.pem"); controllerProcess = await spawnNode("controller:", "../../packages/controller run", /Started controller/); await execCtl("host create-config --id 4 --name host --generate-token"); const relativeFactorioDir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir); - await exec(`node ../../packages/host config set host.factorio_directory ${relativeFactorioDir}`); - await exec("node ../../packages/host config set host.tls_ca ../../test/file/tls/cert.pem"); + await execHost(`config set host.factorio_directory ${relativeFactorioDir}`); + await execHost("config set host.tls_ca ../../test/file/tls/cert.pem"); hostProcess = await spawnNode("host:", "../../packages/host run", /Started host/); From 17db53403a827175c436cd81c474080f60c31cab Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sun, 11 May 2025 00:01:38 +0100 Subject: [PATCH 3/7] Move filepath into config class --- packages/controller/controller.ts | 17 +++++------------ packages/controller/src/Controller.ts | 5 +---- packages/ctl/ctl.ts | 11 ++++++----- packages/host/host.ts | 12 ++++++------ packages/host/src/Host.ts | 15 +++++---------- packages/lib/src/config/classes.ts | 14 ++++++++++---- packages/lib/src/config/definitions.ts | 10 ++++++---- packages/lib/src/shared_commands.ts | 3 +-- test/controller/Controller.js | 2 +- test/controller/WsServer.js | 2 +- test/host.js | 13 +++++++------ test/integration/routing.js | 9 ++------- 12 files changed, 51 insertions(+), 62 deletions(-) diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 45e7c168..5ec98b0c 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -210,7 +210,6 @@ interface InitializeParameters { shouldRun: boolean; clusterLogger: winston.Logger; pluginInfos: lib.PluginNodeEnvInfo[] | null; - controllerConfigPath: string | null; controllerConfig: lib.ControllerConfig | null; } @@ -334,7 +333,6 @@ async function initialize(): Promise { shouldRun, clusterLogger, pluginInfos: null, - controllerConfigPath: null, controllerConfig: null, }; } @@ -344,17 +342,17 @@ async function initialize(): Promise { lib.registerPluginMessages(pluginInfos); lib.addPluginConfigFields(pluginInfos); + let controllerConfig; const controllerConfigPath = args.config; logger.info(`Loading config from ${controllerConfigPath}`); - let controllerConfig = new lib.ControllerConfig("controller"); try { let jsonConfig = JSON.parse(await fs.readFile(controllerConfigPath, { encoding: "utf8" })); - controllerConfig = lib.ControllerConfig.fromJSON(jsonConfig, "controller"); + controllerConfig = lib.ControllerConfig.fromJSON(jsonConfig, "controller", controllerConfigPath); } catch (err: any) { if (err.code === "ENOENT") { logger.info("Config not found, initializing new config"); - controllerConfig = new lib.ControllerConfig("controller"); + controllerConfig = new lib.ControllerConfig("controller", undefined, controllerConfigPath); } else { throw new lib.StartupError(`Failed to load ${controllerConfigPath}: ${err.message}`); @@ -374,9 +372,7 @@ async function initialize(): Promise { } if (command === "config") { - await lib.handleConfigCommand( - args, controllerConfig, controllerConfigPath - ); + await lib.handleConfigCommand(args, controllerConfig); } else if (command === "bootstrap") { await handleBootstrapCommand(args, controllerConfig); @@ -386,7 +382,6 @@ async function initialize(): Promise { args, clusterLogger, pluginInfos, - controllerConfigPath, controllerConfig, shouldRun, }; @@ -402,17 +397,15 @@ async function startup() { shouldRun, clusterLogger, pluginInfos, - controllerConfigPath, controllerConfig, } = await initialize(); - if (!shouldRun || !pluginInfos || !controllerConfigPath || !controllerConfig) { + if (!shouldRun || !pluginInfos || !controllerConfig) { return; } controller = new Controller( clusterLogger, pluginInfos, - controllerConfigPath, controllerConfig, Boolean(args.canRestart), Boolean(args.recovery), diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 925afe93..0c80fa80 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -50,7 +50,6 @@ export default class Controller { clusterLogger: winston.Logger; /** Array of plugin info objects for known plugins */ pluginInfos: lib.PluginNodeEnvInfo[]; - configPath: string; /** Controller config. */ config: lib.ControllerConfig; app: Application; @@ -155,7 +154,6 @@ export default class Controller { constructor( clusterLogger: winston.Logger, pluginInfos: lib.PluginNodeEnvInfo[], - configPath: string, config: lib.ControllerConfig, /** @@ -184,7 +182,6 @@ export default class Controller { ) { this.clusterLogger = clusterLogger; this.pluginInfos = pluginInfos; - this.configPath = configPath; this.config = config; this.app = express(); @@ -586,7 +583,7 @@ export default class Controller { private async _saveDataInternal() { if (this.config.dirty) { this.config.dirty = false; - await lib.safeOutputFile(this.configPath, JSON.stringify(this.config, null, "\t")); + await lib.safeOutputFile(this.config.filepath!, JSON.stringify(this.config, null, "\t")); } await Promise.all([ diff --git a/packages/ctl/ctl.ts b/packages/ctl/ctl.ts index 70d49964..8945b500 100755 --- a/packages/ctl/ctl.ts +++ b/packages/ctl/ctl.ts @@ -209,16 +209,17 @@ async function startControl() { .parse() as CtlArguments ; - logger.verbose(`Loading config from ${args.config}`); let controlConfig; + const controlConfigPath = args.config; + logger.verbose(`Loading config from ${controlConfigPath}`); try { - const jsonConfig = JSON.parse(await fs.readFile(args.config, "utf8")); - controlConfig = lib.ControlConfig.fromJSON(jsonConfig, "control"); + const jsonConfig = JSON.parse(await fs.readFile(controlConfigPath, "utf8")); + controlConfig = lib.ControlConfig.fromJSON(jsonConfig, "control", controlConfigPath); } catch (err: any) { if (err.code === "ENOENT") { logger.verbose("Config not found, initializing new config"); - controlConfig = new lib.ControlConfig("control"); + controlConfig = new lib.ControlConfig("control", undefined, controlConfigPath); } else { throw new lib.StartupError(`Failed to load ${args.config}: ${err.message}`); @@ -232,7 +233,7 @@ async function startControl() { // Handle the control-config command before trying to connect. if (args._[0] === "control-config") { - await lib.handleConfigCommand(args, controlConfig, args.config); + await lib.handleConfigCommand(args, controlConfig); return; } diff --git a/packages/host/host.ts b/packages/host/host.ts index 716c874f..c40df9f1 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -167,16 +167,17 @@ async function startHost() { lib.registerPluginMessages(pluginInfos); lib.addPluginConfigFields(pluginInfos); - logger.info(`Loading config from ${args.config}`); let hostConfig; + const hostConfigPath = args.config; + logger.info(`Loading config from ${hostConfigPath}`); try { - const jsonConfig = JSON.parse(await fs.readFile(args.config, "utf8")); - hostConfig = lib.HostConfig.fromJSON(jsonConfig, "host"); + const jsonConfig = JSON.parse(await fs.readFile(hostConfigPath, "utf8")); + hostConfig = lib.HostConfig.fromJSON(jsonConfig, "host", hostConfigPath); } catch (err: any) { if (err.code === "ENOENT") { logger.info("Config not found, initializing new config"); - hostConfig = new lib.HostConfig("host"); + hostConfig = new lib.HostConfig("host", undefined, hostConfigPath); } else { throw new lib.StartupError(`Failed to load ${args.config}: ${err.message}`); @@ -186,7 +187,7 @@ async function startHost() { hostConfig.set("host.version", version); // Allows tracking last loaded version if (command === "config") { - await lib.handleConfigCommand(args, hostConfig, args.config); + await lib.handleConfigCommand(args, hostConfig); return; } @@ -229,7 +230,6 @@ async function startHost() { let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos); host = new Host( hostConnector, - args.config, hostConfig, tlsCa, pluginInfos, diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 8dfddad9..13616d29 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -48,14 +48,14 @@ async function discoverInstances(instancesDir: string) { try { const jsonConfig = JSON.parse(await fs.readFile(configPath, "utf8")); - instanceConfig = lib.InstanceConfig.fromJSON(jsonConfig, "host"); + instanceConfig = lib.InstanceConfig.fromJSON(jsonConfig, "host", configPath); } catch (err: any) { if (err.code === "ENOENT") { continue; // Ignore folders without config.json } - logger.error(`Error occured while parsing ${configPath}: ${err.message}`); + logger.error(`Error occurred while parsing ${configPath}: ${err.message}`); continue; } @@ -258,7 +258,6 @@ export default class Host extends lib.Link { */ tlsCa?: string; pluginInfos: lib.PluginNodeEnvInfo[]; - configPath: string; config: lib.HostConfig; /** Mapping of plugin name to loaded plugin */ @@ -289,7 +288,6 @@ export default class Host extends lib.Link { constructor( connector: HostConnector, - hostConfigPath: string, hostConfig: lib.HostConfig, tlsCa: string | undefined, pluginInfos: lib.PluginNodeEnvInfo[], @@ -309,7 +307,6 @@ export default class Host extends lib.Link { this.tlsCa = tlsCa; this.pluginInfos = pluginInfos; - this.configPath = hostConfigPath; this.config = hostConfig; this.config.on("fieldChanged", (name, curr, prev) => { @@ -659,6 +656,7 @@ export default class Host extends lib.Link { instanceConfig.update(config, false, "controller"); let instanceDir = await this._createNewInstanceDir(instanceConfig.get("instance.name")); + instanceConfig.filepath = path.join(instanceDir, "instance.json"); logger.info(`Creating ${instanceDir}`); await Instance.populate_folders(instanceDir); @@ -701,10 +699,7 @@ export default class Host extends lib.Link { _warning: "Changes to this file will be overwritten by the controller's copy.", ...instanceInfo.config.toJSON(), }; - await lib.safeOutputFile( - path.join(instanceInfo.path, "instance.json"), - JSON.stringify(warnedOutput, null, "\t") - ); + await lib.safeOutputFile(instanceInfo.config.filepath!, JSON.stringify(warnedOutput, null, "\t")); } async handleInstanceUnassignInternalRequest(request: lib.InstanceUnassignInternalRequest) { @@ -1157,7 +1152,7 @@ export default class Host extends lib.Link { } logger.info("Saving config"); - await lib.safeOutputFile(this.configPath, JSON.stringify(this.config, null, "\t")); + await lib.safeOutputFile(this.config.filepath!, JSON.stringify(this.config, null, "\t")); for (const instanceInfo of this.instanceInfos.values()) { // Save in case host_assigned_game_port changed in the config. await this.saveInstanceConfig(instanceInfo); diff --git a/packages/lib/src/config/classes.ts b/packages/lib/src/config/classes.ts index ab0b056d..dd6c663d 100644 --- a/packages/lib/src/config/classes.ts +++ b/packages/lib/src/config/classes.ts @@ -201,8 +201,9 @@ export class Config< /** Defines the default access locations for the config class */ static defaultAccess: ConfigLocation[] = ["controller", "host", "control"]; - fields: Fields; - _unknownFields: Record = {}; + /** Contains the fields and their values */ + private fields: Fields; + private _unknownFields: Record = {}; /** Set to true when a field in the config is changed. */ dirty = false; @@ -213,13 +214,16 @@ export class Config< * Create a new instance of the given config * * @param location - - * Location to evaluate access for this instance from + * Location to evaluate access for this instance from. * @param fields - * Serialized representation of fields to load. + * @param filepath - + * Filepath for the config to save to. */ constructor( public location: ConfigLocation, fields?: Static, + public filepath?: string, ) { if (typeof location !== "string") { throw new Error("location must be a string"); @@ -306,11 +310,13 @@ export class Config< * * @param json - Serialized config to load. * @param location - Location used for access control. + * @param filepath - Filepath for the config to save to. * @returns Instance of this config */ static fromJSON( json: Static, location: ConfigLocation, + filepath?: string, ) { // migrate: Pre alpha 14 config format if (typeof json === "object" && json !== null && json.groups instanceof Array) { @@ -326,7 +332,7 @@ export class Config< if (!valid) { throw new Error("Invalid config"); } - return new this(location, json); + return new this(location, json, filepath); } toJSON() { diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index b05123f1..f6497df2 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -3,6 +3,8 @@ import * as classes from "./classes"; import type { PluginNodeEnvInfo, PluginWebpackEnvInfo } from "../plugin"; import { Static } from "@sinclair/typebox"; +type configFromJSON = (...args: Parameters) => T; + export interface ControllerConfigFields { "controller.version": string; "controller.name": string; @@ -36,7 +38,7 @@ export interface ControllerConfigFields { * @extends classes.Config */ export class ControllerConfig extends classes.Config { - declare static fromJSON: (json: classes.ConfigSchema, location: classes.ConfigLocation) => ControllerConfig; + declare static fromJSON: configFromJSON; static migrations(config: Static) { if (config.hasOwnProperty("controller.external_address")) { config["controller.public_url"] = config["controller.external_address"]; @@ -243,7 +245,7 @@ export interface HostConfigFields { * @extends classes.Config */ export class HostConfig extends classes.Config { - declare static fromJSON: (json: classes.ConfigSchema, location: classes.ConfigLocation) => HostConfig; + declare static fromJSON: configFromJSON; static fieldDefinitions: classes.ConfigDefs = { "host.version": { type: "string", @@ -375,7 +377,7 @@ export interface InstanceConfigFields { * @extends classes.Config */ export class InstanceConfig extends classes.Config { - declare static fromJSON: (json: classes.ConfigSchema, location: classes.ConfigLocation) => InstanceConfig; + declare static fromJSON: configFromJSON; static migrations(config: Static) { function boolToEnableDisable(name: string) { if (config.hasOwnProperty(name) && typeof config[name] === "boolean") { @@ -568,7 +570,7 @@ export interface ControlConfigFields { * @extends classes.Config */ export class ControlConfig extends classes.Config { - declare static fromJSON: (json: classes.ConfigSchema, location: classes.ConfigLocation) => ControlConfig; + declare static fromJSON: configFromJSON; static fieldDefinitions: classes.ConfigDefs = { "control.controller_url": { description: "URL to connect to the controller at", diff --git a/packages/lib/src/shared_commands.ts b/packages/lib/src/shared_commands.ts index 13be8c32..6197bf52 100644 --- a/packages/lib/src/shared_commands.ts +++ b/packages/lib/src/shared_commands.ts @@ -135,7 +135,6 @@ export function configCommand(yargs: any) { export async function handleConfigCommand( args: Record, instance: libConfig.Config, - configPath: string ) { let command = (args._ as string[])[1]; @@ -165,7 +164,7 @@ export async function handleConfigCommand( try { instance.set(args.field as string, args.value); - await libFileOps.safeOutputFile(configPath, JSON.stringify(instance, null, "\t")); + await libFileOps.safeOutputFile(instance.filepath!, JSON.stringify(instance, null, "\t")); } catch (err) { if (err instanceof libConfig.InvalidField || err instanceof libConfig.InvalidValue) { logger.error(err.message); diff --git a/test/controller/Controller.js b/test/controller/Controller.js index 44ce9b11..1cfc4814 100644 --- a/test/controller/Controller.js +++ b/test/controller/Controller.js @@ -28,7 +28,7 @@ describe("controller/src/Controller", function() { /** @type {Controller} */ let controller, mockInstanceConfig; before(async function() { - controller = new Controller({}, [], "", new ControllerConfig("controller")); + controller = new Controller({}, [], new ControllerConfig("controller")); mockInstanceConfig = new MockInstanceConfig(new Map([ ["instance.id", 100], ["instance.name", "test"], ["factorio.settings", []], ])); diff --git a/test/controller/WsServer.js b/test/controller/WsServer.js index 46f0bd47..7501ec59 100644 --- a/test/controller/WsServer.js +++ b/test/controller/WsServer.js @@ -10,7 +10,7 @@ describe("controller/src/WsServer", function() { before(function() { let config = new ControllerConfig("controller"); config.set("controller.trusted_proxies", "127.0.0.0/8, ::1"); - let controller = new Controller({}, [], "", config); + let controller = new Controller({}, [], config); wsServer = controller.wsServer; }); describe(".remoteAddr()", function() { diff --git a/test/host.js b/test/host.js index f5189d0d..559da84f 100644 --- a/test/host.js +++ b/test/host.js @@ -10,13 +10,14 @@ const { _discoverInstances } = require("@clusterio/host/dist/node/src/Host"); describe("Host testing", function() { describe("discoverInstances()", function() { it("should discover test instance", async function() { - let instancePath = path.join("test", "file", "instances"); - let instanceInfos = await _discoverInstances(instancePath); + const instancePath = path.join("test", "file", "instances"); + const instanceInfos = await _discoverInstances(instancePath); - let referenceConfig = new lib.InstanceConfig("host"); - referenceConfig.set("instance.id", 1); - referenceConfig.set("instance.name", "test"); - referenceConfig.dirty = false; + const configPath = path.join(instancePath, "test", "instance.json"); + const referenceConfig = new lib.InstanceConfig("host", { + "instance.id": 1, + "instance.name": "test", + }, configPath); assert.deepEqual(instanceInfos, new Map([ [1, { diff --git a/test/integration/routing.js b/test/integration/routing.js index f2b17d34..827e6103 100644 --- a/test/integration/routing.js +++ b/test/integration/routing.js @@ -23,7 +23,7 @@ function connectHost(controller, hostId, plugins) { controller.wsServer.hostConnections.set(hostId, hostConnection); const hostConfig = new lib.HostConfig("host"); hostConfig.set("host.id", hostId); - return new Host(hostSide, "invalid", hostConfig, undefined, []); + return new Host(hostSide, hostConfig, undefined, []); } /** @@ -107,12 +107,7 @@ describe("Integration of link routing", function() { let controlB; beforeEach(function() { - controller = new Controller( - {}, - [], - "invalid", - new lib.ControllerConfig("controller") - ); + controller = new Controller({}, [], new lib.ControllerConfig("controller")); hostA = connectHost(controller, 10, []); hostB = connectHost(controller, 20, []); instanceA1 = connectInstance(controller, hostA, 11); From 726a3538c27536e6ca65a47578d4394054387df8 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sun, 11 May 2025 00:44:55 +0100 Subject: [PATCH 4/7] Move save and load into config class --- packages/controller/controller.ts | 7 ++--- packages/controller/src/Controller.ts | 6 +---- packages/host/src/Host.ts | 36 ++++++++++++-------------- packages/lib/src/config/classes.ts | 31 +++++++++++++++++++--- packages/lib/src/config/definitions.ts | 5 ++++ packages/lib/src/shared_commands.ts | 2 +- packages/lib/tsconfig.browser.json | 1 + test/host.js | 1 + 8 files changed, 55 insertions(+), 34 deletions(-) diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 5ec98b0c..3aa41acf 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -346,8 +346,7 @@ async function initialize(): Promise { const controllerConfigPath = args.config; logger.info(`Loading config from ${controllerConfigPath}`); try { - let jsonConfig = JSON.parse(await fs.readFile(controllerConfigPath, { encoding: "utf8" })); - controllerConfig = lib.ControllerConfig.fromJSON(jsonConfig, "controller", controllerConfigPath); + controllerConfig = await lib.ControllerConfig.fromFile("controller", controllerConfigPath); } catch (err: any) { if (err.code === "ENOENT") { @@ -366,9 +365,7 @@ async function initialize(): Promise { let asyncRandomBytes = util.promisify(crypto.randomBytes); let bytes = await asyncRandomBytes(256); controllerConfig.set("controller.auth_secret", bytes.toString("base64")); - await lib.safeOutputFile( - controllerConfigPath, JSON.stringify(controllerConfig, null, "\t") - ); + await controllerConfig.save(); } if (command === "config") { diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 0c80fa80..6b60ac68 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -581,12 +581,8 @@ export default class Controller { }); private async _saveDataInternal() { - if (this.config.dirty) { - this.config.dirty = false; - await lib.safeOutputFile(this.config.filepath!, JSON.stringify(this.config, null, "\t")); - } - await Promise.all([ + this.config.save(), this.systems.save(), this.hosts.save(), this.instances.save(), diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 13616d29..ef363f73 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -43,12 +43,13 @@ async function discoverInstances(instancesDir: string) { let instanceInfos = new Map(); for (let entry of await fs.readdir(instancesDir, { withFileTypes: true })) { if (entry.isDirectory()) { - let instanceConfig = new lib.InstanceConfig("host"); - let configPath = path.join(instancesDir, entry.name, "instance.json"); + let instanceConfig = new lib.InstanceConfig("host", Host.instanceConfigWarning); + const instancePath = path.join(instancesDir, entry.name); + const configPath = path.join(instancePath, "instance.json"); try { - const jsonConfig = JSON.parse(await fs.readFile(configPath, "utf8")); - instanceConfig = lib.InstanceConfig.fromJSON(jsonConfig, "host", configPath); + instanceConfig = await lib.InstanceConfig.fromFile("host", configPath); + instanceConfig.update(Host.instanceConfigWarning, false); // File might be missing the warning } catch (err: any) { if (err.code === "ENOENT") { @@ -64,7 +65,6 @@ async function discoverInstances(instancesDir: string) { continue; } - let instancePath = path.join(instancesDir, entry.name); logger.verbose(`found instance ${instanceConfig.get("instance.name")} in ${instancePath}`); instanceInfos.set(instanceConfig.get("instance.id"), { path: instancePath, @@ -278,6 +278,10 @@ export default class Host extends lib.Link { _disconnecting = false; _shuttingDown = false; + static instanceConfigWarning = { + "_warning": "Changes to this file will be overwritten by the controller's copy.", + }; + static async bootstrap(hostConfig: lib.HostConfig) { const modsDirectory = hostConfig.get("host.mods_directory"); await fs.ensureDir(modsDirectory); @@ -652,7 +656,7 @@ export default class Host extends lib.Link { instanceInfo.config.update(config, true, "controller"); } else { - let instanceConfig = new lib.InstanceConfig("host"); + let instanceConfig = new lib.InstanceConfig("host", Host.instanceConfigWarning); instanceConfig.update(config, false, "controller"); let instanceDir = await this._createNewInstanceDir(instanceConfig.get("instance.name")); @@ -691,15 +695,7 @@ export default class Host extends lib.Link { await this.sendSaveListUpdate(instanceId, path.join(instanceInfo.path, "saves")); // save a copy of the instance config - await this.saveInstanceConfig(instanceInfo); - } - - async saveInstanceConfig(instanceInfo: { path: string, config: lib.InstanceConfig }) { - let warnedOutput = { - _warning: "Changes to this file will be overwritten by the controller's copy.", - ...instanceInfo.config.toJSON(), - }; - await lib.safeOutputFile(instanceInfo.config.filepath!, JSON.stringify(warnedOutput, null, "\t")); + await instanceInfo.config.save(); } async handleInstanceUnassignInternalRequest(request: lib.InstanceUnassignInternalRequest) { @@ -1152,11 +1148,11 @@ export default class Host extends lib.Link { } logger.info("Saving config"); - await lib.safeOutputFile(this.config.filepath!, JSON.stringify(this.config, null, "\t")); - for (const instanceInfo of this.instanceInfos.values()) { - // Save in case host_assigned_game_port changed in the config. - await this.saveInstanceConfig(instanceInfo); - } + await this.config.save(); + // Save host side in case host_assigned_game_port changed in the config. + await Promise.all( + [...this.instanceInfos.values()].map(instanceInfo => instanceInfo.config.save()) + ); try { // Clear silly interval in pidfile library. diff --git a/packages/lib/src/config/classes.ts b/packages/lib/src/config/classes.ts index dd6c663d..bebba07c 100644 --- a/packages/lib/src/config/classes.ts +++ b/packages/lib/src/config/classes.ts @@ -1,11 +1,13 @@ // Configuration classes import { Type, Static } from "@sinclair/typebox"; -import TypedEventEmitter from "../TypedEventEmitter"; +import * as fs from "fs-extra"; +import TypedEventEmitter from "../TypedEventEmitter"; import isDeepStrictEqual from "../is_deep_strict_equal"; import { basicType } from "../helpers"; import * as libSchema from "../schema"; import { StringEnum } from "../data/composites"; +import { safeOutputFile } from "../file_ops"; const ConfigLocation = StringEnum(["controller", "host", "control"]); export type ConfigLocation = Static; @@ -206,7 +208,7 @@ export class Config< private _unknownFields: Record = {}; /** Set to true when a field in the config is changed. */ - dirty = false; + private dirty = false; /** Set to true when a 'restart required' field in the config is changed. */ restartRequired = false; @@ -301,8 +303,31 @@ export class Config< } } - static jsonSchema = ConfigSchema; + /** + * Create config from file + * + * @param location - Location used for access control. + * @param filepath - Filepath for the config to save to. + * @returns Instance of this config + */ + static async fromFile(location: ConfigLocation, filepath: string) { + const fields = await fs.readJson(filepath, { encoding: "utf8" }); + return this.fromJSON(fields, location, filepath); + } + /** Saves this config to file if there are unsaved changes */ + async save() { + if (!this.dirty) { + return; + } + if (!this.filepath) { + throw new Error("Cannot save config which has no filepath"); + } + await safeOutputFile(this.filepath, JSON.stringify(this, null, "\t")); + this.dirty = false; + } + + static jsonSchema = ConfigSchema; static validate = libSchema.compile(this.jsonSchema as any); /** diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index f6497df2..5f670e38 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -4,6 +4,7 @@ import type { PluginNodeEnvInfo, PluginWebpackEnvInfo } from "../plugin"; import { Static } from "@sinclair/typebox"; type configFromJSON = (...args: Parameters) => T; +type configFromFile = (...args: Parameters) => Promise; export interface ControllerConfigFields { "controller.version": string; @@ -39,6 +40,7 @@ export interface ControllerConfigFields { */ export class ControllerConfig extends classes.Config { declare static fromJSON: configFromJSON; + declare static fromFile: configFromFile; static migrations(config: Static) { if (config.hasOwnProperty("controller.external_address")) { config["controller.public_url"] = config["controller.external_address"]; @@ -246,6 +248,7 @@ export interface HostConfigFields { */ export class HostConfig extends classes.Config { declare static fromJSON: configFromJSON; + declare static fromFile: configFromFile; static fieldDefinitions: classes.ConfigDefs = { "host.version": { type: "string", @@ -378,6 +381,7 @@ export interface InstanceConfigFields { */ export class InstanceConfig extends classes.Config { declare static fromJSON: configFromJSON; + declare static fromFile: configFromFile; static migrations(config: Static) { function boolToEnableDisable(name: string) { if (config.hasOwnProperty(name) && typeof config[name] === "boolean") { @@ -571,6 +575,7 @@ export interface ControlConfigFields { */ export class ControlConfig extends classes.Config { declare static fromJSON: configFromJSON; + declare static fromFile: configFromFile; static fieldDefinitions: classes.ConfigDefs = { "control.controller_url": { description: "URL to connect to the controller at", diff --git a/packages/lib/src/shared_commands.ts b/packages/lib/src/shared_commands.ts index 6197bf52..71b8b509 100644 --- a/packages/lib/src/shared_commands.ts +++ b/packages/lib/src/shared_commands.ts @@ -164,7 +164,7 @@ export async function handleConfigCommand( try { instance.set(args.field as string, args.value); - await libFileOps.safeOutputFile(instance.filepath!, JSON.stringify(instance, null, "\t")); + await instance.save(); } catch (err) { if (err instanceof libConfig.InvalidField || err instanceof libConfig.InvalidValue) { logger.error(err.message); diff --git a/packages/lib/tsconfig.browser.json b/packages/lib/tsconfig.browser.json index 0eeb49e4..0825dd20 100644 --- a/packages/lib/tsconfig.browser.json +++ b/packages/lib/tsconfig.browser.json @@ -20,6 +20,7 @@ "./src/schema.ts", "./src/subscriptions.ts", "./src/zip_ops.ts", + "./src/file_ops.ts", "./src/ExponentialBackoff.ts", "./src/RateLimiter.ts", "./src/TypedEventEmitter.ts", diff --git a/test/host.js b/test/host.js index 559da84f..54fd8ffe 100644 --- a/test/host.js +++ b/test/host.js @@ -15,6 +15,7 @@ describe("Host testing", function() { const configPath = path.join(instancePath, "test", "instance.json"); const referenceConfig = new lib.InstanceConfig("host", { + ...Host.instanceConfigWarning, "instance.id": 1, "instance.name": "test", }, configPath); From 515093abd2e8f77e0c72c561e207f4f7c35e7403 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sun, 11 May 2025 00:46:16 +0100 Subject: [PATCH 5/7] Improve error message logged for config load error --- packages/controller/controller.ts | 2 +- packages/ctl/ctl.ts | 2 +- packages/host/host.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 3aa41acf..f986423c 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -354,7 +354,7 @@ async function initialize(): Promise { controllerConfig = new lib.ControllerConfig("controller", undefined, controllerConfigPath); } else { - throw new lib.StartupError(`Failed to load ${controllerConfigPath}: ${err.message}`); + throw new lib.StartupError(`Failed to load ${controllerConfigPath}: ${err.stack ?? err.message ?? err}`); } } diff --git a/packages/ctl/ctl.ts b/packages/ctl/ctl.ts index 8945b500..f2fab5b7 100755 --- a/packages/ctl/ctl.ts +++ b/packages/ctl/ctl.ts @@ -222,7 +222,7 @@ async function startControl() { controlConfig = new lib.ControlConfig("control", undefined, controlConfigPath); } else { - throw new lib.StartupError(`Failed to load ${args.config}: ${err.message}`); + throw new lib.StartupError(`Failed to load ${args.config}: ${err.stack ?? err.message ?? err}`); } } diff --git a/packages/host/host.ts b/packages/host/host.ts index c40df9f1..6e6bb779 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -180,7 +180,7 @@ async function startHost() { hostConfig = new lib.HostConfig("host", undefined, hostConfigPath); } else { - throw new lib.StartupError(`Failed to load ${args.config}: ${err.message}`); + throw new lib.StartupError(`Failed to load ${args.config}: ${err.stack ?? err.message ?? err}`); } } From 117713eda3582de69ead6fbbb742d8044a946918 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sun, 11 May 2025 01:08:28 +0100 Subject: [PATCH 6/7] Fix silent test for alt host --- test/integration/index.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/integration/index.js b/test/integration/index.js index 5d4ee86a..baaaa7a0 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -201,8 +201,15 @@ function getControl() { function spawn(name, cmd, waitFor) { // eslint-disable-next-line node/no-process-env const silent = process.env.SILENT_TEST; + const bootstrap = !controllerProcess || !hostProcess; + function log(...args) { + if (!silent || bootstrap) { + console.log(...args); + } + } + return new Promise((resolve, reject) => { - console.log(cmd); + log(cmd); let parts = cmd.split(" "); let process = child_process.spawn(parts[0], parts.slice(1), { cwd: path.join("temp", "test") }); let stdout = new LineSplitter({ readableObjectMode: true }); @@ -216,10 +223,10 @@ function spawn(name, cmd, waitFor) { } resolve(process); } - console.log(name, line); + log(name, line); }; let onDataErr = line => { - console.log(name, line.toString("utf8")); + log(name, line.toString("utf8")); }; stdout.on("data", onDataOut); stderr.on("data", onDataErr); From f285d7d8034cc9bd3852d8afced511617a1689a2 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Mon, 12 May 2025 21:30:08 +0100 Subject: [PATCH 7/7] Add tests for save and load --- packages/ctl/ctl.ts | 3 +- packages/host/host.ts | 3 +- test/lib/config/classes.js | 167 +++++++++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 4 deletions(-) diff --git a/packages/ctl/ctl.ts b/packages/ctl/ctl.ts index f2fab5b7..ddaf1933 100755 --- a/packages/ctl/ctl.ts +++ b/packages/ctl/ctl.ts @@ -213,8 +213,7 @@ async function startControl() { const controlConfigPath = args.config; logger.verbose(`Loading config from ${controlConfigPath}`); try { - const jsonConfig = JSON.parse(await fs.readFile(controlConfigPath, "utf8")); - controlConfig = lib.ControlConfig.fromJSON(jsonConfig, "control", controlConfigPath); + controlConfig = await lib.ControlConfig.fromFile("control", controlConfigPath); } catch (err: any) { if (err.code === "ENOENT") { diff --git a/packages/host/host.ts b/packages/host/host.ts index 6e6bb779..ec706390 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -171,8 +171,7 @@ async function startHost() { const hostConfigPath = args.config; logger.info(`Loading config from ${hostConfigPath}`); try { - const jsonConfig = JSON.parse(await fs.readFile(hostConfigPath, "utf8")); - hostConfig = lib.HostConfig.fromJSON(jsonConfig, "host", hostConfigPath); + hostConfig = await lib.HostConfig.fromFile("host", hostConfigPath); } catch (err: any) { if (err.code === "ENOENT") { diff --git a/test/lib/config/classes.js b/test/lib/config/classes.js index abe35efe..94b47c69 100644 --- a/test/lib/config/classes.js +++ b/test/lib/config/classes.js @@ -1,5 +1,7 @@ "use strict"; const lib = require("@clusterio/lib"); +const fs = require("fs-extra"); +const path = require("path"); const assert = require("assert").strict; const CA = lib.ConfigAccess; @@ -74,6 +76,10 @@ describe("lib/config/classes", function() { assert.equal(config.get("alpha.foo"), "foo"); assert.equal(config.fields["test.migration"], undefined); }); + it("should accept a filepath", function() { + const testInstance = new TestConfig("local", undefined, "filepath"); + assert.equal(testInstance.filepath, "filepath"); + }); }); describe(".toJSON()", function() { @@ -185,6 +191,167 @@ describe("lib/config/classes", function() { assert.equal(testInstance.fields["test.priv"], undefined); assert.equal(testInstance.fields["test.func"], 42); }); + it("should accept a filepath", function() { + const testInstance = TestConfig.fromJSON({}, "local", "filepath"); + assert.equal(testInstance.filepath, "filepath"); + }); + }); + + describe("static .fromFile", function() { + const filepath = path.join("temp", "test", "config_test.json"); + it("should throw for file and json errors", async function() { + await fs.writeFile(filepath, "abc"); + assert.rejects( + TestConfig.fromFile("local", filepath), + new Error( + `${filepath}: SyntaxError: JSON.parse: unexpected character at line 1 column 1 of the JSON data` + ) + ); + assert.rejects( + TestConfig.fromFile("local", `${filepath}.notExist`), + ); + }); + it("should throw on incorrect input passed", async function() { + await fs.writeFile(filepath, "null"); + await assert.rejects( + TestConfig.fromFile("local", filepath), + new Error("Invalid config") + ); + await fs.writeFile(filepath, "undefined"); + assert.rejects( + TestConfig.fromFile("local", filepath), + new Error("Invalid config") + ); + await fs.writeFile(filepath, "[]"); + assert.rejects( + TestConfig.fromFile("local", filepath), + new Error("Invalid config") + ); + }); + it("should load defaults for missing fields", async function() { + await fs.writeJSON(filepath, { + "alpha.foo": "a", + "test.enum": "a", + }); + const testInstance = await TestConfig.fromFile("local", filepath); + assert.equal(testInstance.get("alpha.foo"), "a"); + assert.deepEqual(testInstance.get("beta.bar"), {}); + assert.equal(testInstance.get("test.enum"), "a"); + assert.equal(testInstance.get("test.test"), null); + assert.equal(testInstance.get("test.func"), 42); + assert.equal(testInstance.filepath, filepath); + }); + it("should load fields", async function() { + await fs.writeJSON(filepath, { + "test.enum": "c", + "test.test": "blah", + "test.func": 22, + "test.bool": null, + "test.json": { valid: true }, + "test.priv": "bar", + }); + const testInstance = await TestConfig.fromFile("local", filepath); + assert.equal(testInstance.get("test.enum"), "c"); + assert.equal(testInstance.get("test.test"), "blah"); + assert.equal(testInstance.get("test.func"), 22); + assert.equal(testInstance.get("test.bool"), null); + assert.equal(testInstance.get("test.priv"), "bar"); + assert.deepEqual(testInstance.get("test.json"), { valid: true }); + assert.equal(testInstance.filepath, filepath); + }); + it("should preserve unknown fields", async function() { + const testFields = { + "extra.blah": true, + "extra.spam": "foobar", + "alpha.foo": "true", + "beta.bar": { value: 20 }, + "test.enum": "a", + "test.test": "blah", + "test.func": 50, + "test.bool": false, + "test.json": {}, + "test.rest": {}, + "test.priv": null, + "test.cred": null, + "test.rdo": null, + "test.alpha": null, + "test.beta": "decay", + "test.gamma": 99, + }; + await fs.writeJson(filepath, testFields); + const testInstance = await TestConfig.fromFile("local", filepath); + assert.deepEqual(testInstance.toJSON(), testFields); + assert.equal(testInstance.filepath, filepath); + }); + it("should ignore inaccessible fields", async function() { + await fs.writeJSON(filepath, { "test.priv": "bad" }); + const testInstance = await TestConfig.fromFile("remote", filepath); + assert.equal(testInstance.fields["test.priv"], undefined); + assert.equal(testInstance.fields["test.func"], 42); + assert.equal(testInstance.filepath, filepath); + }); + }); + + describe(".save()", function() { + const filepath = path.join("temp", "test", "config_test.json"); + beforeEach(async function() { + await fs.remove(filepath); + assert(!await fs.exists(filepath), "File was not removed"); + }); + it("should throw if there is no filepath", async function() { + const testInstance = new TestConfig("local", { "alpha.foo": "a" }); + testInstance.set("alpha.foo", "b"); // Sets the dirty flag + await assert.rejects( + testInstance.save(), + new Error("Cannot save config which has no filepath") + ); + }); + it("should do nothing when not dirty", async function() { + const testInstance = new TestConfig("local", { "alpha.foo": "a" }, filepath); + await testInstance.save(); + assert(!await fs.exists(filepath), "File was created"); + }); + it("should save data to file", async function() { + const testInstance = new TestConfig("local", { "alpha.foo": "a" }, filepath); + testInstance.set("alpha.foo", "b"); // Sets the dirty flag + await testInstance.save(); + const json = await fs.readJSON(filepath); + assert.deepEqual(json, testInstance.toJSON()); + }); + it("should clear the dirty flag after saving", async function() { + const testInstance = new TestConfig("local", { "alpha.foo": "a" }, filepath); + testInstance.set("alpha.foo", "b"); // Sets the dirty flag + await testInstance.save(); + assert(await fs.exists(filepath), "File was not created"); + await fs.remove(filepath); + assert(!await fs.exists(filepath), "File was not removed"); + await testInstance.save(); + assert(!await fs.exists(filepath), "File was created"); + }); + it("should be round trip savable", async function() { + const testInstance = new TestConfig("local", { + "extra.blah": true, + "extra.spam": "foobar", + "alpha.foo": "true", + "beta.bar": { value: 20 }, + "test.enum": "a", + "test.test": "blah", + "test.func": 50, + "test.bool": false, + "test.json": {}, + "test.rest": {}, + "test.priv": null, + "test.cred": null, + "test.rdo": null, + "test.alpha": null, + "test.beta": "decay", + "test.gamma": 99, + }, filepath); + testInstance.set("alpha.foo", "b"); // Sets the dirty flag + await testInstance.save(); + const loadedInstance = await TestConfig.fromFile("local", filepath); + assert.deepEqual(loadedInstance, testInstance); + }); }); describe(".update()", function() {