From 412c2911933ae116e05754b3847f3ce04657aa0d Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sun, 1 Mar 2026 23:58:57 +0000 Subject: [PATCH 01/36] Add recipe notification sync to inventory sync --- plugins/inventory_sync/messages.ts | 1 + plugins/inventory_sync/module/serialize.lua | 65 +++++++++++++++------ 2 files changed, 48 insertions(+), 18 deletions(-) diff --git a/plugins/inventory_sync/messages.ts b/plugins/inventory_sync/messages.ts index 532dfe2f..f2c8fdc4 100644 --- a/plugins/inventory_sync/messages.ts +++ b/plugins/inventory_sync/messages.ts @@ -18,6 +18,7 @@ export type IpcPlayerData = { hotbar?: string[], personal_logistic_slots?: {name:string, min:number, max:number}[], crafting_queue?: any, + recipe_notifications?: string[], } const jsonPlayerData = Type.Object({ diff --git a/plugins/inventory_sync/module/serialize.lua b/plugins/inventory_sync/module/serialize.lua index 989315e9..3e470db0 100644 --- a/plugins/inventory_sync/module/serialize.lua +++ b/plugins/inventory_sync/module/serialize.lua @@ -6,6 +6,7 @@ local serialize = {} local v2_logistic_api = compat.version_ge("2.0.0") local v2_storage_api = compat.version_ge("2.0.0") +local recipe_notifications_api = compat.version_ge("2.0.67") function serialize.serialize_inventories(source, inventories) local serialized = {} @@ -347,22 +348,27 @@ for name, value in pairs(defines.controllers) do controller_to_name[value] = name end --- Players are serialized into a table with the following fields: --- name --- controller: may not be cutscene or editor --- color --- chat_color --- tag --- force --- cheat_mod --- flashlight --- ticks_to_respawn (optional) --- character (optional) --- inventories: non-character inventories --- hotbar: table of string indexes to names (optional) --- personal_logistic_slots (optional) --- crafting_queue (optional) +--- @class SerializedPlayerData +--- @field name string +--- @field controller string +--- @field color Color +--- @field chat_color Color +--- @field tag string +--- @field force string +--- @field cheat_mode boolean +--- @field flashlight boolean +--- @field ticks_to_respawn number +--- @field character table? +--- @field inventories table? +--- @field hotbar table? +--- @field personal_logistic_slots table? +--- @field crafting_queue table? +--- @field recipe_notifications string[]? + +--- @param player LuaPlayer +--- @return SerializedPlayerData function serialize.serialize_player(player) + --- @type SerializedPlayerData local serialized = { controller = controller_to_name[player.controller_type], name = player.name, @@ -385,7 +391,9 @@ function serialize.serialize_player(player) -- Go back to waiting for respawn local character = player.character player.ticks_to_respawn = serialized.ticks_to_respawn - character.destroy() + if character and character.valid then + character.destroy() + end end -- Serialize character @@ -406,7 +414,7 @@ function serialize.serialize_player(player) if not serialized.hotbar then serialized.hotbar = {} end - serialized.hotbar[tostring(i)] = slot.name + serialized.hotbar[tostring(i)] = slot.name --[[@as string]] end end @@ -415,9 +423,20 @@ function serialize.serialize_player(player) serialized.crafting_queue = serialize.serialize_crafting_queue(player) end + -- Serialize recipe notifications + if recipe_notifications_api then + local recipe_notifications = {} + serialized.recipe_notifications = recipe_notifications + for i, recipe in ipairs(player.get_recipe_notifications()) do + recipe_notifications[i] = recipe.name + end + end + return serialized end +--- @param player LuaPlayer +--- @param serialized SerializedPlayerData function serialize.deserialize_player(player, serialized) if player.controller_type ~= defines.controllers[serialized.controller] or serialized.controller == "ghost" then -- If targeting the character or ghost controller then create a character @@ -443,7 +462,9 @@ function serialize.deserialize_player(player, serialized) player.set_controller({ type = defines.controllers.god }) player.set_controller({ type = defines.controllers.ghost }) end - character.destroy() + if character and character.valid then + character.destroy() + end end else @@ -496,6 +517,14 @@ function serialize.deserialize_player(player, serialized) serialize.deserialize_crafting_queue(player, serialized.crafting_queue) end + -- Deserialize recipe notifications + if recipe_notifications_api and serialized.recipe_notifications then + player.clear_recipe_notifications() + local add_notification = player.add_recipe_notification + for _, recipe_name in ipairs(serialized.recipe_notifications) do + add_notification(recipe_name) + end + end end return serialize From 54557a79c57aa0d5a28f86c34039e23ebe6fcb35 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Mon, 2 Mar 2026 16:39:20 +0000 Subject: [PATCH 02/36] Do not add empty array for recipe notifications --- plugins/inventory_sync/module/serialize.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/plugins/inventory_sync/module/serialize.lua b/plugins/inventory_sync/module/serialize.lua index 3e470db0..53b2a9da 100644 --- a/plugins/inventory_sync/module/serialize.lua +++ b/plugins/inventory_sync/module/serialize.lua @@ -426,10 +426,13 @@ function serialize.serialize_player(player) -- Serialize recipe notifications if recipe_notifications_api then local recipe_notifications = {} - serialized.recipe_notifications = recipe_notifications for i, recipe in ipairs(player.get_recipe_notifications()) do recipe_notifications[i] = recipe.name end + if #recipe_notifications > 0 then + -- Do no add if length is zero as this breaks with table_to_json + serialized.recipe_notifications = recipe_notifications + end end return serialized From 3e7b65264e600af9299318897c6b1674e0986385 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Mon, 2 Mar 2026 17:58:45 +0000 Subject: [PATCH 03/36] Invert notifications array to prevent loss of new notifications --- plugins/inventory_sync/module/serialize.lua | 61 ++++++++++++++++----- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/plugins/inventory_sync/module/serialize.lua b/plugins/inventory_sync/module/serialize.lua index 53b2a9da..04ae5d88 100644 --- a/plugins/inventory_sync/module/serialize.lua +++ b/plugins/inventory_sync/module/serialize.lua @@ -343,6 +343,50 @@ function serialize.deserialize_crafting_queue(player, serialized) player.crafting_queue_progress = serialized.crafting_queue_progress end +---@param player LuaPlayer +---@return string? +function serialize.serialize_crafting_notifications(player) + -- Get all unlocked recipes + local recipes = {} + for _, recipe in pairs(player.force.recipes) do + if recipe.enabled and not recipe.hidden then + recipes[recipe.name] = true + end + end + + -- Remove those with notifications + for _, recipe in pairs(player.get_recipe_notifications()) do + recipes[recipe.name] = nil + end + + -- Convert to a list of names + local index = 1 + local recipe_names = {} + for name in pairs(recipes) do + recipe_names[index] = name + index = index + 1 + end + + -- Do no return if length is zero as this breaks with table_to_json + if index > 1 then + return helpers.encode_string(helpers.table_to_json(recipe_names)) + end + + return nil +end + +--- @param player LuaPlayer +--- @param serialized string +function serialize.deserialize_crafting_notifications(player, serialized) + -- Decode the recipe name strings + local recipe_names = helpers.json_to_table(assert(helpers.decode_string(serialized))) + assert(type(recipe_names) == "table", "wrong type decoded from json_to_table") + local clear_notification = player.clear_recipe_notification + for _, recipe_name in pairs(recipe_names) do + clear_notification(recipe_name) + end +end + local controller_to_name = {} for name, value in pairs(defines.controllers) do controller_to_name[value] = name @@ -363,7 +407,7 @@ end --- @field hotbar table? --- @field personal_logistic_slots table? --- @field crafting_queue table? ---- @field recipe_notifications string[]? +--- @field recipe_notifications string? --- @param player LuaPlayer --- @return SerializedPlayerData @@ -425,14 +469,7 @@ function serialize.serialize_player(player) -- Serialize recipe notifications if recipe_notifications_api then - local recipe_notifications = {} - for i, recipe in ipairs(player.get_recipe_notifications()) do - recipe_notifications[i] = recipe.name - end - if #recipe_notifications > 0 then - -- Do no add if length is zero as this breaks with table_to_json - serialized.recipe_notifications = recipe_notifications - end + serialized.recipe_notifications = serialize.serialize_crafting_notifications(player) end return serialized @@ -522,11 +559,7 @@ function serialize.deserialize_player(player, serialized) -- Deserialize recipe notifications if recipe_notifications_api and serialized.recipe_notifications then - player.clear_recipe_notifications() - local add_notification = player.add_recipe_notification - for _, recipe_name in ipairs(serialized.recipe_notifications) do - add_notification(recipe_name) - end + serialize.deserialize_crafting_notifications(player, serialized.recipe_notifications) end end From cdfa9860bb76bf140ee114acfa98492a03223033 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Tue, 3 Mar 2026 20:30:18 +0000 Subject: [PATCH 04/36] Fix IPC type --- plugins/inventory_sync/messages.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inventory_sync/messages.ts b/plugins/inventory_sync/messages.ts index f2c8fdc4..53178b56 100644 --- a/plugins/inventory_sync/messages.ts +++ b/plugins/inventory_sync/messages.ts @@ -18,7 +18,7 @@ export type IpcPlayerData = { hotbar?: string[], personal_logistic_slots?: {name:string, min:number, max:number}[], crafting_queue?: any, - recipe_notifications?: string[], + recipe_notifications?: string, } const jsonPlayerData = Type.Object({ From a106851cdc2212f3ca7d7b0d937f93647cb680b0 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 4 Mar 2026 17:30:28 +0100 Subject: [PATCH 05/36] Enable pnpm shell emulator Some of our scripts assume a POSIX sh like shell to work, and writing any sort midly complex script that works with both cmd and sh is next to impossible. Fortunately pnpm has an option to emulate a sh like shell on Windows. See https://pnpm.io/cli/run#shellemulator for more info. --- pnpm-workspace.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 29bc79e4..01c191a9 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -38,3 +38,4 @@ catalog: catalogMode: strict linkWorkspacePackages: true +shellEmulator: true From fb3fe094666f44d57a0aaa6bb181b2472f0aae3b Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 4 Mar 2026 17:43:32 +0100 Subject: [PATCH 06/36] Remove tlsCa option The migration from phin to fetch in #755 was hampred by the lack of an option to set the CA when using fetch. This meant places that used phin to talk to the controller with the tlsCa option could not be migrated to use fetch. Remove the tlsCa option to unblock this dependency. The replacement for this option is to set the NODE_EXTRA_CA_CERTS environment variable to a path to the certificate file instead, which works with fetch. --- docs/configuration.md | 16 ---------------- docs/setting-up-tls.md | 23 +++++++++++++++++++---- package.json | 2 +- packages/create/create.js | 18 ++++++++++++------ packages/ctl/ctl.ts | 17 +++-------------- packages/ctl/src/commands_controller.ts | 1 - packages/ctl/src/commands_instance.ts | 2 -- packages/ctl/src/commands_mod.ts | 2 -- packages/host/host.ts | 11 +---------- packages/host/src/Host.ts | 9 --------- packages/host/src/Instance.ts | 1 - packages/lib/src/config/definitions.ts | 13 ------------- packages/lib/src/link/connectors.ts | 13 +------------ packages/web_ui/src/bootstrap.tsx | 2 +- test/host.js | 4 ++-- test/integration/clusterio.js | 12 ++++-------- test/integration/index.js | 24 ++++++++++++++++-------- test/integration/link.js | 10 +++------- test/integration/routing.js | 2 +- test/messages/host.test.js | 2 +- test/messages/plugin.test.js | 2 +- 21 files changed, 66 insertions(+), 120 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8d8e73b7..2285e319 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -182,14 +182,6 @@ You can generate an access token with `clusterioctl host generate-token --id ; /** Keep the control connection alive after the command completes. */ @@ -64,12 +62,10 @@ export class Control extends lib.Link { constructor( connector: ControlConnector, controlConfig: lib.ControlConfig, - tlsCa: string | undefined, ctlPlugins: Map ) { super(connector); this.config = controlConfig; - this.tlsCa = tlsCa; this.plugins = ctlPlugins; this.handle(lib.LogMessageEvent, this.handleLogMessageEvent.bind(this)); @@ -299,20 +295,13 @@ async function startControl() { return; } - let tlsCa: string | undefined; - let tlsCaPath = controlConfig.get("control.tls_ca"); - if (tlsCaPath) { - tlsCa = await fs.readFile(tlsCaPath, "utf8"); - } - let controlConnector = new ControlConnector( controlConfig.get("control.controller_url")!, controlConfig.get("control.max_reconnect_delay"), - tlsCa, controlConfig.get("control.controller_token")!, ); - let control = new Control(controlConnector, controlConfig, tlsCa, ctlPlugins); + let control = new Control(controlConnector, controlConfig, ctlPlugins); try { await controlConnector.connect(); } catch (err) { diff --git a/packages/ctl/src/commands_controller.ts b/packages/ctl/src/commands_controller.ts index 9b6481d5..9c2837f6 100644 --- a/packages/ctl/src/commands_controller.ts +++ b/packages/ctl/src/commands_controller.ts @@ -169,7 +169,6 @@ controllerPluginCommands.add(new lib.Command({ let response = await phin<[]>({ url, parse: "json", - core: { ca: control.tlsCa } as object, }); print(asTable(response.body)); }, diff --git a/packages/ctl/src/commands_instance.ts b/packages/ctl/src/commands_instance.ts index 28a4efa4..20d5b92d 100644 --- a/packages/ctl/src/commands_instance.ts +++ b/packages/ctl/src/commands_instance.ts @@ -345,7 +345,6 @@ instanceSaveCommands.add(new lib.Command({ "X-Access-Token": control.config.get("control.controller_token"), "Content-Type": "application/zip", }, - core: { ca: control.tlsCa } as object, data: content, parse: "json", }); @@ -420,7 +419,6 @@ instanceSaveCommands.add(new lib.Command({ url.pathname += `api/stream/${streamId}`; let response = await phin({ url, method: "GET", - core: { ca: control.tlsCa } as object, stream: true, }); diff --git a/packages/ctl/src/commands_mod.ts b/packages/ctl/src/commands_mod.ts index d4f9a3ec..1fff3f42 100644 --- a/packages/ctl/src/commands_mod.ts +++ b/packages/ctl/src/commands_mod.ts @@ -160,7 +160,6 @@ modCommands.add(new lib.Command({ "X-Access-Token": control.config.get("control.controller_token"), "Content-Type": "application/zip", }, - core: { ca: control.tlsCa } as object, data: content, parse: "json", }); @@ -199,7 +198,6 @@ modCommands.add(new lib.Command({ url.pathname += `api/stream/${streamId}`; let response = await phin({ url, method: "GET", - core: { ca: control.tlsCa } as object, stream: true, }); diff --git a/packages/host/host.ts b/packages/host/host.ts index 19127713..8b06fdf0 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -51,13 +51,11 @@ void new lib.Gauge( export class HostConnector extends lib.WebSocketClientConnector { constructor( public hostConfig: lib.HostConfig, - tlsCa: string | undefined, public pluginInfos: lib.PluginNodeEnvInfo[] ) { super( hostConfig.get("host.controller_url"), hostConfig.get("host.max_reconnect_delay"), - tlsCa ); } @@ -230,17 +228,10 @@ async function startHost() { return; } - let tlsCa: string | undefined; - let tlsCaPath = hostConfig.get("host.tls_ca"); - if (tlsCaPath) { - tlsCa = await fs.readFile(tlsCaPath, "utf8"); - } - - let hostConnector = new HostConnector(hostConfig, tlsCa, pluginInfos); + let hostConnector = new HostConnector(hostConfig, pluginInfos); host = new Host( hostConnector, hostConfig, - tlsCa, pluginInfos, Boolean(args.canRestart), Boolean(args.recovery), diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index e59d6593..2b41decf 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -259,10 +259,6 @@ export default class Host extends lib.Link { declare "connector": HostConnector; router = new HostRouter(this); - /** - * Certificate authority used to validate TLS connections to the controller. - */ - tlsCa?: string; pluginInfos: lib.PluginNodeEnvInfo[]; config: lib.HostConfig; @@ -304,7 +300,6 @@ export default class Host extends lib.Link { constructor( connector: HostConnector, hostConfig: lib.HostConfig, - tlsCa: string | undefined, pluginInfos: lib.PluginNodeEnvInfo[], /** * If true indicates that there is a process monitor present that @@ -319,7 +314,6 @@ export default class Host extends lib.Link { public modStore = new lib.ModStore(hostConfig.get("host.mods_directory"), new Map()), ) { super(connector); - this.tlsCa = tlsCa; this.pluginInfos = pluginInfos; this.config = hostConfig; @@ -593,7 +587,6 @@ export default class Host extends lib.Link { url.pathname += `api/stream/${streamId}`; let response = await phin({ url, method: "GET", - core: { ca: this.tlsCa } as object, stream: true, }); @@ -1027,7 +1020,6 @@ export default class Host extends lib.Link { url.pathname += `api/stream/${streamId}`; let response = await phin({ url, method: "GET", - core: { ca: this.tlsCa } as object, stream: true, }); @@ -1082,7 +1074,6 @@ export default class Host extends lib.Link { url.pathname += `api/stream/${streamId}`; phin({ url, method: "PUT", - core: { ca: this.tlsCa } as object, data: content, }).catch(err => { logger.error(`Error pushing save to controller:\n${err.stack}`, this.instanceLogMeta(instanceId)); diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index af5b9ef4..57cf2e03 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -1295,7 +1295,6 @@ end`.replace(/\r?\n/g, " "); let response = await phin({ url, method: "PUT", data: content, - core: { ca: this._host.tlsCa } as object, headers: { "Content-Type": "application/zip", "x-access-token": this._host.config.get("host.controller_token"), diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 18d30609..3b19512b 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -259,7 +259,6 @@ export interface HostConfigFields { "host.instances_directory": string; "host.controller_url": string; "host.controller_token": string; - "host.tls_ca": string | null; "host.public_address": string; "host.factorio_port_range": string; "host.factorio_username": string | null, @@ -327,12 +326,6 @@ export class HostConfig extends classes.Config { type: "string", initialValue: "enter token here", }, - "host.tls_ca": { - description: "Path to Certificate Authority to validate TLS connection to controller against.", - restartRequired: true, - type: "string", - optional: true, - }, "host.public_address": { description: "Public facing address players should connect to in order to join instances on this host", type: "string", @@ -624,7 +617,6 @@ export class InstanceConfig extends classes.Config { export interface ControlConfigFields { "control.controller_url": string | null; "control.controller_token": string | null; - "control.tls_ca": string | null; "control.max_reconnect_delay": number; } @@ -647,11 +639,6 @@ export class ControlConfig extends classes.Config { type: "string", optional: true, }, - "control.tls_ca": { - description: "Path to Certificate Authority to validate TLS connection to controller against.", - type: "string", - optional: true, - }, "control.max_reconnect_delay": { title: "Max Reconnect Delay", description: "Maximum delay to wait before attempting to reconnect WebSocket", diff --git a/packages/lib/src/link/connectors.ts b/packages/lib/src/link/connectors.ts index dd7df5d4..ebed9978 100644 --- a/packages/lib/src/link/connectors.ts +++ b/packages/lib/src/link/connectors.ts @@ -431,7 +431,6 @@ export abstract class WebSocketClientConnector extends WebSocketBaseConnector [p.name, p]))); control.plugins = await loadPlugins(pluginInfos, control); control.inputComponents = inputComponentsFromPlugins(control.plugins); diff --git a/test/host.js b/test/host.js index 63aa61bc..ad7a16aa 100644 --- a/test/host.js +++ b/test/host.js @@ -127,8 +127,8 @@ describe("Host testing", function() { version: hostVersion, }]; const hostConfig = new lib.HostConfig("host", { "host.version": hostVersion }); - const hostConnector = new HostConnector(hostConfig, undefined, pluginInfos); - host = new Host(hostConnector, hostConfig, undefined, pluginInfos); + const hostConnector = new HostConnector(hostConfig, pluginInfos); + host = new Host(hostConnector, hostConfig, pluginInfos); }); it("returns false when no changes are present", async function() { host.pluginInfos[0].hostEntrypoint = true; diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 6052973f..59782207 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -41,7 +41,6 @@ async function startAltHost() { await fs.remove(path.join("temp", "test", "alt-instances")); await fs.remove(path.join("temp", "test", "alt-mods")); await execCtl(`host create-config --id 5 --name alt-host --generate-token --output ${config}`); - await execHost(`--config ${config} config set host.tls_ca ../../test/file/tls/cert.pem`); await execHost(`--config ${config} config set host.instances_directory alt-instances`); const dir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir); await execHost(`--config ${config} config set host.factorio_directory ${dir}`); @@ -168,8 +167,7 @@ describe("Integration of Clusterio", function() { describe("run", function() { it("should handle resume of an active connection", async function() { slowTest(this); - let tlsCa = await fs.readFile("test/file/tls/cert.pem"); - let connectorA = new TestControlConnector(url, 2, tlsCa); + let connectorA = new TestControlConnector(url, 2); connectorA.token = controlToken; let controlA = new TestControl(connectorA); await connectorA.connect(); @@ -177,7 +175,7 @@ describe("Integration of Clusterio", function() { connectorA.stopHeartbeat(); connectorA.on("error", () => {}); - let connectorB = new TestControlConnector(url, 2, tlsCa); + let connectorB = new TestControlConnector(url, 2); connectorB.token = controlToken; connectorB.src = connectorA.src; let controlB = new TestControl(connectorB); @@ -1679,8 +1677,7 @@ describe("Integration of Clusterio", function() { it("should kick existing sessions for the user", async function() { slowTest(this); await getControl().send(new lib.UserCreateRequest("revokee")); - let tlsCa = await fs.readFile("test/file/tls/cert.pem"); - let connector = new TestControlConnector(url, 2, tlsCa); + let connector = new TestControlConnector(url, 2); connector.token = jwt.sign( { aud: "user", user: "revokee" }, Buffer.from("TestSecretDoNotUse", "base64") ); @@ -1709,8 +1706,7 @@ describe("Integration of Clusterio", function() { await execCtl("user set-roles temp Player"); let tempControl; try { - let tlsCa = await fs.readFile("test/file/tls/cert.pem"); - let connector = new TestControlConnector(url, 2, tlsCa); + let connector = new TestControlConnector(url, 2); connector.token = jwt.sign( { aud: "user", user: "temp" }, Buffer.from("TestSecretDoNotUse", "base64") ); diff --git a/test/integration/index.js b/test/integration/index.js index aa6f24ca..fcaf41d0 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -194,11 +194,23 @@ let controllerConfigPath = path.join("temp", "test", "config-controller.json"); let hostConfigPath = path.join("temp", "test", "config-host.json"); let controlConfigPath = path.join("temp", "test", "config-control.json"); +function childOptions(options) { + return { + cwd: path.join("temp", "test"), + env: { + ...process.env, + NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS + ? path.resolve(process.env.NODE_EXTRA_CA_CERTS) + : undefined, + }, + ...options, + }; +} + function exec(command, options = {}) { // Uncomment to show commands run in tests // console.log(command); - options = { cwd: path.join("temp", "test"), ...options }; - return util.promisify(child_process.exec)(command, options); + return util.promisify(child_process.exec)(command, childOptions(options)); } function execController(...args) { @@ -264,7 +276,7 @@ function spawn(name, cmd, waitFor, options = {}) { return new Promise((resolve, reject) => { log(cmd); let parts = cmd.split(" "); - let process = child_process.spawn(parts[0], parts.slice(1), { cwd: path.join("temp", "test"), ...options }); + let process = child_process.spawn(parts[0], parts.slice(1), childOptions(options)); let stdout = new LineSplitter({ readableObjectMode: true }); let stderr = new LineSplitter({ readableObjectMode: true }); let onDataOut = line => { @@ -354,7 +366,6 @@ before(async function() { console.log("Bootstrapping"); await execController("bootstrap create-admin test"); await execController("bootstrap create-ctl-config test"); - await execCtlProcess("control-config set control.tls_ca ../../test/file/tls/cert.pem"); controllerProcess = await spawnNode("controller:", "../../packages/controller run", /Started controller/); @@ -362,12 +373,10 @@ before(async function() { const relativeFactorioDir = path.isAbsolute(factorioDir) ? factorioDir : path.join("..", "..", factorioDir); await execCtlProcess("host create-config --id 4 --name host --generate-token"); await execHost(`config set host.factorio_directory ${relativeFactorioDir}`); - await execHost("config set host.tls_ca ../../test/file/tls/cert.pem"); hostProcess = await spawnNode("host:", "../../packages/host run", /Started host/); - const tlsCa = await fs.readFile("test/file/tls/cert.pem"); - const controlConnector = new TestControlConnector(url, 2, tlsCa); + const controlConnector = new TestControlConnector(url, 2); controlConnector.token = controlToken; control = new TestControl(controlConnector); await controlConnector.connect(); @@ -375,7 +384,6 @@ before(async function() { const initArgs = await initializeCtl(`--plugin-list ${pluginListPath} control-config list`, undefined, true); control.config = initArgs.controlConfig; control.plugins = initArgs.ctlPlugins; - control.tlsCa = tlsCa; const testPack = lib.ModPack.fromJSON({}); testPack.id = 12; diff --git a/test/integration/link.js b/test/integration/link.js index be19acd6..3d75fbfd 100644 --- a/test/integration/link.js +++ b/test/integration/link.js @@ -11,18 +11,14 @@ let token = jwt.sign({ aud: "user", user: "test" }, Buffer.from("TestSecretDoNot let tokenHost = jwt.sign({ aud: "host", host: 0 }, Buffer.from("TestSecretDoNotUse", "base64")); describe("Integration of lib/link", function() { - let tlsCa; let control; let controlConnector; let hostConnector; - before(async function() { - tlsCa = await fs.readFile("test/file/tls/cert.pem"); - }); beforeEach(async function() { - controlConnector = new TestControlConnector(url, 0.2, tlsCa); + controlConnector = new TestControlConnector(url, 0.2); controlConnector.token = token; control = new TestControl(controlConnector, false); - hostConnector = new TestHostConnector(url, 0.2, tlsCa); + hostConnector = new TestHostConnector(url, 0.2); hostConnector.token = tokenHost; hostConnector.hostId = 0; }); @@ -123,7 +119,7 @@ describe("Integration of lib/link", function() { it("should close old connections from the same address", async function() { slowTest(this); - const hostConnectorFirst = new TestHostConnector(url, 0.2, tlsCa); + const hostConnectorFirst = new TestHostConnector(url, 0.2); hostConnectorFirst.token = hostConnector.token; hostConnectorFirst.hostId = hostConnector.hostId; const onceClose = events.once(hostConnectorFirst, "close"); diff --git a/test/integration/routing.js b/test/integration/routing.js index 02c5fcff..a4672305 100644 --- a/test/integration/routing.js +++ b/test/integration/routing.js @@ -23,7 +23,7 @@ function connectHost(controller, hostId, plugins) { controller.wsServer.hostConnections.set(hostId, hostConnection); const hostConfig = new lib.HostConfig("host"); hostConfig.set("host.id", hostId); - return new Host(hostSide, hostConfig, undefined, []); + return new Host(hostSide, hostConfig, []); } /** diff --git a/test/messages/host.test.js b/test/messages/host.test.js index faf3093d..56b1d121 100644 --- a/test/messages/host.test.js +++ b/test/messages/host.test.js @@ -14,7 +14,7 @@ describe("messages/host", function() { lib.Address.fromShorthand({ hostId: 1 }), lib.Address.fromShorthand("controller"), ); - host = new Host(hostConnector, hostConfig, undefined, []); + host = new Host(hostConnector, hostConfig, []); }); describe("HostUpdateRequest", function() { diff --git a/test/messages/plugin.test.js b/test/messages/plugin.test.js index 5b83bc8f..e2c7d9d8 100644 --- a/test/messages/plugin.test.js +++ b/test/messages/plugin.test.js @@ -28,7 +28,7 @@ describe("messages/plugin", function() { lib.Address.fromShorthand({ hostId: 1 }), lib.Address.fromShorthand("controller"), ); - host = new Host(hostConnector, hostConfig, undefined, []); + host = new Host(hostConnector, hostConfig, []); }); describe("PluginDetails", function() { From 8a51e9cf4d380c9230108650969e0c58e88787d6 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Wed, 4 Mar 2026 18:27:07 +0100 Subject: [PATCH 07/36] Use an absolute NODE_EXTRA_CA_CERTS path in test Remove some complexity around correctly handling a relative path to extra CA certs by setting it to an absolute path. --- package.json | 2 +- test/integration/index.js | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 34b2e494..bb44f9cb 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build": "tsc --build", "watch": "tsc --build --watch", "build-mod": "pnpm run -r build-mod --output-dir ../../dist", - "test": "pnpm run build && NODE_EXTRA_CA_CERTS=test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"", + "test": "pnpm run build && NODE_EXTRA_CA_CERTS=$(pwd)/test/file/tls/cert.pem mocha --enable-source-maps --check-leaks -R spec --recursive test \"plugins/*/test/**\" --exclude \"test/manual/**\"", "silent-test": "SILENT_TEST=y pnpm test", "fast-test": "FAST_TEST=y pnpm test", "cover": "nyc pnpm test", diff --git a/test/integration/index.js b/test/integration/index.js index fcaf41d0..696533e7 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -197,12 +197,7 @@ let controlConfigPath = path.join("temp", "test", "config-control.json"); function childOptions(options) { return { cwd: path.join("temp", "test"), - env: { - ...process.env, - NODE_EXTRA_CA_CERTS: process.env.NODE_EXTRA_CA_CERTS - ? path.resolve(process.env.NODE_EXTRA_CA_CERTS) - : undefined, - }, + env: { ...process.env }, ...options, }; } From aef5d5777645665be36c0a7d4797834fd5073b84 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Thu, 5 Mar 2026 01:32:00 +0100 Subject: [PATCH 08/36] Upload test logs for debugging CI failures (#858) The CI often fails with cryptic errors that would be much easier to debug if the logs from Factorio was provided. Upload the cluster log that was written during the tests which includes Factorio logs so that these can be inspected after a test failure. --- .github/workflows/node.js.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 3ba681ab..e029119d 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -40,6 +40,12 @@ jobs: with: files: ./coverage/lcov.info token: ${{ secrets.CODECOV_TOKEN }} + - name: Upload test logs for debugging + if: ${{ !cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: logs-${{ matrix.node-version }}-${{ matrix.factorio-version }} + path: temp/test/logs lint: runs-on: ubuntu-latest From 3f6203f125d74baa32f4d50b8a0b193a404b1834 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Fri, 6 Mar 2026 13:27:45 +0000 Subject: [PATCH 09/36] Add valid recipe check and remove empty table check --- plugins/inventory_sync/module/serialize.lua | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/plugins/inventory_sync/module/serialize.lua b/plugins/inventory_sync/module/serialize.lua index 04ae5d88..c6fed8c2 100644 --- a/plugins/inventory_sync/module/serialize.lua +++ b/plugins/inventory_sync/module/serialize.lua @@ -367,12 +367,7 @@ function serialize.serialize_crafting_notifications(player) index = index + 1 end - -- Do no return if length is zero as this breaks with table_to_json - if index > 1 then - return helpers.encode_string(helpers.table_to_json(recipe_names)) - end - - return nil + return helpers.encode_string(helpers.table_to_json(recipe_names)) end --- @param player LuaPlayer @@ -382,8 +377,11 @@ function serialize.deserialize_crafting_notifications(player, serialized) local recipe_names = helpers.json_to_table(assert(helpers.decode_string(serialized))) assert(type(recipe_names) == "table", "wrong type decoded from json_to_table") local clear_notification = player.clear_recipe_notification + local valid_recipes = compat.prototypes.recipe for _, recipe_name in pairs(recipe_names) do - clear_notification(recipe_name) + if valid_recipes[recipe_name] then + clear_notification(recipe_name) + end end end From 05b6bf8c04c6fd09c5699a645fc90a3cfff9113a Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 6 Mar 2026 15:27:12 +0100 Subject: [PATCH 10/36] Replace remaining usage of phin (#862) Replace the remaining places in the codebase using phin with fetch. With the exception of HttpCloser which uses the http/https request method instead. --- package.json | 1 - packages/ctl/package.json | 1 - packages/ctl/src/commands_controller.ts | 8 ++---- packages/ctl/src/commands_instance.ts | 36 +++++++++++------------ packages/ctl/src/commands_mod.ts | 38 ++++++++++++------------- packages/host/package.json | 1 - packages/host/src/Host.ts | 38 ++++++++++++------------- packages/host/src/Instance.ts | 13 +++++---- plugins/subspace_storage/test/routes.js | 2 +- pnpm-workspace.yaml | 1 - test/controller/HttpCloser.js | 32 ++++++++++++--------- test/integration/clusterio.js | 8 ++---- test/integration/index.js | 14 ++++----- 13 files changed, 91 insertions(+), 102 deletions(-) diff --git a/package.json b/package.json index bb44f9cb..6b2eb80e 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,6 @@ "jszip": "^3.10.1", "mocha": "^11.7.1", "nyc": "^17.1.0", - "phin": "^3.7.1", "typedoc": "^0.28.9", "typescript": "^5.9.2", "yargs": "^18.0.0" diff --git a/packages/ctl/package.json b/packages/ctl/package.json index af4819ed..31d6c371 100644 --- a/packages/ctl/package.json +++ b/packages/ctl/package.json @@ -24,7 +24,6 @@ "set-blocking": "catalog:", "fs-extra": "catalog:", "winston": "catalog:", - "phin": "catalog:", "as-table": "^1.0.55", "yargs": "^17.7.2" }, diff --git a/packages/ctl/src/commands_controller.ts b/packages/ctl/src/commands_controller.ts index 9c2837f6..bb17cbc8 100644 --- a/packages/ctl/src/commands_controller.ts +++ b/packages/ctl/src/commands_controller.ts @@ -1,7 +1,6 @@ import fs from "fs-extra"; import asTableModule from "as-table"; import events from "events"; -import phin from "phin"; import os from "os"; import child_process from "child_process"; @@ -166,11 +165,8 @@ controllerPluginCommands.add(new lib.Command({ handler: async function(args: object, control: Control) { let url = new URL(control.config.get("control.controller_url")!); url.pathname += "api/plugins"; - let response = await phin<[]>({ - url, - parse: "json", - }); - print(asTable(response.body)); + let response = await fetch(url); + print(asTable(await response.json() as [])); }, })); controllerPluginCommands.add(new lib.Command({ diff --git a/packages/ctl/src/commands_instance.ts b/packages/ctl/src/commands_instance.ts index 20d5b92d..d7bfacda 100644 --- a/packages/ctl/src/commands_instance.ts +++ b/packages/ctl/src/commands_instance.ts @@ -2,7 +2,6 @@ import fs from "fs-extra"; import asTableModule from "as-table"; import events from "events"; import path from "path"; -import phin from "phin"; import os from "os"; import child_process from "child_process"; import stream from "stream"; @@ -330,8 +329,6 @@ instanceSaveCommands.add(new lib.Command({ if (!filename.endsWith(".zip")) { throw new lib.CommandError("Save name must end with .zip"); } - // phin doesn't support streaming requests :( - let content = await fs.readFile(args.filepath); let instanceId = await lib.resolveInstance(control, args.instance); let url = new URL(control.config.get("control.controller_url")!); @@ -339,29 +336,32 @@ instanceSaveCommands.add(new lib.Command({ url.searchParams.append("instance_id", String(instanceId)); url.searchParams.append("filename", filename); - let result = await phin<{ errors?: string[], request_errors?: string[], saves?: string[] }>({ - url, method: "POST", + const response = await fetch(url, { + method: "POST", + body: await fs.createReadStream(args.filepath), + duplex: "half", headers: { - "X-Access-Token": control.config.get("control.controller_token"), + "X-Access-Token": control.config.get("control.controller_token")!, "Content-Type": "application/zip", }, - data: content, - parse: "json", }); - for (let error of result.body.errors || []) { + type ResponseType = { errors?: string[], request_errors?: string[], saves?: string[] }; + const result = await response.json() as ResponseType; + + for (let error of result.errors || []) { logger.error(error); } - for (let requestError of result.body.request_errors || []) { + for (let requestError of result.request_errors || []) { logger.error(requestError); } - if (result.body.saves && result.body.saves.length) { - logger.info(`Successfully uploaded as ${result.body.saves[0]}`); + if (result.saves && result.saves.length) { + logger.info(`Successfully uploaded as ${result.saves[0]}`); } - if ((result.body.errors || []).length || (result.body.request_errors || []).length) { + if ((result.errors || []).length || (result.request_errors || []).length) { throw new lib.CommandError("Uploading save failed"); } }, @@ -417,10 +417,10 @@ instanceSaveCommands.add(new lib.Command({ let url = new URL(control.config.get("control.controller_url")!); url.pathname += `api/stream/${streamId}`; - let response = await phin({ - url, method: "GET", - stream: true, - }); + let response = await fetch(url); + if (!response.body) { + throw new lib.CommandError("Got empty response from server."); + } let writeStream; let tempFilename = args.save.replace(/(\.zip)?$/, ".tmp.zip"); @@ -437,7 +437,7 @@ instanceSaveCommands.add(new lib.Command({ } } } - response.pipe(writeStream); + stream.Readable.fromWeb(response.body).pipe(writeStream); await finished(writeStream); let filename = await lib.findUnusedName(".", args.save, ".zip"); diff --git a/packages/ctl/src/commands_mod.ts b/packages/ctl/src/commands_mod.ts index 1fff3f42..a7e9f11e 100644 --- a/packages/ctl/src/commands_mod.ts +++ b/packages/ctl/src/commands_mod.ts @@ -2,7 +2,6 @@ import fs from "fs-extra"; import asTableModule from "as-table"; import events from "events"; import path from "path"; -import phin from "phin"; import stream from "stream"; import util from "util"; @@ -145,39 +144,38 @@ modCommands.add(new lib.Command({ if (!filename.endsWith(".zip")) { throw new lib.CommandError("Mod filename must end with .zip"); } - // phin doesn't support streaming requests :( - let content = await fs.readFile(args.file); let url = new URL(control.config.get("control.controller_url")!); url.pathname += "api/upload-mod"; url.searchParams.append("filename", filename); - let result = await phin< - { errors?: [], request_errors?: [], mods: Parameters[0][]} - >({ - url, method: "POST", + let response = await fetch(url, { + method: "POST", + body: fs.createReadStream(args.file), + duplex: "half", headers: { - "X-Access-Token": control.config.get("control.controller_token"), + "X-Access-Token": control.config.get("control.controller_token")!, "Content-Type": "application/zip", }, - data: content, - parse: "json", }); - for (let error of result.body.errors || []) { + type ResponseType = { errors?: [], request_errors?: [], mods: Parameters[0][]}; + const result = await response.json() as ResponseType; + + for (let error of result.errors || []) { logger.error(error); } - for (let requestError of result.body.request_errors || []) { + for (let requestError of result.request_errors || []) { logger.error(requestError); } - if (result.body.mods && result.body.mods.length) { - const mod = lib.ModInfo.fromJSON(result.body.mods[0]); + if (result.mods && result.mods.length) { + const mod = lib.ModInfo.fromJSON(result.mods[0]); logger.info(`Successfully uploaded ${mod.filename}`); } - if ((result.body.errors || []).length || (result.body.request_errors || []).length) { + if ((result.errors || []).length || (result.request_errors || []).length) { throw new lib.CommandError("Uploading mod failed"); } }, @@ -196,10 +194,10 @@ modCommands.add(new lib.Command({ let url = new URL(control.config.get("control.controller_url")!); url.pathname += `api/stream/${streamId}`; - let response = await phin({ - url, method: "GET", - stream: true, - }); + let response = await fetch(url); + if (!response.body) { + throw new lib.CommandError("Got empty response from server"); + } let writeStream; let filename = `${args.name}_${args.modVersion}.zip`; @@ -217,7 +215,7 @@ modCommands.add(new lib.Command({ } } } - response.pipe(writeStream); + stream.Readable.fromWeb(response.body).pipe(writeStream); await finished(writeStream); await fs.rename(tempFilename, filename); diff --git a/packages/host/package.json b/packages/host/package.json index 638f1501..08fed73a 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -26,7 +26,6 @@ "set-blocking": "catalog:", "fs-extra": "catalog:", "jszip": "catalog:", - "phin": "catalog:", "winston": "catalog:", "winston-daily-rotate-file": "catalog:", "jimp": "^0.22.8", diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 2b41decf..63476061 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -3,7 +3,6 @@ import path from "path"; import events from "events"; import pidusage from "pidusage"; import setBlocking from "set-blocking"; -import phin from "phin"; import stream from "stream"; import util from "util"; @@ -585,17 +584,17 @@ export default class Host extends lib.Link { let url = new URL(this.config.get("host.controller_url")); url.pathname += `api/stream/${streamId}`; - let response = await phin({ - url, method: "GET", - stream: true, - }); + let response = await fetch(url); + if (!response.body) { + throw new Error(`Empty body downloading stream ${streamId} from controller`); + } const file = lib.ModInfo.filename(mod.name, mod.version); const filePath = path.join(this.modStore.modsDirectory, file); const tempFilePath = filePath.replace(/(\.zip)?$/, ".tmp.zip"); const writeStream = fs.createWriteStream(tempFilePath, { flags: "w" }); await events.once(writeStream, "open"); - response.pipe(writeStream); + stream.Readable.fromWeb(response.body).pipe(writeStream); await finished(writeStream); await fs.rename(tempFilePath, filePath); @@ -1018,14 +1017,11 @@ export default class Host extends lib.Link { let url = new URL(this.config.get("host.controller_url")); url.pathname += `api/stream/${streamId}`; - let response = await phin({ - url, method: "GET", - stream: true, - }); + let response = await fetch(url); - if (response.statusCode !== 200) { - let content = await lib.readStream(response); - throw new lib.RequestError(`Stream returned ${response.statusCode}: ${content.toString()}`); + if (response.status !== 200 || !response.body) { + let body = Buffer.from(await response.arrayBuffer()); + throw new lib.RequestError(`Stream returned ${response.status}: ${body.toString()}`); } let savesDir = path.join(instance.path, "saves"); @@ -1044,7 +1040,7 @@ export default class Host extends lib.Link { } } } - response.pipe(writeStream); + stream.Readable.fromWeb(response.body).pipe(writeStream); await finished(writeStream); name = await lib.findUnusedName(savesDir, name, ".zip"); @@ -1059,10 +1055,11 @@ export default class Host extends lib.Link { checkRequestSaveName(name); let instance = this.getRequestInstance(instanceId); - let content: Buffer; + let content = fs.createReadStream(path.join(instance.path, "saves", name)); try { - // phin doesn't support streaming requests :( - content = await fs.readFile(path.join(instance.path, "saves", name)); + // TODO: Due to a bug in fs-extra the ready event is not emitted here. Replace + // "open" with "ready" when fs-extra is replaced with node's fs module. + await events.once(content, "open"); } catch (err: any) { if (err.code === "ENOENT") { throw new lib.RequestError(`${name} does not exist`); @@ -1072,9 +1069,10 @@ export default class Host extends lib.Link { let url = new URL(this.config.get("host.controller_url")); url.pathname += `api/stream/${streamId}`; - phin({ - url, method: "PUT", - data: content, + fetch(url, { + method: "PUT", + body: content, + duplex: "half", }).catch(err => { logger.error(`Error pushing save to controller:\n${err.stack}`, this.instanceLogMeta(instanceId)); }); diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 57cf2e03..952470be 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -1,7 +1,6 @@ import fs from "fs-extra"; import path from "path"; import pidusage from "pidusage"; -import phin from "phin"; import util from "util"; import type { Static } from "@sinclair/typebox"; import { exec } from "child_process"; @@ -1292,16 +1291,18 @@ end`.replace(/\r?\n/g, " "); let url = new URL(this._host.config.get("host.controller_url")); url.pathname += "api/upload-export"; url.searchParams.set("mod_pack_id", String(this.activeModPack.id)); - let response = await phin({ - url, method: "PUT", - data: content, + let response = await fetch(url, { + method: "PUT", + body: content, + duplex: "half", headers: { "Content-Type": "application/zip", "x-access-token": this._host.config.get("host.controller_token"), }, }); - if (response.statusCode !== 200) { - throw Error(`Upload failed: ${response.statusCode} ${response.statusMessage}: ${response.body}`); + if (response.status !== 200) { + const body = Buffer.from(await response.arrayBuffer()); + throw Error(`Upload failed: ${response.status} ${response.statusText}: ${body.toString()}`); } } finally { diff --git a/plugins/subspace_storage/test/routes.js b/plugins/subspace_storage/test/routes.js index 2753b41e..fc4aac5b 100644 --- a/plugins/subspace_storage/test/routes.js +++ b/plugins/subspace_storage/test/routes.js @@ -9,7 +9,7 @@ describe("subspace_storage endpoints", function() { describe("GET /api/inventory", function() { it("should return the controllers current inventory", async function() { let res = await get("/api/inventory"); - let inventory = res.body; + let inventory = await res.json(); assert.equal(typeof inventory, "object", "Inventory should be an object"); return; // XXX Disabled until a suitable test interface is added. diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 01c191a9..11bbe69b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -33,7 +33,6 @@ catalog: "set-blocking": ^2.0.0 "jszip": ^3.10.1 "chalk": ^4.1.2 - "phin": ^3.7.0 "ws": ^8.18.1 catalogMode: strict diff --git a/test/controller/HttpCloser.js b/test/controller/HttpCloser.js index d630600b..d5e34122 100644 --- a/test/controller/HttpCloser.js +++ b/test/controller/HttpCloser.js @@ -4,8 +4,7 @@ const fs = require("fs-extra"); const http = require("http"); const https = require("https"); const util = require("util"); - -const phin = require("phin"); +const events = require("node:events"); const HttpCloser = require("@clusterio/controller/dist/node/src/HttpCloser").default; @@ -38,8 +37,16 @@ function serverSuite(proto) { } async function get(url, options = {}) { - let core = { ...clientOptions, ...(options.core || {}) }; - return await phin({ ...options, url, core }); + const request = api.request(url, { ...clientOptions, ...options }); + request.end(); + const [response] = await events.once(request, "response"); + response.body = await new Promise((resolve, reject) => { + const chunks = []; + response.on("data", chunk => { chunks.push(chunk); }); + response.on("error", err => { reject(err); }); + response.on("end", () => { resolve(Buffer.concat(chunks)); }); + }); + return response; } before(async function() { @@ -101,7 +108,7 @@ function serverSuite(proto) { }); let agent = new api.Agent({ keepAlive: true }); - let request = catched(get(addr, { core: { agent }})); + let request = catched(get(addr, { agent })); await waitForStage1; stage2(); await closer.close(); @@ -121,7 +128,7 @@ function serverSuite(proto) { }); let agent = new api.Agent({ keepAlive: true }); - let request = catched(get(addr, { core: { agent }})); + let request = catched(get(addr, { agent })); await waitForStage1; await closer.close(); @@ -145,7 +152,7 @@ function serverSuite(proto) { let close = catched(closer.close()); let agent = new api.Agent({ keepAlive: true }); - await assert.rejects(get(addr, { core: { agent }}), { code: "ECONNREFUSED" }); + await assert.rejects(get(addr, { agent }), { code: "ECONNREFUSED" }); let response = await request; assert.equal(response.statusCode, 200, "non-200 response status code"); @@ -165,7 +172,7 @@ function serverSuite(proto) { }); let agent = new api.Agent({ keepAlive: true }); - let request = catched(get(addr, { core: { agent }})); + let request = catched(get(addr, { agent })); await waitForStage1; stage2(); await closer.close(); @@ -186,8 +193,8 @@ function serverSuite(proto) { }); let agent = new api.Agent({ keepAlive: true, maxSockets: 1 }); - let request = catched(get(addr, { core: { agent }})); - let droppedRequest = catched(get(addr, { core: { agent }})); + let request = catched(get(addr, { agent })); + let droppedRequest = catched(get(addr, { agent })); await waitForStage1; stage2(); await closer.close(); @@ -212,10 +219,7 @@ function serverSuite(proto) { await closer.close(tick*2); stage2(); - // Due to a bug in phin [https://github.com/ethanent/phin/issues/59] - // the request Promise never settles. - // TODO: uncomment when bug is fixed in phin. - // await assert.rejects(request); + await assert.rejects(request); }); } diff --git a/test/integration/clusterio.js b/test/integration/clusterio.js index 59782207..be234991 100644 --- a/test/integration/clusterio.js +++ b/test/integration/clusterio.js @@ -4,7 +4,6 @@ const fs = require("fs-extra"); const jwt = require("jsonwebtoken"); const path = require("path"); const events = require("events"); -const phin = require("phin"); const lib = require("@clusterio/lib"); const libBuildMod = require("@clusterio/lib/build_mod"); @@ -79,16 +78,15 @@ async function runWithAltHost(callback) { } async function uploadSave(instanceId, name, content) { - return await phin({ - url: `https://localhost:4443/api/upload-save?instance_id=${instanceId}&filename=${name}`, + const uploadUrl = `https://localhost:4443/api/upload-save?instance_id=${instanceId}&filename=${name}`; + return await fetch(uploadUrl, { method: "POST", - core: { rejectUnauthorized: false }, headers: { "X-Access-Token": controlToken, "Content-Type": "application/zip", "Content-Length": String(content.length), }, - data: content, + body: content, }); } diff --git a/test/integration/index.js b/test/integration/index.js index 696533e7..3a11620d 100644 --- a/test/integration/index.js +++ b/test/integration/index.js @@ -4,7 +4,6 @@ const path = require("path"); const fs = require("fs-extra"); const child_process = require("child_process"); const jwt = require("jsonwebtoken"); -const phin = require("phin"); const util = require("util"); const events = require("events"); @@ -148,13 +147,12 @@ function hasFactorio() { } async function get(urlPath) { - let res = await phin({ - method: "GET", - url: `https://localhost:4443${urlPath}`, - core: { rejectUnauthorized: false }, - }); - if (res.statusCode !== 200) { - throw new Error(`Got response code ${res.statusCode}, content: ${res.body}`); + const url = new URL("https://localhost:4443"); + url.pathname = urlPath; + let res = await fetch(url); + if (res.status !== 200) { + const body = Buffer.from(await res.arrayBuffer()); + throw new Error(`Got response code ${res.status}, content: ${body.toString()}`); } return res; } From 4badc2e9c1676976e561a54453abc1f82f96f04a Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Fri, 6 Mar 2026 15:35:49 +0000 Subject: [PATCH 11/36] Fix sync when joining a new server --- plugins/inventory_sync/module/serialize.lua | 31 +++++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/plugins/inventory_sync/module/serialize.lua b/plugins/inventory_sync/module/serialize.lua index c6fed8c2..f2ad87dd 100644 --- a/plugins/inventory_sync/module/serialize.lua +++ b/plugins/inventory_sync/module/serialize.lua @@ -377,10 +377,35 @@ function serialize.deserialize_crafting_notifications(player, serialized) local recipe_names = helpers.json_to_table(assert(helpers.decode_string(serialized))) assert(type(recipe_names) == "table", "wrong type decoded from json_to_table") local clear_notification = player.clear_recipe_notification - local valid_recipes = compat.prototypes.recipe + local add_notification = player.add_recipe_notification + local notifications = {} + + -- Assume all recipes need a notification + for _, recipe in pairs(player.force.recipes) do + if recipe.enabled and not recipe.hidden then + notifications[recipe.name] = true + end + end + + -- Remove recipes the player has cleared for _, recipe_name in pairs(recipe_names) do - if valid_recipes[recipe_name] then - clear_notification(recipe_name) + if notifications[recipe_name] then + notifications[recipe_name] = false + end + end + + -- Clear ones which are ones which are not required + for _, recipe in pairs(player.get_recipe_notifications()) do + if notifications[recipe.name] == false then + clear_notification(recipe.name) + end + notifications[recipe.name] = nil -- Prevents double add below + end + + -- Add any which the player did not have + for recipe_name, should_add in pairs(notifications) do + if should_add then + add_notification(recipe_name) end end end From f2833eec33361cd6863dc2b9e6a8045adb712fee Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Fri, 6 Mar 2026 16:05:57 +0000 Subject: [PATCH 12/36] Add script data to allow persistence of unknown recipes --- .../inventory_sync/module/get_script_data.lua | 1 + .../inventory_sync/module/inventory_sync.lua | 7 ++-- plugins/inventory_sync/module/serialize.lua | 33 +++++++++++++++---- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/plugins/inventory_sync/module/get_script_data.lua b/plugins/inventory_sync/module/get_script_data.lua index eb92ad16..c51a198f 100644 --- a/plugins/inventory_sync/module/get_script_data.lua +++ b/plugins/inventory_sync/module/get_script_data.lua @@ -11,6 +11,7 @@ return function(no_early_return) inventory_sync.players = inventory_sync.players or {} inventory_sync.players_waiting_for_acquire = inventory_sync.players_waiting_for_acquire or {} inventory_sync.players_in_cutscene_to_sync = inventory_sync.players_in_cutscene_to_sync or {} + inventory_sync.failed_deserialization = inventory_sync.failed_deserialization or {} inventory_sync.active_downloads = inventory_sync.active_downloads or {} inventory_sync.finished_downloads = inventory_sync.finished_downloads or {} diff --git a/plugins/inventory_sync/module/inventory_sync.lua b/plugins/inventory_sync/module/inventory_sync.lua index 55a2ce3e..3ec89305 100644 --- a/plugins/inventory_sync/module/inventory_sync.lua +++ b/plugins/inventory_sync/module/inventory_sync.lua @@ -73,7 +73,8 @@ function inventory_sync.serialize_player(player, player_record) end end - local serialized_player = serialize.serialize_player(player) + local failed_deserialization = get_script_data().failed_deserialization[player.name] or {} + local serialized_player = serialize.serialize_player(player, failed_deserialization) serialized_player.generation = player_record.generation return serialized_player @@ -106,7 +107,8 @@ function inventory_sync.deserialize_player(player, finished_record) -- Deserialize downloaded player data local serialized_player = compat.json_to_table(finished_record.data) - serialize.deserialize_player(player, serialized_player) + assert(type(serialized_player) == "table", "wrong type for serialized_player") + script_data.failed_deserialization[player.name] = serialize.deserialize_player(player, serialized_player) -- Restore player position and driving state restore_position(player, finished_record) @@ -385,6 +387,7 @@ inventory_sync.events[defines.events.on_pre_player_left_game] = function(event) player.print("ERROR: Serializing player data failed: " .. result) return end + script_data.failed_deserialization[player.name] = nil script_data.active_uploads[player.name] = { serialized = result, last_attempt = game.ticks_played, diff --git a/plugins/inventory_sync/module/serialize.lua b/plugins/inventory_sync/module/serialize.lua index f2ad87dd..28f4fa06 100644 --- a/plugins/inventory_sync/module/serialize.lua +++ b/plugins/inventory_sync/module/serialize.lua @@ -344,8 +344,9 @@ function serialize.deserialize_crafting_queue(player, serialized) end ---@param player LuaPlayer +---@param failed string[]? ---@return string? -function serialize.serialize_crafting_notifications(player) +function serialize.serialize_crafting_notifications(player, failed) -- Get all unlocked recipes local recipes = {} for _, recipe in pairs(player.force.recipes) do @@ -360,8 +361,8 @@ function serialize.serialize_crafting_notifications(player) end -- Convert to a list of names - local index = 1 - local recipe_names = {} + local recipe_names = failed or {} -- Includes previous recipes if any + local index = #recipe_names + 1 for name in pairs(recipes) do recipe_names[index] = name index = index + 1 @@ -372,6 +373,7 @@ end --- @param player LuaPlayer --- @param serialized string +--- @return string[]? failed Recipe names that do not exist function serialize.deserialize_crafting_notifications(player, serialized) -- Decode the recipe name strings local recipe_names = helpers.json_to_table(assert(helpers.decode_string(serialized))) @@ -379,6 +381,7 @@ function serialize.deserialize_crafting_notifications(player, serialized) local clear_notification = player.clear_recipe_notification local add_notification = player.add_recipe_notification local notifications = {} + local failed = {} -- Assume all recipes need a notification for _, recipe in pairs(player.force.recipes) do @@ -391,6 +394,9 @@ function serialize.deserialize_crafting_notifications(player, serialized) for _, recipe_name in pairs(recipe_names) do if notifications[recipe_name] then notifications[recipe_name] = false + else + -- Recipe does not exist on this server + failed[#failed+1] = recipe_name end end @@ -408,6 +414,9 @@ function serialize.deserialize_crafting_notifications(player, serialized) add_notification(recipe_name) end end + + -- Return failed recipes if any + return next(failed) and failed or nil end local controller_to_name = {} @@ -416,6 +425,7 @@ for name, value in pairs(defines.controllers) do end --- @class SerializedPlayerData +--- @field generation number --- @field name string --- @field controller string --- @field color Color @@ -433,10 +443,12 @@ end --- @field recipe_notifications string? --- @param player LuaPlayer +--- @param failed_deserialization FailedDeserializationPlayerData --- @return SerializedPlayerData -function serialize.serialize_player(player) +function serialize.serialize_player(player, failed_deserialization) --- @type SerializedPlayerData local serialized = { + generation = 0, -- Gets replaced later controller = controller_to_name[player.controller_type], name = player.name, color = player.color, @@ -492,15 +504,21 @@ function serialize.serialize_player(player) -- Serialize recipe notifications if recipe_notifications_api then - serialized.recipe_notifications = serialize.serialize_crafting_notifications(player) + serialized.recipe_notifications = serialize.serialize_crafting_notifications(player, failed_deserialization.recipe_notifications) end return serialized end +--- @class FailedDeserializationPlayerData +--- @field recipe_notifications string[]? + --- @param player LuaPlayer --- @param serialized SerializedPlayerData +--- @return FailedDeserializationPlayerData? function serialize.deserialize_player(player, serialized) + local failed_deserialization = {} + if player.controller_type ~= defines.controllers[serialized.controller] or serialized.controller == "ghost" then -- If targeting the character or ghost controller then create a character if serialized.controller == "character" or serialized.controller == "ghost" then @@ -582,8 +600,11 @@ function serialize.deserialize_player(player, serialized) -- Deserialize recipe notifications if recipe_notifications_api and serialized.recipe_notifications then - serialize.deserialize_crafting_notifications(player, serialized.recipe_notifications) + failed_deserialization.recipe_notifications = + serialize.deserialize_crafting_notifications(player, serialized.recipe_notifications) end + + return next(failed_deserialization) and failed_deserialization or nil end return serialize From 5d9a6cdff634cefbfdccd84a73b8bf2208748c45 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Fri, 6 Mar 2026 17:17:15 +0000 Subject: [PATCH 13/36] Moved nil failed_deserialization to remove_player --- plugins/inventory_sync/module/inventory_sync.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/inventory_sync/module/inventory_sync.lua b/plugins/inventory_sync/module/inventory_sync.lua index 3ec89305..9805601c 100644 --- a/plugins/inventory_sync/module/inventory_sync.lua +++ b/plugins/inventory_sync/module/inventory_sync.lua @@ -36,6 +36,7 @@ local function remove_player(player) local script_data = get_script_data() script_data.players_waiting_for_acquire[player.name] = nil script_data.players_in_cutscene_to_sync[player.name] = nil + script_data.failed_deserialization[player.name] = nil script_data.active_downloads[player.name] = nil script_data.finished_downloads[player.name] = nil script_data.active_uploads[player.name] = nil @@ -387,7 +388,6 @@ inventory_sync.events[defines.events.on_pre_player_left_game] = function(event) player.print("ERROR: Serializing player data failed: " .. result) return end - script_data.failed_deserialization[player.name] = nil script_data.active_uploads[player.name] = { serialized = result, last_attempt = game.ticks_played, From 276c36b6e9321a2db1dfec2828f0248494f88bcb Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 6 Mar 2026 18:50:18 +0100 Subject: [PATCH 14/36] Throw error if lib is imported twice (#863) The lib library was written with the assumption that there's only one copy of it in the whole application and everything imports that one copy. Unfortunately there are quite a few circumstances were package managers end up installing multiple copies of lib and then different parts use different versions of the library. This causes Clusterio to break with strange and hard to track down errors. Most notably permissions and link messages for a plugin may get registered to one copy of lib while the controller uses a different copy and thus does not see or register that what the plugin added exists. Detect this condition when importing @clusterio/lib and throw an error when it has been imported twice in order to prevent the broken installation from causing future hard to debug issues. --- packages/controller/controller.ts | 2 +- packages/controller/src/Controller.ts | 3 +++ packages/host/host.ts | 2 +- packages/host/src/Host.ts | 3 +++ packages/lib/browser.ts | 3 +++ packages/lib/index.ts | 3 +++ packages/lib/src/check_singleton_import.ts | 21 +++++++++++++++++++++ packages/lib/src/errors.ts | 7 +++++++ packages/lib/src/plugin_loader.ts | 3 +++ packages/lib/tsconfig.browser.json | 1 + packages/lib/tsconfig.node.json | 2 +- test/lib/check_singleton_import.test.js | 16 ++++++++++++++++ 12 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 packages/lib/src/check_singleton_import.ts create mode 100644 test/lib/check_singleton_import.test.js diff --git a/packages/controller/controller.ts b/packages/controller/controller.ts index 85b20875..7890c85c 100644 --- a/packages/controller/controller.ts +++ b/packages/controller/controller.ts @@ -490,7 +490,7 @@ I version of clusterio. Expect things to break. I ` ); startup().catch(err => { - if (err instanceof lib.StartupError) { + if (err instanceof lib.StartupError || err.code === "InstallationError") { logger.fatal(` +-------------------------------+ | Unable to to start controller | diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index 7ad03e2c..a5671795 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -1068,6 +1068,9 @@ export default class Controller { this.plugins.set(pluginInfo.name, controllerPlugin); } catch (err: any) { + if (err.code === "InstallationError") { + throw err; + } throw new lib.PluginError(pluginInfo.name, err); } diff --git a/packages/host/host.ts b/packages/host/host.ts index 8b06fdf0..485f2926 100755 --- a/packages/host/host.ts +++ b/packages/host/host.ts @@ -299,7 +299,7 @@ I version of clusterio. Expect things to break. I if (err instanceof lib.WebSocketError) { // Already handled by connector.on("error") within Host - } else if (err instanceof lib.StartupError) { + } else if (err instanceof lib.StartupError || err.code === "InstallationError") { logger.fatal(` +----------------------------------+ | Unable to to start clusteriohost | diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 63476061..7e778c32 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -440,6 +440,9 @@ export default class Host extends lib.Link { this.plugins.set(pluginInfo.name, hostPlugin); } catch (err: any) { + if (err.code === "InstallationError") { + throw err; + } throw new lib.PluginError(pluginInfo.name, err); } diff --git a/packages/lib/browser.ts b/packages/lib/browser.ts index 58c0debf..66b3d5c1 100644 --- a/packages/lib/browser.ts +++ b/packages/lib/browser.ts @@ -16,3 +16,6 @@ export * from "./src/subscriptions"; export { default as ExponentialBackoff } from "./src/ExponentialBackoff"; export { default as RateLimiter } from "./src/RateLimiter"; export { default as ValueCache } from "./src/ValueCache"; + +import { checkSingletonImport } from "./src/check_singleton_import"; +checkSingletonImport(import.meta.url); diff --git a/packages/lib/index.ts b/packages/lib/index.ts index 96a4da80..d8240ffe 100644 --- a/packages/lib/index.ts +++ b/packages/lib/index.ts @@ -38,3 +38,6 @@ export { default as ExponentialBackoff } from "./src/ExponentialBackoff"; export { default as ModStore } from "./src/ModStore"; export { default as RateLimiter } from "./src/RateLimiter"; export { default as ValueCache } from "./src/ValueCache"; + +import { checkSingletonImport } from "./src/check_singleton_import"; +checkSingletonImport(__filename); diff --git a/packages/lib/src/check_singleton_import.ts b/packages/lib/src/check_singleton_import.ts new file mode 100644 index 00000000..6a734128 --- /dev/null +++ b/packages/lib/src/check_singleton_import.ts @@ -0,0 +1,21 @@ +import { InstallationError } from "./errors"; +declare global { + // eslint-disable-next-line vars-on-top + var _clusterioLibImportedFrom: string; +} + +export function checkSingletonImport(location: string) { + if (!global._clusterioLibImportedFrom) { + global._clusterioLibImportedFrom = location; + } else { + throw new InstallationError(`Attempt to import duplicate copy of @clusterio/lib + +Importing more than one copy of @clusterio/lib into the same runtime +will break Clusterio. This typically happens when the package +manager has installed two different versions of @clusterio/lib +and assigned different ones to different components of Clusterio. + +Previously imported @clusterio/lib was located at ${_clusterioLibImportedFrom} +This copy of @clusterio/lib is located at ${location}`); + } +} diff --git a/packages/lib/src/errors.ts b/packages/lib/src/errors.ts index 00071895..e163ef60 100644 --- a/packages/lib/src/errors.ts +++ b/packages/lib/src/errors.ts @@ -10,6 +10,13 @@ export class CommandError extends Error { code = "CommandError"; } +/** + * Thrown if the installation is broken, such as when multiple copies of lib is imported + */ +export class InstallationError extends Error { + code = "InstallationError"; +} + /** * Thrown from requests sent when an error occured handling it */ diff --git a/packages/lib/src/plugin_loader.ts b/packages/lib/src/plugin_loader.ts index 115aef46..37b48ca0 100644 --- a/packages/lib/src/plugin_loader.ts +++ b/packages/lib/src/plugin_loader.ts @@ -45,6 +45,9 @@ export async function loadPluginInfos(pluginList: Map) { pluginInfo = require(pluginPath).plugin; pluginPackage = require(path.posix.join(pluginPath, "package.json")); } catch (err: any) { + if (err.code === "InstallationError") { + throw err; + } throw new libErrors.PluginError(pluginName, err); } diff --git a/packages/lib/tsconfig.browser.json b/packages/lib/tsconfig.browser.json index 05554ac8..97bcd427 100644 --- a/packages/lib/tsconfig.browser.json +++ b/packages/lib/tsconfig.browser.json @@ -7,6 +7,7 @@ "./src/config/*.ts", "./src/data/*.ts", "./src/external/*.ts", + "./src/check_singleton_import.ts", "./src/datastore.ts", "./src/errors.ts", "./src/factorio/exchange_string.ts", diff --git a/packages/lib/tsconfig.node.json b/packages/lib/tsconfig.node.json index a963c8a0..ef6ccd30 100644 --- a/packages/lib/tsconfig.node.json +++ b/packages/lib/tsconfig.node.json @@ -1,6 +1,6 @@ { "extends": "../../tsconfig.node.json", - "include": ["index.ts", "browser.ts", "build_mod.js", "src/**/*.ts", "src/**/*.js"], + "include": ["index.ts", "build_mod.js", "src/**/*.ts", "src/**/*.js"], "compilerOptions": { "allowJs": true, }, diff --git a/test/lib/check_singleton_import.test.js b/test/lib/check_singleton_import.test.js new file mode 100644 index 00000000..c4af4938 --- /dev/null +++ b/test/lib/check_singleton_import.test.js @@ -0,0 +1,16 @@ +const assert = require("assert").strict; +const { checkSingletonImport } = require("@clusterio/lib/dist/node/src/check_singleton_import"); +// Import lib so it sets the global indicating lib has been imported. +require("@clusterio/lib"); + +describe("lib/check_singleton_import", function() { + it("Should throw an installation error if called twice", function() { + assert.throws( + () => checkSingletonImport(__filename), + { + message: /Attempt to import duplicate copy of @clusterio\/lib/, + code: "InstallationError", + } + ); + }); +}); From 101886e5222a5ec4f1edacf68557a479485967cb Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sun, 8 Mar 2026 17:52:35 +0000 Subject: [PATCH 15/36] Downgrade error to a warning as it does not prevent server startup. --- packages/host/src/server.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/host/src/server.ts b/packages/host/src/server.ts index 6152fee6..86cfdfaf 100644 --- a/packages/host/src/server.ts +++ b/packages/host/src/server.ts @@ -887,7 +887,7 @@ export class FactorioServer extends events.EventEmitter { try { this._whitelistWatcher = fs.watch(filePath); } catch (err: any) { - this._logger.error(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`); + this._logger.warn(`Unable to watch whitelist, bidirectional sync will not be available:\n${err}`); return; } From 5b5b2c28f73c24064d9288e20d6054a790b44b49 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Sun, 8 Mar 2026 18:34:21 +0000 Subject: [PATCH 16/36] Always write a whitelist on server start --- packages/host/src/Instance.ts | 11 +++++++++++ test/integration/instance.js | 31 ++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 952470be..67951204 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -866,6 +866,17 @@ end`.replace(/\r?\n/g, " "); this.server.writePath("server-whitelist.json"), JSON.stringify([...this._host.whitelist], null, "\t") ); + } else { + // Always create the whitelist even if unused + // This allows it to be watched for changes without a server restart + try { + // wx will throw an error if the file already exists to prevent overwrite + await fs.writeFile(this.server.writePath("server-whitelist.json"), "\n[]\n", { flag: "wx" }); + } catch (err: any) { + if (err.code !== "EEXIST") { + throw err; + } + } } await this.syncMods(); diff --git a/test/integration/instance.js b/test/integration/instance.js index 2955e001..7cba7de8 100644 --- a/test/integration/instance.js +++ b/test/integration/instance.js @@ -1,11 +1,13 @@ "use strict"; +const path = require("path"); const assert = require("assert").strict; const lib = require("@clusterio/lib"); +const fs = require("fs-extra"); const { testMatrix } = require("../common"); const { slowTest, exec, execCtl, execCtlProcess, sendRcon, getControl, - requiresFactorio, hasFactorio, + requiresFactorio, hasFactorio, instancesDir, } = require("./index"); const instId = 48; @@ -67,6 +69,32 @@ describe("Clusterio Instance", function() { await execCtl(`instance stop ${instAltName}`); await execCtl(`instance delete ${instAltName}`); }); + describe("whitelist creation", function() { + const whitelistPath = path.join(instancesDir, instName, "server-whitelist.json"); + before(async function() { + this.timeout(20000); + await execCtl(`${instSetConfig} factorio.sync_whitelist disabled`); + await execCtl(`${instSetConfig} factorio.enable_whitelist false`); + }); + afterEach(async function() { + this.timeout(20000); + await execCtl(`instance stop ${instName}`); + }); + it("should always create a whitelist json even if unused", async function() { + slowTest(this); + await fs.remove(whitelistPath); + await execCtl(`instance start ${instName}`); + const whitelist = await fs.readJSON(whitelistPath); + assert.deepEqual(whitelist, []); + }); + it("should not overwrite the existing whitelist if sync is disabled", async function() { + slowTest(this); + await fs.writeJSON(whitelistPath, ["test_user"]); + await execCtl(`instance start ${instName}`); + const whitelist = await fs.readJSON(whitelistPath); + assert.deepEqual(whitelist, ["test_user"]); + }); + }); for (const [savePatchingEnabled, scriptCommandsEnabled] of testMatrix([true, false], [true, false])) { const strSavePatching = `save patching: ${savePatchingEnabled ? "Enabled" : "Disabled"}`; const strScriptCommands = `script commands: ${scriptCommandsEnabled ? "Enabled" : "Disabled"}`; @@ -81,6 +109,7 @@ describe("Clusterio Instance", function() { before(async function() { this.timeout(20000); // Create and start a patched save + await execCtl(`${instSetConfig} factorio.enable_whitelist true`); await execCtl(`${instSetConfig} factorio.enable_save_patching ${savePatchingEnabled}`); await execCtl(`${instSetConfig} factorio.enable_script_commands ${scriptCommandsEnabled}`); await execCtl(`instance save create ${instName}`); From ff784cc7724a9420221edbfa6c50cb8cdedea22e Mon Sep 17 00:00:00 2001 From: Ryan Walker Date: Sun, 8 Mar 2026 14:44:42 -0500 Subject: [PATCH 17/36] Differentiate RCON output for improved clarity (#864) * #793 - Differentiate RCON output * -removed
-simplified animation logic. -removed box-shadow in favor of outline * -remove Rube Goldberg machine -removed un needed .css rule * removed redundant null check * added an extra line (2 spaces between imports and rest of file) --------- Co-authored-by: Solar --- packages/web_ui/src/components/InstanceRcon.tsx | 7 +++++-- packages/web_ui/src/style.css | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/web_ui/src/components/InstanceRcon.tsx b/packages/web_ui/src/components/InstanceRcon.tsx index 75245f88..62702787 100644 --- a/packages/web_ui/src/components/InstanceRcon.tsx +++ b/packages/web_ui/src/components/InstanceRcon.tsx @@ -1,4 +1,4 @@ -import React, { useContext, useState } from "react"; +import React, { useContext, useRef, useState } from "react"; import { Input, Typography } from "antd"; import * as lib from "@clusterio/lib"; @@ -17,6 +17,7 @@ export default function InstanceRcon(props: InstanceRconProps) { let control = useContext(ControlContext); let [output, setOutput] = useState(null); let [running, setRunning] = useState(false); + let resultRef = useRef(null); async function sendCommand(command: string) { if (!command) { @@ -31,6 +32,8 @@ export default function InstanceRcon(props: InstanceRconProps) { new lib.InstanceSendRconRequest(command), ); setOutput(result); + const animation = resultRef.current?.children[0]?.getAnimations()[0]; + if (animation) { animation.currentTime = 0; } } finally { setRunning(false); } @@ -39,7 +42,7 @@ export default function InstanceRcon(props: InstanceRconProps) { return <> {output && <> Rcon result - {output} + {output} } Date: Mon, 9 Mar 2026 13:54:08 +0100 Subject: [PATCH 18/36] Fix circular import breaking --dev mode A circular import caused code to attempt to use symbols before they were defined. Due to the --dev mode on the controller running with different chunck boundaries this only appeared in the --dev mode. --- packages/lib/src/external/FactorioVersions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/lib/src/external/FactorioVersions.ts b/packages/lib/src/external/FactorioVersions.ts index 45125433..0fe9b8c6 100644 --- a/packages/lib/src/external/FactorioVersions.ts +++ b/packages/lib/src/external/FactorioVersions.ts @@ -1,5 +1,5 @@ import { Static, Type } from "@sinclair/typebox"; -import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data"; +import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data/version"; const ARCHIVE_URL = "https://factorio.com/download/archive"; From f56605600fb4d29157297ff06d33136060c64b17 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 9 Mar 2026 14:12:40 +0100 Subject: [PATCH 19/36] Fix host bricked on Factorio version check If the FactorioVersionRequest throws the instance and host ends up bricked as the instance is left in an invalid starting state that can't be stopped. --- packages/host/src/Host.ts | 1 + packages/host/src/Instance.ts | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index 7e778c32..b096a889 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -74,6 +74,7 @@ async function discoverInstances(instancesDir: string) { return instances; } +// Requests that wake up the instance is responsible for updating the instance state in case of errors. const instanceStartingMessages = new Set([ lib.InstanceStartRequest.name, lib.InstanceLoadScenarioRequest.name, diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 952470be..782b26a2 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -637,9 +637,10 @@ end`.replace(/\r?\n/g, " "); } async init(pluginInfos: lib.PluginNodeEnvInfo[]) { + // After changing the status this function is responsible for calling notifyExit if it throws. this.notifyStatus("starting"); - const factorioVersions = await this.sendTo("controller", new lib.FactorioVersionsRequest()); try { + const factorioVersions = await this.sendTo("controller", new lib.FactorioVersionsRequest()); await this._loadStats(); await this.server.checkForUpdates(factorioVersions); await this.server.init(); From 25d98d2ab355e0e17ca7b28c6cc64a19476d5096 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 9 Mar 2026 14:54:19 +0100 Subject: [PATCH 20/36] Fix error starting instance on unlisted versions The FactorioServer.checkForUpdates function would wrongfully assume that the version running would always be listed on Wube's archive page, and throw an error if this was not the case. Ignore version checking logic if no suitable latest version could be found. --- packages/host/src/server.ts | 4 + test/host/server.js | 370 +++++++-------------- test/lib/external/FactorioVersions.test.js | 6 + 3 files changed, 124 insertions(+), 256 deletions(-) diff --git a/packages/host/src/server.ts b/packages/host/src/server.ts index 6152fee6..c51538d1 100644 --- a/packages/host/src/server.ts +++ b/packages/host/src/server.ts @@ -978,6 +978,10 @@ export class FactorioServer extends events.EventEmitter { .sort((v1, v2) => versionOrder(v1.version, v2.version)) [0]; + if (!latestVersion) { + return; + } + const installedVersions = await listFactorioVersions(this._factorioDir); if (!installedVersions.versions.has(latestVersion.version)) { const v = latestVersion.version; diff --git a/test/host/server.js b/test/host/server.js index 28d21054..70f9cff8 100644 --- a/test/host/server.js +++ b/test/host/server.js @@ -342,264 +342,122 @@ describe("host/server", function() { }); }); - describe("full version", function() { - it("should do nothing when there is no newer version", async function() { - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1.1"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); + for (const [suite, target] of [ + ["full version", "0.1.1"], + ["partial version", "0.1"], + ["latest version", "latest"], + ]) { + /* eslint-disable no-loop-func */ + describe("full version", function() { + it("should do nothing when there are no versions", async function() { + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target; + await server.checkForUpdates([]); - assert.equal(fetchCalledWith, null); + assert.equal(fetchCalledWith, null); + }); + it("should do nothing when no version matches", async function() { + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target; + await server.checkForUpdates([{ + stable: true, + version: "0.2.1", + headlessUrl: "test1", + }, { + stable: false, + version: "0.2.0", + headlessUrl: "test2", + }]); + + assert.equal(fetchCalledWith, null); + }); + it("should do nothing when there is no newer version", async function() { + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target; + await server.checkForUpdates([{ + stable: true, + version: "0.1.1", + headlessUrl: "test1", + }, { + stable: false, + version: "0.1.0", + headlessUrl: "test2", + }]); + + assert.equal(fetchCalledWith, null); + }); + it("should do nothing for direct installs", async function() { + server._factorioDir = path.join("test", "file", "factorio", "0.1.1"); + server._targetVersion = target === "0.1.1" ? "0.1.5" : target; + await server.checkForUpdates([{ + stable: true, + version: "0.1.5", + headlessUrl: "test1", + }, { + stable: true, + version: "0.1.1", + headlessUrl: "test1", + }, { + stable: false, + version: "0.1.0", + headlessUrl: "test2", + }]); + + assert.equal(fetchCalledWith, null); + }); + it("should do nothing when on windows", async function() { + let logLine = null; + server._logger = { info: line => { logLine = line; } }; + Object.defineProperty(process, "platform", { value: "win32" }); + + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target === "0.1.1" ? "0.1.5" : target; + await server.checkForUpdates([{ + stable: true, + version: "0.1.5", + headlessUrl: "test1", + }, { + stable: true, + version: "0.1.1", + headlessUrl: "test2", + }, { + stable: false, + version: "0.1.0", + headlessUrl: "test3", + }]); + + assert.equal(fetchCalledWith, null); + assert.ok(logLine !== null); + assert.ok(logLine.endsWith("but must be manually downloaded")); + }); + it("should do attempt to download on linux", async function() { + let logLine = null; + server._logger = { info: line => { logLine = line; } }; + Object.defineProperty(process, "platform", { value: "linux" }); + + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target === "0.1.1" ? "0.1.5" : target; + await assert.rejects(server.checkForUpdates([{ + stable: true, + version: "0.1.5", + headlessUrl: "test1", + }, { + stable: true, + version: "0.1.1", + headlessUrl: "test2", + }, { + stable: false, + version: "0.1.0", + headlessUrl: "test3", + }]), new Error("Failed to fetch test1: -1 Fetch called")); + + assert.equal(fetchCalledWith, "test1"); + assert.ok(logLine !== null); + assert.ok(logLine.endsWith("starting download...")); + }); }); - it("should do nothing for direct installs", async function() { - server._factorioDir = path.join("test", "file", "factorio", "0.1.1"); - server._targetVersion = "0.1.5"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); - - assert.equal(fetchCalledWith, null); - }); - it("should do nothing when on windows", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "win32" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1.5"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]); - - assert.equal(fetchCalledWith, null); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("but must be manually downloaded")); - }); - it("should do attempt to download on linux", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "linux" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1.5"; - await assert.rejects(server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]), new Error("Failed to fetch test1: -1 Fetch called")); - - assert.equal(fetchCalledWith, "test1"); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("starting download...")); - }); - }); - describe("partial version", function() { - it("should do nothing when there is no newer version", async function() { - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); - - assert.equal(fetchCalledWith, null); - }); - it("should do nothing for direct installs", async function() { - server._factorioDir = path.join("test", "file", "factorio", "0.1.1"); - server._targetVersion = "0.1"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); - - assert.equal(fetchCalledWith, null); - }); - it("should do nothing when on windows", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "win32" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]); - - assert.equal(fetchCalledWith, null); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("but must be manually downloaded")); - }); - it("should do attempt to download on linux", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "linux" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "0.1"; - await assert.rejects(server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]), new Error("Failed to fetch test1: -1 Fetch called")); - - assert.equal(fetchCalledWith, "test1"); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("starting download...")); - }); - }); - describe("latest version", function() { - it("should do nothing when there is no newer version", async function() { - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "latest"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); - - assert.equal(fetchCalledWith, null); - }); - it("should do nothing for direct installs", async function() { - server._factorioDir = path.join("test", "file", "factorio", "0.1.1"); - server._targetVersion = "latest"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test2", - }]); - - assert.equal(fetchCalledWith, null); - }); - it("should do nothing when on windows", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "win32" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "latest"; - await server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]); - - assert.equal(fetchCalledWith, null); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("but must be manually downloaded")); - }); - it("should do attempt to download on linux", async function() { - let logLine = null; - server._logger = { info: line => { logLine = line; } }; - Object.defineProperty(process, "platform", { value: "linux" }); - - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = "latest"; - await assert.rejects(server.checkForUpdates([{ - stable: true, - version: "0.1.5", - headlessUrl: "test1", - }, { - stable: true, - version: "0.1.1", - headlessUrl: "test2", - }, { - stable: false, - version: "0.1.0", - headlessUrl: "test3", - }]), new Error("Failed to fetch test1: -1 Fetch called")); - - assert.equal(fetchCalledWith, "test1"); - assert.ok(logLine !== null); - assert.ok(logLine.endsWith("starting download...")); - }); - }); + /* eslint-enable no-loop-func */ + } it("should download a version correctly (live api)", async function() { slowTest(this); if (_platform !== "linux") { diff --git a/test/lib/external/FactorioVersions.test.js b/test/lib/external/FactorioVersions.test.js index 9058e6e0..c2e4f49a 100644 --- a/test/lib/external/FactorioVersions.test.js +++ b/test/lib/external/FactorioVersions.test.js @@ -121,6 +121,12 @@ describe("FactorioVersions", function() { assert.equal(versions.filter(v => v.stable).length, 48); assert.equal(versions.filter(v => !v.stable).length, 57); }); + it("parses versions from empty html file", async function() { + mockFetch(""); + + const versions = await fetchFactorioVersions(); + assert.equal(versions.length, 0); + }); it("parses versions from live api", async function() { slowTest(this); global.fetch = originalFetch; From 2f70db4b7e92ab8427521fd77d72082c7f1f61fa Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 9 Mar 2026 15:16:46 +0100 Subject: [PATCH 21/36] Ignore errors fetching Factorio versions If an error occurs trying to fetch the list of Factorio versions log a warning and then return an empty list instead of preventing the system from operating. --- packages/controller/src/Controller.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/controller/src/Controller.ts b/packages/controller/src/Controller.ts index a5671795..6e1f8a78 100644 --- a/packages/controller/src/Controller.ts +++ b/packages/controller/src/Controller.ts @@ -112,7 +112,14 @@ export default class Controller { // Cache for mod portal requests modPortalCache = new Map(); // Cache for factorio versions - factorioVersions = new lib.ValueCache(lib.fetchFactorioVersions); + factorioVersions = new lib.ValueCache(async () => { + try { + return await lib.fetchFactorioVersions(); + } catch (err: any) { + logger.warn(`Failed to retrieve Factorio versions ${err.message}`); + return []; + } + }); static async bootstrap(config: lib.ControllerConfig) { let databaseDirectory = config.get("controller.database_directory"); From 41b9eb1005ee55c274ea4076d0d30123f223243f Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 9 Mar 2026 15:32:43 +0100 Subject: [PATCH 22/36] Avoid redirect in fetchFactorioVersions Speed up fetching of result by avoiding a roundtrip due to path being redirected to a path ending in a /. --- packages/lib/src/external/FactorioVersions.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/lib/src/external/FactorioVersions.ts b/packages/lib/src/external/FactorioVersions.ts index 0fe9b8c6..86360b39 100644 --- a/packages/lib/src/external/FactorioVersions.ts +++ b/packages/lib/src/external/FactorioVersions.ts @@ -1,7 +1,7 @@ import { Static, Type } from "@sinclair/typebox"; import { FullVersionSchema, integerFullVersion, isFullVersion } from "../data/version"; -const ARCHIVE_URL = "https://factorio.com/download/archive"; +const ARCHIVE_URL = "https://factorio.com/download/archive/"; /** * Represents the parsed result of a factorio version From 2399fb9d15d8dda5b146c183947e0370c6e9d6c2 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 9 Mar 2026 15:58:35 +0100 Subject: [PATCH 23/36] Fix NpmButton fetching config on every render The useEffect updates list has the major pitfall that passing an object via for example a prop like foo={{ bar: 1 }} to it causes it to to detect it as a change on every re-render due to the newly created object comparing unqual to the previously passed in object. Unpack the object to prevent the identity from changing every time the metrics update on the Host page. --- packages/web_ui/src/components/NpmButton.tsx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/web_ui/src/components/NpmButton.tsx b/packages/web_ui/src/components/NpmButton.tsx index 8696cac1..aa01c7b4 100644 --- a/packages/web_ui/src/components/NpmButton.tsx +++ b/packages/web_ui/src/components/NpmButton.tsx @@ -82,7 +82,11 @@ function NpmPluginInstall({ setApplyAction, form, target }: NpmActionProps) { ; } -export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: boolean, disabled?: boolean }) { +export function NpmButton(props: { + target: "controller" | { hostId: number }, + canRestart?: boolean, + disabled?: boolean +}) { const control = useContext(ControlContext); const account = useAccount(); const [open, setOpen] = useState(false); @@ -93,6 +97,7 @@ export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: bo remoteUpdates: true, pluginUpdates: true, pluginInstall: true, }); + const hostId = props.target === "controller" ? undefined : props.target.hostId; useEffect(() => { if (props.target === "controller") { if (!account.hasPermission("core.controller.get_config")) { @@ -121,7 +126,7 @@ export function NpmButton(props: { target: lib.AddressShorthand, canRestart?: bo }); }).catch(notifyErrorHandler("Failed to fetch host config for remote updates")); } - }, [props.target]); + }, [hostId]); function onValuesChange({ action } : { action?: string }) { if (action) { From b0595a6502af20a8d1ea4abab6c45cfaddf7f8e5 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 9 Mar 2026 16:20:21 +0100 Subject: [PATCH 24/36] Fix identical updatedAtMs values in datastore The assumption with the subscription system is that new updates will always increment the updatedAtMs value to something greater than what has been seen before. This might not happen if the system clock is adjusted, or updates are sent so fast that Date.now() returns the same value. Fix these edge cases so that the timestamp is always incremented. --- packages/lib/src/datastore.ts | 45 ++++++++++++++--- test/controller/User.test.js | 6 +++ test/lib/datastore.js | 91 +++++++++++++++++++++++++++++------ 3 files changed, 121 insertions(+), 21 deletions(-) diff --git a/packages/lib/src/datastore.ts b/packages/lib/src/datastore.ts index 0dae1a57..48b9d15b 100644 --- a/packages/lib/src/datastore.ts +++ b/packages/lib/src/datastore.ts @@ -290,18 +290,50 @@ export class KeyValueDatastore< export class SubscribableDatastore< V extends SubscribableValue, > extends Datastore { + /** Last time an update occurred */ + lastUpdatedAtMs: number = 0; + + private static maxUpdatedAtMs(values: Iterable) { + let max = 0; + for (const value of values) { + // Prevent one NaN, undefined, +Inf or similar from breaking the datastore. + if (Number.isFinite(value.updatedAtMs)) { + max = Math.max(max, value.updatedAtMs); + } + } + return max; + } + + private incrementLastUpdatedMs() { + this.lastUpdatedAtMs = Math.max(Date.now(), this.lastUpdatedAtMs + 1); + } + + constructor( + provider: DatastoreProvider = new MemoryDatastoreProvider(), + data = new Map(), + ) { + super(provider, data); + this.lastUpdatedAtMs = SubscribableDatastore.maxUpdatedAtMs(this.data.values()); + } + + async load() { + await super.load(); + this.lastUpdatedAtMs = SubscribableDatastore.maxUpdatedAtMs(this.data.values()); + } + // Set the value in the datastore, be careful of race conditions if you await any functions before calling set set(value: V) { + this.incrementLastUpdatedMs(); this.data.set(value.id, value); - value.updatedAtMs = Date.now(); + value.updatedAtMs = this.lastUpdatedAtMs; this.touch([value]); } // Set many values at once from an array of key value pairs setMany(values: V[]) { - const nowMs = Date.now(); + this.incrementLastUpdatedMs(); for (const value of values) { - value.updatedAtMs = nowMs; + value.updatedAtMs = this.lastUpdatedAtMs; this.data.set(value.id, value); } this.touch(values); @@ -309,18 +341,19 @@ export class SubscribableDatastore< // Delete a value in the datastore delete(value: V) { + this.incrementLastUpdatedMs(); this.data.delete(value.id); - value.updatedAtMs = Date.now(); + value.updatedAtMs = this.lastUpdatedAtMs; value.isDeleted = true; this.touch([value]); } // Delete many values at once from an array of values deleteMany(values: V[]) { - const nowMs = Date.now(); + this.incrementLastUpdatedMs(); for (const value of values) { value.isDeleted = true; - value.updatedAtMs = nowMs; + value.updatedAtMs = this.lastUpdatedAtMs; this.data.delete(value.id); } this.touch(values); diff --git a/test/controller/User.test.js b/test/controller/User.test.js index 3e5ea4ab..cf605280 100644 --- a/test/controller/User.test.js +++ b/test/controller/User.test.js @@ -37,6 +37,12 @@ describe("controller/User", function () { user.saveRecord(); assert(user.updatedAtMs > prev); }); + it("should always increment updatedAtMs even if called in quick succession", function () { + user.saveRecord(); + let prev = user.updatedAtMs; + user.saveRecord(); + assert(user.updatedAtMs > prev, "updatedAtMs did not increase"); + }); }); describe(".set()", function () { diff --git a/test/lib/datastore.js b/test/lib/datastore.js index 93b53277..ba2cf116 100644 --- a/test/lib/datastore.js +++ b/test/lib/datastore.js @@ -5,13 +5,6 @@ const lib = require("@clusterio/lib"); const fs = require("fs-extra"); const path = require("path"); -async function incrementDateMs() { - const prev = Date.now(); - do { - await lib.wait(1); - } while (prev >= Date.now()); -} - class MockDatastoreProvider extends lib.DatastoreProvider { constructor() { super(); @@ -571,12 +564,31 @@ describe("lib/datastore", function() { eventsRaised = []; datastoreProvider = new MockDatastoreProvider(); datastoreProvider.value = new Map([ - ["foo", {id: "foo"}], ["bar", {id: "bar"}], ["baz", {id: "baz"}], + ["foo", {id: "foo", updatedAtMs: 10}], ["bar", {id: "bar"}], ["baz", {id: "baz"}], ]); datastore = new lib.SubscribableDatastore(datastoreProvider, datastoreProvider.value); datastore.on("update", updates => eventsRaised.push(updates)); }); + function forceFuture() { + const future = {id: "future"}; + for (let i = 1; i < 100; i++) { + datastore.set(future); + } + return future.updatedAtMs; + } + + describe("lastUpdateAtMs", function() { + it("is set to the highest value after constructing a new instance", function() { + assert(datastore.lastUpdatedAtMs === 10, `${datastore.lastUpdatedAtMs} !== 10`); + }); + it("is set to the highest value after loading from a provider", async function() { + const future = Date.now() + 2000; + datastore.provider.value = ["future", {id: "future", updatedAtMs: future}]; + await datastore.load(); + assert(datastore.lastUpdatedAtMs === future, `${datastore.lastUpdatedAtMs} !== ${future}`); + }); + }); describe("set", function() { it("can set a value", function() { const value = { id: "test" }; @@ -598,12 +610,21 @@ describe("lib/datastore", function() { const value = { id: "test" }; datastore.set(value); assert.notEqual(value.updatedAtMs, undefined); - assert(value.updatedAtMs > 0, "updatedAtMs greater than zero"); + assert(value.updatedAtMs > 0, "updatedAtMs not greater than zero"); + }); + it("increments the updatedAtMs property", async function() { + const value = { id: "test" }; + datastore.set(value); const prev = value.updatedAtMs; - await incrementDateMs(); datastore.set(value); assert(value.updatedAtMs > prev, "updatedAtMs must increase"); }); + it("increments the updatedAtMs property past existing values", async function() { + const future = forceFuture(); + const value = { id: "test" }; + datastore.set(value); + assert(value.updatedAtMs > future, "updatedAtMs must increase"); + }); }); describe("setMany", function() { @@ -632,11 +653,22 @@ describe("lib/datastore", function() { assert.notEqual(updates[1].updatedAtMs, undefined); assert(updates[0].updatedAtMs > 0, "updatedAtMs greater than zero"); assert(updates[1].updatedAtMs > 0, "updatedAtMs greater than zero"); - const prev = [updates[0].updatedAtMs, updates[1].updatedAtMs]; - await incrementDateMs(); + }); + it("increments the updatedAtMs property", async function() { + const value = { id: "test1" }; + datastore.set(value); + const prev = value.updatedAtMs; + const updates = [{ id: "test1" }, { id: "test2" }]; datastore.setMany(updates); - assert(updates[0].updatedAtMs > prev[0], "updatedAtMs must increase"); - assert(updates[1].updatedAtMs > prev[1], "updatedAtMs must increase"); + assert(updates[0].updatedAtMs > prev, "updatedAtMs must increase"); + assert(updates[1].updatedAtMs > prev, "updatedAtMs must increase"); + }); + it("increments the updatedAtMs property past existing values", async function() { + const future = forceFuture(); + const updates = [{ id: "test1" }, { id: "test2" }]; + datastore.setMany(updates); + assert(updates[0].updatedAtMs > future, "updatedAtMs must increase"); + assert(updates[1].updatedAtMs > future, "updatedAtMs must increase"); }); }); @@ -667,7 +699,20 @@ describe("lib/datastore", function() { const value = { id: "test" }; datastore.delete(value); assert.notEqual(value.updatedAtMs, undefined); - assert(value.updatedAtMs > 0, "updatedAtMs greater than zero"); + assert(value.updatedAtMs > 0, "updatedAtMs not greater than zero"); + }); + it("increments the updatedAtMs property", function() { + const value = { id: "test" }; + datastore.set(value); + const prev = value.updatedAtMs; + datastore.delete(value); + assert(value.updatedAtMs > prev, "updatedAtMs must increase"); + }); + it("increments the updatedAtMs property past existing values", function() { + const future = forceFuture(); + const value = { id: "test" }; + datastore.delete(value); + assert(value.updatedAtMs > future, "updatedAtMs must increase"); }); it("sets the isDeleted property", function() { const value = { id: "test" }; @@ -704,6 +749,22 @@ describe("lib/datastore", function() { assert(values[0].updatedAtMs > 0, "updatedAtMs greater than zero"); assert(values[1].updatedAtMs > 0, "updatedAtMs greater than zero"); }); + it("increments the updatedAtMs property", function() { + const value = { id: "foo" }; + datastore.set(value); + const prev = value.updatedAtMs; + const values = [{id: "foo", updatedAtMs: prev}, {id: "bar", updatedAtMs: prev}]; + datastore.deleteMany(values); + assert(values[0].updatedAtMs > prev, "updatedAtMs must increase"); + assert(values[1].updatedAtMs > prev, "updatedAtMs must increase"); + }); + it("increments the updatedAtMs property past existing values", function() { + const future = forceFuture(); + const values = [{id: "foo"}, {id: "bar"}]; + datastore.deleteMany(values); + assert(values[0].updatedAtMs > future, "updatedAtMs must increase"); + assert(values[1].updatedAtMs > future, "updatedAtMs must increase"); + }); it("sets the isDeleted property", function() { const values = [{id: "foo"}, {id: "bar"}]; datastore.deleteMany(values); From f6eb55163fa7e823085df4a24bf976a995df0b23 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 9 Mar 2026 19:07:10 +0100 Subject: [PATCH 25/36] Fix metrics broken by comma and equal in labels The logic for turning a key back into the labelles that defined it would not account for the labels values possibly including a comma or equal sign, which in turn broke the logic for handling these metrics. Fix by properly parsing the keys. --- packages/lib/src/prometheus.ts | 21 +++++++++++++++------ test/lib/prometheus.js | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/packages/lib/src/prometheus.ts b/packages/lib/src/prometheus.ts index 2a55ddd7..becdf0cb 100644 --- a/packages/lib/src/prometheus.ts +++ b/packages/lib/src/prometheus.ts @@ -272,18 +272,27 @@ function labelsToKey(labels: LabelValues, metricLabels: string[]) { function keyToLabels(key: string) { let labels = new Map(); - if (key !== "") { - for (let pair of key.split(",")) { - let [name, value] = pair.split("=", 2); - if (!name || value === undefined) { - continue; + let pos = 0; + let matchLabel = /([^=]+)="(([^\\"]|\\["n\\])+)"/y; + while (pos < key.length) { + if (pos !== 0) { + if (key[pos] !== ",") { + throw Error(`Expected , at pos ${pos} in "${key}"`); } + pos += 1; + } + matchLabel.lastIndex = pos; + const match = matchLabel.exec(key); + if (match) { + let [, name, value] = match; labels.set(name, value - .slice(1, -1) .replace(/\\"/g, "\"") .replace(/\\n/g, "\n") .replace(/\\\\/g, "\\") ); + pos += match[0].length; + } else { + throw new Error(`Failed to parse label at pos ${pos} in "${key}"`); } } diff --git a/test/lib/prometheus.js b/test/lib/prometheus.js index b0f24ca3..ade1b09f 100644 --- a/test/lib/prometheus.js +++ b/test/lib/prometheus.js @@ -651,8 +651,21 @@ describe("lib/prometheus", function() { let gauge = new prometheus.Gauge( "test", "Help", { register: false, labels: ["a", "b"] } ); - gauge.labels('x\\\\\n\"\"\n\\', "2"); - gauge.removeAll({ a: 'x\\\\\n\"\"\n\\' }); + + let labelValue = ""; + for (let i = 0; i < 128; i++) { + const char = String.fromCodePoint(i); + if (!/[1-9B-Zb-z]/.test(char)) { // exclude boring characters + labelValue += char; + } + } + labelValue += "©💣"; + + gauge.labels(labelValue, "2"); + gauge.removeAll({ a: labelValue }); + assert(gauge._values.size === 0, "Matching labels remained"); + gauge.labels("1", labelValue); + gauge.removeAll({ a: "1" }); assert(gauge._values.size === 0, "Matching labels remained"); }); }); From 05ae607a6a4ef3d5cb8183d2b0fb0fe721ce95a1 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 9 Mar 2026 20:09:07 +0100 Subject: [PATCH 26/36] Fix tests for checkForUpdates Fix incorrect suite name and attempting to test no version matches using the "latest" version which always matches in the tests for the FactorioServer.checkForUpdates method. --- test/host/server.js | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/test/host/server.js b/test/host/server.js index 70f9cff8..13de4e75 100644 --- a/test/host/server.js +++ b/test/host/server.js @@ -348,7 +348,7 @@ describe("host/server", function() { ["latest version", "latest"], ]) { /* eslint-disable no-loop-func */ - describe("full version", function() { + describe(suite, function() { it("should do nothing when there are no versions", async function() { server._factorioDir = path.join("test", "file", "factorio"); server._targetVersion = target; @@ -356,21 +356,23 @@ describe("host/server", function() { assert.equal(fetchCalledWith, null); }); - it("should do nothing when no version matches", async function() { - server._factorioDir = path.join("test", "file", "factorio"); - server._targetVersion = target; - await server.checkForUpdates([{ - stable: true, - version: "0.2.1", - headlessUrl: "test1", - }, { - stable: false, - version: "0.2.0", - headlessUrl: "test2", - }]); + if (target !== "latest") { + it("should do nothing when no version matches", async function() { + server._factorioDir = path.join("test", "file", "factorio"); + server._targetVersion = target; + await server.checkForUpdates([{ + stable: true, + version: "0.2.1", + headlessUrl: "test1", + }, { + stable: false, + version: "0.2.0", + headlessUrl: "test2", + }]); - assert.equal(fetchCalledWith, null); - }); + assert.equal(fetchCalledWith, null); + }); + } it("should do nothing when there is no newer version", async function() { server._factorioDir = path.join("test", "file", "factorio"); server._targetVersion = target; From b75e2da48002a37349ca0eed1f391aaa202baf67 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Tue, 10 Mar 2026 00:34:41 +0000 Subject: [PATCH 27/36] Add missing awaits to Instance.prepare --- packages/host/src/Instance.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/host/src/Instance.ts b/packages/host/src/Instance.ts index 67951204..9b9509ca 100644 --- a/packages/host/src/Instance.ts +++ b/packages/host/src/Instance.ts @@ -844,7 +844,7 @@ end`.replace(/\r?\n/g, " "); if (this.config.get("factorio.sync_adminlist") !== "disabled") { this.logger.verbose("Writing server-adminlist.json"); - lib.safeOutputFile( + await lib.safeOutputFile( this.server.writePath("server-adminlist.json"), JSON.stringify([...this._host.adminlist], null, "\t") ); @@ -852,7 +852,7 @@ end`.replace(/\r?\n/g, " "); if (this.config.get("factorio.sync_banlist") !== "disabled") { this.logger.verbose("Writing server-banlist.json"); - lib.safeOutputFile( + await lib.safeOutputFile( this.server.writePath("server-banlist.json"), JSON.stringify([...this._host.banlist].map( ([username, reason]) => ({ username, reason }) @@ -862,7 +862,7 @@ end`.replace(/\r?\n/g, " "); if (this.config.get("factorio.sync_whitelist") !== "disabled") { this.logger.verbose("Writing server-whitelist.json"); - lib.safeOutputFile( + await lib.safeOutputFile( this.server.writePath("server-whitelist.json"), JSON.stringify([...this._host.whitelist], null, "\t") ); From 30f552847ab4a8bef7ffd32b36610bc8de228e80 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 10 Mar 2026 02:13:13 +0100 Subject: [PATCH 28/36] Move FactorioVersionsRequest to BaseConnection No reason to duplicate this in both the instance and host connection if the implementation is identical in both. --- packages/controller/src/BaseConnection.ts | 5 +++++ packages/controller/src/ControlConnection.ts | 5 ----- packages/controller/src/HostConnection.ts | 5 ----- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/packages/controller/src/BaseConnection.ts b/packages/controller/src/BaseConnection.ts index 8efbf14c..07b753f8 100644 --- a/packages/controller/src/BaseConnection.ts +++ b/packages/controller/src/BaseConnection.ts @@ -32,6 +32,7 @@ export default class BaseConnection extends lib.Link { this.handle(lib.ModPackGetRequest, this.handleModPackGetRequest.bind(this)); this.handle(lib.ModPackGetDefaultRequest, this.handleModPackGetDefaultRequest.bind(this)); this.handle(lib.ModDownloadRequest, this.handleModDownloadRequest.bind(this)); + this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this)); this.handle(lib.SubscriptionRequest, this.handleSubscriptionRequest.bind(this)); this.connector.on("close", () => { @@ -106,4 +107,8 @@ export default class BaseConnection extends lib.Link { async handleSubscriptionRequest(request: lib.SubscriptionRequest, src: lib.Address, dst: lib.Address) { return await this._controller.subscriptions.handleRequest(this, request, src, dst); } + + async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) { + return await this._controller.factorioVersions.get(request.maxAgeMs); + } } diff --git a/packages/controller/src/ControlConnection.ts b/packages/controller/src/ControlConnection.ts index 673a9277..d1ac875c 100644 --- a/packages/controller/src/ControlConnection.ts +++ b/packages/controller/src/ControlConnection.ts @@ -127,7 +127,6 @@ export default class ControlConnection extends BaseConnection { this.handle(lib.PluginUpdateRequest, this.handlePluginUpdateRequest.bind(this)); this.handle(lib.PluginInstallRequest, this.handlePluginInstallRequest.bind(this)); this.handle(lib.DebugDumpWsRequest, this.handleDebugDumpWsRequest.bind(this)); - this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this)); } validateIngress(message: lib.MessageRequest | lib.MessageEvent) { @@ -1240,10 +1239,6 @@ export default class ControlConnection extends BaseConnection { this.connector._socket!.clusterio_ignore_dump = true; this._controller.debugEvents.on("message", this.ws_dumper); } - - async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) { - return await this._controller.factorioVersions.get(request.maxAgeMs); - } } module.exports = ControlConnection; diff --git a/packages/controller/src/HostConnection.ts b/packages/controller/src/HostConnection.ts index 66601e20..e7132925 100644 --- a/packages/controller/src/HostConnection.ts +++ b/packages/controller/src/HostConnection.ts @@ -81,7 +81,6 @@ export default class HostConnection extends BaseConnection { this.handle(lib.InstanceSaveDetailsUpdatesEvent, this.handleInstanceSaveDetailsUpdatesEvent.bind(this)); this.handle(lib.LogMessageEvent, this.handleLogMessageEvent.bind(this)); this.handle(lib.InstancePlayerUpdateEvent, this.handleInstancePlayerUpdateEvent.bind(this)); - this.handle(lib.FactorioVersionsRequest, this.handleFactorioVersionsRequest.bind(this)); this.snoopEvent(lib.InstanceAdminlistUpdateEvent, this.snoopInstanceAdminlistUpdateEvent.bind(this)); this.snoopEvent(lib.InstanceBanlistUpdateEvent, this.snoopInstanceBanlistUpdateEvent.bind(this)); this.snoopEvent(lib.InstanceWhitelistUpdateEvent, this.snoopInstanceWhitelistUpdateEvent.bind(this)); @@ -359,10 +358,6 @@ export default class HostConnection extends BaseConnection { }); } - async handleFactorioVersionsRequest(request: lib.FactorioVersionsRequest) { - return await this._controller.factorioVersions.get(request.maxAgeMs); - } - async snoopInstanceAdminlistUpdateEvent(event: lib.InstanceAdminlistUpdateEvent, src: lib.Address) { if (src.type !== lib.Address.instance) { return; From 66c066616b37c80e0b087e1da472cb3157fa2691 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 10 Mar 2026 02:14:44 +0100 Subject: [PATCH 29/36] Add test for Controller.factorioVersions failure Check that errors from fetchFactorioVersions is correctly handled in the fetch method provided by the Controller class. --- test/controller/Controller.js | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/controller/Controller.js b/test/controller/Controller.js index ae45a11b..7b0d4127 100644 --- a/test/controller/Controller.js +++ b/test/controller/Controller.js @@ -252,6 +252,28 @@ describe("controller/src/Controller", function() { assert.equal(controller.config.restartRequired, true); }); }); + + describe(".factorioVersions", function() { + const _fetch = global.fetch; + const fetchCalls = []; + before(function() { + global.fetch = async (url) => { + fetchCalls.push(url); + return { ok: false, status: 503, statusText: "Service Unavailable" }; + }; + }); + it("should handle fetch failure", async function() { + await assert.deepEqual( + await controller.factorioVersions.get(0), + [], + ); + assert(fetchCalls.length === 1, "expected one fetch call"); + assert(fetchCalls[0].indexOf("/download") !== -1, "fetch call to be to a URL with /download in it"); + }); + after(function() { + global.fetch = _fetch; + }); + }); }); describe("finalise", function() { describe(".finaliseHosts()", function() { From 86a648e759ef40b3b167565726a44e697017588e Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 10 Mar 2026 02:17:36 +0100 Subject: [PATCH 30/36] Detail requirements for instance waking requests These need to make sure instance status is correctly handled, otherwise they may cause the instance or host to become inoperable. --- packages/host/src/Host.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/host/src/Host.ts b/packages/host/src/Host.ts index b096a889..c3f2466e 100644 --- a/packages/host/src/Host.ts +++ b/packages/host/src/Host.ts @@ -74,7 +74,10 @@ async function discoverInstances(instancesDir: string) { return instances; } -// Requests that wake up the instance is responsible for updating the instance state in case of errors. +// Requests that wake up instances are responsible for making sure the +// instance status is correctly updated even if they fail with an error. This +// means they should typically use try catch on anything that can throw +// and call notifyExit upon any errors. const instanceStartingMessages = new Set([ lib.InstanceStartRequest.name, lib.InstanceLoadScenarioRequest.name, From 60eb2389a370341028882d3e6cf18615d29acfdb Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 10 Mar 2026 02:19:01 +0100 Subject: [PATCH 31/36] Test every ASCII character in prometheus test The a and A felt lonely being the only letters included. Add the other letters in the ASCII character set as there was no good reason to leave them out. --- test/lib/prometheus.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/lib/prometheus.js b/test/lib/prometheus.js index ade1b09f..8ef24147 100644 --- a/test/lib/prometheus.js +++ b/test/lib/prometheus.js @@ -653,12 +653,11 @@ describe("lib/prometheus", function() { ); let labelValue = ""; + // Include every ASCII codepoint for (let i = 0; i < 128; i++) { - const char = String.fromCodePoint(i); - if (!/[1-9B-Zb-z]/.test(char)) { // exclude boring characters - labelValue += char; - } + labelValue += String.fromCodePoint(i); } + // Include Unicode codepoints from the basic and supplementary planes. labelValue += "©💣"; gauge.labels(labelValue, "2"); From 76099d2a42b29e1ba42e3e155c0eb0b23b552343 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Tue, 10 Mar 2026 02:24:03 +0100 Subject: [PATCH 32/36] Document unreachable code paths in keyToLabels These were added to as a diagnostic in case the logic turns out to be broken. They should not be possible to reach as any value produced by labelsToKeys will not cause the error conditions to occur. --- packages/lib/src/prometheus.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/lib/src/prometheus.ts b/packages/lib/src/prometheus.ts index becdf0cb..84999093 100644 --- a/packages/lib/src/prometheus.ts +++ b/packages/lib/src/prometheus.ts @@ -276,13 +276,16 @@ function keyToLabels(key: string) { let matchLabel = /([^=]+)="(([^\\"]|\\["n\\])+)"/y; while (pos < key.length) { if (pos !== 0) { + /* istanbul ignore if */ if (key[pos] !== ",") { + // Should be unreachable throw Error(`Expected , at pos ${pos} in "${key}"`); } pos += 1; } matchLabel.lastIndex = pos; const match = matchLabel.exec(key); + /* istanbul ignore else */ if (match) { let [, name, value] = match; labels.set(name, value @@ -292,6 +295,7 @@ function keyToLabels(key: string) { ); pos += match[0].length; } else { + // Should be unreachable throw new Error(`Failed to parse label at pos ${pos} in "${key}"`); } } From 90c742ace645d03123c6202cf419363c8affa3cd Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Tue, 10 Mar 2026 19:03:18 +0000 Subject: [PATCH 33/36] 2.0.0-alpha.23 --- CHANGELOG.md | 60 ++++++++++++++++++++++++ packages/controller/package.json | 2 +- packages/create/package.json | 2 +- packages/ctl/package.json | 2 +- packages/host/package.json | 2 +- packages/lib/package.json | 2 +- packages/web_ui/package.json | 2 +- plugins/global_chat/package.json | 2 +- plugins/inventory_sync/package.json | 2 +- plugins/player_auth/package.json | 2 +- plugins/research_sync/package.json | 2 +- plugins/statistics_exporter/package.json | 2 +- plugins/subspace_storage/package.json | 2 +- 13 files changed, 72 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbbc9cba..62e82d2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,66 @@ Many thanks to the following for contributing to this release: [@username](https://github.com/username) --> +## Version 2.0.0-alpha.23 + +### Major Features + +- Added automatic headless download and update on linux. [#330](https://github.com/clusterio/clusterio/issues/330) +- Added recipe notification sync to inventory sync. [#506](https://github.com/clusterio/clusterio/issues/506) + +### Features + +- Added config lock files to prevent process duplication. [#815](https://github.com/clusterio/clusterio/issues/815) +- Added process and system start times to system metrics. [#826](https://github.com/clusterio/clusterio/issues/826) +- Added icons to status tags for host and instances. [#827](https://github.com/clusterio/clusterio/issues/827) +- Extended export-data to produce separate spritesheets for recipes, signals, technologies, planets, qualities, entities, and static UI icons. [#838](https://github.com/clusterio/clusterio/pull/838) +- Added Icon Reference page to web interface at `/icon-reference` for browsing all exported icons. [#838](https://github.com/clusterio/clusterio/pull/838) +- Added visual flash to RCON result box to confirm command execution, even when the output is identical to the previous result. [#793](https://github.com/clusterio/clusterio/issues/793) + +### Changes + +- Login screen now displays the cluster name. [#818](https://github.com/clusterio/clusterio/issues/818) +- Instance versions can now be set using Major.Minor without specifing a patch version. [#820](https://github.com/clusterio/clusterio/issues/820) +- Versions for factorio and modpacks can now be selected using a drop down menu. [#820](https://github.com/clusterio/clusterio/issues/820) +- The controller will now refuse to start if no users were loaded from file. [#814](https://github.com/clusterio/clusterio/issues/814) +- Improved wording and display of the generate host token modal. [#828](https://github.com/clusterio/clusterio/issues/828) +- Installations where muliple conflicting clusterio/lib packages have been installed will now fail with an error [#863](https://github.com/clusterio/clusterio/issues/863). +- Export completion now logs per-category icon counts. [#838](https://github.com/clusterio/clusterio/pull/838) + +### Fixes + +- Removed debug log statement from equipment grid serialization. [#XXX](https://github.com/clusterio/clusterio/pull/XXX) +- Fix that the documentation had a dead link to details of the message passing protocol (#812) +- Fixed case where achievements were not disabled. [#817](https://github.com/clusterio/clusterio/pull/817) +- Fix headless download on linux install. [#815](https://github.com/clusterio/clusterio/issues/816) +- Fix option `--no-download-headless` being ignored. [#815](https://github.com/clusterio/clusterio/issues/816) +- Fixed action buttons overflowing on narrow screens across the web interface. [#810](https://github.com/clusterio/clusterio/issues/810) +- Fixed player logistics failing to load on servers without unlocked logistics networks. [#1234](https://github.com/clusterio/clusterio/issues/1234) +- Fixed mod dependency parser breaking with inconsistent spaces. [#834](https://github.com/clusterio/clusterio/issues/834) +- No longer attempts to sync inventory in single player. [#835](https://github.com/clusterio/clusterio/issues/835) +- Fixed tests failing when the configured factorio_directory contains stuff other than Factorio installs [#540](https://github.com/clusterio/clusterio/issues/540). +- Always create a whitelist file so it can be watched for changes. [#865](https://github.com/clusterio/clusterio/issues/865) +- Added missing await statements which could cause an instance to start with an outdated admin/ban/whitelist. [#869](https://github.com/clusterio/clusterio/issues/869) +- Fixed metrics breaking when labels contain comma or equal signs [#861](https://github.com/clusterio/clusterio/issues/861) +- Fixed Host page show repeated errors when viewing an offline local host [#848](https://github.com/clusterio/clusterio/issues/848) +- Fixed bad mod paths in icon specifications crashing the export. [#841](https://github.com/clusterio/clusterio/issues/841) +- Fixed prototypes with no icon data causing errors during export. [#841](https://github.com/clusterio/clusterio/issues/841) + +### Breaking Changes + +- Custom permission methods should use `IUser` rather than `IControllerUser` [#837](https://github.com/clusterio/clusterio/pull/837) +- `controller.userManager` has been refactored, see PR for details. [#837](https://github.com/clusterio/clusterio/pull/837) +- `controller.instances` has been refactored, see PR for details. [#837](https://github.com/clusterio/clusterio/pull/837) +- `User/Instance/HostInfo` renamed to `User/Instance/HostRecord` [#837](https://github.com/clusterio/clusterio/pull/837) +- Removed `host.tls_ca` and `control.tls_ca` config options, see [section on self signed certificates](https://github.com/clusterio/clusterio/blob/master/docs/setting-up-tls.md#creating-and-using-a-self-signed-certificate) for the replacement for this option [#857](https://github.com/clusterio/clusterio/issues/857) + +Many thanks to the following for contributing to this release: +[@Cooldude2606](https://github.com/Cooldude2606) +[@Daniel](https://github.com/Danielv123) +[@Double](https://github.com/DoubleThoughtTheProgrammer) +[@Hornwitser](https://github.com/Hornwitser) +[@Ryan Walker](https://github.com/solarcloud7) + ## Version 2.0.0-alpha.22 ### Major Features diff --git a/packages/controller/package.json b/packages/controller/package.json index 9f350bd6..81872cb8 100644 --- a/packages/controller/package.json +++ b/packages/controller/package.json @@ -3,7 +3,7 @@ "description": "Implementation of Clusterio controller", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build && webpack-cli --mode production", diff --git a/packages/create/package.json b/packages/create/package.json index b26a0e45..4ea788f6 100644 --- a/packages/create/package.json +++ b/packages/create/package.json @@ -3,7 +3,7 @@ "description": "Installer for clusterio", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "main": "create.js", "bin": { "clusteriocreate": "./create.js" diff --git a/packages/ctl/package.json b/packages/ctl/package.json index 31d6c371..07da9b6b 100644 --- a/packages/ctl/package.json +++ b/packages/ctl/package.json @@ -3,7 +3,7 @@ "description": "Clusterio command line management tool", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.22", + "version": "2.0.0-alpha.23", "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build", diff --git a/packages/host/package.json b/packages/host/package.json index 08fed73a..ee8bbfd6 100644 --- a/packages/host/package.json +++ b/packages/host/package.json @@ -3,7 +3,7 @@ "description": "Implementation of Clusterio host server", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "main": "dist/node/index.js", "scripts": { "prepare": "tsc --build", diff --git a/packages/lib/package.json b/packages/lib/package.json index 74e4f560..27c475b7 100644 --- a/packages/lib/package.json +++ b/packages/lib/package.json @@ -1,7 +1,7 @@ { "name": "@clusterio/lib", "description": "Shared library for Clusterio", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "repository": "https://github.com/clusterio/clusterio", "license": "MIT", "scripts": { diff --git a/packages/web_ui/package.json b/packages/web_ui/package.json index 7cfe8161..8a63bdc5 100644 --- a/packages/web_ui/package.json +++ b/packages/web_ui/package.json @@ -3,7 +3,7 @@ "description": "Clusterio web interface implementation", "author": "Hornwitser", "license": "MIT", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "browser": "src/index.ts", "main": "dist/browser/src/index.js", "scripts": { diff --git a/plugins/global_chat/package.json b/plugins/global_chat/package.json index 14f0f38d..11119db4 100644 --- a/plugins/global_chat/package.json +++ b/plugins/global_chat/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-global_chat", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin forwarding between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", diff --git a/plugins/inventory_sync/package.json b/plugins/inventory_sync/package.json index 73e69188..d8a165ad 100644 --- a/plugins/inventory_sync/package.json +++ b/plugins/inventory_sync/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-inventory_sync", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin forwarding between Factorio servers", "author": "Danielv ", "homepage": "https://github.com/clusterio/factorioClusterio#readme", diff --git a/plugins/player_auth/package.json b/plugins/player_auth/package.json index 358f04c4..edc2f632 100644 --- a/plugins/player_auth/package.json +++ b/plugins/player_auth/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-player_auth", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin authenticating logged in players to the web interface", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", diff --git a/plugins/research_sync/package.json b/plugins/research_sync/package.json index 29704b02..426c17eb 100644 --- a/plugins/research_sync/package.json +++ b/plugins/research_sync/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-research_sync", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin for synchronizing research between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", diff --git a/plugins/statistics_exporter/package.json b/plugins/statistics_exporter/package.json index 683cf207..ba8df382 100644 --- a/plugins/statistics_exporter/package.json +++ b/plugins/statistics_exporter/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-statistics_exporter", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin for gathering statistics from Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", diff --git a/plugins/subspace_storage/package.json b/plugins/subspace_storage/package.json index 63187fe0..9177e837 100644 --- a/plugins/subspace_storage/package.json +++ b/plugins/subspace_storage/package.json @@ -1,6 +1,6 @@ { "name": "@clusterio/plugin-subspace_storage", - "version": "2.0.0-alpha.22b", + "version": "2.0.0-alpha.23", "description": "Clusterio plugin for sharing storage between Factorio servers", "author": "Hornwitser ", "homepage": "https://github.com/clusterio/clusterio#readme", From bbd3cc7fdb997d84a359f12b5ae491c548820a40 Mon Sep 17 00:00:00 2001 From: Cooldude2606 <25043174+Cooldude2606@users.noreply.github.com> Date: Tue, 10 Mar 2026 19:13:29 +0000 Subject: [PATCH 34/36] Remove pnpm version from publish workflow --- .github/workflows/publish.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 854a9358..0ff3ffa5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,8 +13,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 - with: - version: 8 - uses: actions/setup-node@v4 with: node-version: 22.x @@ -33,8 +31,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: pnpm/action-setup@v4 - with: - version: 8 - uses: actions/setup-node@v4 with: node-version: 22.x From ab5162e581cb4bea7e20383941c4d1c1fb5c30b4 Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Fri, 13 Mar 2026 15:33:24 +0100 Subject: [PATCH 35/36] Add changelog script parsing entries from PRs (#871) Add script which uses GitHub's REST API to fetch issues and pull requests that have changed since the last release, parse out the changelog entries from merged pull requests, and print a formatted version of the changelog suitable for copying into the release description, the CHANGELOG.md file and the Discord announcement. The script expects Pull Requests to have a `### Changelog` section in them with a code block containing changelog entries. As an added bonus the script can parse out `#` references so that you do not need to write it out in the ugly `[#id](long-url-with-id-repeated)` format. Updates documentation and pull request template to describe the changelog format and usage of the formatting script. Includes improvements from review and a second look. --- .github/pull_request_template.md | 15 +- docs/contributing.md | 2 + docs/release-workflow.md | 4 +- package.json | 1 + pnpm-workspace.yaml | 30 ++-- scripts/github_api.mts | 136 ++++++++++++++++++ scripts/pr_to_changelog.mts | 233 +++++++++++++++++++++++++++++++ 7 files changed, 400 insertions(+), 21 deletions(-) create mode 100644 scripts/github_api.mts create mode 100644 scripts/pr_to_changelog.mts diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index c374ea3e..c91ff2ee 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,10 +1,15 @@ -Describe why and what has been changed by this pullrequest, feel free to also ask for input on decisions you are unsure about etc. -The changelog lines below will be copied directly from merged pull requests to the changelog on new releases. Changes are expected to preferrentially link to the associated issues. If there are no relevant issues, link to the pull request itself. + + +### Changelog ``` ### Features -- Added doodad to web interface. [#999](https://github.com/clusterio/clusterio/issues/999) +- Added doodad to web interface. #999 ### Changes - ... @@ -12,6 +17,6 @@ The changelog lines below will be copied directly from merged pull requests to t ### Fixes - ... -### Breaking changes +### Breaking Changes - ... ``` diff --git a/docs/contributing.md b/docs/contributing.md index 6a5fe1da..a35de4fc 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -137,6 +137,8 @@ You can also pass the `--set-upstream` (or `-u`) flag to make `git push` do the After your feature branch have been merged you should consider it final. If there are more changes you want to have merged or mistakes you need to fix you must start a new feature branch and submit a new pull request. +Pull request descriptions should contain a `### Changelog` section with additions for the changelog contained inside a code block, see the [CHANGELOG.md](/CHANGELOG.md) file for the formatting of changelog entries and possible sections to use. +If there are no changelog entries to add, remove the code block and start the changelog section with the text `None.`, followed by an optional reason for why it is not is needed. ### Editing your Feature Branch diff --git a/docs/release-workflow.md b/docs/release-workflow.md index 8c0adf44..691afec2 100644 --- a/docs/release-workflow.md +++ b/docs/release-workflow.md @@ -5,7 +5,9 @@ - CHANGELOG.md in the repository root - The changelog.txt file in mod directories -Pull requests will usually have suggestions for changelog entries. +Pull requests will usually have suggestions for changelog entries. These can be automatically parsed and formatted by running `node scripts/pr_to_changelog.mts`, though you will have to manually sort through which should go in the changelog in the repository root and which should go in the changelogs in mod directories. + +If the script fails due to GitHub's 60/hour public access rate limit, you can set the `GH_TOKEN` environment variable to a fine-grained personal access token which has a much higher rate limits. ## Packages diff --git a/package.json b/package.json index 6b2eb80e..023ee107 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@eslint/js": "^9.32.0", "@stylistic/eslint-plugin": "^5.2.2", "@swc/core": "^1.13.3", + "@types/node": "catalog:", "@typescript-eslint/eslint-plugin": "^8.38.0", "@typescript-eslint/parser": "^8.38.0", "eslint": "^9.32.0", diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 11bbe69b..fc8d08c6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -6,33 +6,33 @@ packages: - "test/" catalog: - "@types/node": ^20.17.19 - "@types/fs-extra": ^11.0.1 - "@types/express": ^4.17.17 - "@types/finalhandler": ^1.2.0 - "@types/jsonwebtoken": ^9.0.9 - "@types/set-blocking": ^2.0.0 - "@types/react": ^18.3.18 - "@types/ws": ^8.5.14 "@sinclair/typebox": ^0.30.4 "@swc/core": ^1.13.3 + "@types/express": ^4.17.17 + "@types/finalhandler": ^1.2.0 + "@types/fs-extra": ^11.0.1 + "@types/jsonwebtoken": ^9.0.9 + "@types/node": ^20.17.19 + "@types/react": ^18.3.18 + "@types/set-blocking": ^2.0.0 + "@types/ws": ^8.5.14 "antd": ^5.24.2 + "chalk": ^4.1.2 + "express": ^5.1.0 + "fs-extra": ^8.1.0 + "jsonwebtoken": ^9.0.2 + "jszip": ^3.10.1 "react": ^18.2.0 "react-dom": ^18.2.0 "react-router-dom": ^6.29.0 + "set-blocking": ^2.0.0 "typescript": ^5.7.3 "webpack": ^5.98.0 "webpack-cli": ^5.1.4 "webpack-merge": ^5.9.0 - "fs-extra": ^8.1.0 - "express": ^5.1.0 "winston": ^3.17.0 - "winston-transport": ^4.9.0 "winston-daily-rotate-file": ^4.7.1 - "jsonwebtoken": ^9.0.2 - "set-blocking": ^2.0.0 - "jszip": ^3.10.1 - "chalk": ^4.1.2 + "winston-transport": ^4.9.0 "ws": ^8.18.1 catalogMode: strict diff --git a/scripts/github_api.mts b/scripts/github_api.mts new file mode 100644 index 00000000..3b4f16ea --- /dev/null +++ b/scripts/github_api.mts @@ -0,0 +1,136 @@ +export async function githubFetch( + path: string, + query: Record= {}, + init: RequestInit & { headers?: Record } = {}, +) { + const url = new URL("https://api.github.com/"); + url.pathname = path; + for (const [name, value] of Object.entries(query)) { + url.searchParams.set(name, value); + } + init = { + ...init, + headers: { + ...init.headers as Record ?? {}, + "Accept": "application/vnd.github+json", + } + }; + if (process.env.GH_TOKEN) { + init.headers!["Authorization"] = `Bearer ${process.env.GH_TOKEN}`; + } + console.log("Fetching", url.href); + const response = await fetch(url, init); + if (!response.ok) { + console.log(response.headers); + throw new Error(`GitHub replied with ${response.status} ${response.statusText}: ${await response.text()}`); + } + if (!response.headers.get("Content-Type")?.startsWith("application/json")) { + throw new Error( + `GitHub replied with ${response.status} ${response.statusText}: ${response.headers.get("Content-Type")}` + ); + } + return response; +} + +export async function githubFetchJson( + path: string, + query: Record = {}, + init: RequestInit & { headers?: Record } = {}, +): Promise { + const response = await githubFetch(path, query, init); + return await response.json(); +} + +export async function githubFetchJsonPaginated( + path: string, + query: Record = {}, + init: RequestInit & { headers?: Record } = {}, + shouldContinue: (result: T[]) => boolean = () => true, +): Promise { + const data: T[] = []; + let pagesRemaining = true; + query = { + ...query, + per_page: "100", + }; + while (pagesRemaining) { + const response = await githubFetch(path, query, init); + const responseItems = await response.json(); + data.push(...responseItems); + const linkHeader = response.headers.get("Link"); + pagesRemaining = linkHeader?.includes(`rel="next"`) ?? false; + if (pagesRemaining) { + pagesRemaining = shouldContinue(responseItems); + } + if (pagesRemaining) { + query = { + ...query, + page: String((query.page ? (Number(query.page)) : 1) + 1), + } + } + } + return data; +} + +// Minimal types that include only what is used and useful for our scripts. See +// https://docs.github.com/en/rest for complete reference. +export interface PullRequest { + number: number, + url: string, + html_url: string, + state: string, + title: string, + body: string | null, + created_at: string, + updated_at: string, + closed_at: string | null, + merged_at: string | null, + merge_commit_sha: string | null, +} + +export interface Issue { + number: number, + title: string, + url: string, + html_url: string, + state: string, + body?: string | null, + pull_request?: { + url: string, + html_url: string, + merged_at?: string | null, + }, +} + +export interface Release { + tag_name: string, + target_commitish: string, + name: string | null, + body: string | null, + created_at: string, + published_at: string | null, + updated_at?: string | null, +} + +export interface Commit { + sha: string, + commit: { + message: string, + }, + author: null | {} | { + name?: null | string, + login: string, + type: string, + }, + parents: { + sha: string, + }[], +} + +export interface Reference { + ref: string, + object: { + type: string, + sha: string, + }, +} diff --git a/scripts/pr_to_changelog.mts b/scripts/pr_to_changelog.mts new file mode 100644 index 00000000..d974937f --- /dev/null +++ b/scripts/pr_to_changelog.mts @@ -0,0 +1,233 @@ +import { + type Release, + type Issue, + githubFetchJson, + githubFetchJsonPaginated, +} from "./github_api.mts"; + +type Changelog = Record; + +const changelogSections = [ + "Major Features", + "Features", + "Fixes", + "Changes", + "Breaking Changes", + "Meta", +]; + +// Generic result codes +const success = 0; +const failure = 1; + +type ParserState = { + pos: number, + lines: string[], +}; + +function createParser(lines: string[]): ParserState { + return { pos: 0, lines }; +} + +function atEnd(parser: ParserState) { + return parser.pos >= parser.lines.length; +} + +function currentLine(parser: ParserState) { + if (atEnd(parser)) { + throw new Error("Attempt to access line past end"); + } + return parser.lines[parser.pos]; +} + +function skipToLine( + parser: ParserState, + pattern: RegExp +) { + for (; parser.pos < parser.lines.length; parser.pos++) { + if (pattern.test(currentLine(parser))) { + return success; + } + } + return failure; +} + +function skipEmptyLines(parser: ParserState) { + while (parser.pos < parser.lines.length && /^ *$/.test(currentLine(parser))) { + parser.pos += 1; + } +} + +function extractPullRequestChangelog( + parser: ParserState, +): + | [typeof success, string[]] + | [typeof failure, undefined, string] +{ + const headerFail = skipToLine(parser, /^###? Changelog *$/); + if (headerFail) { + return [failure, , "Header not found"]; + } + parser.pos += 1; + skipEmptyLines(parser); + if (!atEnd(parser) && /^None/i.test(currentLine(parser))) { + parser.pos += 1; + return [success, []]; + } + const blockStartFail = skipToLine(parser, /```/); + if (blockStartFail) { + return [failure, , "Code block not found after header"]; + } + parser.pos += 1 + const blockStartPos = parser.pos; + const blockEndFail = skipToLine(parser, /```/); + if (blockEndFail) { + return [failure, , "End of code block not found after header"]; + } + return [success, parser.lines.slice(blockStartPos, parser.pos)]; +} + +function parseChangelog(parser: ParserState, pr: Issue, issues: Issue[]): [Changelog, string[]] { + let section: string | undefined; + let changelog: Changelog = Object.create(null); + let warnings: string[] = []; + while (!atEnd(parser)) { + const line = currentLine(parser); + parser.pos += 1; + if (line.trim() === "") { + continue; + } + if (line.startsWith("###")) { + section = line.slice(4).trim(); + if (!changelogSections.includes(section)) { + warnings.push( + `Changelog section ${section} on line ${parser.pos} is not one of ${changelogSections.join(", ")}` + ); + } + changelog[section] = changelog[section] ?? []; + continue; + } + if (!section) { + warnings.push(`Unexpected content "${line}" outside of a changelog section on line ${parser.pos}`); + continue; + } + if (line.startsWith("- ")) { + const entry = line + .slice(2) + .trim() + .replace(/\[#(\d+)\]\(https:\/\/github.com\/clusterio\/clusterio\/(issues|pull)\/\d+\)/g, "#$1"); + let hasRef = false + for (const ref of entry.matchAll(/#(\d+)/g)) { + hasRef = true; + const id = Number.parseInt(ref[1], 10); + const refIssue = issues.find(issue => issue.number === id); + if (!refIssue) { + warnings.push( + `Issue reference ${ref[0]} on line ${parser.pos} does not exist ` + + "or has not been updated since the last release" + ); + } else if (refIssue.pull_request && refIssue.number !== pr.number) { + warnings.push( + `Pull request reference ${ref[0]} on line ${parser.pos} is to a ` + + "different Pull request than this one." + ); + } + } + if (!hasRef) { + warnings.push(`Changelog entry on line ${parser.pos} does not reference an issue`); + } + changelog[section].push(entry); + continue; + } + warnings.push(`Unexpected content "${line}" on line ${parser.pos}`); + } + return [changelog, warnings]; +} + +function changelogFromPullRequests(pullRequests: Issue[], issues: Issue[]) { + const mergedChangelog: Changelog = Object.create(null); + for (const pr of pullRequests) { + const [ + changelogFail, + changelogLines, + changelogFailReason + ] = extractPullRequestChangelog(createParser(pr.body!.split(/\r?\n/))); + if (changelogFail) { + console.error(`Failed to extract changelog in ${pr.html_url}: ${changelogFailReason}`) + continue; + } + const [changelog, warnings] = parseChangelog(createParser(changelogLines), pr, issues); + for (const warning of warnings) { + console.warn(`Warning in changelog for ${pr.html_url}: ${warning}`) + } + for (const [section, items] of Object.entries(changelog)) { + mergedChangelog[section] = [...mergedChangelog[section] ?? [], ...items]; + } + } + return mergedChangelog; +} + +async function fetchLastRelease() { + // Assumes releases are ordered by their creation or tag time in descending order. + const releases = await githubFetchJson("/repos/clusterio/clusterio/releases"); + const latest = releases.find(r => r.target_commitish === "master"); + if (!latest) { + throw new Error("Unable to find latest release on master"); + } + return latest; +} + +async function fetchIssuesUpdatedSince(since: string) { + return await githubFetchJsonPaginated( + "/repos/clusterio/clusterio/issues", + { state: "all", since, sort: "updated", direction: "asc" }, + ); +} + +function printMarkdown(changelog: Changelog, issues: Issue[], refText: (issue: Issue) => string) { + const sections = new Set([...changelogSections, ...Object.keys(changelog)]); + for (const section of sections) { + if (section in changelog) { + console.log(`### ${section}`); + const entries = changelog[section]; + console.log(entries.map(text => ( + `- ${text.replaceAll(/#(\d+)/g, (ref, idAsText) => { + const id = Number.parseInt(idAsText, 10); + if (!Number.isFinite(id)) { + return ref; + } + const issue = issues.find(issue => issue.number === id); + if (!issue) { + return ref; + } + return refText(issue); + })}` + )).join("\n")); + console.log(); + } + } +} + +async function main() { + const lastRelease = await fetchLastRelease(); + const issues = await fetchIssuesUpdatedSince(lastRelease.created_at); + + const pullRequests = issues.filter(issue => ( + issue.pull_request + && issue.pull_request.merged_at + && issue.pull_request.merged_at > lastRelease.created_at + )); + + const changelog = changelogFromPullRequests(pullRequests, issues); + + console.log(); + console.log("=== Github markdown ==="); + printMarkdown(changelog, issues, issue => `clusterio/clusterio#${issue.number}`); + console.log(); + console.log("=== Discord markdown ==="); + printMarkdown(changelog, issues, issue => `[#${issue.number}](<${issue.html_url}>`); +} + +if (import.meta.main) { + main().catch(console.error); +} From f4ee2005e3813c515fda0c56ab3a8106cf202b5f Mon Sep 17 00:00:00 2001 From: Hornwitser Date: Mon, 16 Mar 2026 13:38:41 +0100 Subject: [PATCH 36/36] Fix bad changelog entries for alpha 23 [skip ci] (#873) --- CHANGELOG.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62e82d2f..22515dcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,29 +43,29 @@ Many thanks to the following for contributing to this release: - Added config lock files to prevent process duplication. [#815](https://github.com/clusterio/clusterio/issues/815) - Added process and system start times to system metrics. [#826](https://github.com/clusterio/clusterio/issues/826) - Added icons to status tags for host and instances. [#827](https://github.com/clusterio/clusterio/issues/827) -- Extended export-data to produce separate spritesheets for recipes, signals, technologies, planets, qualities, entities, and static UI icons. [#838](https://github.com/clusterio/clusterio/pull/838) +- Extended export-data to produce icons for recipes, signals, technologies, planets, qualities and entities. [#838](https://github.com/clusterio/clusterio/pull/838) - Added Icon Reference page to web interface at `/icon-reference` for browsing all exported icons. [#838](https://github.com/clusterio/clusterio/pull/838) - Added visual flash to RCON result box to confirm command execution, even when the output is identical to the previous result. [#793](https://github.com/clusterio/clusterio/issues/793) ### Changes - Login screen now displays the cluster name. [#818](https://github.com/clusterio/clusterio/issues/818) -- Instance versions can now be set using Major.Minor without specifing a patch version. [#820](https://github.com/clusterio/clusterio/issues/820) -- Versions for factorio and modpacks can now be selected using a drop down menu. [#820](https://github.com/clusterio/clusterio/issues/820) +- Instance versions can now be set using Major.Minor without specifying a patch version. [#820](https://github.com/clusterio/clusterio/issues/820) +- Versions for factorio and mod packs can now be selected using a drop down menu. [#820](https://github.com/clusterio/clusterio/issues/820) - The controller will now refuse to start if no users were loaded from file. [#814](https://github.com/clusterio/clusterio/issues/814) - Improved wording and display of the generate host token modal. [#828](https://github.com/clusterio/clusterio/issues/828) -- Installations where muliple conflicting clusterio/lib packages have been installed will now fail with an error [#863](https://github.com/clusterio/clusterio/issues/863). +- Installations where multiple conflicting clusterio/lib packages have been installed will now fail with an error [#863](https://github.com/clusterio/clusterio/issues/863). - Export completion now logs per-category icon counts. [#838](https://github.com/clusterio/clusterio/pull/838) ### Fixes -- Removed debug log statement from equipment grid serialization. [#XXX](https://github.com/clusterio/clusterio/pull/XXX) -- Fix that the documentation had a dead link to details of the message passing protocol (#812) +- Removed debug log statement from equipment grid serialization. [#807](https://github.com/clusterio/clusterio/pull/807) +- Fix that the documentation had a dead link to details of the message passing protocol [#812](https://github.com/clusterio/clusterio/issues/812) - Fixed case where achievements were not disabled. [#817](https://github.com/clusterio/clusterio/pull/817) -- Fix headless download on linux install. [#815](https://github.com/clusterio/clusterio/issues/816) -- Fix option `--no-download-headless` being ignored. [#815](https://github.com/clusterio/clusterio/issues/816) +- Fix headless download on linux install. [#816](https://github.com/clusterio/clusterio/issues/816) +- Fix option `--no-download-headless` being ignored. [#816](https://github.com/clusterio/clusterio/issues/816) - Fixed action buttons overflowing on narrow screens across the web interface. [#810](https://github.com/clusterio/clusterio/issues/810) -- Fixed player logistics failing to load on servers without unlocked logistics networks. [#1234](https://github.com/clusterio/clusterio/issues/1234) +- Fixed player logistics failing to load on servers without unlocked logistics networks. [#806](https://github.com/clusterio/clusterio/issues/806) - Fixed mod dependency parser breaking with inconsistent spaces. [#834](https://github.com/clusterio/clusterio/issues/834) - No longer attempts to sync inventory in single player. [#835](https://github.com/clusterio/clusterio/issues/835) - Fixed tests failing when the configured factorio_directory contains stuff other than Factorio installs [#540](https://github.com/clusterio/clusterio/issues/540).