From 19569aefa4c87e2ca939f0e0acb6b159f1ec91c7 Mon Sep 17 00:00:00 2001 From: bbassie <17990055+bbassie@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:42:41 +0000 Subject: [PATCH] Measure transfer_min_speed in KiB/s and raise it to 64 The guard measures a proxy stream on completed writes to the receiver. The OS buffers a couple of MiB per connection and only takes more once a large part has been sent, so a slow receiver can go a long time with no write completing. At the old default of 30 KiB per 30 s, downloads well above that could be closed, and the slow receiver test failed every run on Linux. controller.transfer_min_progress (KiB per 30 s) is replaced by controller.transfer_min_speed (KiB/s), defaulting to 64. At that speed the measured 1.6 MB bursts come within the 30 second check. Configs on the old default move to the new one, other values are converted. The slow receiver test checks every 3 s instead of 500 ms, as with a 500 ms check it fails at any minimum. Co-Authored-By: Claude Opus 5.5 --- docs/configuration.md | 10 +++-- packages/controller/src/routes.ts | 39 +++++++++-------- packages/lib/src/config/definitions.ts | 18 +++++--- test/controller/routes.js | 60 +++++++++++++++----------- test/lib/config/migrations.js | 10 +++++ test/mock.js | 2 +- 6 files changed, 85 insertions(+), 54 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 0fa69826..325f65fb 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -241,17 +241,19 @@ Must be greater than 0. Defaults to 15. -### controller.transfer_min_progress +### controller.transfer_min_speed -Amount in KiB a request to the controller has to move every 30 seconds, requests moving less are closed. +Speed in KiB/s a request to the controller has to move at, measured over 30 seconds, requests moving slower are closed. This closes uploads and downloads that have stalled or are sent too slowly to ever finish, while there is no limit on how long a transfer making progress can take. It applies to saves, mods and exports uploaded to the controller, proxy streams through it, and the body of any other request. Time spent waiting for a proxy stream to start is not counted, see controller.proxy_stream_timeout. -Data is passed on in chunks of up to 64 KiB and progress is counted per chunk, so a proxy stream moving less than about that in 30 seconds may also be closed. +Setting it lower than the default is not recommended. +Downloads through a proxy stream are measured on what the controller has handed to the OS. +The OS can buffer a couple of MiB per connection and only accepts more once a large part of that has been sent, so at lower values a download that keeps moving can go 30 seconds without the controller passing anything on and be closed. Set to 0 for no limit. Must be 0 or greater. -Defaults to 30. +Defaults to 64. ### controller.factorio_username diff --git a/packages/controller/src/routes.ts b/packages/controller/src/routes.ts index 48348726..6670e4de 100644 --- a/packages/controller/src/routes.ts +++ b/packages/controller/src/routes.ts @@ -419,7 +419,7 @@ export interface ProxyStream { timeout: ReturnType; closed: boolean; /** Bytes the receiver has taken, set once the stream is flowing */ - progress?: () => number; + bytesTaken?: () => number; } export async function createProxyStream(app: Application): Promise { @@ -473,8 +473,8 @@ const answeredUploadDrainMs = 1000; * so the connection is closed if the body hasn't ended shortly after the * response. Closing it right away could make the client lose the response. * - * The connection is also closed if the request moved less than - * controller.transfer_min_progress during a check interval, so that a + * The connection is also closed if the request moved slower than + * controller.transfer_min_speed during a check interval, so that a * request which stalled or is trickled in does not stay open forever while * a slow one can take the time it needs. For a request body this is what * was received of it. For both ends of a proxy stream it is what the @@ -501,33 +501,34 @@ function guardRequest(checkIntervalMs: number) { }); }); - const received = () => socket.bytesRead; - let measure: (() => number) | undefined = received; - let last = received(); + const bytesReceived = () => socket.bytesRead; + let measuredBytes: (() => number) | undefined = bytesReceived; + let lastBytes = bytesReceived(); const check = setInterval(() => { const stream: ProxyStream | undefined = res.locals.proxyStream; - let current: (() => number) | undefined; + let currentBytes: (() => number) | undefined; if (stream && !stream.closed) { - current = stream.progress; + currentBytes = stream.bytesTaken; } else if (!req.complete) { - current = received; + currentBytes = bytesReceived; } - const minProgress = req.app.locals.controller.config.get("controller.transfer_min_progress"); - if (current !== measure || !current || !minProgress) { + const minSpeedKiBps = req.app.locals.controller.config.get("controller.transfer_min_speed"); + if (currentBytes !== measuredBytes || !currentBytes || !minSpeedKiBps) { // What to measure changed, start over from here. - measure = current; - last = current ? current() : 0; + measuredBytes = currentBytes; + lastBytes = currentBytes ? currentBytes() : 0; return; } - const progress = current(); - if (progress - last < minProgress * 1024) { + const bytes = currentBytes(); + const intervalS = checkIntervalMs / 1000; + if (bytes - lastBytes < minSpeedKiBps * 1024 * intervalS) { logger.warn( - `Closing ${req.method} ${req.path}, it moved less than ${minProgress} KiB ` + - `in ${checkIntervalMs / 1000} seconds` + `Closing ${req.method} ${req.path}, it moved slower than ${minSpeedKiBps} KiB/s ` + + `over ${intervalS} seconds` ); socket.destroy(); } - last = progress; + lastBytes = bytes; }, checkIntervalMs); res.once("close", () => clearInterval(check)); next(); @@ -567,7 +568,7 @@ function startStream(res: Response, stream: ProxyStream) { source.pipe(res); stream.flowing = true; // Counts completed writes, so data waiting in a buffer is not progress. - stream.progress = () => socket.bytesWritten - socket.writableLength; + stream.bytesTaken = () => socket.bytesWritten - socket.writableLength; // Pipe does not pass on a source that was cut short, the receiver would wait forever. const onSourceClose = () => { if (!res.writableEnded) { diff --git a/packages/lib/src/config/definitions.ts b/packages/lib/src/config/definitions.ts index 86dd1b84..299ea628 100644 --- a/packages/lib/src/config/definitions.ts +++ b/packages/lib/src/config/definitions.ts @@ -31,7 +31,7 @@ export interface ControllerConfigFields { "controller.max_subscription_filters": number; "controller.system_metrics_interval": number; "controller.proxy_stream_timeout": number; - "controller.transfer_min_progress": number; + "controller.transfer_min_speed": number; "controller.factorio_username": string | null; "controller.factorio_token": string | null; "controller.share_factorio_credentials_with_hosts": boolean; @@ -63,6 +63,14 @@ export class ControllerConfig extends classes.Config { config["controller.public_url"] = config["controller.external_address"]; delete config["controller.external_address"]; } + if (config.hasOwnProperty("controller.transfer_min_progress")) { + // Was KiB per 30 s, the old default is left out so the new one applies + const kib = config["controller.transfer_min_progress"]; + if (kib !== 30) { + config["controller.transfer_min_speed"] = typeof kib === "number" ? kib / 30 : kib; + } + delete config["controller.transfer_min_progress"]; + } return config; } @@ -224,11 +232,11 @@ export class ControllerConfig extends classes.Config { initialValue: 15, validator: validators.greaterThanZero, }, - "controller.transfer_min_progress": { - title: "Transfer Minimum Progress", - description: "KiB an upload or download has to move every 30 seconds to not be closed, 0 for no limit.", + "controller.transfer_min_speed": { + title: "Transfer Minimum Speed", + description: "KiB/s an upload or download has to move at over 30 seconds to not be closed, 0 for no limit.", type: "number", - initialValue: 30, + initialValue: 64, validator: validators.greaterThanEqualZero, }, "controller.session_timeout": { diff --git a/test/controller/routes.js b/test/controller/routes.js index 32669064..e868b748 100644 --- a/test/controller/routes.js +++ b/test/controller/routes.js @@ -179,7 +179,7 @@ describe("controller/src/routes", function() { return status; } it("should close a transfer that stops making progress", async function() { - controller.mockConfigEntries.set("controller.transfer_min_progress", 1); + controller.mockConfigEntries.set("controller.transfer_min_speed", 2); const stream = await routes.createProxyStream(controller.app); const request = http.request( `${endpoint}/${stream.id}`, { method: "PUT", headers: { "Content-Length": 1024 * 1024 } } @@ -196,7 +196,7 @@ describe("controller/src/routes", function() { }); it("should close a transfer that is trickled in", async function() { this.timeout(10000); - controller.mockConfigEntries.set("controller.transfer_min_progress", 4); + controller.mockConfigEntries.set("controller.transfer_min_speed", 8); const stream = await routes.createProxyStream(controller.app); const start = Date.now(); const upload = putPaced(stream.id, 200, 50); @@ -208,7 +208,7 @@ describe("controller/src/routes", function() { }); it("should not close a slow transfer that makes progress", async function() { this.timeout(10000); - controller.mockConfigEntries.set("controller.transfer_min_progress", 1); + controller.mockConfigEntries.set("controller.transfer_min_speed", 2); const stream = await routes.createProxyStream(controller.app); const upload = putPaced(stream.id, 1024, 20); const download = fetch(`${endpoint}/${stream.id}`).then(response => response.arrayBuffer()); @@ -217,29 +217,39 @@ describe("controller/src/routes", function() { }); it("should not close a transfer held back by a slow receiver", async function() { this.timeout(20000); - controller.mockConfigEntries.set("controller.transfer_min_progress", 16); - const size = 4 * 1024 * 1024; - const stream = await routes.createProxyStream(controller.app); - const upload = fetch(`${endpoint}/${stream.id}`, { method: "PUT", body: Buffer.alloc(size) }); - const received = await new Promise((resolve, reject) => { - http.get(`${endpoint}/${stream.id}`, response => { - let bytes = 0; - response.on("data", chunk => { - bytes += chunk.length; - response.pause(); - setTimeout(() => response.resume(), 50); - }); - response.on("end", () => resolve(bytes)); - response.on("error", reject); - }).on("error", reject); - }); - assert.equal(received, size); - assert.equal((await upload).status, 200); + // The OS takes over a second to free buffered data at this speed, longer than the usual 500 ms checks. + const slowController = new mock.MockController(); + routes.addRouteHandlers(slowController.app, 3000); + const slowServer = http.createServer(routes.httpServerOptions, slowController.app); + slowServer.listen(0, "localhost"); + await events.once(slowServer, "listening"); + const slowEndpoint = `http://localhost:${slowServer.address().port}/api/stream`; + try { + const size = 8 * 1024 * 1024; + const stream = await routes.createProxyStream(slowController.app); + const upload = fetch(`${slowEndpoint}/${stream.id}`, { method: "PUT", body: Buffer.alloc(size) }); + const received = await new Promise((resolve, reject) => { + http.get(`${slowEndpoint}/${stream.id}`, response => { + let bytes = 0; + response.on("data", chunk => { + bytes += chunk.length; + response.pause(); + setTimeout(() => response.resume(), 50); + }); + response.on("end", () => resolve(bytes)); + response.on("error", reject); + }).on("error", reject); + }); + assert.equal(received, size); + assert.equal((await upload).status, 200); + } finally { + slowServer.close(); + } }); it("should not count the time waiting for the stream to start", async function() { this.timeout(10000); controller.mockConfigEntries.set("controller.proxy_stream_timeout", 3); - controller.mockConfigEntries.set("controller.transfer_min_progress", 1); + controller.mockConfigEntries.set("controller.transfer_min_speed", 2); const stream = await routes.createProxyStream(controller.app); const download = fetch(`${endpoint}/${stream.id}`); await lib.wait(1500); @@ -1257,7 +1267,7 @@ describe("controller/src/routes", function() { }); it("should close an upload that is trickled in", async function() { this.timeout(10000); - controller.mockConfigEntries.set("controller.transfer_min_progress", 4); + controller.mockConfigEntries.set("controller.transfer_min_speed", 8); const modsDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-mods-")); try { controller.mockConfigEntries.set("controller.mods_directory", modsDirectory); @@ -1277,7 +1287,7 @@ describe("controller/src/routes", function() { }); it("should close a request to another route that is trickled in", async function() { this.timeout(10000); - controller.mockConfigEntries.set("controller.transfer_min_progress", 4); + controller.mockConfigEntries.set("controller.transfer_min_speed", 8); controller.app.post("/test", express.raw({ type: "*/*", limit: "2mb" }), (req, res) => res.sendStatus(200)); const result = await trickle(`http://localhost:${port}/test`, { "Content-Type": "text/plain" }); assert.equal(result.status, undefined); @@ -1285,7 +1295,7 @@ describe("controller/src/routes", function() { }); it("should not close a trickled in request when the limit is 0", async function() { this.timeout(10000); - controller.mockConfigEntries.set("controller.transfer_min_progress", 0); + controller.mockConfigEntries.set("controller.transfer_min_speed", 0); controller.app.post("/test", express.raw({ type: "*/*", limit: "2mb" }), (req, res) => res.sendStatus(200)); const result = await trickle(`http://localhost:${port}/test`, { "Content-Type": "text/plain" }); assert(result.duration >= 5000, `closed after ${result.duration} ms`); diff --git a/test/lib/config/migrations.js b/test/lib/config/migrations.js index 72447983..e50e5e47 100644 --- a/test/lib/config/migrations.js +++ b/test/lib/config/migrations.js @@ -15,6 +15,16 @@ describe("lib/config/definitions/migrations", function() { assert.equal(config.fields["controller.external_address"], undefined, "external_address was not removed"); }); + it("should migrate 'transfer_min_progress' to 'transfer_min_speed'", function() { // 2.1.2 + const migrate = value => lib.ControllerConfig.fromJSON({ + "controller.transfer_min_progress": value, + }, "controller"); + assert.equal(migrate(30).get("controller.transfer_min_speed"), 64, "old default was kept"); + assert.equal(migrate(0).get("controller.transfer_min_speed"), 0, "no limit was not kept"); + assert.equal(migrate(3000).get("controller.transfer_min_speed"), 100, "value was not converted"); + assert.equal(migrate(30).fields["controller.transfer_min_progress"], undefined, + "transfer_min_progress was not removed"); + }); }); describe("Host Config", function() { diff --git a/test/mock.js b/test/mock.js index 32b5270b..e1be239e 100644 --- a/test/mock.js +++ b/test/mock.js @@ -175,7 +175,7 @@ export class MockController { ["controller.public_url", "test"], ["controller.auth_secret", "TestSecretDoNotUse"], ["controller.proxy_stream_timeout", 1], - ["controller.transfer_min_progress", 30], + ["controller.transfer_min_speed", 64], ["controller.metrics_timeout", 1], ["controller.host_pairing_timeout", 3600], ["controller.host_pairing_max_pending", 50],