Merge pull request #1237 from bbassie/fix/stream-slow-receiver-test

Measure transfer_min_speed in KiB/s and raise it to 64
This commit is contained in:
Cooldude2606 authored and GitHub committed 2026-11-20 22:33:08 +00:00
commit a49841f3fc
6 files changed
+154 -60

No files matched your search

+7 -4
View File
@@ -241,17 +241,20 @@ Must be greater than 0.
Defaults to 15. Defaults to 15.
### controller.transfer_min_progress ### controller.transfer_min_speed
Amount in KiB a request to the controller has to move every 30 seconds, requests moving less are closed. Speed in KiB/s at which a request to the controller is never closed for moving too slowly.
This closes uploads and downloads that have stalled or are sent too slowly to ever finish, while there is no limit on how long a transfer making progress can take. This closes uploads and downloads that have stalled or are sent too slowly to ever finish, while there is no limit on how long a transfer making progress can take.
It applies to saves, mods and exports uploaded to the controller, proxy streams through it, and the body of any other request. It applies to saves, mods and exports uploaded to the controller, proxy streams through it, and the body of any other request.
Time spent waiting for a proxy stream to start is not counted, see controller.proxy_stream_timeout. Time spent waiting for a proxy stream to start is not counted, see controller.proxy_stream_timeout.
Data is passed on in chunks of up to 64 KiB and progress is counted per chunk, so a proxy stream moving less than about that in 30 seconds may also be closed. Requests are checked every 60 seconds and must have moved this speed times 60 seconds, less 2 MiB, since the last check.
The 2 MiB covers the OS sending data in bursts, it can buffer a couple of MiB per connection and only accept more once a large part of that has gone out.
So at the default a request moving slower than about 30 KiB/s is closed, and one moving between that and 64 KiB/s may be.
Set below about 34 KiB/s, only requests that moved nothing in 60 seconds are closed.
Set to 0 for no limit. Set to 0 for no limit.
Must be 0 or greater. Must be 0 or greater.
Defaults to 30. Defaults to 64.
### controller.factorio_username ### controller.factorio_username
+34 -22
View File
@@ -419,7 +419,7 @@ export interface ProxyStream {
timeout: ReturnType<typeof setTimeout>; timeout: ReturnType<typeof setTimeout>;
closed: boolean; closed: boolean;
/** Bytes the receiver has taken, set once the stream is flowing */ /** Bytes the receiver has taken, set once the stream is flowing */
progress?: () => number; bytesTaken?: () => number;
} }
export async function createProxyStream(app: Application): Promise<ProxyStream> { export async function createProxyStream(app: Application): Promise<ProxyStream> {
@@ -473,18 +473,23 @@ const answeredUploadDrainMs = 1000;
* so the connection is closed if the body hasn't ended shortly after the * so the connection is closed if the body hasn't ended shortly after the
* response. Closing it right away could make the client lose the response. * response. Closing it right away could make the client lose the response.
* *
* The connection is also closed if the request moved less than * The connection is also closed if the request moved too little during a
* controller.transfer_min_progress during a check interval, so that a * check interval for controller.transfer_min_speed, so that a
* request which stalled or is trickled in does not stay open forever while * request which stalled or is trickled in does not stay open forever while
* a slow one can take the time it needs. For a request body this is what * a slow one can take the time it needs. For a request body this is what
* was received of it. For both ends of a proxy stream it is what the * was received of it. For both ends of a proxy stream it is what the
* receiver has taken, as the sender is held back when the receiver is the * receiver has taken, as the sender is held back when the receiver is the
* slow one. Time spent waiting for a proxy stream to start is not counted, * slow one. Time spent waiting for a proxy stream to start is not counted,
* controller.proxy_stream_timeout limits that. * controller.proxy_stream_timeout limits that.
*
* The OS hands data on in bursts, so a transfer at the minimum speed can
* move up to a burst less than that in a check. The burst allowance is
* taken off what a check requires so such a transfer is never closed.
* @param checkIntervalMs - Interval to check the progress of requests on. * @param checkIntervalMs - Interval to check the progress of requests on.
* @param burstAllowanceBytes - Bytes taken off what a check requires.
* @returns Express middleware. * @returns Express middleware.
*/ */
function guardRequest(checkIntervalMs: number) { function guardRequest(checkIntervalMs: number, burstAllowanceBytes: number) {
return function(req: Request, res: Response, next: any) { return function(req: Request, res: Response, next: any) {
const socket = req.socket; const socket = req.socket;
res.once("finish", () => { res.once("finish", () => {
@@ -501,33 +506,36 @@ function guardRequest(checkIntervalMs: number) {
}); });
}); });
const received = () => socket.bytesRead; const bytesReceived = () => socket.bytesRead;
let measure: (() => number) | undefined = received; let measuredBytes: (() => number) | undefined = bytesReceived;
let last = received(); let lastBytes = bytesReceived();
const check = setInterval(() => { const check = setInterval(() => {
const stream: ProxyStream | undefined = res.locals.proxyStream; const stream: ProxyStream | undefined = res.locals.proxyStream;
let current: (() => number) | undefined; let currentBytes: (() => number) | undefined;
if (stream && !stream.closed) { if (stream && !stream.closed) {
current = stream.progress; currentBytes = stream.bytesTaken;
} else if (!req.complete) { } else if (!req.complete) {
current = received; currentBytes = bytesReceived;
} }
const minProgress = req.app.locals.controller.config.get("controller.transfer_min_progress"); const minSpeedKiBps = req.app.locals.controller.config.get("controller.transfer_min_speed");
if (current !== measure || !current || !minProgress) { if (currentBytes !== measuredBytes || !currentBytes || !minSpeedKiBps) {
// What to measure changed, start over from here. // What to measure changed, start over from here.
measure = current; measuredBytes = currentBytes;
last = current ? current() : 0; lastBytes = currentBytes ? currentBytes() : 0;
return; return;
} }
const progress = current(); const bytes = currentBytes();
if (progress - last < minProgress * 1024) { const intervalS = checkIntervalMs / 1000;
// At least 1 so a transfer that moved nothing is closed at any speed setting.
const requiredBytes = Math.max(minSpeedKiBps * 1024 * intervalS - burstAllowanceBytes, 1);
if (bytes - lastBytes < requiredBytes) {
logger.warn( logger.warn(
`Closing ${req.method} ${req.path}, it moved less than ${minProgress} KiB ` + `Closing ${req.method} ${req.path}, it moved ${bytes - lastBytes} bytes in ${intervalS} seconds, ` +
`in ${checkIntervalMs / 1000} seconds` `less than the ${requiredBytes} required for ${minSpeedKiBps} KiB/s`
); );
socket.destroy(); socket.destroy();
} }
last = progress; lastBytes = bytes;
}, checkIntervalMs); }, checkIntervalMs);
res.once("close", () => clearInterval(check)); res.once("close", () => clearInterval(check));
next(); next();
@@ -567,7 +575,7 @@ function startStream(res: Response, stream: ProxyStream) {
source.pipe(res); source.pipe(res);
stream.flowing = true; stream.flowing = true;
// Counts completed writes, so data waiting in a buffer is not progress. // Counts completed writes, so data waiting in a buffer is not progress.
stream.progress = () => socket.bytesWritten - socket.writableLength; stream.bytesTaken = () => socket.bytesWritten - socket.writableLength;
// Pipe does not pass on a source that was cut short, the receiver would wait forever. // Pipe does not pass on a source that was cut short, the receiver would wait forever.
const onSourceClose = () => { const onSourceClose = () => {
if (!res.writableEnded) { if (!res.writableEnded) {
@@ -856,8 +864,12 @@ async function uploadMod(req: Request, res: Response) {
} }
export function addRouteHandlers(app: Application, progressCheckIntervalMs = 30_000) { export function addRouteHandlers(
app.use(guardRequest(progressCheckIntervalMs)); app: Application,
progressCheckIntervalMs = 60_000,
burstAllowanceBytes = 2 * 1024 * 1024,
) {
app.use(guardRequest(progressCheckIntervalMs, burstAllowanceBytes));
app.get("/metrics", app.get("/metrics",
validateUserToken, validateUserToken,
(req:Request, res:Response, next:any) => getMetrics(req, res).catch(next) (req:Request, res:Response, next:any) => getMetrics(req, res).catch(next)
+13 -5
View File
@@ -31,7 +31,7 @@ export interface ControllerConfigFields {
"controller.max_subscription_filters": number; "controller.max_subscription_filters": number;
"controller.system_metrics_interval": number; "controller.system_metrics_interval": number;
"controller.proxy_stream_timeout": number; "controller.proxy_stream_timeout": number;
"controller.transfer_min_progress": number; "controller.transfer_min_speed": number;
"controller.factorio_username": string | null; "controller.factorio_username": string | null;
"controller.factorio_token": string | null; "controller.factorio_token": string | null;
"controller.share_factorio_credentials_with_hosts": boolean; "controller.share_factorio_credentials_with_hosts": boolean;
@@ -63,6 +63,14 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
config["controller.public_url"] = config["controller.external_address"]; config["controller.public_url"] = config["controller.external_address"];
delete config["controller.external_address"]; delete config["controller.external_address"];
} }
if (config.hasOwnProperty("controller.transfer_min_progress")) {
// Was KiB per 30 s, the old default is left out so the new one applies
const kib = config["controller.transfer_min_progress"];
if (kib !== 30) {
config["controller.transfer_min_speed"] = typeof kib === "number" ? kib / 30 : kib;
}
delete config["controller.transfer_min_progress"];
}
return config; return config;
} }
@@ -224,11 +232,11 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
initialValue: 15, initialValue: 15,
validator: validators.greaterThanZero, validator: validators.greaterThanZero,
}, },
"controller.transfer_min_progress": { "controller.transfer_min_speed": {
title: "Transfer Minimum Progress", title: "Transfer Minimum Speed",
description: "KiB an upload or download has to move every 30 seconds to not be closed, 0 for no limit.", description: "KiB/s at which an upload or download is never closed for moving too slowly, 0 for no limit.",
type: "number", type: "number",
initialValue: 30, initialValue: 64,
validator: validators.greaterThanEqualZero, validator: validators.greaterThanEqualZero,
}, },
"controller.session_timeout": { "controller.session_timeout": {
+89 -28
View File
@@ -22,8 +22,8 @@ describe("controller/src/routes", function() {
let port; let port;
beforeEach(async function() { beforeEach(async function() {
controller = new mock.MockController(); controller = new mock.MockController();
// Check the progress of requests every 500 ms instead of every 30 seconds. // Check the progress of requests every 500 ms without a burst allowance.
routes.addRouteHandlers(controller.app, 500); routes.addRouteHandlers(controller.app, 500, 0);
server = http.createServer(routes.httpServerOptions, controller.app); server = http.createServer(routes.httpServerOptions, controller.app);
server.listen(0, "localhost"); server.listen(0, "localhost");
await events.once(server, "listening"); await events.once(server, "listening");
@@ -179,7 +179,7 @@ describe("controller/src/routes", function() {
return status; return status;
} }
it("should close a transfer that stops making progress", async function() { it("should close a transfer that stops making progress", async function() {
controller.mockConfigEntries.set("controller.transfer_min_progress", 1); controller.mockConfigEntries.set("controller.transfer_min_speed", 2);
const stream = await routes.createProxyStream(controller.app); const stream = await routes.createProxyStream(controller.app);
const request = http.request( const request = http.request(
`${endpoint}/${stream.id}`, { method: "PUT", headers: { "Content-Length": 1024 * 1024 } } `${endpoint}/${stream.id}`, { method: "PUT", headers: { "Content-Length": 1024 * 1024 } }
@@ -196,7 +196,7 @@ describe("controller/src/routes", function() {
}); });
it("should close a transfer that is trickled in", async function() { it("should close a transfer that is trickled in", async function() {
this.timeout(10000); this.timeout(10000);
controller.mockConfigEntries.set("controller.transfer_min_progress", 4); controller.mockConfigEntries.set("controller.transfer_min_speed", 8);
const stream = await routes.createProxyStream(controller.app); const stream = await routes.createProxyStream(controller.app);
const start = Date.now(); const start = Date.now();
const upload = putPaced(stream.id, 200, 50); const upload = putPaced(stream.id, 200, 50);
@@ -208,7 +208,7 @@ describe("controller/src/routes", function() {
}); });
it("should not close a slow transfer that makes progress", async function() { it("should not close a slow transfer that makes progress", async function() {
this.timeout(10000); this.timeout(10000);
controller.mockConfigEntries.set("controller.transfer_min_progress", 1); controller.mockConfigEntries.set("controller.transfer_min_speed", 2);
const stream = await routes.createProxyStream(controller.app); const stream = await routes.createProxyStream(controller.app);
const upload = putPaced(stream.id, 1024, 20); const upload = putPaced(stream.id, 1024, 20);
const download = fetch(`${endpoint}/${stream.id}`).then(response => response.arrayBuffer()); const download = fetch(`${endpoint}/${stream.id}`).then(response => response.arrayBuffer());
@@ -216,30 +216,41 @@ describe("controller/src/routes", function() {
assert.equal((await download).byteLength, 20 * 1024); assert.equal((await download).byteLength, 20 * 1024);
}); });
it("should not close a transfer held back by a slow receiver", async function() { it("should not close a transfer held back by a slow receiver", async function() {
this.timeout(20000); this.timeout(40000);
controller.mockConfigEntries.set("controller.transfer_min_progress", 16); // The receiver takes about 1.2 MB/s, which the OS hands on in bursts of over a MB.
const size = 4 * 1024 * 1024; const slowController = new mock.MockController();
const stream = await routes.createProxyStream(controller.app); slowController.mockConfigEntries.set("controller.transfer_min_speed", 1024);
const upload = fetch(`${endpoint}/${stream.id}`, { method: "PUT", body: Buffer.alloc(size) }); routes.addRouteHandlers(slowController.app, 3000);
const received = await new Promise((resolve, reject) => { const slowServer = http.createServer(routes.httpServerOptions, slowController.app);
http.get(`${endpoint}/${stream.id}`, response => { slowServer.listen(0, "localhost");
let bytes = 0; await events.once(slowServer, "listening");
response.on("data", chunk => { const slowEndpoint = `http://localhost:${slowServer.address().port}/api/stream`;
bytes += chunk.length; try {
response.pause(); const size = 16 * 1024 * 1024;
setTimeout(() => response.resume(), 50); const stream = await routes.createProxyStream(slowController.app);
}); const upload = fetch(`${slowEndpoint}/${stream.id}`, { method: "PUT", body: Buffer.alloc(size) });
response.on("end", () => resolve(bytes)); const received = await new Promise((resolve, reject) => {
response.on("error", reject); http.get(`${slowEndpoint}/${stream.id}`, response => {
}).on("error", reject); let bytes = 0;
}); response.on("data", chunk => {
assert.equal(received, size); bytes += chunk.length;
assert.equal((await upload).status, 200); response.pause();
setTimeout(() => response.resume(), 50);
});
response.on("end", () => resolve(bytes));
response.on("error", reject);
}).on("error", reject);
});
assert.equal(received, size);
assert.equal((await upload).status, 200);
} finally {
slowServer.close();
}
}); });
it("should not count the time waiting for the stream to start", async function() { it("should not count the time waiting for the stream to start", async function() {
this.timeout(10000); this.timeout(10000);
controller.mockConfigEntries.set("controller.proxy_stream_timeout", 3); controller.mockConfigEntries.set("controller.proxy_stream_timeout", 3);
controller.mockConfigEntries.set("controller.transfer_min_progress", 1); controller.mockConfigEntries.set("controller.transfer_min_speed", 2);
const stream = await routes.createProxyStream(controller.app); const stream = await routes.createProxyStream(controller.app);
const download = fetch(`${endpoint}/${stream.id}`); const download = fetch(`${endpoint}/${stream.id}`);
await lib.wait(1500); await lib.wait(1500);
@@ -1257,7 +1268,7 @@ describe("controller/src/routes", function() {
}); });
it("should close an upload that is trickled in", async function() { it("should close an upload that is trickled in", async function() {
this.timeout(10000); this.timeout(10000);
controller.mockConfigEntries.set("controller.transfer_min_progress", 4); controller.mockConfigEntries.set("controller.transfer_min_speed", 8);
const modsDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-mods-")); const modsDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "clusterio-mods-"));
try { try {
controller.mockConfigEntries.set("controller.mods_directory", modsDirectory); controller.mockConfigEntries.set("controller.mods_directory", modsDirectory);
@@ -1277,15 +1288,65 @@ describe("controller/src/routes", function() {
}); });
it("should close a request to another route that is trickled in", async function() { it("should close a request to another route that is trickled in", async function() {
this.timeout(10000); this.timeout(10000);
controller.mockConfigEntries.set("controller.transfer_min_progress", 4); controller.mockConfigEntries.set("controller.transfer_min_speed", 8);
controller.app.post("/test", express.raw({ type: "*/*", limit: "2mb" }), (req, res) => res.sendStatus(200)); controller.app.post("/test", express.raw({ type: "*/*", limit: "2mb" }), (req, res) => res.sendStatus(200));
const result = await trickle(`http://localhost:${port}/test`, { "Content-Type": "text/plain" }); const result = await trickle(`http://localhost:${port}/test`, { "Content-Type": "text/plain" });
assert.equal(result.status, undefined); assert.equal(result.status, undefined);
assert(result.duration < 2000, `closed after ${result.duration} ms`); assert(result.duration < 2000, `closed after ${result.duration} ms`);
}); });
describe("with a burst allowance", function() {
let burstServer, burstUrl;
beforeEach(async function() {
// Each check requires 256 KiB/s * 0.5 s - 64 KiB = 64 KiB.
const burstController = new mock.MockController();
burstController.mockConfigEntries.set("controller.transfer_min_speed", 256);
routes.addRouteHandlers(burstController.app, 500, 64 * 1024);
burstController.app.post("/test", express.raw({ type: "*/*", limit: "2mb" }), (req, res) => {
res.sendStatus(200);
});
burstServer = http.createServer(routes.httpServerOptions, burstController.app);
burstServer.listen(0, "localhost");
await events.once(burstServer, "listening");
burstUrl = `http://localhost:${burstServer.address().port}/test`;
});
afterEach(function() {
burstServer.close();
});
async function sendBursts(burstBytes) {
const bursts = 12;
const request = http.request(burstUrl, {
method: "POST", headers: { "Content-Type": "text/plain", "Content-Length": burstBytes * bursts },
});
let status;
request.on("response", response => {
status = response.statusCode;
response.resume();
});
request.on("error", () => {});
const closed = new Promise(resolve => request.on("close", resolve));
for (let i = 0; i < bursts && !request.closed; i++) {
request.write(Buffer.alloc(burstBytes));
await lib.wait(300);
}
if (!request.closed) {
request.end();
}
await closed;
return status;
}
it("should not close a request at the minimum speed arriving in bursts", async function() {
this.timeout(10000);
// 78 KiB every 300 ms is 260 KiB/s, a check can see only one burst.
assert.equal(await sendBursts(78 * 1024), 200);
});
it("should close a request below the minimum speed arriving in bursts", async function() {
this.timeout(10000);
assert.equal(await sendBursts(39 * 1024), undefined);
});
});
it("should not close a trickled in request when the limit is 0", async function() { it("should not close a trickled in request when the limit is 0", async function() {
this.timeout(10000); this.timeout(10000);
controller.mockConfigEntries.set("controller.transfer_min_progress", 0); controller.mockConfigEntries.set("controller.transfer_min_speed", 0);
controller.app.post("/test", express.raw({ type: "*/*", limit: "2mb" }), (req, res) => res.sendStatus(200)); controller.app.post("/test", express.raw({ type: "*/*", limit: "2mb" }), (req, res) => res.sendStatus(200));
const result = await trickle(`http://localhost:${port}/test`, { "Content-Type": "text/plain" }); const result = await trickle(`http://localhost:${port}/test`, { "Content-Type": "text/plain" });
assert(result.duration >= 5000, `closed after ${result.duration} ms`); assert(result.duration >= 5000, `closed after ${result.duration} ms`);
+10
View File
@@ -15,6 +15,16 @@ describe("lib/config/definitions/migrations", function() {
assert.equal(config.fields["controller.external_address"], undefined, assert.equal(config.fields["controller.external_address"], undefined,
"external_address was not removed"); "external_address was not removed");
}); });
it("should migrate 'transfer_min_progress' to 'transfer_min_speed'", function() { // 2.1.2
const migrate = value => lib.ControllerConfig.fromJSON({
"controller.transfer_min_progress": value,
}, "controller");
assert.equal(migrate(30).get("controller.transfer_min_speed"), 64, "old default was kept");
assert.equal(migrate(0).get("controller.transfer_min_speed"), 0, "no limit was not kept");
assert.equal(migrate(3000).get("controller.transfer_min_speed"), 100, "value was not converted");
assert.equal(migrate(30).fields["controller.transfer_min_progress"], undefined,
"transfer_min_progress was not removed");
});
}); });
describe("Host Config", function() { describe("Host Config", function() {
+1 -1
View File
@@ -175,7 +175,7 @@ export class MockController {
["controller.public_url", "test"], ["controller.public_url", "test"],
["controller.auth_secret", "TestSecretDoNotUse"], ["controller.auth_secret", "TestSecretDoNotUse"],
["controller.proxy_stream_timeout", 1], ["controller.proxy_stream_timeout", 1],
["controller.transfer_min_progress", 30], ["controller.transfer_min_speed", 64],
["controller.metrics_timeout", 1], ["controller.metrics_timeout", 1],
["controller.host_pairing_timeout", 3600], ["controller.host_pairing_timeout", 3600],
["controller.host_pairing_max_pending", 50], ["controller.host_pairing_max_pending", 50],