mirror of
https://github.com/clusterio/clusterio.git
synced 2026-11-21 08:03:13 +09:00
Take a burst allowance off what a transfer check requires
controller.transfer_min_speed is now the speed that is never closed. A check requires that speed over 60 s less 2 MiB, which covers the bursts the OS sends data in, so at the default of 64 KiB/s the cutoff is about 30 KiB/s. The requirement is at least 1 byte, so a transfer that moved nothing is still closed at low settings. The slow receiver test sends 16 MiB so that it spans real checks, and two upload tests with set bursts cover the allowance. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
42bc31e406
commit
2e273dc630
4 files changed
+81
-18
No files matched your search
@@ -243,13 +243,14 @@ Defaults to 15.
|
||||
|
||||
### controller.transfer_min_speed
|
||||
|
||||
Speed in KiB/s a request to the controller has to move at, measured over 60 seconds, requests moving slower are closed.
|
||||
Speed in KiB/s at which a request to the controller is never closed for moving too slowly.
|
||||
This closes uploads and downloads that have stalled or are sent too slowly to ever finish, while there is no limit on how long a transfer making progress can take.
|
||||
It applies to saves, mods and exports uploaded to the controller, proxy streams through it, and the body of any other request.
|
||||
Time spent waiting for a proxy stream to start is not counted, see controller.proxy_stream_timeout.
|
||||
Setting it lower than the default is not recommended.
|
||||
Downloads through a proxy stream are measured on what the controller has handed to the OS.
|
||||
The OS can buffer a couple of MiB per connection and only accepts more once a large part of that has been sent, so at lower values a download that keeps moving can go 60 seconds without the controller passing anything on and be closed.
|
||||
Requests are checked every 60 seconds and must have moved this speed times 60 seconds, less 2 MiB, since the last check.
|
||||
The 2 MiB covers the OS sending data in bursts, it can buffer a couple of MiB per connection and only accept more once a large part of that has gone out.
|
||||
So at the default a request moving slower than about 30 KiB/s is closed, and one moving between that and 64 KiB/s may be.
|
||||
Set below about 34 KiB/s, only requests that moved nothing in 60 seconds are closed.
|
||||
Set to 0 for no limit.
|
||||
Must be 0 or greater.
|
||||
|
||||
|
||||
@@ -473,18 +473,23 @@ const answeredUploadDrainMs = 1000;
|
||||
* so the connection is closed if the body hasn't ended shortly after the
|
||||
* response. Closing it right away could make the client lose the response.
|
||||
*
|
||||
* The connection is also closed if the request moved slower than
|
||||
* controller.transfer_min_speed during a check interval, so that a
|
||||
* The connection is also closed if the request moved too little during a
|
||||
* check interval for controller.transfer_min_speed, so that a
|
||||
* request which stalled or is trickled in does not stay open forever while
|
||||
* a slow one can take the time it needs. For a request body this is what
|
||||
* was received of it. For both ends of a proxy stream it is what the
|
||||
* receiver has taken, as the sender is held back when the receiver is the
|
||||
* slow one. Time spent waiting for a proxy stream to start is not counted,
|
||||
* controller.proxy_stream_timeout limits that.
|
||||
*
|
||||
* The OS hands data on in bursts, so a transfer at the minimum speed can
|
||||
* move up to a burst less than that in a check. The burst allowance is
|
||||
* taken off what a check requires so such a transfer is never closed.
|
||||
* @param checkIntervalMs - Interval to check the progress of requests on.
|
||||
* @param burstAllowanceBytes - Bytes taken off what a check requires.
|
||||
* @returns Express middleware.
|
||||
*/
|
||||
function guardRequest(checkIntervalMs: number) {
|
||||
function guardRequest(checkIntervalMs: number, burstAllowanceBytes: number) {
|
||||
return function(req: Request, res: Response, next: any) {
|
||||
const socket = req.socket;
|
||||
res.once("finish", () => {
|
||||
@@ -521,10 +526,12 @@ function guardRequest(checkIntervalMs: number) {
|
||||
}
|
||||
const bytes = currentBytes();
|
||||
const intervalS = checkIntervalMs / 1000;
|
||||
if (bytes - lastBytes < minSpeedKiBps * 1024 * intervalS) {
|
||||
// At least 1 so a transfer that moved nothing is closed at any speed setting.
|
||||
const requiredBytes = Math.max(minSpeedKiBps * 1024 * intervalS - burstAllowanceBytes, 1);
|
||||
if (bytes - lastBytes < requiredBytes) {
|
||||
logger.warn(
|
||||
`Closing ${req.method} ${req.path}, it moved slower than ${minSpeedKiBps} KiB/s ` +
|
||||
`over ${intervalS} seconds`
|
||||
`Closing ${req.method} ${req.path}, it moved ${bytes - lastBytes} bytes in ${intervalS} seconds, ` +
|
||||
`less than the ${requiredBytes} required for ${minSpeedKiBps} KiB/s`
|
||||
);
|
||||
socket.destroy();
|
||||
}
|
||||
@@ -857,8 +864,12 @@ async function uploadMod(req: Request, res: Response) {
|
||||
}
|
||||
|
||||
|
||||
export function addRouteHandlers(app: Application, progressCheckIntervalMs = 60_000) {
|
||||
app.use(guardRequest(progressCheckIntervalMs));
|
||||
export function addRouteHandlers(
|
||||
app: Application,
|
||||
progressCheckIntervalMs = 60_000,
|
||||
burstAllowanceBytes = 2 * 1024 * 1024,
|
||||
) {
|
||||
app.use(guardRequest(progressCheckIntervalMs, burstAllowanceBytes));
|
||||
app.get("/metrics",
|
||||
validateUserToken,
|
||||
(req:Request, res:Response, next:any) => getMetrics(req, res).catch(next)
|
||||
|
||||
@@ -234,7 +234,7 @@ export class ControllerConfig extends classes.Config<ControllerConfigFields> {
|
||||
},
|
||||
"controller.transfer_min_speed": {
|
||||
title: "Transfer Minimum Speed",
|
||||
description: "KiB/s an upload or download has to move at over 60 seconds to not be closed, 0 for no limit.",
|
||||
description: "KiB/s at which an upload or download is never closed for moving too slowly, 0 for no limit.",
|
||||
type: "number",
|
||||
initialValue: 64,
|
||||
validator: validators.greaterThanEqualZero,
|
||||
|
||||
@@ -22,8 +22,8 @@ describe("controller/src/routes", function() {
|
||||
let port;
|
||||
beforeEach(async function() {
|
||||
controller = new mock.MockController();
|
||||
// Check the progress of requests every 500 ms instead of every 30 seconds.
|
||||
routes.addRouteHandlers(controller.app, 500);
|
||||
// Check the progress of requests every 500 ms without a burst allowance.
|
||||
routes.addRouteHandlers(controller.app, 500, 0);
|
||||
server = http.createServer(routes.httpServerOptions, controller.app);
|
||||
server.listen(0, "localhost");
|
||||
await events.once(server, "listening");
|
||||
@@ -216,16 +216,17 @@ describe("controller/src/routes", function() {
|
||||
assert.equal((await download).byteLength, 20 * 1024);
|
||||
});
|
||||
it("should not close a transfer held back by a slow receiver", async function() {
|
||||
this.timeout(20000);
|
||||
// The OS takes over a second to free buffered data at this speed, longer than the usual 500 ms checks.
|
||||
this.timeout(40000);
|
||||
// The receiver takes about 1.2 MB/s, which the OS hands on in bursts of over a MB.
|
||||
const slowController = new mock.MockController();
|
||||
slowController.mockConfigEntries.set("controller.transfer_min_speed", 1024);
|
||||
routes.addRouteHandlers(slowController.app, 3000);
|
||||
const slowServer = http.createServer(routes.httpServerOptions, slowController.app);
|
||||
slowServer.listen(0, "localhost");
|
||||
await events.once(slowServer, "listening");
|
||||
const slowEndpoint = `http://localhost:${slowServer.address().port}/api/stream`;
|
||||
try {
|
||||
const size = 8 * 1024 * 1024;
|
||||
const size = 16 * 1024 * 1024;
|
||||
const stream = await routes.createProxyStream(slowController.app);
|
||||
const upload = fetch(`${slowEndpoint}/${stream.id}`, { method: "PUT", body: Buffer.alloc(size) });
|
||||
const received = await new Promise((resolve, reject) => {
|
||||
@@ -1293,6 +1294,56 @@ describe("controller/src/routes", function() {
|
||||
assert.equal(result.status, undefined);
|
||||
assert(result.duration < 2000, `closed after ${result.duration} ms`);
|
||||
});
|
||||
describe("with a burst allowance", function() {
|
||||
let burstServer, burstUrl;
|
||||
beforeEach(async function() {
|
||||
// Each check requires 256 KiB/s * 0.5 s - 64 KiB = 64 KiB.
|
||||
const burstController = new mock.MockController();
|
||||
burstController.mockConfigEntries.set("controller.transfer_min_speed", 256);
|
||||
routes.addRouteHandlers(burstController.app, 500, 64 * 1024);
|
||||
burstController.app.post("/test", express.raw({ type: "*/*", limit: "2mb" }), (req, res) => {
|
||||
res.sendStatus(200);
|
||||
});
|
||||
burstServer = http.createServer(routes.httpServerOptions, burstController.app);
|
||||
burstServer.listen(0, "localhost");
|
||||
await events.once(burstServer, "listening");
|
||||
burstUrl = `http://localhost:${burstServer.address().port}/test`;
|
||||
});
|
||||
afterEach(function() {
|
||||
burstServer.close();
|
||||
});
|
||||
async function sendBursts(burstBytes) {
|
||||
const bursts = 12;
|
||||
const request = http.request(burstUrl, {
|
||||
method: "POST", headers: { "Content-Type": "text/plain", "Content-Length": burstBytes * bursts },
|
||||
});
|
||||
let status;
|
||||
request.on("response", response => {
|
||||
status = response.statusCode;
|
||||
response.resume();
|
||||
});
|
||||
request.on("error", () => {});
|
||||
const closed = new Promise(resolve => request.on("close", resolve));
|
||||
for (let i = 0; i < bursts && !request.closed; i++) {
|
||||
request.write(Buffer.alloc(burstBytes));
|
||||
await lib.wait(300);
|
||||
}
|
||||
if (!request.closed) {
|
||||
request.end();
|
||||
}
|
||||
await closed;
|
||||
return status;
|
||||
}
|
||||
it("should not close a request at the minimum speed arriving in bursts", async function() {
|
||||
this.timeout(10000);
|
||||
// 78 KiB every 300 ms is 260 KiB/s, a check can see only one burst.
|
||||
assert.equal(await sendBursts(78 * 1024), 200);
|
||||
});
|
||||
it("should close a request below the minimum speed arriving in bursts", async function() {
|
||||
this.timeout(10000);
|
||||
assert.equal(await sendBursts(39 * 1024), undefined);
|
||||
});
|
||||
});
|
||||
it("should not close a trickled in request when the limit is 0", async function() {
|
||||
this.timeout(10000);
|
||||
controller.mockConfigEntries.set("controller.transfer_min_speed", 0);
|
||||
|
||||
Reference in new issue
Block a user