-
-
Notifications
You must be signed in to change notification settings - Fork 640
fix: fix backpressure when using TLS #1752
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
wellwelwel
merged 8 commits into
sidorares:master
from
vobarian:fix/tls-stream-backpressure
Sep 1, 2025
+168
−2
Merged
Changes from 2 commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
8ec58bc
Fix backpressure when using TLS
1b67ed3
Skip load data infile backpressure test with compression
d516c9c
Merge branch 'master' into fix/tls-stream-backpressure
wellwelwel f6781f1
Merge branch 'master' into fix/tls-stream-backpressure
wellwelwel 0a8219e
Tests use compressed protocol only when MYSQL_USE_COMPRESSION=1; prev…
8093e06
Update backpressure tests using poku functions; tighten threshold on …
3f2e2b7
Update test/integration/connection/test-backpressure-load-data-infile…
wellwelwel 2931957
Increase delay for observing backpressure to account for environment …
vobarian File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
109 changes: 109 additions & 0 deletions
109
test/integration/connection/test-backpressure-load-data-infile.test.cjs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,109 @@ | ||
'use strict'; | ||
|
||
const { assert, test, log } = require('poku'); | ||
const common = require('../../common.test.cjs'); | ||
const { Readable, Duplex } = require('stream'); | ||
const Net = require('node:net'); | ||
const driver = require('../../../index.js'); | ||
const { setTimeout } = require('node:timers/promises'); | ||
wellwelwel marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
if (common.config.compress) { | ||
console.log('skipping test with compression; load data infile backpressure is not working with compression'); | ||
Check failure on line 11 in test/integration/connection/test-backpressure-load-data-infile.test.cjs
|
||
process.exit(0); | ||
wellwelwel marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
|
||
class BigInput extends Readable { | ||
count = 0; | ||
MAX_EXPECTED_ROWS = 100_000; | ||
onStart = null; | ||
|
||
_read() { | ||
if (this.onStart) { | ||
this.onStart(); | ||
this.onStart = null; | ||
} | ||
|
||
if (this.count < this.MAX_EXPECTED_ROWS) { | ||
this.count++; | ||
const row = `${this.count}\n`; | ||
this.push(row); | ||
} else { | ||
this.push(null); | ||
} | ||
} | ||
} | ||
|
||
test('load data infile backpressure on local stream', async () => { | ||
const config = common.config; | ||
const netStream = Net.connect(config.port, config.host); | ||
netStream.setNoDelay(true); | ||
await new Promise((resolve, reject) => | ||
netStream.once('connect', resolve).once('error', reject) | ||
); | ||
|
||
class NetworkInterceptor extends Duplex { | ||
simulateWriteBackpressure = false; | ||
|
||
constructor() { | ||
super({ writableHighWaterMark: 65536 }); | ||
netStream.on('data', (data) => { | ||
const continueReading = this.push(data); | ||
if (!continueReading) { | ||
netStream.pause(); | ||
} | ||
}); | ||
netStream.on('error', (err) => this.destroy(err)); | ||
} | ||
|
||
_read() { | ||
netStream.resume(); | ||
} | ||
|
||
_write(chunk, encoding, callback) { | ||
netStream.write(chunk, encoding, (err) => { | ||
if (err) { | ||
callback(err); | ||
} else if (!this.simulateWriteBackpressure) { | ||
callback(); | ||
} | ||
}); | ||
} | ||
} | ||
|
||
const interceptor = new NetworkInterceptor(); | ||
const connection = driver.createConnection({ | ||
...config, | ||
multipleStatements: true, | ||
stream: interceptor, | ||
}); | ||
|
||
try { | ||
wellwelwel marked this conversation as resolved.
Show resolved
Hide resolved
|
||
const bigInput = new BigInput(); | ||
bigInput.onStart = () => (interceptor.simulateWriteBackpressure = true); | ||
|
||
connection.query( | ||
{ | ||
sql: ` | ||
set global local_infile = 1; | ||
create temporary table test_load_data_backpressure (id varchar(100)); | ||
load data local infile "_" replace into table test_load_data_backpressure; | ||
`, | ||
infileStreamFactory: () => bigInput, | ||
}, | ||
(err, result) => { | ||
if (err) throw err; | ||
log('Load complete', result); | ||
} | ||
); | ||
|
||
await setTimeout(100); // allow time for backpressure to take effect | ||
|
||
assert.ok( | ||
bigInput.count < bigInput.MAX_EXPECTED_ROWS, | ||
`expected backpressure to stop infile stream at less than ${bigInput.MAX_EXPECTED_ROWS} rows (read ${bigInput.count} rows)` | ||
); | ||
} finally { | ||
connection.close(); | ||
netStream.destroy(); | ||
} | ||
}); |
56 changes: 56 additions & 0 deletions
56
test/integration/connection/test-backpressure-result-streaming.test.cjs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
'use strict'; | ||
|
||
const { assert, test } = require('poku'); | ||
const common = require('../../common.test.cjs'); | ||
const timers = require('node:timers'); | ||
|
||
test('result event backpressure with pause/resume', async () => { | ||
const connection = common.createConnection({ | ||
multipleStatements: true, | ||
}); | ||
try { | ||
// in case wrapping with TLS, get the underlying socket first so we can see actual number of bytes read | ||
const originalSocket = connection.stream; | ||
|
||
// the full result set will be over 6 MB | ||
const largeQuery = ` | ||
SET SESSION cte_max_recursion_depth = 100000; | ||
WITH RECURSIVE cte (n, s) AS ( | ||
SELECT 1, 'this is just to cause more bytes transferred for each row' | ||
UNION ALL | ||
SELECT n + 1, s | ||
FROM cte | ||
WHERE n < 100000 | ||
) | ||
SELECT * FROM cte; | ||
`; | ||
|
||
let resultRowsCount = 0; | ||
await new Promise((resolve, reject) => | ||
connection | ||
.query(largeQuery) | ||
.on('result', (row) => { | ||
resultRowsCount++; | ||
if (row.n === 1) { | ||
connection.pause(); | ||
resolve(); | ||
} | ||
}) | ||
.on('error', reject) | ||
); | ||
|
||
// if backpressure is not working, the bytes received will grow during this time, even though connection is paused | ||
await timers.promises.setTimeout(500); | ||
|
||
assert.equal(resultRowsCount, 2, 'stop receiving result rows when paused'); | ||
|
||
// if backpressure is working, there should be less than 1 MB received; | ||
// experimentally it appears to be around 100 KB but may vary if buffer sizes change | ||
assert.ok( | ||
originalSocket.bytesRead < 1000000, | ||
`Received ${originalSocket.bytesRead} bytes on paused connection` | ||
); | ||
} finally { | ||
connection.close(); | ||
} | ||
}); |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.