@@ -1003,6 +1003,13 @@ static void mptcp_wmem_uncharge(struct sock *sk, int size)
1003
1003
msk -> wmem_reserved += size ;
1004
1004
}
1005
1005
1006
+ static void __mptcp_mem_reclaim_partial (struct sock * sk )
1007
+ {
1008
+ lockdep_assert_held_once (& sk -> sk_lock .slock );
1009
+ __mptcp_update_wmem (sk );
1010
+ sk_mem_reclaim_partial (sk );
1011
+ }
1012
+
1006
1013
static void mptcp_mem_reclaim_partial (struct sock * sk )
1007
1014
{
1008
1015
struct mptcp_sock * msk = mptcp_sk (sk );
@@ -1094,12 +1101,8 @@ static void __mptcp_clean_una(struct sock *sk)
1094
1101
msk -> recovery = false;
1095
1102
1096
1103
out :
1097
- if (cleaned ) {
1098
- if (tcp_under_memory_pressure (sk )) {
1099
- __mptcp_update_wmem (sk );
1100
- sk_mem_reclaim_partial (sk );
1101
- }
1102
- }
1104
+ if (cleaned && tcp_under_memory_pressure (sk ))
1105
+ __mptcp_mem_reclaim_partial (sk );
1103
1106
1104
1107
if (snd_una == READ_ONCE (msk -> snd_nxt ) && !msk -> recovery ) {
1105
1108
if (mptcp_timer_pending (sk ) && !mptcp_data_fin_enabled (msk ))
@@ -1179,6 +1182,7 @@ struct mptcp_sendmsg_info {
1179
1182
u16 limit ;
1180
1183
u16 sent ;
1181
1184
unsigned int flags ;
1185
+ bool data_lock_held ;
1182
1186
};
1183
1187
1184
1188
static int mptcp_check_allowed_size (struct mptcp_sock * msk , u64 data_seq ,
@@ -1250,17 +1254,17 @@ static bool __mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp)
1250
1254
return false;
1251
1255
}
1252
1256
1253
- static bool mptcp_must_reclaim_memory (struct sock * sk , struct sock * ssk )
1257
+ static bool mptcp_alloc_tx_skb (struct sock * sk , struct sock * ssk , bool data_lock_held )
1254
1258
{
1255
- return !ssk -> sk_tx_skb_cache &&
1256
- tcp_under_memory_pressure (sk );
1257
- }
1259
+ gfp_t gfp = data_lock_held ? GFP_ATOMIC : sk -> sk_allocation ;
1258
1260
1259
- static bool mptcp_alloc_tx_skb (struct sock * sk , struct sock * ssk )
1260
- {
1261
- if (unlikely (mptcp_must_reclaim_memory (sk , ssk )))
1262
- mptcp_mem_reclaim_partial (sk );
1263
- return __mptcp_alloc_tx_skb (sk , ssk , sk -> sk_allocation );
1261
+ if (unlikely (tcp_under_memory_pressure (sk ))) {
1262
+ if (data_lock_held )
1263
+ __mptcp_mem_reclaim_partial (sk );
1264
+ else
1265
+ mptcp_mem_reclaim_partial (sk );
1266
+ }
1267
+ return __mptcp_alloc_tx_skb (sk , ssk , gfp );
1264
1268
}
1265
1269
1266
1270
/* note: this always recompute the csum on the whole skb, even
@@ -1284,7 +1288,7 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
1284
1288
bool zero_window_probe = false;
1285
1289
struct mptcp_ext * mpext = NULL ;
1286
1290
struct sk_buff * skb , * tail ;
1287
- bool can_collapse = false;
1291
+ bool must_collapse = false;
1288
1292
int size_bias = 0 ;
1289
1293
int avail_size ;
1290
1294
size_t ret = 0 ;
@@ -1304,16 +1308,24 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
1304
1308
* SSN association set here
1305
1309
*/
1306
1310
mpext = skb_ext_find (skb , SKB_EXT_MPTCP );
1307
- can_collapse = (info -> size_goal - skb -> len > 0 ) &&
1308
- mptcp_skb_can_collapse_to (data_seq , skb , mpext );
1309
- if (!can_collapse ) {
1311
+ if (!mptcp_skb_can_collapse_to (data_seq , skb , mpext )) {
1310
1312
TCP_SKB_CB (skb )-> eor = 1 ;
1311
- } else {
1313
+ goto alloc_skb ;
1314
+ }
1315
+
1316
+ must_collapse = (info -> size_goal - skb -> len > 0 ) &&
1317
+ (skb_shinfo (skb )-> nr_frags < sysctl_max_skb_frags );
1318
+ if (must_collapse ) {
1312
1319
size_bias = skb -> len ;
1313
1320
avail_size = info -> size_goal - skb -> len ;
1314
1321
}
1315
1322
}
1316
1323
1324
+ alloc_skb :
1325
+ if (!must_collapse && !ssk -> sk_tx_skb_cache &&
1326
+ !mptcp_alloc_tx_skb (sk , ssk , info -> data_lock_held ))
1327
+ return 0 ;
1328
+
1317
1329
/* Zero window and all data acked? Probe. */
1318
1330
avail_size = mptcp_check_allowed_size (msk , data_seq , avail_size );
1319
1331
if (avail_size == 0 ) {
@@ -1343,7 +1355,6 @@ static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
1343
1355
if (skb == tail ) {
1344
1356
TCP_SKB_CB (tail )-> tcp_flags &= ~TCPHDR_PSH ;
1345
1357
mpext -> data_len += ret ;
1346
- WARN_ON_ONCE (!can_collapse );
1347
1358
WARN_ON_ONCE (zero_window_probe );
1348
1359
goto out ;
1349
1360
}
@@ -1530,15 +1541,6 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags)
1530
1541
if (ssk != prev_ssk )
1531
1542
lock_sock (ssk );
1532
1543
1533
- /* keep it simple and always provide a new skb for the
1534
- * subflow, even if we will not use it when collapsing
1535
- * on the pending one
1536
- */
1537
- if (!mptcp_alloc_tx_skb (sk , ssk )) {
1538
- mptcp_push_release (sk , ssk , & info );
1539
- goto out ;
1540
- }
1541
-
1542
1544
ret = mptcp_sendmsg_frag (sk , ssk , dfrag , & info );
1543
1545
if (ret <= 0 ) {
1544
1546
mptcp_push_release (sk , ssk , & info );
@@ -1571,7 +1573,9 @@ void __mptcp_push_pending(struct sock *sk, unsigned int flags)
1571
1573
static void __mptcp_subflow_push_pending (struct sock * sk , struct sock * ssk )
1572
1574
{
1573
1575
struct mptcp_sock * msk = mptcp_sk (sk );
1574
- struct mptcp_sendmsg_info info ;
1576
+ struct mptcp_sendmsg_info info = {
1577
+ .data_lock_held = true,
1578
+ };
1575
1579
struct mptcp_data_frag * dfrag ;
1576
1580
struct sock * xmit_ssk ;
1577
1581
int len , copied = 0 ;
@@ -1597,13 +1601,6 @@ static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk)
1597
1601
goto out ;
1598
1602
}
1599
1603
1600
- if (unlikely (mptcp_must_reclaim_memory (sk , ssk ))) {
1601
- __mptcp_update_wmem (sk );
1602
- sk_mem_reclaim_partial (sk );
1603
- }
1604
- if (!__mptcp_alloc_tx_skb (sk , ssk , GFP_ATOMIC ))
1605
- goto out ;
1606
-
1607
1604
ret = mptcp_sendmsg_frag (sk , ssk , dfrag , & info );
1608
1605
if (ret <= 0 )
1609
1606
goto out ;
@@ -2409,9 +2406,6 @@ static void __mptcp_retrans(struct sock *sk)
2409
2406
info .sent = 0 ;
2410
2407
info .limit = READ_ONCE (msk -> csum_enabled ) ? dfrag -> data_len : dfrag -> already_sent ;
2411
2408
while (info .sent < info .limit ) {
2412
- if (!mptcp_alloc_tx_skb (sk , ssk ))
2413
- break ;
2414
-
2415
2409
ret = mptcp_sendmsg_frag (sk , ssk , dfrag , & info );
2416
2410
if (ret <= 0 )
2417
2411
break ;
0 commit comments