@@ -894,10 +894,6 @@ ice_add_xdp_frag(struct ice_rx_ring *rx_ring, struct xdp_buff *xdp,
894
894
__skb_fill_page_desc_noacc (sinfo , sinfo -> nr_frags ++ , rx_buf -> page ,
895
895
rx_buf -> page_offset , size );
896
896
sinfo -> xdp_frags_size += size ;
897
- /* remember frag count before XDP prog execution; bpf_xdp_adjust_tail()
898
- * can pop off frags but driver has to handle it on its own
899
- */
900
- rx_ring -> nr_frags = sinfo -> nr_frags ;
901
897
902
898
if (page_is_pfmemalloc (rx_buf -> page ))
903
899
xdp_buff_set_frag_pfmemalloc (xdp );
@@ -968,20 +964,20 @@ ice_get_rx_buf(struct ice_rx_ring *rx_ring, const unsigned int size,
968
964
/**
969
965
* ice_get_pgcnts - grab page_count() for gathered fragments
970
966
* @rx_ring: Rx descriptor ring to store the page counts on
967
+ * @ntc: the next to clean element (not included in this frame!)
971
968
*
972
969
* This function is intended to be called right before running XDP
973
970
* program so that the page recycling mechanism will be able to take
974
971
* a correct decision regarding underlying pages; this is done in such
975
972
* way as XDP program can change the refcount of page
976
973
*/
977
- static void ice_get_pgcnts (struct ice_rx_ring * rx_ring )
974
+ static void ice_get_pgcnts (struct ice_rx_ring * rx_ring , unsigned int ntc )
978
975
{
979
- u32 nr_frags = rx_ring -> nr_frags + 1 ;
980
976
u32 idx = rx_ring -> first_desc ;
981
977
struct ice_rx_buf * rx_buf ;
982
978
u32 cnt = rx_ring -> count ;
983
979
984
- for ( int i = 0 ; i < nr_frags ; i ++ ) {
980
+ while ( idx != ntc ) {
985
981
rx_buf = & rx_ring -> rx_buf [idx ];
986
982
rx_buf -> pgcnt = page_count (rx_buf -> page );
987
983
@@ -1154,62 +1150,51 @@ ice_put_rx_buf(struct ice_rx_ring *rx_ring, struct ice_rx_buf *rx_buf)
1154
1150
}
1155
1151
1156
1152
/**
1157
- * ice_put_rx_mbuf - ice_put_rx_buf() caller, for all frame frags
1153
+ * ice_put_rx_mbuf - ice_put_rx_buf() caller, for all buffers in frame
1158
1154
* @rx_ring: Rx ring with all the auxiliary data
1159
1155
* @xdp: XDP buffer carrying linear + frags part
1160
- * @xdp_xmit: XDP_TX/XDP_REDIRECT verdict storage
1161
- * @ntc: a current next_to_clean value to be stored at rx_ring
1156
+ * @ntc: the next to clean element (not included in this frame!)
1162
1157
* @verdict: return code from XDP program execution
1163
1158
*
1164
- * Walk through gathered fragments and satisfy internal page
1165
- * recycle mechanism; we take here an action related to verdict
1166
- * returned by XDP program;
1159
+ * Called after XDP program is completed, or on error with verdict set to
1160
+ * ICE_XDP_CONSUMED.
1161
+ *
1162
+ * Walk through buffers from first_desc to the end of the frame, releasing
1163
+ * buffers and satisfying internal page recycle mechanism. The action depends
1164
+ * on verdict from XDP program.
1167
1165
*/
1168
1166
static void ice_put_rx_mbuf (struct ice_rx_ring * rx_ring , struct xdp_buff * xdp ,
1169
- u32 * xdp_xmit , u32 ntc , u32 verdict )
1167
+ u32 ntc , u32 verdict )
1170
1168
{
1171
- u32 nr_frags = rx_ring -> nr_frags + 1 ;
1172
1169
u32 idx = rx_ring -> first_desc ;
1173
1170
u32 cnt = rx_ring -> count ;
1174
- u32 post_xdp_frags = 1 ;
1175
1171
struct ice_rx_buf * buf ;
1176
- int i ;
1172
+ u32 xdp_frags = 0 ;
1173
+ int i = 0 ;
1177
1174
1178
1175
if (unlikely (xdp_buff_has_frags (xdp )))
1179
- post_xdp_frags + = xdp_get_shared_info_from_buff (xdp )-> nr_frags ;
1176
+ xdp_frags = xdp_get_shared_info_from_buff (xdp )-> nr_frags ;
1180
1177
1181
- for ( i = 0 ; i < post_xdp_frags ; i ++ ) {
1178
+ while ( idx != ntc ) {
1182
1179
buf = & rx_ring -> rx_buf [idx ];
1180
+ if (++ idx == cnt )
1181
+ idx = 0 ;
1183
1182
1184
- if (verdict & (ICE_XDP_TX | ICE_XDP_REDIR )) {
1183
+ /* An XDP program could release fragments from the end of the
1184
+ * buffer. For these, we need to keep the pagecnt_bias as-is.
1185
+ * To do this, only adjust pagecnt_bias for fragments up to
1186
+ * the total remaining after the XDP program has run.
1187
+ */
1188
+ if (verdict != ICE_XDP_CONSUMED )
1185
1189
ice_rx_buf_adjust_pg_offset (buf , xdp -> frame_sz );
1186
- * xdp_xmit |= verdict ;
1187
- } else if (verdict & ICE_XDP_CONSUMED ) {
1190
+ else if (i ++ <= xdp_frags )
1188
1191
buf -> pagecnt_bias ++ ;
1189
- } else if (verdict == ICE_XDP_PASS ) {
1190
- ice_rx_buf_adjust_pg_offset (buf , xdp -> frame_sz );
1191
- }
1192
1192
1193
1193
ice_put_rx_buf (rx_ring , buf );
1194
-
1195
- if (++ idx == cnt )
1196
- idx = 0 ;
1197
- }
1198
- /* handle buffers that represented frags released by XDP prog;
1199
- * for these we keep pagecnt_bias as-is; refcount from struct page
1200
- * has been decremented within XDP prog and we do not have to increase
1201
- * the biased refcnt
1202
- */
1203
- for (; i < nr_frags ; i ++ ) {
1204
- buf = & rx_ring -> rx_buf [idx ];
1205
- ice_put_rx_buf (rx_ring , buf );
1206
- if (++ idx == cnt )
1207
- idx = 0 ;
1208
1194
}
1209
1195
1210
1196
xdp -> data = NULL ;
1211
1197
rx_ring -> first_desc = ntc ;
1212
- rx_ring -> nr_frags = 0 ;
1213
1198
}
1214
1199
1215
1200
/**
@@ -1317,6 +1302,10 @@ static int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget)
1317
1302
/* retrieve a buffer from the ring */
1318
1303
rx_buf = ice_get_rx_buf (rx_ring , size , ntc );
1319
1304
1305
+ /* Increment ntc before calls to ice_put_rx_mbuf() */
1306
+ if (++ ntc == cnt )
1307
+ ntc = 0 ;
1308
+
1320
1309
if (!xdp -> data ) {
1321
1310
void * hard_start ;
1322
1311
@@ -1325,24 +1314,23 @@ static int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget)
1325
1314
xdp_prepare_buff (xdp , hard_start , offset , size , !!offset );
1326
1315
xdp_buff_clear_frags_flag (xdp );
1327
1316
} else if (ice_add_xdp_frag (rx_ring , xdp , rx_buf , size )) {
1328
- ice_put_rx_mbuf (rx_ring , xdp , NULL , ntc , ICE_XDP_CONSUMED );
1317
+ ice_put_rx_mbuf (rx_ring , xdp , ntc , ICE_XDP_CONSUMED );
1329
1318
break ;
1330
1319
}
1331
- if (++ ntc == cnt )
1332
- ntc = 0 ;
1333
1320
1334
1321
/* skip if it is NOP desc */
1335
1322
if (ice_is_non_eop (rx_ring , rx_desc ))
1336
1323
continue ;
1337
1324
1338
- ice_get_pgcnts (rx_ring );
1325
+ ice_get_pgcnts (rx_ring , ntc );
1339
1326
xdp_verdict = ice_run_xdp (rx_ring , xdp , xdp_prog , xdp_ring , rx_desc );
1340
1327
if (xdp_verdict == ICE_XDP_PASS )
1341
1328
goto construct_skb ;
1342
1329
total_rx_bytes += xdp_get_buff_len (xdp );
1343
1330
total_rx_pkts ++ ;
1344
1331
1345
- ice_put_rx_mbuf (rx_ring , xdp , & xdp_xmit , ntc , xdp_verdict );
1332
+ ice_put_rx_mbuf (rx_ring , xdp , ntc , xdp_verdict );
1333
+ xdp_xmit |= xdp_verdict & (ICE_XDP_TX | ICE_XDP_REDIR );
1346
1334
1347
1335
continue ;
1348
1336
construct_skb :
@@ -1355,7 +1343,7 @@ static int ice_clean_rx_irq(struct ice_rx_ring *rx_ring, int budget)
1355
1343
rx_ring -> ring_stats -> rx_stats .alloc_buf_failed ++ ;
1356
1344
xdp_verdict = ICE_XDP_CONSUMED ;
1357
1345
}
1358
- ice_put_rx_mbuf (rx_ring , xdp , & xdp_xmit , ntc , xdp_verdict );
1346
+ ice_put_rx_mbuf (rx_ring , xdp , ntc , xdp_verdict );
1359
1347
1360
1348
if (!skb )
1361
1349
break ;
0 commit comments