Skip to content

Commit 67f5789

Browse files
Bob Pearsonjgunthorpe
Bob Pearson
authored and
jgunthorpe
committed
RDMA/rxe: Merge request and complete tasks
Currently the rxe driver has three work queue tasks per qp. These are the req.task, comp.task and resp.task which call rxe_requester(), rxe_completer() and rxe_responder() respectively directly or on work queues. Each of these subroutines checks to see if there is work to be performed on the send queue or on the response packet queue or the request packet queue and will run until there is no work remaining or yield the cpu and reschedule itself until there is no work remaining. This commit combines the req.task and comp.task into a single send.task and renames the resp.task to the recv.task. The combined send.task calls rxe_requester() and rxe_completer() serially and continues until all work on both the send queue and the response packet queue are done. In various benchmarks the performance is either improved or left the same. At high scale there is a significant reduction in the load on the cpu. This is the first step in combining these two tasks. Once they are serialized cross rescheduling of req.task and comp.task can be more efficiently handled by just letting the send.task continue to run. This will be done in the next several patches. Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Bob Pearson <[email protected]> Signed-off-by: Jason Gunthorpe <[email protected]>
1 parent ff30e45 commit 67f5789

File tree

10 files changed

+63
-55
lines changed

10 files changed

+63
-55
lines changed

‎drivers/infiniband/sw/rxe/rxe_comp.c

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ void retransmit_timer(struct timer_list *t)
122122
spin_lock_irqsave(&qp->state_lock, flags);
123123
if (qp->valid) {
124124
qp->comp.timeout = 1;
125-
rxe_sched_task(&qp->comp.task);
125+
rxe_sched_task(&qp->send_task);
126126
}
127127
spin_unlock_irqrestore(&qp->state_lock, flags);
128128
}
@@ -133,14 +133,14 @@ void rxe_comp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb)
133133

134134
must_sched = skb_queue_len(&qp->resp_pkts) > 0;
135135
if (must_sched != 0)
136-
rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_COMPLETER_SCHED);
136+
rxe_counter_inc(SKB_TO_PKT(skb)->rxe, RXE_CNT_SENDER_SCHED);
137137

138138
skb_queue_tail(&qp->resp_pkts, skb);
139139

140140
if (must_sched)
141-
rxe_sched_task(&qp->comp.task);
141+
rxe_sched_task(&qp->send_task);
142142
else
143-
rxe_run_task(&qp->comp.task);
143+
rxe_run_task(&qp->send_task);
144144
}
145145

146146
static inline enum comp_state get_wqe(struct rxe_qp *qp,
@@ -325,7 +325,7 @@ static inline enum comp_state check_ack(struct rxe_qp *qp,
325325
qp->comp.psn = pkt->psn;
326326
if (qp->req.wait_psn) {
327327
qp->req.wait_psn = 0;
328-
rxe_sched_task(&qp->req.task);
328+
rxe_sched_task(&qp->send_task);
329329
}
330330
}
331331
return COMPST_ERROR_RETRY;
@@ -476,7 +476,7 @@ static void do_complete(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
476476
*/
477477
if (qp->req.wait_fence) {
478478
qp->req.wait_fence = 0;
479-
rxe_sched_task(&qp->req.task);
479+
rxe_sched_task(&qp->send_task);
480480
}
481481
}
482482

@@ -515,7 +515,7 @@ static inline enum comp_state complete_ack(struct rxe_qp *qp,
515515
if (qp->req.need_rd_atomic) {
516516
qp->comp.timeout_retry = 0;
517517
qp->req.need_rd_atomic = 0;
518-
rxe_sched_task(&qp->req.task);
518+
rxe_sched_task(&qp->send_task);
519519
}
520520
}
521521

@@ -541,7 +541,7 @@ static inline enum comp_state complete_wqe(struct rxe_qp *qp,
541541

542542
if (qp->req.wait_psn) {
543543
qp->req.wait_psn = 0;
544-
rxe_sched_task(&qp->req.task);
544+
rxe_sched_task(&qp->send_task);
545545
}
546546
}
547547

@@ -737,7 +737,7 @@ int rxe_completer(struct rxe_qp *qp)
737737

738738
if (qp->req.wait_psn) {
739739
qp->req.wait_psn = 0;
740-
rxe_sched_task(&qp->req.task);
740+
rxe_sched_task(&qp->send_task);
741741
}
742742

743743
state = COMPST_DONE;
@@ -792,7 +792,7 @@ int rxe_completer(struct rxe_qp *qp)
792792
RXE_CNT_COMP_RETRY);
793793
qp->req.need_retry = 1;
794794
qp->comp.started_retry = 1;
795-
rxe_sched_task(&qp->req.task);
795+
rxe_sched_task(&qp->send_task);
796796
}
797797
goto done;
798798

‎drivers/infiniband/sw/rxe/rxe_hw_counters.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ static const struct rdma_stat_desc rxe_counter_descs[] = {
1414
[RXE_CNT_RCV_RNR].name = "rcvd_rnr_err",
1515
[RXE_CNT_SND_RNR].name = "send_rnr_err",
1616
[RXE_CNT_RCV_SEQ_ERR].name = "rcvd_seq_err",
17-
[RXE_CNT_COMPLETER_SCHED].name = "ack_deferred",
17+
[RXE_CNT_SENDER_SCHED].name = "ack_deferred",
1818
[RXE_CNT_RETRY_EXCEEDED].name = "retry_exceeded_err",
1919
[RXE_CNT_RNR_RETRY_EXCEEDED].name = "retry_rnr_exceeded_err",
2020
[RXE_CNT_COMP_RETRY].name = "completer_retry_err",

‎drivers/infiniband/sw/rxe/rxe_hw_counters.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ enum rxe_counters {
1818
RXE_CNT_RCV_RNR,
1919
RXE_CNT_SND_RNR,
2020
RXE_CNT_RCV_SEQ_ERR,
21-
RXE_CNT_COMPLETER_SCHED,
21+
RXE_CNT_SENDER_SCHED,
2222
RXE_CNT_RETRY_EXCEEDED,
2323
RXE_CNT_RNR_RETRY_EXCEEDED,
2424
RXE_CNT_COMP_RETRY,

‎drivers/infiniband/sw/rxe/rxe_loc.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,8 @@ void rxe_dealloc(struct ib_device *ib_dev);
164164

165165
int rxe_completer(struct rxe_qp *qp);
166166
int rxe_requester(struct rxe_qp *qp);
167-
int rxe_responder(struct rxe_qp *qp);
167+
int rxe_sender(struct rxe_qp *qp);
168+
int rxe_receiver(struct rxe_qp *qp);
168169

169170
/* rxe_icrc.c */
170171
int rxe_icrc_init(struct rxe_dev *rxe);

‎drivers/infiniband/sw/rxe/rxe_net.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ static void rxe_skb_tx_dtor(struct sk_buff *skb)
351351

352352
if (unlikely(qp->need_req_skb &&
353353
skb_out < RXE_INFLIGHT_SKBS_PER_QP_LOW))
354-
rxe_sched_task(&qp->req.task);
354+
rxe_sched_task(&qp->send_task);
355355

356356
rxe_put(qp);
357357
}
@@ -443,7 +443,7 @@ int rxe_xmit_packet(struct rxe_qp *qp, struct rxe_pkt_info *pkt,
443443
if ((qp_type(qp) != IB_QPT_RC) &&
444444
(pkt->mask & RXE_END_MASK)) {
445445
pkt->wqe->state = wqe_state_done;
446-
rxe_sched_task(&qp->comp.task);
446+
rxe_sched_task(&qp->send_task);
447447
}
448448

449449
rxe_counter_inc(rxe, RXE_CNT_SENT_PKTS);

‎drivers/infiniband/sw/rxe/rxe_qp.c

Lines changed: 17 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -265,8 +265,7 @@ static int rxe_qp_init_req(struct rxe_dev *rxe, struct rxe_qp *qp,
265265
qp->req.opcode = -1;
266266
qp->comp.opcode = -1;
267267

268-
rxe_init_task(&qp->req.task, qp, rxe_requester);
269-
rxe_init_task(&qp->comp.task, qp, rxe_completer);
268+
rxe_init_task(&qp->send_task, qp, rxe_sender);
270269

271270
qp->qp_timeout_jiffies = 0; /* Can't be set for UD/UC in modify_qp */
272271
if (init->qp_type == IB_QPT_RC) {
@@ -337,7 +336,7 @@ static int rxe_qp_init_resp(struct rxe_dev *rxe, struct rxe_qp *qp,
337336
return err;
338337
}
339338

340-
rxe_init_task(&qp->resp.task, qp, rxe_responder);
339+
rxe_init_task(&qp->recv_task, qp, rxe_receiver);
341340

342341
qp->resp.opcode = OPCODE_NONE;
343342
qp->resp.msn = 0;
@@ -514,14 +513,12 @@ int rxe_qp_chk_attr(struct rxe_dev *rxe, struct rxe_qp *qp,
514513
static void rxe_qp_reset(struct rxe_qp *qp)
515514
{
516515
/* stop tasks from running */
517-
rxe_disable_task(&qp->resp.task);
518-
rxe_disable_task(&qp->comp.task);
519-
rxe_disable_task(&qp->req.task);
516+
rxe_disable_task(&qp->recv_task);
517+
rxe_disable_task(&qp->send_task);
520518

521519
/* drain work and packet queuesc */
522-
rxe_requester(qp);
523-
rxe_completer(qp);
524-
rxe_responder(qp);
520+
rxe_sender(qp);
521+
rxe_receiver(qp);
525522

526523
if (qp->rq.queue)
527524
rxe_queue_reset(qp->rq.queue);
@@ -548,9 +545,8 @@ static void rxe_qp_reset(struct rxe_qp *qp)
548545
cleanup_rd_atomic_resources(qp);
549546

550547
/* reenable tasks */
551-
rxe_enable_task(&qp->resp.task);
552-
rxe_enable_task(&qp->comp.task);
553-
rxe_enable_task(&qp->req.task);
548+
rxe_enable_task(&qp->recv_task);
549+
rxe_enable_task(&qp->send_task);
554550
}
555551

556552
/* move the qp to the error state */
@@ -562,9 +558,8 @@ void rxe_qp_error(struct rxe_qp *qp)
562558
qp->attr.qp_state = IB_QPS_ERR;
563559

564560
/* drain work and packet queues */
565-
rxe_sched_task(&qp->resp.task);
566-
rxe_sched_task(&qp->comp.task);
567-
rxe_sched_task(&qp->req.task);
561+
rxe_sched_task(&qp->recv_task);
562+
rxe_sched_task(&qp->send_task);
568563
spin_unlock_irqrestore(&qp->state_lock, flags);
569564
}
570565

@@ -575,8 +570,7 @@ static void rxe_qp_sqd(struct rxe_qp *qp, struct ib_qp_attr *attr,
575570

576571
spin_lock_irqsave(&qp->state_lock, flags);
577572
qp->attr.sq_draining = 1;
578-
rxe_sched_task(&qp->comp.task);
579-
rxe_sched_task(&qp->req.task);
573+
rxe_sched_task(&qp->send_task);
580574
spin_unlock_irqrestore(&qp->state_lock, flags);
581575
}
582576

@@ -821,19 +815,15 @@ static void rxe_qp_do_cleanup(struct work_struct *work)
821815
del_timer_sync(&qp->rnr_nak_timer);
822816
}
823817

824-
if (qp->resp.task.func)
825-
rxe_cleanup_task(&qp->resp.task);
818+
if (qp->recv_task.func)
819+
rxe_cleanup_task(&qp->recv_task);
826820

827-
if (qp->req.task.func)
828-
rxe_cleanup_task(&qp->req.task);
829-
830-
if (qp->comp.task.func)
831-
rxe_cleanup_task(&qp->comp.task);
821+
if (qp->send_task.func)
822+
rxe_cleanup_task(&qp->send_task);
832823

833824
/* flush out any receive wr's or pending requests */
834-
rxe_requester(qp);
835-
rxe_completer(qp);
836-
rxe_responder(qp);
825+
rxe_sender(qp);
826+
rxe_receiver(qp);
837827

838828
if (qp->sq.queue)
839829
rxe_queue_cleanup(qp->sq.queue);

‎drivers/infiniband/sw/rxe/rxe_req.c

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ void rnr_nak_timer(struct timer_list *t)
108108
/* request a send queue retry */
109109
qp->req.need_retry = 1;
110110
qp->req.wait_for_rnr_timer = 0;
111-
rxe_sched_task(&qp->req.task);
111+
rxe_sched_task(&qp->send_task);
112112
}
113113
spin_unlock_irqrestore(&qp->state_lock, flags);
114114
}
@@ -659,7 +659,7 @@ static int rxe_do_local_ops(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
659659
* which can lead to a deadlock. So go ahead and complete
660660
* it now.
661661
*/
662-
rxe_sched_task(&qp->comp.task);
662+
rxe_sched_task(&qp->send_task);
663663

664664
return 0;
665665
}
@@ -786,7 +786,7 @@ int rxe_requester(struct rxe_qp *qp)
786786
qp->req.wqe_index);
787787
wqe->state = wqe_state_done;
788788
wqe->status = IB_WC_SUCCESS;
789-
rxe_sched_task(&qp->comp.task);
789+
rxe_sched_task(&qp->send_task);
790790
goto done;
791791
}
792792
payload = mtu;
@@ -855,7 +855,7 @@ int rxe_requester(struct rxe_qp *qp)
855855
*/
856856
qp->need_req_skb = 1;
857857

858-
rxe_sched_task(&qp->req.task);
858+
rxe_sched_task(&qp->send_task);
859859
goto exit;
860860
}
861861

@@ -878,3 +878,20 @@ int rxe_requester(struct rxe_qp *qp)
878878
out:
879879
return ret;
880880
}
881+
882+
int rxe_sender(struct rxe_qp *qp)
883+
{
884+
int req_ret;
885+
int comp_ret;
886+
887+
/* process the send queue */
888+
req_ret = rxe_requester(qp);
889+
890+
/* process the response queue */
891+
comp_ret = rxe_completer(qp);
892+
893+
/* exit the task loop if both requester and completer
894+
* are ready
895+
*/
896+
return (req_ret && comp_ret) ? -EAGAIN : 0;
897+
}

‎drivers/infiniband/sw/rxe/rxe_resp.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,9 @@ void rxe_resp_queue_pkt(struct rxe_qp *qp, struct sk_buff *skb)
5858
(skb_queue_len(&qp->req_pkts) > 1);
5959

6060
if (must_sched)
61-
rxe_sched_task(&qp->resp.task);
61+
rxe_sched_task(&qp->recv_task);
6262
else
63-
rxe_run_task(&qp->resp.task);
63+
rxe_run_task(&qp->recv_task);
6464
}
6565

6666
static inline enum resp_states get_req(struct rxe_qp *qp,
@@ -1485,7 +1485,7 @@ static void flush_recv_queue(struct rxe_qp *qp, bool notify)
14851485
qp->resp.wqe = NULL;
14861486
}
14871487

1488-
int rxe_responder(struct rxe_qp *qp)
1488+
int rxe_receiver(struct rxe_qp *qp)
14891489
{
14901490
struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
14911491
enum resp_states state;

‎drivers/infiniband/sw/rxe/rxe_verbs.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -905,7 +905,7 @@ static int rxe_post_send_kernel(struct rxe_qp *qp,
905905

906906
/* kickoff processing of any posted wqes */
907907
if (good)
908-
rxe_sched_task(&qp->req.task);
908+
rxe_sched_task(&qp->send_task);
909909

910910
return err;
911911
}
@@ -935,7 +935,7 @@ static int rxe_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
935935

936936
if (qp->is_user) {
937937
/* Utilize process context to do protocol processing */
938-
rxe_run_task(&qp->req.task);
938+
rxe_run_task(&qp->send_task);
939939
} else {
940940
err = rxe_post_send_kernel(qp, wr, bad_wr);
941941
if (err)
@@ -1045,7 +1045,7 @@ static int rxe_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
10451045

10461046
spin_lock_irqsave(&qp->state_lock, flags);
10471047
if (qp_state(qp) == IB_QPS_ERR)
1048-
rxe_sched_task(&qp->resp.task);
1048+
rxe_sched_task(&qp->recv_task);
10491049
spin_unlock_irqrestore(&qp->state_lock, flags);
10501050

10511051
return err;

‎drivers/infiniband/sw/rxe/rxe_verbs.h

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,6 @@ struct rxe_req_info {
113113
int need_retry;
114114
int wait_for_rnr_timer;
115115
int noack_pkts;
116-
struct rxe_task task;
117116
};
118117

119118
struct rxe_comp_info {
@@ -124,7 +123,6 @@ struct rxe_comp_info {
124123
int started_retry;
125124
u32 retry_cnt;
126125
u32 rnr_retry;
127-
struct rxe_task task;
128126
};
129127

130128
enum rdatm_res_state {
@@ -196,7 +194,6 @@ struct rxe_resp_info {
196194
unsigned int res_head;
197195
unsigned int res_tail;
198196
struct resp_res *res;
199-
struct rxe_task task;
200197
};
201198

202199
struct rxe_qp {
@@ -229,6 +226,9 @@ struct rxe_qp {
229226
struct sk_buff_head req_pkts;
230227
struct sk_buff_head resp_pkts;
231228

229+
struct rxe_task send_task;
230+
struct rxe_task recv_task;
231+
232232
struct rxe_req_info req;
233233
struct rxe_comp_info comp;
234234
struct rxe_resp_info resp;

0 commit comments

Comments
 (0)