@@ -129,9 +129,6 @@ struct smb_direct_transport {
129
129
spinlock_t recvmsg_queue_lock ;
130
130
struct list_head recvmsg_queue ;
131
131
132
- spinlock_t empty_recvmsg_queue_lock ;
133
- struct list_head empty_recvmsg_queue ;
134
-
135
132
int send_credit_target ;
136
133
atomic_t send_credits ;
137
134
spinlock_t lock_new_recv_credits ;
@@ -268,40 +265,19 @@ smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t)
268
265
static void put_recvmsg (struct smb_direct_transport * t ,
269
266
struct smb_direct_recvmsg * recvmsg )
270
267
{
271
- ib_dma_unmap_single (t -> cm_id -> device , recvmsg -> sge .addr ,
272
- recvmsg -> sge .length , DMA_FROM_DEVICE );
268
+ if (likely (recvmsg -> sge .length != 0 )) {
269
+ ib_dma_unmap_single (t -> cm_id -> device ,
270
+ recvmsg -> sge .addr ,
271
+ recvmsg -> sge .length ,
272
+ DMA_FROM_DEVICE );
273
+ recvmsg -> sge .length = 0 ;
274
+ }
273
275
274
276
spin_lock (& t -> recvmsg_queue_lock );
275
277
list_add (& recvmsg -> list , & t -> recvmsg_queue );
276
278
spin_unlock (& t -> recvmsg_queue_lock );
277
279
}
278
280
279
- static struct
280
- smb_direct_recvmsg * get_empty_recvmsg (struct smb_direct_transport * t )
281
- {
282
- struct smb_direct_recvmsg * recvmsg = NULL ;
283
-
284
- spin_lock (& t -> empty_recvmsg_queue_lock );
285
- if (!list_empty (& t -> empty_recvmsg_queue )) {
286
- recvmsg = list_first_entry (& t -> empty_recvmsg_queue ,
287
- struct smb_direct_recvmsg , list );
288
- list_del (& recvmsg -> list );
289
- }
290
- spin_unlock (& t -> empty_recvmsg_queue_lock );
291
- return recvmsg ;
292
- }
293
-
294
- static void put_empty_recvmsg (struct smb_direct_transport * t ,
295
- struct smb_direct_recvmsg * recvmsg )
296
- {
297
- ib_dma_unmap_single (t -> cm_id -> device , recvmsg -> sge .addr ,
298
- recvmsg -> sge .length , DMA_FROM_DEVICE );
299
-
300
- spin_lock (& t -> empty_recvmsg_queue_lock );
301
- list_add_tail (& recvmsg -> list , & t -> empty_recvmsg_queue );
302
- spin_unlock (& t -> empty_recvmsg_queue_lock );
303
- }
304
-
305
281
static void enqueue_reassembly (struct smb_direct_transport * t ,
306
282
struct smb_direct_recvmsg * recvmsg ,
307
283
int data_length )
@@ -386,9 +362,6 @@ static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id)
386
362
spin_lock_init (& t -> recvmsg_queue_lock );
387
363
INIT_LIST_HEAD (& t -> recvmsg_queue );
388
364
389
- spin_lock_init (& t -> empty_recvmsg_queue_lock );
390
- INIT_LIST_HEAD (& t -> empty_recvmsg_queue );
391
-
392
365
init_waitqueue_head (& t -> wait_send_pending );
393
366
atomic_set (& t -> send_pending , 0 );
394
367
@@ -548,13 +521,13 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
548
521
t = recvmsg -> transport ;
549
522
550
523
if (wc -> status != IB_WC_SUCCESS || wc -> opcode != IB_WC_RECV ) {
524
+ put_recvmsg (t , recvmsg );
551
525
if (wc -> status != IB_WC_WR_FLUSH_ERR ) {
552
526
pr_err ("Recv error. status='%s (%d)' opcode=%d\n" ,
553
527
ib_wc_status_msg (wc -> status ), wc -> status ,
554
528
wc -> opcode );
555
529
smb_direct_disconnect_rdma_connection (t );
556
530
}
557
- put_empty_recvmsg (t , recvmsg );
558
531
return ;
559
532
}
560
533
@@ -568,15 +541,16 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
568
541
switch (recvmsg -> type ) {
569
542
case SMB_DIRECT_MSG_NEGOTIATE_REQ :
570
543
if (wc -> byte_len < sizeof (struct smb_direct_negotiate_req )) {
571
- put_empty_recvmsg (t , recvmsg );
544
+ put_recvmsg (t , recvmsg );
545
+ smb_direct_disconnect_rdma_connection (t );
572
546
return ;
573
547
}
574
548
t -> negotiation_requested = true;
575
549
t -> full_packet_received = true;
576
550
t -> status = SMB_DIRECT_CS_CONNECTED ;
577
551
enqueue_reassembly (t , recvmsg , 0 );
578
552
wake_up_interruptible (& t -> wait_status );
579
- break ;
553
+ return ;
580
554
case SMB_DIRECT_MSG_DATA_TRANSFER : {
581
555
struct smb_direct_data_transfer * data_transfer =
582
556
(struct smb_direct_data_transfer * )recvmsg -> packet ;
@@ -585,15 +559,17 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
585
559
586
560
if (wc -> byte_len <
587
561
offsetof(struct smb_direct_data_transfer , padding )) {
588
- put_empty_recvmsg (t , recvmsg );
562
+ put_recvmsg (t , recvmsg );
563
+ smb_direct_disconnect_rdma_connection (t );
589
564
return ;
590
565
}
591
566
592
567
data_length = le32_to_cpu (data_transfer -> data_length );
593
568
if (data_length ) {
594
569
if (wc -> byte_len < sizeof (struct smb_direct_data_transfer ) +
595
570
(u64 )data_length ) {
596
- put_empty_recvmsg (t , recvmsg );
571
+ put_recvmsg (t , recvmsg );
572
+ smb_direct_disconnect_rdma_connection (t );
597
573
return ;
598
574
}
599
575
@@ -605,16 +581,11 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
605
581
else
606
582
t -> full_packet_received = true;
607
583
608
- enqueue_reassembly (t , recvmsg , (int )data_length );
609
- wake_up_interruptible (& t -> wait_reassembly_queue );
610
-
611
584
spin_lock (& t -> receive_credit_lock );
612
585
receive_credits = -- (t -> recv_credits );
613
586
avail_recvmsg_count = t -> count_avail_recvmsg ;
614
587
spin_unlock (& t -> receive_credit_lock );
615
588
} else {
616
- put_empty_recvmsg (t , recvmsg );
617
-
618
589
spin_lock (& t -> receive_credit_lock );
619
590
receive_credits = -- (t -> recv_credits );
620
591
avail_recvmsg_count = ++ (t -> count_avail_recvmsg );
@@ -636,11 +607,23 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc)
636
607
if (is_receive_credit_post_required (receive_credits , avail_recvmsg_count ))
637
608
mod_delayed_work (smb_direct_wq ,
638
609
& t -> post_recv_credits_work , 0 );
639
- break ;
610
+
611
+ if (data_length ) {
612
+ enqueue_reassembly (t , recvmsg , (int )data_length );
613
+ wake_up_interruptible (& t -> wait_reassembly_queue );
614
+ } else
615
+ put_recvmsg (t , recvmsg );
616
+
617
+ return ;
640
618
}
641
- default :
642
- break ;
643
619
}
620
+
621
+ /*
622
+ * This is an internal error!
623
+ */
624
+ WARN_ON_ONCE (recvmsg -> type != SMB_DIRECT_MSG_DATA_TRANSFER );
625
+ put_recvmsg (t , recvmsg );
626
+ smb_direct_disconnect_rdma_connection (t );
644
627
}
645
628
646
629
static int smb_direct_post_recv (struct smb_direct_transport * t ,
@@ -670,6 +653,7 @@ static int smb_direct_post_recv(struct smb_direct_transport *t,
670
653
ib_dma_unmap_single (t -> cm_id -> device ,
671
654
recvmsg -> sge .addr , recvmsg -> sge .length ,
672
655
DMA_FROM_DEVICE );
656
+ recvmsg -> sge .length = 0 ;
673
657
smb_direct_disconnect_rdma_connection (t );
674
658
return ret ;
675
659
}
@@ -811,26 +795,16 @@ static void smb_direct_post_recv_credits(struct work_struct *work)
811
795
struct smb_direct_recvmsg * recvmsg ;
812
796
int receive_credits , credits = 0 ;
813
797
int ret ;
814
- int use_free = 1 ;
815
798
816
799
spin_lock (& t -> receive_credit_lock );
817
800
receive_credits = t -> recv_credits ;
818
801
spin_unlock (& t -> receive_credit_lock );
819
802
820
803
if (receive_credits < t -> recv_credit_target ) {
821
804
while (true) {
822
- if (use_free )
823
- recvmsg = get_free_recvmsg (t );
824
- else
825
- recvmsg = get_empty_recvmsg (t );
826
- if (!recvmsg ) {
827
- if (use_free ) {
828
- use_free = 0 ;
829
- continue ;
830
- } else {
831
- break ;
832
- }
833
- }
805
+ recvmsg = get_free_recvmsg (t );
806
+ if (!recvmsg )
807
+ break ;
834
808
835
809
recvmsg -> type = SMB_DIRECT_MSG_DATA_TRANSFER ;
836
810
recvmsg -> first_segment = false;
@@ -1806,8 +1780,6 @@ static void smb_direct_destroy_pools(struct smb_direct_transport *t)
1806
1780
1807
1781
while ((recvmsg = get_free_recvmsg (t )))
1808
1782
mempool_free (recvmsg , t -> recvmsg_mempool );
1809
- while ((recvmsg = get_empty_recvmsg (t )))
1810
- mempool_free (recvmsg , t -> recvmsg_mempool );
1811
1783
1812
1784
mempool_destroy (t -> recvmsg_mempool );
1813
1785
t -> recvmsg_mempool = NULL ;
@@ -1863,6 +1835,7 @@ static int smb_direct_create_pools(struct smb_direct_transport *t)
1863
1835
if (!recvmsg )
1864
1836
goto err ;
1865
1837
recvmsg -> transport = t ;
1838
+ recvmsg -> sge .length = 0 ;
1866
1839
list_add (& recvmsg -> list , & t -> recvmsg_queue );
1867
1840
}
1868
1841
t -> count_avail_recvmsg = t -> recv_credit_max ;
0 commit comments