@@ -77,11 +77,13 @@ void amdgpu_fence_slab_fini(void)
77
77
* Cast helper
78
78
*/
79
79
static const struct dma_fence_ops amdgpu_fence_ops ;
80
+ static const struct dma_fence_ops amdgpu_job_fence_ops ;
80
81
static inline struct amdgpu_fence * to_amdgpu_fence (struct dma_fence * f )
81
82
{
82
83
struct amdgpu_fence * __f = container_of (f , struct amdgpu_fence , base );
83
84
84
- if (__f -> base .ops == & amdgpu_fence_ops )
85
+ if (__f -> base .ops == & amdgpu_fence_ops ||
86
+ __f -> base .ops == & amdgpu_job_fence_ops )
85
87
return __f ;
86
88
87
89
return NULL ;
@@ -158,19 +160,18 @@ int amdgpu_fence_emit(struct amdgpu_ring *ring, struct dma_fence **f, struct amd
158
160
}
159
161
160
162
seq = ++ ring -> fence_drv .sync_seq ;
161
- if (job != NULL && job -> job_run_counter ) {
163
+ if (job && job -> job_run_counter ) {
162
164
/* reinit seq for resubmitted jobs */
163
165
fence -> seqno = seq ;
164
166
} else {
165
- dma_fence_init (fence , & amdgpu_fence_ops ,
166
- & ring -> fence_drv .lock ,
167
- adev -> fence_context + ring -> idx ,
168
- seq );
169
- }
170
-
171
- if (job != NULL ) {
172
- /* mark this fence has a parent job */
173
- set_bit (AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT , & fence -> flags );
167
+ if (job )
168
+ dma_fence_init (fence , & amdgpu_job_fence_ops ,
169
+ & ring -> fence_drv .lock ,
170
+ adev -> fence_context + ring -> idx , seq );
171
+ else
172
+ dma_fence_init (fence , & amdgpu_fence_ops ,
173
+ & ring -> fence_drv .lock ,
174
+ adev -> fence_context + ring -> idx , seq );
174
175
}
175
176
176
177
amdgpu_ring_emit_fence (ring , ring -> fence_drv .gpu_addr ,
@@ -620,6 +621,25 @@ void amdgpu_fence_driver_hw_init(struct amdgpu_device *adev)
620
621
}
621
622
}
622
623
624
+ /**
625
+ * amdgpu_fence_driver_clear_job_fences - clear job embedded fences of ring
626
+ *
627
+ * @ring: fence of the ring to be cleared
628
+ *
629
+ */
630
+ void amdgpu_fence_driver_clear_job_fences (struct amdgpu_ring * ring )
631
+ {
632
+ int i ;
633
+ struct dma_fence * old , * * ptr ;
634
+
635
+ for (i = 0 ; i <= ring -> fence_drv .num_fences_mask ; i ++ ) {
636
+ ptr = & ring -> fence_drv .fences [i ];
637
+ old = rcu_dereference_protected (* ptr , 1 );
638
+ if (old && old -> ops == & amdgpu_job_fence_ops )
639
+ RCU_INIT_POINTER (* ptr , NULL );
640
+ }
641
+ }
642
+
623
643
/**
624
644
* amdgpu_fence_driver_force_completion - force signal latest fence of ring
625
645
*
@@ -643,16 +663,14 @@ static const char *amdgpu_fence_get_driver_name(struct dma_fence *fence)
643
663
644
664
static const char * amdgpu_fence_get_timeline_name (struct dma_fence * f )
645
665
{
646
- struct amdgpu_ring * ring ;
666
+ return (const char * )to_amdgpu_fence (f )-> ring -> name ;
667
+ }
647
668
648
- if (test_bit (AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT , & f -> flags )) {
649
- struct amdgpu_job * job = container_of (f , struct amdgpu_job , hw_fence );
669
+ static const char * amdgpu_job_fence_get_timeline_name (struct dma_fence * f )
670
+ {
671
+ struct amdgpu_job * job = container_of (f , struct amdgpu_job , hw_fence );
650
672
651
- ring = to_amdgpu_ring (job -> base .sched );
652
- } else {
653
- ring = to_amdgpu_fence (f )-> ring ;
654
- }
655
- return (const char * )ring -> name ;
673
+ return (const char * )to_amdgpu_ring (job -> base .sched )-> name ;
656
674
}
657
675
658
676
/**
@@ -665,18 +683,25 @@ static const char *amdgpu_fence_get_timeline_name(struct dma_fence *f)
665
683
*/
666
684
static bool amdgpu_fence_enable_signaling (struct dma_fence * f )
667
685
{
668
- struct amdgpu_ring * ring ;
686
+ if (!timer_pending (& to_amdgpu_fence (f )-> ring -> fence_drv .fallback_timer ))
687
+ amdgpu_fence_schedule_fallback (to_amdgpu_fence (f )-> ring );
669
688
670
- if ( test_bit ( AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT , & f -> flags )) {
671
- struct amdgpu_job * job = container_of ( f , struct amdgpu_job , hw_fence );
689
+ return true;
690
+ }
672
691
673
- ring = to_amdgpu_ring (job -> base .sched );
674
- } else {
675
- ring = to_amdgpu_fence (f )-> ring ;
676
- }
692
+ /**
693
+ * amdgpu_job_fence_enable_signaling - enable signalling on job fence
694
+ * @f: fence
695
+ *
696
+ * This is the simliar function with amdgpu_fence_enable_signaling above, it
697
+ * only handles the job embedded fence.
698
+ */
699
+ static bool amdgpu_job_fence_enable_signaling (struct dma_fence * f )
700
+ {
701
+ struct amdgpu_job * job = container_of (f , struct amdgpu_job , hw_fence );
677
702
678
- if (!timer_pending (& ring -> fence_drv .fallback_timer ))
679
- amdgpu_fence_schedule_fallback (ring );
703
+ if (!timer_pending (& to_amdgpu_ring ( job -> base . sched ) -> fence_drv .fallback_timer ))
704
+ amdgpu_fence_schedule_fallback (to_amdgpu_ring ( job -> base . sched ) );
680
705
681
706
return true;
682
707
}
@@ -692,19 +717,23 @@ static void amdgpu_fence_free(struct rcu_head *rcu)
692
717
{
693
718
struct dma_fence * f = container_of (rcu , struct dma_fence , rcu );
694
719
695
- if (test_bit (AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT , & f -> flags )) {
696
- /* free job if fence has a parent job */
697
- struct amdgpu_job * job ;
698
-
699
- job = container_of (f , struct amdgpu_job , hw_fence );
700
- kfree (job );
701
- } else {
702
720
/* free fence_slab if it's separated fence*/
703
- struct amdgpu_fence * fence ;
721
+ kmem_cache_free (amdgpu_fence_slab , to_amdgpu_fence (f ));
722
+ }
704
723
705
- fence = to_amdgpu_fence (f );
706
- kmem_cache_free (amdgpu_fence_slab , fence );
707
- }
724
+ /**
725
+ * amdgpu_job_fence_free - free up the job with embedded fence
726
+ *
727
+ * @rcu: RCU callback head
728
+ *
729
+ * Free up the job with embedded fence after the RCU grace period.
730
+ */
731
+ static void amdgpu_job_fence_free (struct rcu_head * rcu )
732
+ {
733
+ struct dma_fence * f = container_of (rcu , struct dma_fence , rcu );
734
+
735
+ /* free job if fence has a parent job */
736
+ kfree (container_of (f , struct amdgpu_job , hw_fence ));
708
737
}
709
738
710
739
/**
@@ -720,13 +749,32 @@ static void amdgpu_fence_release(struct dma_fence *f)
720
749
call_rcu (& f -> rcu , amdgpu_fence_free );
721
750
}
722
751
752
+ /**
753
+ * amdgpu_job_fence_release - callback that job embedded fence can be freed
754
+ *
755
+ * @f: fence
756
+ *
757
+ * This is the simliar function with amdgpu_fence_release above, it
758
+ * only handles the job embedded fence.
759
+ */
760
+ static void amdgpu_job_fence_release (struct dma_fence * f )
761
+ {
762
+ call_rcu (& f -> rcu , amdgpu_job_fence_free );
763
+ }
764
+
723
765
static const struct dma_fence_ops amdgpu_fence_ops = {
724
766
.get_driver_name = amdgpu_fence_get_driver_name ,
725
767
.get_timeline_name = amdgpu_fence_get_timeline_name ,
726
768
.enable_signaling = amdgpu_fence_enable_signaling ,
727
769
.release = amdgpu_fence_release ,
728
770
};
729
771
772
+ static const struct dma_fence_ops amdgpu_job_fence_ops = {
773
+ .get_driver_name = amdgpu_fence_get_driver_name ,
774
+ .get_timeline_name = amdgpu_job_fence_get_timeline_name ,
775
+ .enable_signaling = amdgpu_job_fence_enable_signaling ,
776
+ .release = amdgpu_job_fence_release ,
777
+ };
730
778
731
779
/*
732
780
* Fence debugfs
0 commit comments