Skip to content

Commit ded946f

Browse files
jiadozhualexdeucher
jiadozhu
authored and
alexdeucher
committed
drm/amdgpu: Introduce gfx software ring (v9)
The software ring is created to support priority context while there is only one hardware queue for gfx. Every software ring has its fence driver and could be used as an ordinary ring for the GPU scheduler. Multiple software rings are bound to a real ring with the ring muxer. The packages committed on the software ring are copied to the real ring. v2: Use array to store software ring entry. v3: Remove unnecessary prints. v4: Remove amdgpu_ring_sw_init/fini functions, using gtt for sw ring buffer for later dma copy optimization. v5: Allocate ring entry dynamically in the muxer. v6: Update comments for the ring muxer. v7: Modify for function naming. v8: Combine software ring functions into amdgpu_ring_mux.c v9: Use kernel-doc comment on the get_rptr function. Cc: Christian Koenig <[email protected]> Cc: Luben Tuikov <[email protected]> Cc: Andrey Grodzovsky <[email protected]> Cc: Michel Dänzer <[email protected]> Signed-off-by: Jiadong.Zhu <[email protected]> Acked-by: Huang Rui <[email protected]> Acked-by: Luben Tuikov <[email protected]> Acked-by: Christian König <[email protected]> Signed-off-by: Alex Deucher <[email protected]>
1 parent 30ebe41 commit ded946f

File tree

5 files changed

+306
-1
lines changed

5 files changed

+306
-1
lines changed

‎drivers/gpu/drm/amd/amdgpu/Makefile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,8 @@ amdgpu-y += amdgpu_device.o amdgpu_kms.o \
5858
amdgpu_vm_sdma.o amdgpu_discovery.o amdgpu_ras_eeprom.o amdgpu_nbio.o \
5959
amdgpu_umc.o smu_v11_0_i2c.o amdgpu_fru_eeprom.o amdgpu_rap.o \
6060
amdgpu_fw_attestation.o amdgpu_securedisplay.o \
61-
amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o
61+
amdgpu_eeprom.o amdgpu_mca.o amdgpu_psp_ta.o amdgpu_lsdma.o \
62+
amdgpu_ring_mux.o
6263

6364
amdgpu-$(CONFIG_PROC_FS) += amdgpu_fdinfo.o
6465

‎drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include "amdgpu_imu.h"
3434
#include "soc15.h"
3535
#include "amdgpu_ras.h"
36+
#include "amdgpu_ring_mux.h"
3637

3738
/* GFX current status */
3839
#define AMDGPU_GFX_NORMAL_MODE 0x00000000L
@@ -352,6 +353,8 @@ struct amdgpu_gfx {
352353
struct amdgpu_gfx_ras *ras;
353354

354355
bool is_poweron;
356+
357+
struct amdgpu_ring_mux muxer;
355358
};
356359

357360
#define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))

‎drivers/gpu/drm/amd/amdgpu/amdgpu_ring.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,10 @@ struct amdgpu_ring {
279279
bool is_mes_queue;
280280
uint32_t hw_queue_id;
281281
struct amdgpu_mes_ctx_data *mes_ctx;
282+
283+
bool is_sw_ring;
284+
unsigned int entry_index;
285+
282286
};
283287

284288
#define amdgpu_ring_parse_cs(r, p, job, ib) ((r)->funcs->parse_cs((p), (job), (ib)))

‎drivers/gpu/drm/amd/amdgpu/amdgpu_ring_mux.c

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
/*
2+
* Copyright 2022 Advanced Micro Devices, Inc.
3+
*
4+
* Permission is hereby granted, free of charge, to any person obtaining a
5+
* copy of this software and associated documentation files (the "Software"),
6+
* to deal in the Software without restriction, including without limitation
7+
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+
* and/or sell copies of the Software, and to permit persons to whom the
9+
* Software is furnished to do so, subject to the following conditions:
10+
*
11+
* The above copyright notice and this permission notice shall be included in
12+
* all copies or substantial portions of the Software.
13+
*
14+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+
* OTHER DEALINGS IN THE SOFTWARE.
21+
*
22+
*/
23+
#include <linux/slab.h>
24+
#include <drm/drm_print.h>
25+
26+
#include "amdgpu_ring_mux.h"
27+
#include "amdgpu_ring.h"
28+
#include "amdgpu.h"
29+
30+
#define AMDGPU_MUX_RESUBMIT_JIFFIES_TIMEOUT (HZ / 2)
31+
32+
int amdgpu_ring_mux_init(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring,
33+
unsigned int entry_size)
34+
{
35+
mux->real_ring = ring;
36+
mux->num_ring_entries = 0;
37+
mux->ring_entry = kcalloc(entry_size, sizeof(struct amdgpu_mux_entry), GFP_KERNEL);
38+
if (!mux->ring_entry)
39+
return -ENOMEM;
40+
41+
mux->ring_entry_size = entry_size;
42+
spin_lock_init(&mux->lock);
43+
44+
return 0;
45+
}
46+
47+
void amdgpu_ring_mux_fini(struct amdgpu_ring_mux *mux)
48+
{
49+
kfree(mux->ring_entry);
50+
mux->ring_entry = NULL;
51+
mux->num_ring_entries = 0;
52+
mux->ring_entry_size = 0;
53+
}
54+
55+
int amdgpu_ring_mux_add_sw_ring(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
56+
{
57+
struct amdgpu_mux_entry *e;
58+
59+
if (mux->num_ring_entries >= mux->ring_entry_size) {
60+
DRM_ERROR("add sw ring exceeding max entry size\n");
61+
return -ENOENT;
62+
}
63+
64+
e = &mux->ring_entry[mux->num_ring_entries];
65+
ring->entry_index = mux->num_ring_entries;
66+
e->ring = ring;
67+
68+
mux->num_ring_entries += 1;
69+
return 0;
70+
}
71+
72+
static inline struct amdgpu_mux_entry *amdgpu_ring_mux_sw_entry(struct amdgpu_ring_mux *mux,
73+
struct amdgpu_ring *ring)
74+
{
75+
return ring->entry_index < mux->ring_entry_size ?
76+
&mux->ring_entry[ring->entry_index] : NULL;
77+
}
78+
79+
/* copy packages on sw ring range[begin, end) */
80+
static void amdgpu_ring_mux_copy_pkt_from_sw_ring(struct amdgpu_ring_mux *mux,
81+
struct amdgpu_ring *ring,
82+
u64 s_start, u64 s_end)
83+
{
84+
u64 start, end;
85+
struct amdgpu_ring *real_ring = mux->real_ring;
86+
87+
start = s_start & ring->buf_mask;
88+
end = s_end & ring->buf_mask;
89+
90+
if (start == end) {
91+
DRM_ERROR("no more data copied from sw ring\n");
92+
return;
93+
}
94+
if (start > end) {
95+
amdgpu_ring_alloc(real_ring, (ring->ring_size >> 2) + end - start);
96+
amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start],
97+
(ring->ring_size >> 2) - start);
98+
amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[0], end);
99+
} else {
100+
amdgpu_ring_alloc(real_ring, end - start);
101+
amdgpu_ring_write_multiple(real_ring, (void *)&ring->ring[start], end - start);
102+
}
103+
}
104+
105+
void amdgpu_ring_mux_set_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, u64 wptr)
106+
{
107+
struct amdgpu_mux_entry *e;
108+
109+
e = amdgpu_ring_mux_sw_entry(mux, ring);
110+
if (!e) {
111+
DRM_ERROR("cannot find entry for sw ring\n");
112+
return;
113+
}
114+
115+
spin_lock(&mux->lock);
116+
e->sw_cptr = e->sw_wptr;
117+
e->sw_wptr = wptr;
118+
e->start_ptr_in_hw_ring = mux->real_ring->wptr;
119+
120+
amdgpu_ring_mux_copy_pkt_from_sw_ring(mux, ring, e->sw_cptr, wptr);
121+
e->end_ptr_in_hw_ring = mux->real_ring->wptr;
122+
amdgpu_ring_commit(mux->real_ring);
123+
124+
spin_unlock(&mux->lock);
125+
}
126+
127+
u64 amdgpu_ring_mux_get_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
128+
{
129+
struct amdgpu_mux_entry *e;
130+
131+
e = amdgpu_ring_mux_sw_entry(mux, ring);
132+
if (!e) {
133+
DRM_ERROR("cannot find entry for sw ring\n");
134+
return 0;
135+
}
136+
137+
return e->sw_wptr;
138+
}
139+
140+
/**
141+
* amdgpu_ring_mux_get_rptr - get the readptr of the software ring
142+
* @mux: the multiplexer the software rings attach to
143+
* @ring: the software ring of which we calculate the readptr
144+
*
145+
* The return value of the readptr is not precise while the other rings could
146+
* write data onto the real ring buffer.After overwriting on the real ring, we
147+
* can not decide if our packages have been excuted or not read yet. However,
148+
* this function is only called by the tools such as umr to collect the latest
149+
* packages for the hang analysis. We assume the hang happens near our latest
150+
* submit. Thus we could use the following logic to give the clue:
151+
* If the readptr is between start and end, then we return the copy pointer
152+
* plus the distance from start to readptr. If the readptr is before start, we
153+
* return the copy pointer. Lastly, if the readptr is past end, we return the
154+
* write pointer.
155+
*/
156+
u64 amdgpu_ring_mux_get_rptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring)
157+
{
158+
struct amdgpu_mux_entry *e;
159+
u64 readp, offset, start, end;
160+
161+
e = amdgpu_ring_mux_sw_entry(mux, ring);
162+
if (!e) {
163+
DRM_ERROR("no sw entry found!\n");
164+
return 0;
165+
}
166+
167+
readp = amdgpu_ring_get_rptr(mux->real_ring);
168+
169+
start = e->start_ptr_in_hw_ring & mux->real_ring->buf_mask;
170+
end = e->end_ptr_in_hw_ring & mux->real_ring->buf_mask;
171+
if (start > end) {
172+
if (readp <= end)
173+
readp += mux->real_ring->ring_size >> 2;
174+
end += mux->real_ring->ring_size >> 2;
175+
}
176+
177+
if (start <= readp && readp <= end) {
178+
offset = readp - start;
179+
e->sw_rptr = (e->sw_cptr + offset) & ring->buf_mask;
180+
} else if (readp < start) {
181+
e->sw_rptr = e->sw_cptr;
182+
} else {
183+
/* end < readptr */
184+
e->sw_rptr = e->sw_wptr;
185+
}
186+
187+
return e->sw_rptr;
188+
}
189+
190+
u64 amdgpu_sw_ring_get_rptr_gfx(struct amdgpu_ring *ring)
191+
{
192+
struct amdgpu_device *adev = ring->adev;
193+
struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
194+
195+
WARN_ON(!ring->is_sw_ring);
196+
return amdgpu_ring_mux_get_rptr(mux, ring);
197+
}
198+
199+
u64 amdgpu_sw_ring_get_wptr_gfx(struct amdgpu_ring *ring)
200+
{
201+
struct amdgpu_device *adev = ring->adev;
202+
struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
203+
204+
WARN_ON(!ring->is_sw_ring);
205+
return amdgpu_ring_mux_get_wptr(mux, ring);
206+
}
207+
208+
void amdgpu_sw_ring_set_wptr_gfx(struct amdgpu_ring *ring)
209+
{
210+
struct amdgpu_device *adev = ring->adev;
211+
struct amdgpu_ring_mux *mux = &adev->gfx.muxer;
212+
213+
WARN_ON(!ring->is_sw_ring);
214+
amdgpu_ring_mux_set_wptr(mux, ring, ring->wptr);
215+
}
216+
217+
/* Override insert_nop to prevent emitting nops to the software rings */
218+
void amdgpu_sw_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count)
219+
{
220+
WARN_ON(!ring->is_sw_ring);
221+
}

‎drivers/gpu/drm/amd/amdgpu/amdgpu_ring_mux.h

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Copyright 2022 Advanced Micro Devices, Inc.
3+
*
4+
* Permission is hereby granted, free of charge, to any person obtaining a
5+
* copy of this software and associated documentation files (the "Software"),
6+
* to deal in the Software without restriction, including without limitation
7+
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8+
* and/or sell copies of the Software, and to permit persons to whom the
9+
* Software is furnished to do so, subject to the following conditions:
10+
*
11+
* The above copyright notice and this permission notice shall be included in
12+
* all copies or substantial portions of the Software.
13+
*
14+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17+
* THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18+
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19+
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20+
* OTHER DEALINGS IN THE SOFTWARE.
21+
*
22+
*/
23+
24+
#ifndef __AMDGPU_RING_MUX__
25+
#define __AMDGPU_RING_MUX__
26+
27+
#include <linux/timer.h>
28+
#include <linux/spinlock.h>
29+
#include "amdgpu_ring.h"
30+
31+
struct amdgpu_ring;
32+
/**
33+
* struct amdgpu_mux_entry - the entry recording software rings copying information.
34+
* @ring: the pointer to the software ring.
35+
* @start_ptr_in_hw_ring: last start location copied to in the hardware ring.
36+
* @end_ptr_in_hw_ring: last end location copied to in the hardware ring.
37+
* @sw_cptr: the position of the copy pointer in the sw ring.
38+
* @sw_rptr: the read pointer in software ring.
39+
* @sw_wptr: the write pointer in software ring.
40+
*/
41+
struct amdgpu_mux_entry {
42+
struct amdgpu_ring *ring;
43+
u64 start_ptr_in_hw_ring;
44+
u64 end_ptr_in_hw_ring;
45+
u64 sw_cptr;
46+
u64 sw_rptr;
47+
u64 sw_wptr;
48+
};
49+
50+
struct amdgpu_ring_mux {
51+
struct amdgpu_ring *real_ring;
52+
53+
struct amdgpu_mux_entry *ring_entry;
54+
unsigned int num_ring_entries;
55+
unsigned int ring_entry_size;
56+
/*the lock for copy data from different software rings*/
57+
spinlock_t lock;
58+
};
59+
60+
int amdgpu_ring_mux_init(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring,
61+
unsigned int entry_size);
62+
void amdgpu_ring_mux_fini(struct amdgpu_ring_mux *mux);
63+
int amdgpu_ring_mux_add_sw_ring(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring);
64+
void amdgpu_ring_mux_set_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring, u64 wptr);
65+
u64 amdgpu_ring_mux_get_wptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring);
66+
u64 amdgpu_ring_mux_get_rptr(struct amdgpu_ring_mux *mux, struct amdgpu_ring *ring);
67+
68+
u64 amdgpu_sw_ring_get_rptr_gfx(struct amdgpu_ring *ring);
69+
u64 amdgpu_sw_ring_get_wptr_gfx(struct amdgpu_ring *ring);
70+
void amdgpu_sw_ring_set_wptr_gfx(struct amdgpu_ring *ring);
71+
72+
void amdgpu_sw_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count);
73+
void amdgpu_sw_ring_ib_begin(struct amdgpu_ring *ring);
74+
void amdgpu_sw_ring_ib_end(struct amdgpu_ring *ring);
75+
76+
#endif

0 commit comments

Comments
 (0)