Skip to content

Commit 984fee6

Browse files
airliedairlied
committed
Merge branch 'drm-etnaviv-next' of git://git.pengutronix.de/git/lst/linux into drm-next
Notable changes: - correctness fixes to the GPU cache flushing when switching execution state and when powering down the GPU - reduction of time spent in hardirq-off context - placement improvements to the GPU DMA linear window, allowing the driver to properly work on i.MX6 systems with more than 2GB of RAM * 'drm-etnaviv-next' of git://git.pengutronix.de/git/lst/linux: drm: etnaviv: clean up submit_bo() drm: etnaviv: clean up vram_mapping submission/retire path drm: etnaviv: improve readability of command insertion to ring buffer drm: etnaviv: clean up GPU command submission drm: etnaviv: use previous GPU pipe state when pipe switching drm: etnaviv: flush all GPU caches when stopping GPU drm: etnaviv: track current execution state drm: etnaviv: extract arming of semaphore drm: etnaviv: extract replacement of WAIT command drm: etnaviv: extract command ring reservation drm/etnaviv: move GPU linear window to end of DMA window drm/etnaviv: move runtime PM balance into retire worker
2 parents 507d44a + 8779aa8 commit 984fee6

File tree

9 files changed

+239
-159
lines changed

9 files changed

+239
-159
lines changed

‎drivers/gpu/drm/etnaviv/etnaviv_buffer.c

Lines changed: 139 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
#include "common.xml.h"
2323
#include "state.xml.h"
24+
#include "state_3d.xml.h"
2425
#include "cmdstream.xml.h"
2526

2627
/*
@@ -85,28 +86,31 @@ static inline void CMD_STALL(struct etnaviv_cmdbuf *buffer,
8586
OUT(buffer, VIV_FE_STALL_TOKEN_FROM(from) | VIV_FE_STALL_TOKEN_TO(to));
8687
}
8788

88-
static void etnaviv_cmd_select_pipe(struct etnaviv_cmdbuf *buffer, u8 pipe)
89+
static inline void CMD_SEM(struct etnaviv_cmdbuf *buffer, u32 from, u32 to)
8990
{
90-
u32 flush;
91-
u32 stall;
91+
CMD_LOAD_STATE(buffer, VIVS_GL_SEMAPHORE_TOKEN,
92+
VIVS_GL_SEMAPHORE_TOKEN_FROM(from) |
93+
VIVS_GL_SEMAPHORE_TOKEN_TO(to));
94+
}
95+
96+
static void etnaviv_cmd_select_pipe(struct etnaviv_gpu *gpu,
97+
struct etnaviv_cmdbuf *buffer, u8 pipe)
98+
{
99+
u32 flush = 0;
92100

93101
/*
94102
* This assumes that if we're switching to 2D, we're switching
95103
* away from 3D, and vice versa. Hence, if we're switching to
96104
* the 2D core, we need to flush the 3D depth and color caches,
97105
* otherwise we need to flush the 2D pixel engine cache.
98106
*/
99-
if (pipe == ETNA_PIPE_2D)
100-
flush = VIVS_GL_FLUSH_CACHE_DEPTH | VIVS_GL_FLUSH_CACHE_COLOR;
101-
else
107+
if (gpu->exec_state == ETNA_PIPE_2D)
102108
flush = VIVS_GL_FLUSH_CACHE_PE2D;
103-
104-
stall = VIVS_GL_SEMAPHORE_TOKEN_FROM(SYNC_RECIPIENT_FE) |
105-
VIVS_GL_SEMAPHORE_TOKEN_TO(SYNC_RECIPIENT_PE);
109+
else if (gpu->exec_state == ETNA_PIPE_3D)
110+
flush = VIVS_GL_FLUSH_CACHE_DEPTH | VIVS_GL_FLUSH_CACHE_COLOR;
106111

107112
CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_CACHE, flush);
108-
CMD_LOAD_STATE(buffer, VIVS_GL_SEMAPHORE_TOKEN, stall);
109-
113+
CMD_SEM(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
110114
CMD_STALL(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
111115

112116
CMD_LOAD_STATE(buffer, VIVS_GL_PIPE_SELECT,
@@ -131,6 +135,36 @@ static void etnaviv_buffer_dump(struct etnaviv_gpu *gpu,
131135
ptr, len * 4, 0);
132136
}
133137

138+
/*
139+
* Safely replace the WAIT of a waitlink with a new command and argument.
140+
* The GPU may be executing this WAIT while we're modifying it, so we have
141+
* to write it in a specific order to avoid the GPU branching to somewhere
142+
* else. 'wl_offset' is the offset to the first byte of the WAIT command.
143+
*/
144+
static void etnaviv_buffer_replace_wait(struct etnaviv_cmdbuf *buffer,
145+
unsigned int wl_offset, u32 cmd, u32 arg)
146+
{
147+
u32 *lw = buffer->vaddr + wl_offset;
148+
149+
lw[1] = arg;
150+
mb();
151+
lw[0] = cmd;
152+
mb();
153+
}
154+
155+
/*
156+
* Ensure that there is space in the command buffer to contiguously write
157+
* 'cmd_dwords' 64-bit words into the buffer, wrapping if necessary.
158+
*/
159+
static u32 etnaviv_buffer_reserve(struct etnaviv_gpu *gpu,
160+
struct etnaviv_cmdbuf *buffer, unsigned int cmd_dwords)
161+
{
162+
if (buffer->user_size + cmd_dwords * sizeof(u64) > buffer->size)
163+
buffer->user_size = 0;
164+
165+
return gpu_va(gpu, buffer) + buffer->user_size;
166+
}
167+
134168
u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu)
135169
{
136170
struct etnaviv_cmdbuf *buffer = gpu->buffer;
@@ -147,81 +181,79 @@ u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu)
147181
void etnaviv_buffer_end(struct etnaviv_gpu *gpu)
148182
{
149183
struct etnaviv_cmdbuf *buffer = gpu->buffer;
184+
unsigned int waitlink_offset = buffer->user_size - 16;
185+
u32 link_target, flush = 0;
150186

151-
/* Replace the last WAIT with an END */
152-
buffer->user_size -= 16;
153-
154-
CMD_END(buffer);
155-
mb();
187+
if (gpu->exec_state == ETNA_PIPE_2D)
188+
flush = VIVS_GL_FLUSH_CACHE_PE2D;
189+
else if (gpu->exec_state == ETNA_PIPE_3D)
190+
flush = VIVS_GL_FLUSH_CACHE_DEPTH |
191+
VIVS_GL_FLUSH_CACHE_COLOR |
192+
VIVS_GL_FLUSH_CACHE_TEXTURE |
193+
VIVS_GL_FLUSH_CACHE_TEXTUREVS |
194+
VIVS_GL_FLUSH_CACHE_SHADER_L2;
195+
196+
if (flush) {
197+
unsigned int dwords = 7;
198+
199+
link_target = etnaviv_buffer_reserve(gpu, buffer, dwords);
200+
201+
CMD_SEM(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
202+
CMD_STALL(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
203+
CMD_LOAD_STATE(buffer, VIVS_GL_FLUSH_CACHE, flush);
204+
if (gpu->exec_state == ETNA_PIPE_3D)
205+
CMD_LOAD_STATE(buffer, VIVS_TS_FLUSH_CACHE,
206+
VIVS_TS_FLUSH_CACHE_FLUSH);
207+
CMD_SEM(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
208+
CMD_STALL(buffer, SYNC_RECIPIENT_FE, SYNC_RECIPIENT_PE);
209+
CMD_END(buffer);
210+
211+
etnaviv_buffer_replace_wait(buffer, waitlink_offset,
212+
VIV_FE_LINK_HEADER_OP_LINK |
213+
VIV_FE_LINK_HEADER_PREFETCH(dwords),
214+
link_target);
215+
} else {
216+
/* Replace the last link-wait with an "END" command */
217+
etnaviv_buffer_replace_wait(buffer, waitlink_offset,
218+
VIV_FE_END_HEADER_OP_END, 0);
219+
}
156220
}
157221

222+
/* Append a command buffer to the ring buffer. */
158223
void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
159224
struct etnaviv_cmdbuf *cmdbuf)
160225
{
161226
struct etnaviv_cmdbuf *buffer = gpu->buffer;
162-
u32 *lw = buffer->vaddr + buffer->user_size - 16;
163-
u32 back, link_target, link_size, reserve_size, extra_size = 0;
227+
unsigned int waitlink_offset = buffer->user_size - 16;
228+
u32 return_target, return_dwords;
229+
u32 link_target, link_dwords;
164230

165231
if (drm_debug & DRM_UT_DRIVER)
166232
etnaviv_buffer_dump(gpu, buffer, 0, 0x50);
167233

234+
link_target = gpu_va(gpu, cmdbuf);
235+
link_dwords = cmdbuf->size / 8;
236+
168237
/*
169-
* If we need to flush the MMU prior to submitting this buffer, we
170-
* will need to append a mmu flush load state, followed by a new
238+
* If we need maintanence prior to submitting this buffer, we will
239+
* need to append a mmu flush load state, followed by a new
171240
* link to this buffer - a total of four additional words.
172241
*/
173242
if (gpu->mmu->need_flush || gpu->switch_context) {
243+
u32 target, extra_dwords;
244+
174245
/* link command */
175-
extra_size += 2;
246+
extra_dwords = 1;
247+
176248
/* flush command */
177249
if (gpu->mmu->need_flush)
178-
extra_size += 2;
250+
extra_dwords += 1;
251+
179252
/* pipe switch commands */
180253
if (gpu->switch_context)
181-
extra_size += 8;
182-
}
254+
extra_dwords += 4;
183255

184-
reserve_size = (6 + extra_size) * 4;
185-
186-
/*
187-
* if we are going to completely overflow the buffer, we need to wrap.
188-
*/
189-
if (buffer->user_size + reserve_size > buffer->size)
190-
buffer->user_size = 0;
191-
192-
/* save offset back into main buffer */
193-
back = buffer->user_size + reserve_size - 6 * 4;
194-
link_target = gpu_va(gpu, buffer) + buffer->user_size;
195-
link_size = 6;
196-
197-
/* Skip over any extra instructions */
198-
link_target += extra_size * sizeof(u32);
199-
200-
if (drm_debug & DRM_UT_DRIVER)
201-
pr_info("stream link to 0x%08x @ 0x%08x %p\n",
202-
link_target, gpu_va(gpu, cmdbuf), cmdbuf->vaddr);
203-
204-
/* jump back from cmd to main buffer */
205-
CMD_LINK(cmdbuf, link_size, link_target);
206-
207-
link_target = gpu_va(gpu, cmdbuf);
208-
link_size = cmdbuf->size / 8;
209-
210-
211-
212-
if (drm_debug & DRM_UT_DRIVER) {
213-
print_hex_dump(KERN_INFO, "cmd ", DUMP_PREFIX_OFFSET, 16, 4,
214-
cmdbuf->vaddr, cmdbuf->size, 0);
215-
216-
pr_info("link op: %p\n", lw);
217-
pr_info("link addr: %p\n", lw + 1);
218-
pr_info("addr: 0x%08x\n", link_target);
219-
pr_info("back: 0x%08x\n", gpu_va(gpu, buffer) + back);
220-
pr_info("event: %d\n", event);
221-
}
222-
223-
if (gpu->mmu->need_flush || gpu->switch_context) {
224-
u32 new_target = gpu_va(gpu, buffer) + buffer->user_size;
256+
target = etnaviv_buffer_reserve(gpu, buffer, extra_dwords);
225257

226258
if (gpu->mmu->need_flush) {
227259
/* Add the MMU flush */
@@ -236,32 +268,59 @@ void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, unsigned int event,
236268
}
237269

238270
if (gpu->switch_context) {
239-
etnaviv_cmd_select_pipe(buffer, cmdbuf->exec_state);
271+
etnaviv_cmd_select_pipe(gpu, buffer, cmdbuf->exec_state);
272+
gpu->exec_state = cmdbuf->exec_state;
240273
gpu->switch_context = false;
241274
}
242275

243-
/* And the link to the first buffer */
244-
CMD_LINK(buffer, link_size, link_target);
276+
/* And the link to the submitted buffer */
277+
CMD_LINK(buffer, link_dwords, link_target);
245278

246279
/* Update the link target to point to above instructions */
247-
link_target = new_target;
248-
link_size = extra_size;
280+
link_target = target;
281+
link_dwords = extra_dwords;
249282
}
250283

251-
/* trigger event */
284+
/*
285+
* Append a LINK to the submitted command buffer to return to
286+
* the ring buffer. return_target is the ring target address.
287+
* We need three dwords: event, wait, link.
288+
*/
289+
return_dwords = 3;
290+
return_target = etnaviv_buffer_reserve(gpu, buffer, return_dwords);
291+
CMD_LINK(cmdbuf, return_dwords, return_target);
292+
293+
/*
294+
* Append event, wait and link pointing back to the wait
295+
* command to the ring buffer.
296+
*/
252297
CMD_LOAD_STATE(buffer, VIVS_GL_EVENT, VIVS_GL_EVENT_EVENT_ID(event) |
253298
VIVS_GL_EVENT_FROM_PE);
254-
255-
/* append WAIT/LINK to main buffer */
256299
CMD_WAIT(buffer);
257-
CMD_LINK(buffer, 2, gpu_va(gpu, buffer) + (buffer->user_size - 4));
300+
CMD_LINK(buffer, 2, return_target + 8);
258301

259-
/* Change WAIT into a LINK command; write the address first. */
260-
*(lw + 1) = link_target;
261-
mb();
262-
*(lw) = VIV_FE_LINK_HEADER_OP_LINK |
263-
VIV_FE_LINK_HEADER_PREFETCH(link_size);
264-
mb();
302+
if (drm_debug & DRM_UT_DRIVER)
303+
pr_info("stream link to 0x%08x @ 0x%08x %p\n",
304+
return_target, gpu_va(gpu, cmdbuf), cmdbuf->vaddr);
305+
306+
if (drm_debug & DRM_UT_DRIVER) {
307+
print_hex_dump(KERN_INFO, "cmd ", DUMP_PREFIX_OFFSET, 16, 4,
308+
cmdbuf->vaddr, cmdbuf->size, 0);
309+
310+
pr_info("link op: %p\n", buffer->vaddr + waitlink_offset);
311+
pr_info("addr: 0x%08x\n", link_target);
312+
pr_info("back: 0x%08x\n", return_target);
313+
pr_info("event: %d\n", event);
314+
}
315+
316+
/*
317+
* Kick off the submitted command by replacing the previous
318+
* WAIT with a link to the address in the ring buffer.
319+
*/
320+
etnaviv_buffer_replace_wait(buffer, waitlink_offset,
321+
VIV_FE_LINK_HEADER_OP_LINK |
322+
VIV_FE_LINK_HEADER_PREFETCH(link_dwords),
323+
link_target);
265324

266325
if (drm_debug & DRM_UT_DRIVER)
267326
etnaviv_buffer_dump(gpu, buffer, 0, 0x50);

‎drivers/gpu/drm/etnaviv/etnaviv_drv.h

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,6 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data,
7575
int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma);
7676
int etnaviv_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf);
7777
int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset);
78-
int etnaviv_gem_get_iova(struct etnaviv_gpu *gpu,
79-
struct drm_gem_object *obj, u32 *iova);
80-
void etnaviv_gem_put_iova(struct etnaviv_gpu *gpu, struct drm_gem_object *obj);
8178
struct sg_table *etnaviv_gem_prime_get_sg_table(struct drm_gem_object *obj);
8279
void *etnaviv_gem_prime_vmap(struct drm_gem_object *obj);
8380
void etnaviv_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr);

‎drivers/gpu/drm/etnaviv/etnaviv_gem.c

Lines changed: 31 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -260,8 +260,32 @@ etnaviv_gem_get_vram_mapping(struct etnaviv_gem_object *obj,
260260
return NULL;
261261
}
262262

263-
int etnaviv_gem_get_iova(struct etnaviv_gpu *gpu,
264-
struct drm_gem_object *obj, u32 *iova)
263+
void etnaviv_gem_mapping_reference(struct etnaviv_vram_mapping *mapping)
264+
{
265+
struct etnaviv_gem_object *etnaviv_obj = mapping->object;
266+
267+
drm_gem_object_reference(&etnaviv_obj->base);
268+
269+
mutex_lock(&etnaviv_obj->lock);
270+
WARN_ON(mapping->use == 0);
271+
mapping->use += 1;
272+
mutex_unlock(&etnaviv_obj->lock);
273+
}
274+
275+
void etnaviv_gem_mapping_unreference(struct etnaviv_vram_mapping *mapping)
276+
{
277+
struct etnaviv_gem_object *etnaviv_obj = mapping->object;
278+
279+
mutex_lock(&etnaviv_obj->lock);
280+
WARN_ON(mapping->use == 0);
281+
mapping->use -= 1;
282+
mutex_unlock(&etnaviv_obj->lock);
283+
284+
drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
285+
}
286+
287+
struct etnaviv_vram_mapping *etnaviv_gem_mapping_get(
288+
struct drm_gem_object *obj, struct etnaviv_gpu *gpu)
265289
{
266290
struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
267291
struct etnaviv_vram_mapping *mapping;
@@ -329,28 +353,12 @@ int etnaviv_gem_get_iova(struct etnaviv_gpu *gpu,
329353
out:
330354
mutex_unlock(&etnaviv_obj->lock);
331355

332-
if (!ret) {
333-
/* Take a reference on the object */
334-
drm_gem_object_reference(obj);
335-
*iova = mapping->iova;
336-
}
337-
338-
return ret;
339-
}
340-
341-
void etnaviv_gem_put_iova(struct etnaviv_gpu *gpu, struct drm_gem_object *obj)
342-
{
343-
struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj);
344-
struct etnaviv_vram_mapping *mapping;
345-
346-
mutex_lock(&etnaviv_obj->lock);
347-
mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, gpu->mmu);
348-
349-
WARN_ON(mapping->use == 0);
350-
mapping->use -= 1;
351-
mutex_unlock(&etnaviv_obj->lock);
356+
if (ret)
357+
return ERR_PTR(ret);
352358

353-
drm_gem_object_unreference_unlocked(obj);
359+
/* Take a reference on the object */
360+
drm_gem_object_reference(obj);
361+
return mapping;
354362
}
355363

356364
void *etnaviv_gem_vmap(struct drm_gem_object *obj)

0 commit comments

Comments
 (0)