Skip to content

Commit 99ef4db

Browse files
chuckleveramschuma-ntap
chucklever
authored and
amschuma-ntap
committed
xprtrdma: Replace DMA_BIDIRECTIONAL
The use of DMA_BIDIRECTIONAL is discouraged by DMA-API.txt. Fortunately, xprtrdma now knows which direction I/O is going as soon as it allocates each regbuf. The RPC Call and Reply buffers are no longer the same regbuf. They can each be labeled correctly now. The RPC Reply buffer is never part of either a Send or Receive WR, but it can be part of Reply chunk, which is mapped and registered via ->ro_map . So it is not DMA mapped when it is allocated (DMA_NONE), to avoid a double- mapping. Since Receive buffers are no longer DMA_BIDIRECTIONAL and their contents are never modified by the host CPU, DMA-API-HOWTO.txt suggests that a DMA sync before posting each buffer should be unnecessary. (See my_card_interrupt_handler). Signed-off-by: Chuck Lever <chuck.lever@oracle.com> Signed-off-by: Anna Schumaker <Anna.Schumaker@Netapp.com>
1 parent 08cf2ef commit 99ef4db

File tree

4 files changed

+36
-35
lines changed

4 files changed

+36
-35
lines changed

‎net/sunrpc/xprtrdma/backchannel.c

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,14 @@ static int rpcrdma_bc_setup_rqst(struct rpcrdma_xprt *r_xprt,
4545
return PTR_ERR(req);
4646
req->rl_backchannel = true;
4747

48-
rb = rpcrdma_alloc_regbuf(ia, RPCRDMA_HDRBUF_SIZE, GFP_KERNEL);
48+
rb = rpcrdma_alloc_regbuf(ia, RPCRDMA_HDRBUF_SIZE,
49+
DMA_TO_DEVICE, GFP_KERNEL);
4950
if (IS_ERR(rb))
5051
goto out_fail;
5152
req->rl_rdmabuf = rb;
5253

5354
size = r_xprt->rx_data.inline_rsize;
54-
rb = rpcrdma_alloc_regbuf(ia, size, GFP_KERNEL);
55+
rb = rpcrdma_alloc_regbuf(ia, size, DMA_TO_DEVICE, GFP_KERNEL);
5556
if (IS_ERR(rb))
5657
goto out_fail;
5758
req->rl_sendbuf = rb;

‎net/sunrpc/xprtrdma/transport.c

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -490,7 +490,7 @@ rpcrdma_get_rdmabuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
490490
if (req->rl_rdmabuf)
491491
return true;
492492

493-
rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, flags);
493+
rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, DMA_TO_DEVICE, flags);
494494
if (IS_ERR(rb))
495495
return false;
496496

@@ -517,7 +517,8 @@ rpcrdma_get_sendbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
517517
return true;
518518

519519
min_size = max_t(size_t, size, r_xprt->rx_data.inline_wsize);
520-
rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, min_size, flags);
520+
rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, min_size,
521+
DMA_TO_DEVICE, flags);
521522
if (IS_ERR(rb))
522523
return false;
523524

@@ -547,7 +548,7 @@ rpcrdma_get_recvbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
547548
if (req->rl_recvbuf && rdmab_length(req->rl_recvbuf) >= size)
548549
return true;
549550

550-
rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, flags);
551+
rb = rpcrdma_alloc_regbuf(&r_xprt->rx_ia, size, DMA_NONE, flags);
551552
if (IS_ERR(rb))
552553
return false;
553554

‎net/sunrpc/xprtrdma/verbs.c

Lines changed: 26 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -866,7 +866,7 @@ rpcrdma_create_rep(struct rpcrdma_xprt *r_xprt)
866866
goto out;
867867

868868
rep->rr_rdmabuf = rpcrdma_alloc_regbuf(ia, cdata->inline_rsize,
869-
GFP_KERNEL);
869+
DMA_FROM_DEVICE, GFP_KERNEL);
870870
if (IS_ERR(rep->rr_rdmabuf)) {
871871
rc = PTR_ERR(rep->rr_rdmabuf);
872872
goto out_free;
@@ -1172,27 +1172,24 @@ rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
11721172
spin_unlock(&buffers->rb_lock);
11731173
}
11741174

1175-
/*
1176-
* Wrappers for internal-use kmalloc memory registration, used by buffer code.
1177-
*/
1178-
11791175
/**
1180-
* rpcrdma_alloc_regbuf - kmalloc and register memory for SEND/RECV buffers
1176+
* rpcrdma_alloc_regbuf - allocate and DMA-map memory for SEND/RECV buffers
11811177
* @ia: controlling rpcrdma_ia
11821178
* @size: size of buffer to be allocated, in bytes
1179+
* @direction: direction of data movement
11831180
* @flags: GFP flags
11841181
*
1185-
* Returns pointer to private header of an area of internally
1186-
* registered memory, or an ERR_PTR. The registered buffer follows
1187-
* the end of the private header.
1182+
* Returns an ERR_PTR, or a pointer to a regbuf, which is a
1183+
* contiguous memory region that is DMA mapped persistently, and
1184+
* is registered for local I/O.
11881185
*
11891186
* xprtrdma uses a regbuf for posting an outgoing RDMA SEND, or for
1190-
* receiving the payload of RDMA RECV operations. regbufs are not
1191-
* used for RDMA READ/WRITE operations, thus are registered only for
1192-
* LOCAL access.
1187+
* receiving the payload of RDMA RECV operations. During Long Calls
1188+
* or Replies they may be registered externally via ro_map.
11931189
*/
11941190
struct rpcrdma_regbuf *
1195-
rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags)
1191+
rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size,
1192+
enum dma_data_direction direction, gfp_t flags)
11961193
{
11971194
struct rpcrdma_regbuf *rb;
11981195
struct ib_sge *iov;
@@ -1201,15 +1198,20 @@ rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags)
12011198
if (rb == NULL)
12021199
goto out;
12031200

1201+
rb->rg_direction = direction;
12041202
iov = &rb->rg_iov;
1205-
iov->addr = ib_dma_map_single(ia->ri_device,
1206-
(void *)rb->rg_base, size,
1207-
DMA_BIDIRECTIONAL);
1208-
if (ib_dma_mapping_error(ia->ri_device, iov->addr))
1209-
goto out_free;
1210-
12111203
iov->length = size;
12121204
iov->lkey = ia->ri_pd->local_dma_lkey;
1205+
1206+
if (direction != DMA_NONE) {
1207+
iov->addr = ib_dma_map_single(ia->ri_device,
1208+
(void *)rb->rg_base,
1209+
rdmab_length(rb),
1210+
rb->rg_direction);
1211+
if (ib_dma_mapping_error(ia->ri_device, iov->addr))
1212+
goto out_free;
1213+
}
1214+
12131215
return rb;
12141216

12151217
out_free:
@@ -1226,14 +1228,14 @@ rpcrdma_alloc_regbuf(struct rpcrdma_ia *ia, size_t size, gfp_t flags)
12261228
void
12271229
rpcrdma_free_regbuf(struct rpcrdma_ia *ia, struct rpcrdma_regbuf *rb)
12281230
{
1229-
struct ib_sge *iov;
1230-
12311231
if (!rb)
12321232
return;
12331233

1234-
iov = &rb->rg_iov;
1235-
ib_dma_unmap_single(ia->ri_device,
1236-
iov->addr, iov->length, DMA_BIDIRECTIONAL);
1234+
if (rb->rg_direction != DMA_NONE) {
1235+
ib_dma_unmap_single(ia->ri_device, rdmab_addr(rb),
1236+
rdmab_length(rb), rb->rg_direction);
1237+
}
1238+
12371239
kfree(rb);
12381240
}
12391241

@@ -1305,11 +1307,6 @@ rpcrdma_ep_post_recv(struct rpcrdma_ia *ia,
13051307
recv_wr.sg_list = &rep->rr_rdmabuf->rg_iov;
13061308
recv_wr.num_sge = 1;
13071309

1308-
ib_dma_sync_single_for_cpu(ia->ri_device,
1309-
rdmab_addr(rep->rr_rdmabuf),
1310-
rdmab_length(rep->rr_rdmabuf),
1311-
DMA_BIDIRECTIONAL);
1312-
13131310
rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);
13141311
if (rc)
13151312
goto out_postrecv;

‎net/sunrpc/xprtrdma/xprt_rdma.h

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ struct rpcrdma_ep {
113113

114114
struct rpcrdma_regbuf {
115115
struct ib_sge rg_iov;
116+
enum dma_data_direction rg_direction;
116117
__be32 rg_base[0] __attribute__ ((aligned(256)));
117118
};
118119

@@ -477,7 +478,8 @@ void rpcrdma_recv_buffer_put(struct rpcrdma_rep *);
477478
void rpcrdma_defer_mr_recovery(struct rpcrdma_mw *);
478479

479480
struct rpcrdma_regbuf *rpcrdma_alloc_regbuf(struct rpcrdma_ia *,
480-
size_t, gfp_t);
481+
size_t, enum dma_data_direction,
482+
gfp_t);
481483
void rpcrdma_free_regbuf(struct rpcrdma_ia *,
482484
struct rpcrdma_regbuf *);
483485

0 commit comments

Comments
 (0)