Skip to content

Commit 009a82f

Browse files
author
Trond Myklebust
committed
SUNRPC: Micro-optimise when the task is known not to be sleeping
In cases where we know the task is not sleeping, try to optimise away the indirect call to task->tk_action() by replacing it with a direct call. Only change tail calls, to allow gcc to perform tail call elimination. Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>
1 parent 03e51d3 commit 009a82f

File tree

2 files changed

+73
-34
lines changed

2 files changed

+73
-34
lines changed

‎include/linux/sunrpc/sched.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,4 +304,12 @@ rpc_clnt_swap_deactivate(struct rpc_clnt *clnt)
304304
}
305305
#endif /* CONFIG_SUNRPC_SWAP */
306306

307+
static inline bool
308+
rpc_task_need_resched(const struct rpc_task *task)
309+
{
310+
if (RPC_IS_QUEUED(task) || task->tk_callback)
311+
return true;
312+
return false;
313+
}
314+
307315
#endif /* _LINUX_SUNRPC_SCHED_H_ */

‎net/sunrpc/clnt.c

Lines changed: 65 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1540,6 +1540,7 @@ call_start(struct rpc_task *task)
15401540
clnt->cl_stats->rpccnt++;
15411541
task->tk_action = call_reserve;
15421542
rpc_task_set_transport(task, clnt);
1543+
call_reserve(task);
15431544
}
15441545

15451546
/*
@@ -1553,6 +1554,9 @@ call_reserve(struct rpc_task *task)
15531554
task->tk_status = 0;
15541555
task->tk_action = call_reserveresult;
15551556
xprt_reserve(task);
1557+
if (rpc_task_need_resched(task))
1558+
return;
1559+
call_reserveresult(task);
15561560
}
15571561

15581562
static void call_retry_reserve(struct rpc_task *task);
@@ -1575,6 +1579,7 @@ call_reserveresult(struct rpc_task *task)
15751579
if (status >= 0) {
15761580
if (task->tk_rqstp) {
15771581
task->tk_action = call_refresh;
1582+
call_refresh(task);
15781583
return;
15791584
}
15801585

@@ -1600,6 +1605,7 @@ call_reserveresult(struct rpc_task *task)
16001605
/* fall through */
16011606
case -EAGAIN: /* woken up; retry */
16021607
task->tk_action = call_retry_reserve;
1608+
call_retry_reserve(task);
16031609
return;
16041610
case -EIO: /* probably a shutdown */
16051611
break;
@@ -1622,6 +1628,9 @@ call_retry_reserve(struct rpc_task *task)
16221628
task->tk_status = 0;
16231629
task->tk_action = call_reserveresult;
16241630
xprt_retry_reserve(task);
1631+
if (rpc_task_need_resched(task))
1632+
return;
1633+
call_reserveresult(task);
16251634
}
16261635

16271636
/*
@@ -1636,6 +1645,9 @@ call_refresh(struct rpc_task *task)
16361645
task->tk_status = 0;
16371646
task->tk_client->cl_stats->rpcauthrefresh++;
16381647
rpcauth_refreshcred(task);
1648+
if (rpc_task_need_resched(task))
1649+
return;
1650+
call_refreshresult(task);
16391651
}
16401652

16411653
/*
@@ -1654,6 +1666,7 @@ call_refreshresult(struct rpc_task *task)
16541666
case 0:
16551667
if (rpcauth_uptodatecred(task)) {
16561668
task->tk_action = call_allocate;
1669+
call_allocate(task);
16571670
return;
16581671
}
16591672
/* Use rate-limiting and a max number of retries if refresh
@@ -1672,6 +1685,7 @@ call_refreshresult(struct rpc_task *task)
16721685
task->tk_cred_retry--;
16731686
dprintk("RPC: %5u %s: retry refresh creds\n",
16741687
task->tk_pid, __func__);
1688+
call_refresh(task);
16751689
return;
16761690
}
16771691
dprintk("RPC: %5u %s: refresh creds failed with error %d\n",
@@ -1697,8 +1711,10 @@ call_allocate(struct rpc_task *task)
16971711
task->tk_status = 0;
16981712
task->tk_action = call_encode;
16991713

1700-
if (req->rq_buffer)
1714+
if (req->rq_buffer) {
1715+
call_encode(task);
17011716
return;
1717+
}
17021718

17031719
if (proc->p_proc != 0) {
17041720
BUG_ON(proc->p_arglen == 0);
@@ -1719,8 +1735,12 @@ call_allocate(struct rpc_task *task)
17191735

17201736
status = xprt->ops->buf_alloc(task);
17211737
xprt_inject_disconnect(xprt);
1722-
if (status == 0)
1738+
if (status == 0) {
1739+
if (rpc_task_need_resched(task))
1740+
return;
1741+
call_encode(task);
17231742
return;
1743+
}
17241744
if (status != -ENOMEM) {
17251745
rpc_exit(task, status);
17261746
return;
@@ -1803,12 +1823,8 @@ call_encode(struct rpc_task *task)
18031823
xprt_request_enqueue_receive(task);
18041824
xprt_request_enqueue_transmit(task);
18051825
out:
1806-
task->tk_action = call_transmit;
1807-
/* Check that the connection is OK */
1808-
if (!xprt_bound(task->tk_xprt))
1809-
task->tk_action = call_bind;
1810-
else if (!xprt_connected(task->tk_xprt))
1811-
task->tk_action = call_connect;
1826+
task->tk_action = call_bind;
1827+
call_bind(task);
18121828
}
18131829

18141830
/*
@@ -1842,14 +1858,17 @@ call_bind(struct rpc_task *task)
18421858
return;
18431859
}
18441860

1861+
if (xprt_bound(xprt)) {
1862+
task->tk_action = call_connect;
1863+
call_connect(task);
1864+
return;
1865+
}
1866+
18451867
dprint_status(task);
18461868

1847-
task->tk_action = call_connect;
1848-
if (!xprt_bound(xprt)) {
1849-
task->tk_action = call_bind_status;
1850-
task->tk_timeout = xprt->bind_timeout;
1851-
xprt->ops->rpcbind(task);
1852-
}
1869+
task->tk_action = call_bind_status;
1870+
task->tk_timeout = xprt->bind_timeout;
1871+
xprt->ops->rpcbind(task);
18531872
}
18541873

18551874
/*
@@ -1869,6 +1888,7 @@ call_bind_status(struct rpc_task *task)
18691888
dprint_status(task);
18701889
task->tk_status = 0;
18711890
task->tk_action = call_connect;
1891+
call_connect(task);
18721892
return;
18731893
}
18741894

@@ -1949,21 +1969,24 @@ call_connect(struct rpc_task *task)
19491969
return;
19501970
}
19511971

1972+
if (xprt_connected(xprt)) {
1973+
task->tk_action = call_transmit;
1974+
call_transmit(task);
1975+
return;
1976+
}
1977+
19521978
dprintk("RPC: %5u call_connect xprt %p %s connected\n",
19531979
task->tk_pid, xprt,
19541980
(xprt_connected(xprt) ? "is" : "is not"));
19551981

1956-
task->tk_action = call_transmit;
1957-
if (!xprt_connected(xprt)) {
1958-
task->tk_action = call_connect_status;
1959-
if (task->tk_status < 0)
1960-
return;
1961-
if (task->tk_flags & RPC_TASK_NOCONNECT) {
1962-
rpc_exit(task, -ENOTCONN);
1963-
return;
1964-
}
1965-
xprt_connect(task);
1982+
task->tk_action = call_connect_status;
1983+
if (task->tk_status < 0)
1984+
return;
1985+
if (task->tk_flags & RPC_TASK_NOCONNECT) {
1986+
rpc_exit(task, -ENOTCONN);
1987+
return;
19661988
}
1989+
xprt_connect(task);
19671990
}
19681991

19691992
/*
@@ -2016,6 +2039,7 @@ call_connect_status(struct rpc_task *task)
20162039
case 0:
20172040
clnt->cl_stats->netreconn++;
20182041
task->tk_action = call_transmit;
2042+
call_transmit(task);
20192043
return;
20202044
}
20212045
rpc_exit(task, status);
@@ -2040,19 +2064,20 @@ call_transmit(struct rpc_task *task)
20402064
dprint_status(task);
20412065

20422066
task->tk_action = call_transmit_status;
2067+
if (!xprt_prepare_transmit(task))
2068+
return;
2069+
task->tk_status = 0;
20432070
if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
2044-
if (!xprt_prepare_transmit(task))
2071+
if (!xprt_connected(task->tk_xprt)) {
2072+
task->tk_status = -ENOTCONN;
20452073
return;
2046-
task->tk_status = 0;
2047-
if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
2048-
if (!xprt_connected(task->tk_xprt)) {
2049-
task->tk_status = -ENOTCONN;
2050-
return;
2051-
}
2052-
xprt_transmit(task);
20532074
}
2075+
xprt_transmit(task);
20542076
}
20552077
xprt_end_transmit(task);
2078+
if (rpc_task_need_resched(task))
2079+
return;
2080+
call_transmit_status(task);
20562081
}
20572082

20582083
/*
@@ -2067,8 +2092,12 @@ call_transmit_status(struct rpc_task *task)
20672092
* Common case: success. Force the compiler to put this
20682093
* test first.
20692094
*/
2070-
if (task->tk_status == 0) {
2071-
xprt_request_wait_receive(task);
2095+
if (rpc_task_transmitted(task)) {
2096+
if (task->tk_status == 0)
2097+
xprt_request_wait_receive(task);
2098+
if (rpc_task_need_resched(task))
2099+
return;
2100+
call_status(task);
20722101
return;
20732102
}
20742103

@@ -2129,6 +2158,7 @@ call_bc_encode(struct rpc_task *task)
21292158
{
21302159
xprt_request_enqueue_transmit(task);
21312160
task->tk_action = call_bc_transmit;
2161+
call_bc_transmit(task);
21322162
}
21332163

21342164
/*
@@ -2219,6 +2249,7 @@ call_status(struct rpc_task *task)
22192249
status = task->tk_status;
22202250
if (status >= 0) {
22212251
task->tk_action = call_decode;
2252+
call_decode(task);
22222253
return;
22232254
}
22242255

0 commit comments

Comments
 (0)