Skip to content

Commit cbf5500

Browse files
Eliezer Tamirdavem330
Eliezer Tamir
authored and
davem330
committed
net: rename low latency sockets functions to busy poll
Rename functions in include/net/ll_poll.h to busy wait. Clarify documentation about expected power use increase. Rename POLL_LL to POLL_BUSY_LOOP. Add need_resched() testing to poll/select busy loops. Note, that in select and poll can_busy_poll is dynamic and is updated continuously to reflect the existence of supported sockets with valid queue information. Signed-off-by: Eliezer Tamir <eliezer.tamir@linux.intel.com> Signed-off-by: David S. Miller <davem@davemloft.net>
1 parent c7e8e8a commit cbf5500

File tree

7 files changed

+80
-61
lines changed

7 files changed

+80
-61
lines changed

‎Documentation/sysctl/net.txt

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -53,22 +53,24 @@ Default: 64
5353
low_latency_read
5454
----------------
5555
Low latency busy poll timeout for socket reads. (needs CONFIG_NET_LL_RX_POLL)
56-
Approximate time in us to spin waiting for packets on the device queue.
56+
Approximate time in us to busy loop waiting for packets on the device queue.
5757
This sets the default value of the SO_LL socket option.
58-
Can be set or overridden per socket by setting socket option SO_LL.
59-
Recommended value is 50. May increase power usage.
58+
Can be set or overridden per socket by setting socket option SO_LL, which is
59+
the preferred method of enabling.
60+
If you need to enable the feature globally via sysctl, a value of 50 is recommended.
61+
Will increase power usage.
6062
Default: 0 (off)
6163

6264
low_latency_poll
6365
----------------
6466
Low latency busy poll timeout for poll and select. (needs CONFIG_NET_LL_RX_POLL)
65-
Approximate time in us to spin waiting for packets on the device queue.
67+
Approximate time in us to busy loop waiting for events.
6668
Recommended value depends on the number of sockets you poll on.
6769
For several sockets 50, for several hundreds 100.
6870
For more than that you probably want to use epoll.
6971
Note that only sockets with SO_LL set will be busy polled, so you want to either
7072
selectively set SO_LL on those sockets or set sysctl.net.low_latency_read globally.
71-
May increase power usage.
73+
Will increase power usage.
7274
Default: 0 (off)
7375

7476
rmem_default

‎fs/select.c

Lines changed: 37 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -402,9 +402,9 @@ int do_select(int n, fd_set_bits *fds, struct timespec *end_time)
402402
poll_table *wait;
403403
int retval, i, timed_out = 0;
404404
unsigned long slack = 0;
405-
unsigned int ll_flag = ll_get_flag();
406-
u64 ll_start = ll_start_time(ll_flag);
407-
u64 ll_time = ll_run_time();
405+
unsigned int busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0;
406+
u64 busy_start = busy_loop_start_time(busy_flag);
407+
u64 busy_end = busy_loop_end_time();
408408

409409
rcu_read_lock();
410410
retval = max_select_fd(n, fds);
@@ -427,7 +427,7 @@ int do_select(int n, fd_set_bits *fds, struct timespec *end_time)
427427
retval = 0;
428428
for (;;) {
429429
unsigned long *rinp, *routp, *rexp, *inp, *outp, *exp;
430-
bool can_ll = false;
430+
bool can_busy_loop = false;
431431

432432
inp = fds->in; outp = fds->out; exp = fds->ex;
433433
rinp = fds->res_in; routp = fds->res_out; rexp = fds->res_ex;
@@ -456,7 +456,7 @@ int do_select(int n, fd_set_bits *fds, struct timespec *end_time)
456456
mask = DEFAULT_POLLMASK;
457457
if (f_op && f_op->poll) {
458458
wait_key_set(wait, in, out,
459-
bit, ll_flag);
459+
bit, busy_flag);
460460
mask = (*f_op->poll)(f.file, wait);
461461
}
462462
fdput(f);
@@ -475,11 +475,18 @@ int do_select(int n, fd_set_bits *fds, struct timespec *end_time)
475475
retval++;
476476
wait->_qproc = NULL;
477477
}
478-
if (mask & POLL_LL)
479-
can_ll = true;
480478
/* got something, stop busy polling */
481-
if (retval)
482-
ll_flag = 0;
479+
if (retval) {
480+
can_busy_loop = false;
481+
busy_flag = 0;
482+
483+
/*
484+
* only remember a returned
485+
* POLL_BUSY_LOOP if we asked for it
486+
*/
487+
} else if (busy_flag & mask)
488+
can_busy_loop = true;
489+
483490
}
484491
}
485492
if (res_in)
@@ -498,8 +505,9 @@ int do_select(int n, fd_set_bits *fds, struct timespec *end_time)
498505
break;
499506
}
500507

501-
/* only if on, have sockets with POLL_LL and not out of time */
502-
if (ll_flag && can_ll && can_poll_ll(ll_start, ll_time))
508+
/* only if found POLL_BUSY_LOOP sockets && not out of time */
509+
if (!need_resched() && can_busy_loop &&
510+
busy_loop_range(busy_start, busy_end))
503511
continue;
504512

505513
/*
@@ -734,7 +742,8 @@ struct poll_list {
734742
* if pwait->_qproc is non-NULL.
735743
*/
736744
static inline unsigned int do_pollfd(struct pollfd *pollfd, poll_table *pwait,
737-
bool *can_ll, unsigned int ll_flag)
745+
bool *can_busy_poll,
746+
unsigned int busy_flag)
738747
{
739748
unsigned int mask;
740749
int fd;
@@ -748,10 +757,10 @@ static inline unsigned int do_pollfd(struct pollfd *pollfd, poll_table *pwait,
748757
mask = DEFAULT_POLLMASK;
749758
if (f.file->f_op && f.file->f_op->poll) {
750759
pwait->_key = pollfd->events|POLLERR|POLLHUP;
751-
pwait->_key |= ll_flag;
760+
pwait->_key |= busy_flag;
752761
mask = f.file->f_op->poll(f.file, pwait);
753-
if (mask & POLL_LL)
754-
*can_ll = true;
762+
if (mask & busy_flag)
763+
*can_busy_poll = true;
755764
}
756765
/* Mask out unneeded events. */
757766
mask &= pollfd->events | POLLERR | POLLHUP;
@@ -770,9 +779,10 @@ static int do_poll(unsigned int nfds, struct poll_list *list,
770779
ktime_t expire, *to = NULL;
771780
int timed_out = 0, count = 0;
772781
unsigned long slack = 0;
773-
unsigned int ll_flag = ll_get_flag();
774-
u64 ll_start = ll_start_time(ll_flag);
775-
u64 ll_time = ll_run_time();
782+
unsigned int busy_flag = net_busy_loop_on() ? POLL_BUSY_LOOP : 0;
783+
u64 busy_start = busy_loop_start_time(busy_flag);
784+
u64 busy_end = busy_loop_end_time();
785+
776786

777787
/* Optimise the no-wait case */
778788
if (end_time && !end_time->tv_sec && !end_time->tv_nsec) {
@@ -785,7 +795,7 @@ static int do_poll(unsigned int nfds, struct poll_list *list,
785795

786796
for (;;) {
787797
struct poll_list *walk;
788-
bool can_ll = false;
798+
bool can_busy_loop = false;
789799

790800
for (walk = list; walk != NULL; walk = walk->next) {
791801
struct pollfd * pfd, * pfd_end;
@@ -800,10 +810,13 @@ static int do_poll(unsigned int nfds, struct poll_list *list,
800810
* this. They'll get immediately deregistered
801811
* when we break out and return.
802812
*/
803-
if (do_pollfd(pfd, pt, &can_ll, ll_flag)) {
813+
if (do_pollfd(pfd, pt, &can_busy_loop,
814+
busy_flag)) {
804815
count++;
805816
pt->_qproc = NULL;
806-
ll_flag = 0;
817+
/* found something, stop busy polling */
818+
busy_flag = 0;
819+
can_busy_loop = false;
807820
}
808821
}
809822
}
@@ -820,8 +833,9 @@ static int do_poll(unsigned int nfds, struct poll_list *list,
820833
if (count || timed_out)
821834
break;
822835

823-
/* only if on, have sockets with POLL_LL and not out of time */
824-
if (ll_flag && can_ll && can_poll_ll(ll_start, ll_time))
836+
/* only if found POLL_BUSY_LOOP sockets && not out of time */
837+
if (!need_resched() && can_busy_loop &&
838+
busy_loop_range(busy_start, busy_end))
825839
continue;
826840

827841
/*

‎include/net/ll_poll.h

Lines changed: 24 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -37,17 +37,17 @@ extern unsigned int sysctl_net_ll_poll __read_mostly;
3737
#define LL_FLUSH_FAILED -1
3838
#define LL_FLUSH_BUSY -2
3939

40-
static inline unsigned int ll_get_flag(void)
40+
static inline bool net_busy_loop_on(void)
4141
{
42-
return sysctl_net_ll_poll ? POLL_LL : 0;
42+
return sysctl_net_ll_poll;
4343
}
4444

4545
/* a wrapper to make debug_smp_processor_id() happy
4646
* we can use sched_clock() because we don't care much about precision
4747
* we only care that the average is bounded
4848
*/
4949
#ifdef CONFIG_DEBUG_PREEMPT
50-
static inline u64 ll_sched_clock(void)
50+
static inline u64 busy_loop_sched_clock(void)
5151
{
5252
u64 rc;
5353

@@ -58,7 +58,7 @@ static inline u64 ll_sched_clock(void)
5858
return rc;
5959
}
6060
#else /* CONFIG_DEBUG_PREEMPT */
61-
static inline u64 ll_sched_clock(void)
61+
static inline u64 busy_loop_sched_clock(void)
6262
{
6363
return sched_clock();
6464
}
@@ -67,46 +67,48 @@ static inline u64 ll_sched_clock(void)
6767
/* we don't mind a ~2.5% imprecision so <<10 instead of *1000
6868
* sk->sk_ll_usec is a u_int so this can't overflow
6969
*/
70-
static inline u64 ll_sk_run_time(struct sock *sk)
70+
static inline u64 sk_busy_loop_end_time(struct sock *sk)
7171
{
7272
return (u64)ACCESS_ONCE(sk->sk_ll_usec) << 10;
7373
}
7474

7575
/* in poll/select we use the global sysctl_net_ll_poll value
7676
* only call sched_clock() if enabled
7777
*/
78-
static inline u64 ll_run_time(void)
78+
static inline u64 busy_loop_end_time(void)
7979
{
8080
return (u64)ACCESS_ONCE(sysctl_net_ll_poll) << 10;
8181
}
8282

83-
/* if flag is not set we don't need to know the time */
84-
static inline u64 ll_start_time(unsigned int flag)
83+
/* if flag is not set we don't need to know the time
84+
* so we want to avoid a potentially expensive sched_clock()
85+
*/
86+
static inline u64 busy_loop_start_time(unsigned int flag)
8587
{
86-
return flag ? ll_sched_clock() : 0;
88+
return flag ? busy_loop_sched_clock() : 0;
8789
}
8890

89-
static inline bool sk_valid_ll(struct sock *sk)
91+
static inline bool sk_can_busy_loop(struct sock *sk)
9092
{
9193
return sk->sk_ll_usec && sk->sk_napi_id &&
9294
!need_resched() && !signal_pending(current);
9395
}
9496

9597
/* careful! time_in_range64 will evaluate now twice */
96-
static inline bool can_poll_ll(u64 start_time, u64 run_time)
98+
static inline bool busy_loop_range(u64 start_time, u64 run_time)
9799
{
98-
u64 now = ll_sched_clock();
100+
u64 now = busy_loop_sched_clock();
99101

100102
return time_in_range64(now, start_time, start_time + run_time);
101103
}
102104

103105
/* when used in sock_poll() nonblock is known at compile time to be true
104106
* so the loop and end_time will be optimized out
105107
*/
106-
static inline bool sk_poll_ll(struct sock *sk, int nonblock)
108+
static inline bool sk_busy_loop(struct sock *sk, int nonblock)
107109
{
108-
u64 start_time = ll_start_time(!nonblock);
109-
u64 run_time = ll_sk_run_time(sk);
110+
u64 start_time = busy_loop_start_time(!nonblock);
111+
u64 end_time = sk_busy_loop_end_time(sk);
110112
const struct net_device_ops *ops;
111113
struct napi_struct *napi;
112114
int rc = false;
@@ -137,7 +139,7 @@ static inline bool sk_poll_ll(struct sock *sk, int nonblock)
137139
LINUX_MIB_LOWLATENCYRXPACKETS, rc);
138140

139141
} while (!nonblock && skb_queue_empty(&sk->sk_receive_queue) &&
140-
can_poll_ll(start_time, run_time));
142+
busy_loop_range(start_time, end_time));
141143

142144
rc = !skb_queue_empty(&sk->sk_receive_queue);
143145
out:
@@ -158,27 +160,27 @@ static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb)
158160
}
159161

160162
#else /* CONFIG_NET_LL_RX_POLL */
161-
static inline unsigned long ll_get_flag(void)
163+
static inline unsigned long net_busy_loop_on(void)
162164
{
163165
return 0;
164166
}
165167

166-
static inline u64 ll_start_time(unsigned int flag)
168+
static inline u64 busy_loop_start_time(unsigned int flag)
167169
{
168170
return 0;
169171
}
170172

171-
static inline u64 ll_run_time(void)
173+
static inline u64 busy_loop_end_time(void)
172174
{
173175
return 0;
174176
}
175177

176-
static inline bool sk_valid_ll(struct sock *sk)
178+
static inline bool sk_can_busy_loop(struct sock *sk)
177179
{
178180
return false;
179181
}
180182

181-
static inline bool sk_poll_ll(struct sock *sk, int nonblock)
183+
static inline bool sk_busy_poll(struct sock *sk, int nonblock)
182184
{
183185
return false;
184186
}
@@ -191,7 +193,7 @@ static inline void sk_mark_ll(struct sock *sk, struct sk_buff *skb)
191193
{
192194
}
193195

194-
static inline bool can_poll_ll(u64 start_time, u64 run_time)
196+
static inline bool busy_loop_range(u64 start_time, u64 run_time)
195197
{
196198
return false;
197199
}

‎include/uapi/asm-generic/poll.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@
3030

3131
#define POLLFREE 0x4000 /* currently only for epoll */
3232

33-
#define POLL_LL 0x8000
33+
#define POLL_BUSY_LOOP 0x8000
3434

3535
struct pollfd {
3636
int fd;

‎net/core/datagram.c

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,8 @@ struct sk_buff *__skb_recv_datagram(struct sock *sk, unsigned int flags,
208208
}
209209
spin_unlock_irqrestore(&queue->lock, cpu_flags);
210210

211-
if (sk_valid_ll(sk) && sk_poll_ll(sk, flags & MSG_DONTWAIT))
211+
if (sk_can_busy_loop(sk) &&
212+
sk_busy_loop(sk, flags & MSG_DONTWAIT))
212213
continue;
213214

214215
/* User doesn't want to wait */

‎net/ipv4/tcp.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1554,9 +1554,9 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
15541554
struct sk_buff *skb;
15551555
u32 urg_hole = 0;
15561556

1557-
if (sk_valid_ll(sk) && skb_queue_empty(&sk->sk_receive_queue)
1558-
&& (sk->sk_state == TCP_ESTABLISHED))
1559-
sk_poll_ll(sk, nonblock);
1557+
if (sk_can_busy_loop(sk) && skb_queue_empty(&sk->sk_receive_queue) &&
1558+
(sk->sk_state == TCP_ESTABLISHED))
1559+
sk_busy_loop(sk, nonblock);
15601560

15611561
lock_sock(sk);
15621562

‎net/socket.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1148,24 +1148,24 @@ EXPORT_SYMBOL(sock_create_lite);
11481148
/* No kernel lock held - perfect */
11491149
static unsigned int sock_poll(struct file *file, poll_table *wait)
11501150
{
1151-
unsigned int ll_flag = 0;
1151+
unsigned int busy_flag = 0;
11521152
struct socket *sock;
11531153

11541154
/*
11551155
* We can't return errors to poll, so it's either yes or no.
11561156
*/
11571157
sock = file->private_data;
11581158

1159-
if (sk_valid_ll(sock->sk)) {
1159+
if (sk_can_busy_loop(sock->sk)) {
11601160
/* this socket can poll_ll so tell the system call */
1161-
ll_flag = POLL_LL;
1161+
busy_flag = POLL_BUSY_LOOP;
11621162

11631163
/* once, only if requested by syscall */
1164-
if (wait && (wait->_key & POLL_LL))
1165-
sk_poll_ll(sock->sk, 1);
1164+
if (wait && (wait->_key & POLL_BUSY_LOOP))
1165+
sk_busy_loop(sock->sk, 1);
11661166
}
11671167

1168-
return ll_flag | sock->ops->poll(file, sock, wait);
1168+
return busy_flag | sock->ops->poll(file, sock, wait);
11691169
}
11701170

11711171
static int sock_mmap(struct file *file, struct vm_area_struct *vma)

0 commit comments

Comments
 (0)