Skip to content

Commit 49116f2

Browse files
KAGA-KOKOKAGA-KOKO
committed
x86/vdso: Introduce and use vgtod_ts
It's desired to support more clocks in the VDSO, e.g. CLOCK_TAI. This results either in indirect calls due to the larger switch case, which then requires retpolines or when the compiler is forced to avoid jump tables it results in even more conditionals. To avoid both variants which are bad for performance the high resolution functions and the coarse grained functions will be collapsed into one for each. That requires to store the clock specific base time in an array. Introcude struct vgtod_ts for storage and convert the data store, the update function and the individual clock functions over to use it. The new storage does not longer use gtod_long_t for seconds depending on 32 or 64 bit compile because this needs to be the full 64bit value even for 32bit when a Y2038 function is added. No point in keeping the distinction alive in the internal representation. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Acked-by: Andy Lutomirski <luto@kernel.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Matt Rickard <matt@softrans.com.au> Cc: Stephen Boyd <sboyd@kernel.org> Cc: John Stultz <john.stultz@linaro.org> Cc: Florian Weimer <fweimer@redhat.com> Cc: "K. Y. Srinivasan" <kys@microsoft.com> Cc: Vitaly Kuznetsov <vkuznets@redhat.com> Cc: devel@linuxdriverproject.org Cc: virtualization@lists.linux-foundation.org Cc: Paolo Bonzini <pbonzini@redhat.com> Cc: Arnd Bergmann <arnd@arndb.de> Cc: Juergen Gross <jgross@suse.com> Link: https://lkml.kernel.org/r/20180917130707.324679401@linutronix.de
1 parent 77e9c67 commit 49116f2

File tree

3 files changed

+60
-49
lines changed

3 files changed

+60
-49
lines changed

‎arch/x86/entry/vdso/vclock_gettime.c

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -208,15 +208,16 @@ notrace static inline u64 vgetsns(int *mode)
208208
/* Code size doesn't matter (vdso is 4k anyway) and this is faster. */
209209
notrace static int __always_inline do_realtime(struct timespec *ts)
210210
{
211+
struct vgtod_ts *base = &gtod->basetime[CLOCK_REALTIME];
211212
unsigned int seq;
212213
u64 ns;
213214
int mode;
214215

215216
do {
216217
seq = gtod_read_begin(gtod);
217218
mode = gtod->vclock_mode;
218-
ts->tv_sec = gtod->wall_time_sec;
219-
ns = gtod->wall_time_snsec;
219+
ts->tv_sec = base->sec;
220+
ns = base->nsec;
220221
ns += vgetsns(&mode);
221222
ns >>= gtod->shift;
222223
} while (unlikely(gtod_read_retry(gtod, seq)));
@@ -229,15 +230,16 @@ notrace static int __always_inline do_realtime(struct timespec *ts)
229230

230231
notrace static int __always_inline do_monotonic(struct timespec *ts)
231232
{
233+
struct vgtod_ts *base = &gtod->basetime[CLOCK_MONOTONIC];
232234
unsigned int seq;
233235
u64 ns;
234236
int mode;
235237

236238
do {
237239
seq = gtod_read_begin(gtod);
238240
mode = gtod->vclock_mode;
239-
ts->tv_sec = gtod->monotonic_time_sec;
240-
ns = gtod->monotonic_time_snsec;
241+
ts->tv_sec = base->sec;
242+
ns = base->nsec;
241243
ns += vgetsns(&mode);
242244
ns >>= gtod->shift;
243245
} while (unlikely(gtod_read_retry(gtod, seq)));
@@ -250,21 +252,25 @@ notrace static int __always_inline do_monotonic(struct timespec *ts)
250252

251253
notrace static void do_realtime_coarse(struct timespec *ts)
252254
{
255+
struct vgtod_ts *base = &gtod->basetime[CLOCK_REALTIME_COARSE];
253256
unsigned int seq;
257+
254258
do {
255259
seq = gtod_read_begin(gtod);
256-
ts->tv_sec = gtod->wall_time_coarse_sec;
257-
ts->tv_nsec = gtod->wall_time_coarse_nsec;
260+
ts->tv_sec = base->sec;
261+
ts->tv_nsec = base->nsec;
258262
} while (unlikely(gtod_read_retry(gtod, seq)));
259263
}
260264

261265
notrace static void do_monotonic_coarse(struct timespec *ts)
262266
{
267+
struct vgtod_ts *base = &gtod->basetime[CLOCK_MONOTONIC_COARSE];
263268
unsigned int seq;
269+
264270
do {
265271
seq = gtod_read_begin(gtod);
266-
ts->tv_sec = gtod->monotonic_time_coarse_sec;
267-
ts->tv_nsec = gtod->monotonic_time_coarse_nsec;
272+
ts->tv_sec = base->sec;
273+
ts->tv_nsec = base->nsec;
268274
} while (unlikely(gtod_read_retry(gtod, seq)));
269275
}
270276

@@ -320,7 +326,7 @@ int gettimeofday(struct timeval *, struct timezone *)
320326
notrace time_t __vdso_time(time_t *t)
321327
{
322328
/* This is atomic on x86 so we don't need any locks. */
323-
time_t result = READ_ONCE(gtod->wall_time_sec);
329+
time_t result = READ_ONCE(gtod->basetime[CLOCK_REALTIME].sec);
324330

325331
if (t)
326332
*t = result;

‎arch/x86/entry/vsyscall/vsyscall_gtod.c

Lines changed: 24 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ void update_vsyscall(struct timekeeper *tk)
3131
{
3232
int vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode;
3333
struct vsyscall_gtod_data *vdata = &vsyscall_gtod_data;
34+
struct vgtod_ts *base;
35+
u64 nsec;
3436

3537
/* Mark the new vclock used. */
3638
BUILD_BUG_ON(VCLOCK_MAX >= 32);
@@ -45,34 +47,33 @@ void update_vsyscall(struct timekeeper *tk)
4547
vdata->mult = tk->tkr_mono.mult;
4648
vdata->shift = tk->tkr_mono.shift;
4749

48-
vdata->wall_time_sec = tk->xtime_sec;
49-
vdata->wall_time_snsec = tk->tkr_mono.xtime_nsec;
50+
base = &vdata->basetime[CLOCK_REALTIME];
51+
base->sec = tk->xtime_sec;
52+
base->nsec = tk->tkr_mono.xtime_nsec;
5053

51-
vdata->monotonic_time_sec = tk->xtime_sec
52-
+ tk->wall_to_monotonic.tv_sec;
53-
vdata->monotonic_time_snsec = tk->tkr_mono.xtime_nsec
54-
+ ((u64)tk->wall_to_monotonic.tv_nsec
55-
<< tk->tkr_mono.shift);
56-
while (vdata->monotonic_time_snsec >=
57-
(((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
58-
vdata->monotonic_time_snsec -=
59-
((u64)NSEC_PER_SEC) << tk->tkr_mono.shift;
60-
vdata->monotonic_time_sec++;
54+
base = &vdata->basetime[CLOCK_MONOTONIC];
55+
base->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
56+
nsec = tk->tkr_mono.xtime_nsec;
57+
nsec += ((u64)tk->wall_to_monotonic.tv_nsec << tk->tkr_mono.shift);
58+
while (nsec >= (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) {
59+
nsec -= ((u64)NSEC_PER_SEC) << tk->tkr_mono.shift;
60+
base->sec++;
6161
}
62+
base->nsec = nsec;
6263

63-
vdata->wall_time_coarse_sec = tk->xtime_sec;
64-
vdata->wall_time_coarse_nsec = (long)(tk->tkr_mono.xtime_nsec >>
65-
tk->tkr_mono.shift);
64+
base = &vdata->basetime[CLOCK_REALTIME_COARSE];
65+
base->sec = tk->xtime_sec;
66+
base->nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
6667

67-
vdata->monotonic_time_coarse_sec =
68-
vdata->wall_time_coarse_sec + tk->wall_to_monotonic.tv_sec;
69-
vdata->monotonic_time_coarse_nsec =
70-
vdata->wall_time_coarse_nsec + tk->wall_to_monotonic.tv_nsec;
71-
72-
while (vdata->monotonic_time_coarse_nsec >= NSEC_PER_SEC) {
73-
vdata->monotonic_time_coarse_nsec -= NSEC_PER_SEC;
74-
vdata->monotonic_time_coarse_sec++;
68+
base = &vdata->basetime[CLOCK_MONOTONIC_COARSE];
69+
base->sec = tk->xtime_sec + tk->wall_to_monotonic.tv_sec;
70+
nsec = tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift;
71+
nsec += tk->wall_to_monotonic.tv_nsec;
72+
while (nsec >= NSEC_PER_SEC) {
73+
nsec -= NSEC_PER_SEC;
74+
base->sec++;
7575
}
76+
base->nsec = nsec;
7677

7778
gtod_write_end(vdata);
7879
}

‎arch/x86/include/asm/vgtod.h

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,33 +5,37 @@
55
#include <linux/compiler.h>
66
#include <linux/clocksource.h>
77

8+
#include <uapi/linux/time.h>
9+
810
#ifdef BUILD_VDSO32_64
911
typedef u64 gtod_long_t;
1012
#else
1113
typedef unsigned long gtod_long_t;
1214
#endif
15+
16+
struct vgtod_ts {
17+
u64 sec;
18+
u64 nsec;
19+
};
20+
21+
#define VGTOD_BASES (CLOCK_MONOTONIC_COARSE + 1)
22+
#define VGTOD_HRES (BIT(CLOCK_REALTIME) | BIT(CLOCK_MONOTONIC))
23+
#define VGTOD_COARSE (BIT(CLOCK_REALTIME_COARSE) | BIT(CLOCK_MONOTONIC_COARSE))
24+
1325
/*
1426
* vsyscall_gtod_data will be accessed by 32 and 64 bit code at the same time
1527
* so be carefull by modifying this structure.
1628
*/
1729
struct vsyscall_gtod_data {
18-
unsigned int seq;
19-
20-
int vclock_mode;
21-
u64 cycle_last;
22-
u64 mask;
23-
u32 mult;
24-
u32 shift;
25-
26-
/* open coded 'struct timespec' */
27-
u64 wall_time_snsec;
28-
gtod_long_t wall_time_sec;
29-
gtod_long_t monotonic_time_sec;
30-
u64 monotonic_time_snsec;
31-
gtod_long_t wall_time_coarse_sec;
32-
gtod_long_t wall_time_coarse_nsec;
33-
gtod_long_t monotonic_time_coarse_sec;
34-
gtod_long_t monotonic_time_coarse_nsec;
30+
unsigned int seq;
31+
32+
int vclock_mode;
33+
u64 cycle_last;
34+
u64 mask;
35+
u32 mult;
36+
u32 shift;
37+
38+
struct vgtod_ts basetime[VGTOD_BASES];
3539

3640
int tz_minuteswest;
3741
int tz_dsttime;

0 commit comments

Comments
 (0)