diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-04-14 10:53:44 -0700 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-04-14 10:53:44 -0700 |
| commit | f21f7b5162e9dbde6d3d5ce727d4ca2552d76ce9 (patch) | |
| tree | 2c1d858605001adedeff10f66f031e20da1db34d /arch/sparc | |
| parent | c1fe867b5bf9c57ab7856486d342720e2b205eed (diff) | |
| parent | 7138a8698a39e81eb153e05500823fff76d5b3bd (diff) | |
Merge tag 'timers-vdso-2026-04-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull vdso updates from Thomas Gleixner:
- Make the handling of compat functions consistent and more robust
- Rework the underlying data store so that it is dynamically allocated,
which allows the conversion of the last holdout SPARC64 to the
generic VDSO implementation
- Rework the SPARC64 VDSO to utilize the generic implementation
- Mop up the left overs of the non-generic VDSO support in the core
code
- Expand the VDSO selftest and make them more robust
- Allow time namespaces to be enabled independently of the generic VDSO
support, which was not possible before due to SPARC64 not using it
- Various cleanups and improvements in the related code
* tag 'timers-vdso-2026-04-12' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (51 commits)
timens: Use task_lock guard in timens_get*()
timens: Use mutex guard in proc_timens_set_offset()
timens: Simplify some calls to put_time_ns()
timens: Add a __free() wrapper for put_time_ns()
timens: Remove dependency on the vDSO
vdso/timens: Move functions to new file
selftests: vDSO: vdso_test_correctness: Add a test for time()
selftests: vDSO: vdso_test_correctness: Use facilities from parse_vdso.c
selftests: vDSO: vdso_test_correctness: Handle different tv_usec types
selftests: vDSO: vdso_test_correctness: Drop SYS_getcpu fallbacks
selftests: vDSO: vdso_test_gettimeofday: Remove nolibc checks
Revert "selftests: vDSO: parse_vdso: Use UAPI headers instead of libc headers"
random: vDSO: Remove ifdeffery
random: vDSO: Trim vDSO includes
vdso/datapage: Trim down unnecessary includes
vdso/datapage: Remove inclusion of gettimeofday.h
vdso/helpers: Explicitly include vdso/processor.h
vdso/gettimeofday: Add explicit includes
random: vDSO: Add explicit includes
MIPS: vdso: Explicitly include asm/vdso/vdso.h
...
Diffstat (limited to 'arch/sparc')
| -rw-r--r-- | arch/sparc/Kconfig | 3 | ||||
| -rw-r--r-- | arch/sparc/include/asm/clocksource.h | 9 | ||||
| -rw-r--r-- | arch/sparc/include/asm/processor.h | 3 | ||||
| -rw-r--r-- | arch/sparc/include/asm/processor_32.h | 2 | ||||
| -rw-r--r-- | arch/sparc/include/asm/processor_64.h | 25 | ||||
| -rw-r--r-- | arch/sparc/include/asm/vdso.h | 2 | ||||
| -rw-r--r-- | arch/sparc/include/asm/vdso/clocksource.h | 10 | ||||
| -rw-r--r-- | arch/sparc/include/asm/vdso/gettimeofday.h | 184 | ||||
| -rw-r--r-- | arch/sparc/include/asm/vdso/processor.h | 41 | ||||
| -rw-r--r-- | arch/sparc/include/asm/vdso/vsyscall.h | 10 | ||||
| -rw-r--r-- | arch/sparc/include/asm/vvar.h | 75 | ||||
| -rw-r--r-- | arch/sparc/kernel/Makefile | 1 | ||||
| -rw-r--r-- | arch/sparc/kernel/time_64.c | 6 | ||||
| -rw-r--r-- | arch/sparc/kernel/vdso.c | 69 | ||||
| -rw-r--r-- | arch/sparc/vdso/Makefile | 11 | ||||
| -rw-r--r-- | arch/sparc/vdso/vclock_gettime.c | 380 | ||||
| -rw-r--r-- | arch/sparc/vdso/vdso-layout.lds.S | 26 | ||||
| -rw-r--r-- | arch/sparc/vdso/vdso.lds.S | 2 | ||||
| -rw-r--r-- | arch/sparc/vdso/vdso2c.c | 24 | ||||
| -rw-r--r-- | arch/sparc/vdso/vdso2c.h | 45 | ||||
| -rw-r--r-- | arch/sparc/vdso/vdso32/vdso32.lds.S | 4 | ||||
| -rw-r--r-- | arch/sparc/vdso/vma.c | 274 |
22 files changed, 306 insertions, 900 deletions
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 8699be91fca9..a6b787efc2c4 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -104,7 +104,6 @@ config SPARC64 select ARCH_USE_QUEUED_RWLOCKS select ARCH_USE_QUEUED_SPINLOCKS select GENERIC_TIME_VSYSCALL - select ARCH_CLOCKSOURCE_DATA select ARCH_HAS_PTE_SPECIAL select PCI_DOMAINS if PCI select ARCH_HAS_GIGANTIC_PAGE @@ -115,6 +114,8 @@ config SPARC64 select ARCH_SUPPORTS_SCHED_SMT if SMP select ARCH_SUPPORTS_SCHED_MC if SMP select ARCH_HAS_LAZY_MMU_MODE + select HAVE_GENERIC_VDSO + select GENERIC_GETTIMEOFDAY config ARCH_PROC_KCORE_TEXT def_bool y diff --git a/arch/sparc/include/asm/clocksource.h b/arch/sparc/include/asm/clocksource.h index d63ef224befe..68303ad26eb2 100644 --- a/arch/sparc/include/asm/clocksource.h +++ b/arch/sparc/include/asm/clocksource.h @@ -5,13 +5,4 @@ #ifndef _ASM_SPARC_CLOCKSOURCE_H #define _ASM_SPARC_CLOCKSOURCE_H -/* VDSO clocksources */ -#define VCLOCK_NONE 0 /* Nothing userspace can do. */ -#define VCLOCK_TICK 1 /* Use %tick. */ -#define VCLOCK_STICK 2 /* Use %stick. */ - -struct arch_clocksource_data { - int vclock_mode; -}; - #endif /* _ASM_SPARC_CLOCKSOURCE_H */ diff --git a/arch/sparc/include/asm/processor.h b/arch/sparc/include/asm/processor.h index 18295ea625dd..e34de956519a 100644 --- a/arch/sparc/include/asm/processor.h +++ b/arch/sparc/include/asm/processor.h @@ -1,6 +1,9 @@ /* SPDX-License-Identifier: GPL-2.0 */ #ifndef ___ASM_SPARC_PROCESSOR_H #define ___ASM_SPARC_PROCESSOR_H + +#include <asm/vdso/processor.h> + #if defined(__sparc__) && defined(__arch64__) #include <asm/processor_64.h> #else diff --git a/arch/sparc/include/asm/processor_32.h b/arch/sparc/include/asm/processor_32.h index ba8b70ffec08..a074d313f4f8 100644 --- a/arch/sparc/include/asm/processor_32.h +++ b/arch/sparc/include/asm/processor_32.h @@ -91,8 +91,6 @@ unsigned long __get_wchan(struct task_struct *); extern struct task_struct *last_task_used_math; int do_mathemu(struct pt_regs *regs, struct task_struct *fpt); -#define cpu_relax() barrier() - extern void (*sparc_idle)(void); #endif diff --git a/arch/sparc/include/asm/processor_64.h b/arch/sparc/include/asm/processor_64.h index 321859454ca4..485070495263 100644 --- a/arch/sparc/include/asm/processor_64.h +++ b/arch/sparc/include/asm/processor_64.h @@ -182,31 +182,6 @@ unsigned long __get_wchan(struct task_struct *task); #define KSTK_EIP(tsk) (task_pt_regs(tsk)->tpc) #define KSTK_ESP(tsk) (task_pt_regs(tsk)->u_regs[UREG_FP]) -/* Please see the commentary in asm/backoff.h for a description of - * what these instructions are doing and how they have been chosen. - * To make a long story short, we are trying to yield the current cpu - * strand during busy loops. - */ -#ifdef BUILD_VDSO -#define cpu_relax() asm volatile("\n99:\n\t" \ - "rd %%ccr, %%g0\n\t" \ - "rd %%ccr, %%g0\n\t" \ - "rd %%ccr, %%g0\n\t" \ - ::: "memory") -#else /* ! BUILD_VDSO */ -#define cpu_relax() asm volatile("\n99:\n\t" \ - "rd %%ccr, %%g0\n\t" \ - "rd %%ccr, %%g0\n\t" \ - "rd %%ccr, %%g0\n\t" \ - ".section .pause_3insn_patch,\"ax\"\n\t"\ - ".word 99b\n\t" \ - "wr %%g0, 128, %%asr27\n\t" \ - "nop\n\t" \ - "nop\n\t" \ - ".previous" \ - ::: "memory") -#endif - /* Prefetch support. This is tuned for UltraSPARC-III and later. * UltraSPARC-I will treat these as nops, and UltraSPARC-II has * a shallower prefetch queue than later chips. diff --git a/arch/sparc/include/asm/vdso.h b/arch/sparc/include/asm/vdso.h index 59e79d35cd73..f08562d10215 100644 --- a/arch/sparc/include/asm/vdso.h +++ b/arch/sparc/include/asm/vdso.h @@ -8,8 +8,6 @@ struct vdso_image { void *data; unsigned long size; /* Always a multiple of PAGE_SIZE */ - - long sym_vvar_start; /* Negative offset to the vvar area */ }; #ifdef CONFIG_SPARC64 diff --git a/arch/sparc/include/asm/vdso/clocksource.h b/arch/sparc/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..007aa8ceaf52 --- /dev/null +++ b/arch/sparc/include/asm/vdso/clocksource.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_CLOCKSOURCE_H +#define __ASM_VDSO_CLOCKSOURCE_H + +/* VDSO clocksources */ +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_TICK, \ + VDSO_CLOCKMODE_STICK + +#endif /* __ASM_VDSO_CLOCKSOURCE_H */ diff --git a/arch/sparc/include/asm/vdso/gettimeofday.h b/arch/sparc/include/asm/vdso/gettimeofday.h new file mode 100644 index 000000000000..b0c80c8a28bb --- /dev/null +++ b/arch/sparc/include/asm/vdso/gettimeofday.h @@ -0,0 +1,184 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright 2006 Andi Kleen, SUSE Labs. + */ + +#ifndef _ASM_SPARC_VDSO_GETTIMEOFDAY_H +#define _ASM_SPARC_VDSO_GETTIMEOFDAY_H + +#include <uapi/linux/time.h> +#include <uapi/linux/unistd.h> + +#include <vdso/align.h> +#include <vdso/clocksource.h> +#include <vdso/datapage.h> +#include <vdso/page.h> + +#include <linux/types.h> + +#ifdef CONFIG_SPARC64 +static __always_inline u64 vread_tick(void) +{ + u64 ret; + + __asm__ __volatile__("rd %%tick, %0" : "=r" (ret)); + return ret; +} + +static __always_inline u64 vread_tick_stick(void) +{ + u64 ret; + + __asm__ __volatile__("rd %%asr24, %0" : "=r" (ret)); + return ret; +} +#else +static __always_inline u64 vdso_shift_ns(u64 val, u32 amt) +{ + u64 ret; + + __asm__ __volatile__("sllx %H1, 32, %%g1\n\t" + "srl %L1, 0, %L1\n\t" + "or %%g1, %L1, %%g1\n\t" + "srlx %%g1, %2, %L0\n\t" + "srlx %L0, 32, %H0" + : "=r" (ret) + : "r" (val), "r" (amt) + : "g1"); + return ret; +} +#define vdso_shift_ns vdso_shift_ns + +static __always_inline u64 vread_tick(void) +{ + register unsigned long long ret asm("o4"); + + __asm__ __volatile__("rd %%tick, %L0\n\t" + "srlx %L0, 32, %H0" + : "=r" (ret)); + return ret; +} + +static __always_inline u64 vread_tick_stick(void) +{ + register unsigned long long ret asm("o4"); + + __asm__ __volatile__("rd %%asr24, %L0\n\t" + "srlx %L0, 32, %H0" + : "=r" (ret)); + return ret; +} +#endif + +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode, const struct vdso_time_data *vd) +{ + if (likely(clock_mode == VDSO_CLOCKMODE_STICK)) + return vread_tick_stick(); + else + return vread_tick(); +} + +#ifdef CONFIG_SPARC64 +#define SYSCALL_STRING \ + "ta 0x6d;" \ + "bcs,a 1f;" \ + " sub %%g0, %%o0, %%o0;" \ + "1:" +#else +#define SYSCALL_STRING \ + "ta 0x10;" \ + "bcs,a 1f;" \ + " sub %%g0, %%o0, %%o0;" \ + "1:" +#endif + +#define SYSCALL_CLOBBERS \ + "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \ + "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \ + "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \ + "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \ + "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \ + "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \ + "cc", "memory" + +#ifdef CONFIG_SPARC64 + +static __always_inline +long clock_gettime_fallback(clockid_t clock, struct __kernel_timespec *ts) +{ + register long num __asm__("g1") = __NR_clock_gettime; + register long o0 __asm__("o0") = clock; + register long o1 __asm__("o1") = (long) ts; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +#else /* !CONFIG_SPARC64 */ + +static __always_inline +long clock_gettime_fallback(clockid_t clock, struct __kernel_timespec *ts) +{ + register long num __asm__("g1") = __NR_clock_gettime64; + register long o0 __asm__("o0") = clock; + register long o1 __asm__("o1") = (long) ts; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +static __always_inline +long clock_gettime32_fallback(clockid_t clock, struct old_timespec32 *ts) +{ + register long num __asm__("g1") = __NR_clock_gettime; + register long o0 __asm__("o0") = clock; + register long o1 __asm__("o1") = (long) ts; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +#endif /* CONFIG_SPARC64 */ + +static __always_inline +long gettimeofday_fallback(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + register long num __asm__("g1") = __NR_gettimeofday; + register long o0 __asm__("o0") = (long) tv; + register long o1 __asm__("o1") = (long) tz; + + __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), + "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); + return o0; +} + +static __always_inline const struct vdso_time_data *__arch_get_vdso_u_time_data(void) +{ + unsigned long ret; + + /* + * SPARC does not support native PC-relative code relocations. + * Calculate the address manually, works for 32 and 64 bit code. + */ + __asm__ __volatile__( + "1:\n" + "call 3f\n" // Jump over the embedded data and set up %o7 + "nop\n" // Delay slot + "2:\n" + ".word vdso_u_time_data - .\n" // Embedded offset to external symbol + "3:\n" + "add %%o7, 2b - 1b, %%o7\n" // Point %o7 to the embedded offset + "ldsw [%%o7], %0\n" // Load the offset + "add %0, %%o7, %0\n" // Calculate the absolute address + : "=r" (ret) + : + : "o7"); + + return (const struct vdso_time_data *)ret; +} +#define __arch_get_vdso_u_time_data __arch_get_vdso_u_time_data + +#endif /* _ASM_SPARC_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/sparc/include/asm/vdso/processor.h b/arch/sparc/include/asm/vdso/processor.h new file mode 100644 index 000000000000..f7a9adc807f7 --- /dev/null +++ b/arch/sparc/include/asm/vdso/processor.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SPARC_VDSO_PROCESSOR_H +#define _ASM_SPARC_VDSO_PROCESSOR_H + +#include <linux/compiler.h> + +#if defined(__arch64__) + +/* Please see the commentary in asm/backoff.h for a description of + * what these instructions are doing and how they have been chosen. + * To make a long story short, we are trying to yield the current cpu + * strand during busy loops. + */ +#ifdef BUILD_VDSO +#define cpu_relax() asm volatile("\n99:\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + ::: "memory") +#else /* ! BUILD_VDSO */ +#define cpu_relax() asm volatile("\n99:\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + "rd %%ccr, %%g0\n\t" \ + ".section .pause_3insn_patch,\"ax\"\n\t"\ + ".word 99b\n\t" \ + "wr %%g0, 128, %%asr27\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + ".previous" \ + ::: "memory") +#endif /* BUILD_VDSO */ + +#else /* ! __arch64__ */ + +#define cpu_relax() barrier() + +#endif /* __arch64__ */ + +#endif /* _ASM_SPARC_VDSO_PROCESSOR_H */ diff --git a/arch/sparc/include/asm/vdso/vsyscall.h b/arch/sparc/include/asm/vdso/vsyscall.h new file mode 100644 index 000000000000..8bfe703fedc5 --- /dev/null +++ b/arch/sparc/include/asm/vdso/vsyscall.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _ASM_SPARC_VDSO_VSYSCALL_H +#define _ASM_SPARC_VDSO_VSYSCALL_H + +#define __VDSO_PAGES 4 + +#include <asm-generic/vdso/vsyscall.h> + +#endif /* _ASM_SPARC_VDSO_VSYSCALL_H */ diff --git a/arch/sparc/include/asm/vvar.h b/arch/sparc/include/asm/vvar.h deleted file mode 100644 index 6eaf5cfcaae1..000000000000 --- a/arch/sparc/include/asm/vvar.h +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. - */ - -#ifndef _ASM_SPARC_VVAR_DATA_H -#define _ASM_SPARC_VVAR_DATA_H - -#include <asm/clocksource.h> -#include <asm/processor.h> -#include <asm/barrier.h> -#include <linux/time.h> -#include <linux/types.h> - -struct vvar_data { - unsigned int seq; - - int vclock_mode; - struct { /* extract of a clocksource struct */ - u64 cycle_last; - u64 mask; - int mult; - int shift; - } clock; - /* open coded 'struct timespec' */ - u64 wall_time_sec; - u64 wall_time_snsec; - u64 monotonic_time_snsec; - u64 monotonic_time_sec; - u64 monotonic_time_coarse_sec; - u64 monotonic_time_coarse_nsec; - u64 wall_time_coarse_sec; - u64 wall_time_coarse_nsec; - - int tz_minuteswest; - int tz_dsttime; -}; - -extern struct vvar_data *vvar_data; -extern int vdso_fix_stick; - -static inline unsigned int vvar_read_begin(const struct vvar_data *s) -{ - unsigned int ret; - -repeat: - ret = READ_ONCE(s->seq); - if (unlikely(ret & 1)) { - cpu_relax(); - goto repeat; - } - smp_rmb(); /* Finish all reads before we return seq */ - return ret; -} - -static inline int vvar_read_retry(const struct vvar_data *s, - unsigned int start) -{ - smp_rmb(); /* Finish all reads before checking the value of seq */ - return unlikely(s->seq != start); -} - -static inline void vvar_write_begin(struct vvar_data *s) -{ - ++s->seq; - smp_wmb(); /* Makes sure that increment of seq is reflected */ -} - -static inline void vvar_write_end(struct vvar_data *s) -{ - smp_wmb(); /* Makes the value of seq current before we increment */ - ++s->seq; -} - - -#endif /* _ASM_SPARC_VVAR_DATA_H */ diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile index 22170d4f8e06..497b5714fa8f 100644 --- a/arch/sparc/kernel/Makefile +++ b/arch/sparc/kernel/Makefile @@ -41,7 +41,6 @@ obj-$(CONFIG_SPARC32) += systbls_32.o obj-y += time_$(BITS).o obj-$(CONFIG_SPARC32) += windows.o obj-y += cpu.o -obj-$(CONFIG_SPARC64) += vdso.o obj-$(CONFIG_SPARC32) += devices.o obj-y += ptrace_$(BITS).o obj-y += unaligned_$(BITS).o diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index b32f27f929d1..87b267043ccd 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -838,14 +838,14 @@ void __init time_init_early(void) if (tlb_type == spitfire) { if (is_hummingbird()) { init_tick_ops(&hbtick_operations); - clocksource_tick.archdata.vclock_mode = VCLOCK_NONE; + clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_NONE; } else { init_tick_ops(&tick_operations); - clocksource_tick.archdata.vclock_mode = VCLOCK_TICK; + clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_TICK; } } else { init_tick_ops(&stick_operations); - clocksource_tick.archdata.vclock_mode = VCLOCK_STICK; + clocksource_tick.vdso_clock_mode = VDSO_CLOCKMODE_STICK; } } diff --git a/arch/sparc/kernel/vdso.c b/arch/sparc/kernel/vdso.c deleted file mode 100644 index 0e27437eb97b..000000000000 --- a/arch/sparc/kernel/vdso.c +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE - * Copyright 2003 Andi Kleen, SuSE Labs. - * - * Thanks to hpa@transmeta.com for some useful hint. - * Special thanks to Ingo Molnar for his early experience with - * a different vsyscall implementation for Linux/IA32 and for the name. - */ - -#include <linux/time.h> -#include <linux/timekeeper_internal.h> - -#include <asm/vvar.h> - -void update_vsyscall_tz(void) -{ - if (unlikely(vvar_data == NULL)) - return; - - vvar_data->tz_minuteswest = sys_tz.tz_minuteswest; - vvar_data->tz_dsttime = sys_tz.tz_dsttime; -} - -void update_vsyscall(struct timekeeper *tk) -{ - struct vvar_data *vdata = vvar_data; - - if (unlikely(vdata == NULL)) - return; - - vvar_write_begin(vdata); - vdata->vclock_mode = tk->tkr_mono.clock->archdata.vclock_mode; - vdata->clock.cycle_last = tk->tkr_mono.cycle_last; - vdata->clock.mask = tk->tkr_mono.mask; - vdata->clock.mult = tk->tkr_mono.mult; - vdata->clock.shift = tk->tkr_mono.shift; - - vdata->wall_time_sec = tk->xtime_sec; - vdata->wall_time_snsec = tk->tkr_mono.xtime_nsec; - - vdata->monotonic_time_sec = tk->xtime_sec + - tk->wall_to_monotonic.tv_sec; - vdata->monotonic_time_snsec = tk->tkr_mono.xtime_nsec + - (tk->wall_to_monotonic.tv_nsec << - tk->tkr_mono.shift); - - while (vdata->monotonic_time_snsec >= - (((u64)NSEC_PER_SEC) << tk->tkr_mono.shift)) { - vdata->monotonic_time_snsec -= - ((u64)NSEC_PER_SEC) << tk->tkr_mono.shift; - vdata->monotonic_time_sec++; - } - - vdata->wall_time_coarse_sec = tk->xtime_sec; - vdata->wall_time_coarse_nsec = - (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift); - - vdata->monotonic_time_coarse_sec = - vdata->wall_time_coarse_sec + tk->wall_to_monotonic.tv_sec; - vdata->monotonic_time_coarse_nsec = - vdata->wall_time_coarse_nsec + tk->wall_to_monotonic.tv_nsec; - - while (vdata->monotonic_time_coarse_nsec >= NSEC_PER_SEC) { - vdata->monotonic_time_coarse_nsec -= NSEC_PER_SEC; - vdata->monotonic_time_coarse_sec++; - } - - vvar_write_end(vdata); -} diff --git a/arch/sparc/vdso/Makefile b/arch/sparc/vdso/Makefile index 683b2d408224..83fb2aca59cb 100644 --- a/arch/sparc/vdso/Makefile +++ b/arch/sparc/vdso/Makefile @@ -3,6 +3,9 @@ # Building vDSO images for sparc. # +# Include the generic Makefile to check the built vDSO: +include $(srctree)/lib/vdso/Makefile.include + # files to link into the vdso vobjs-y := vdso-note.o vclock_gettime.o @@ -90,6 +93,9 @@ KBUILD_CFLAGS_32 += -DDISABLE_BRANCH_PROFILING KBUILD_CFLAGS_32 += -mv8plus $(obj)/vdso32.so.dbg: KBUILD_CFLAGS = $(KBUILD_CFLAGS_32) +CHECKFLAGS_32 := $(filter-out -m64 -D__sparc_v9__ -D__arch64__, $(CHECKFLAGS)) -m32 +$(obj)/vdso32.so.dbg: CHECKFLAGS = $(CHECKFLAGS_32) + $(obj)/vdso32.so.dbg: FORCE \ $(obj)/vdso32/vdso32.lds \ $(obj)/vdso32/vclock_gettime.o \ @@ -102,6 +108,7 @@ $(obj)/vdso32.so.dbg: FORCE \ quiet_cmd_vdso = VDSO $@ cmd_vdso = $(LD) -nostdlib -o $@ \ $(VDSO_LDFLAGS) $(VDSO_LDFLAGS_$(filter %.lds,$(^F))) \ - -T $(filter %.lds,$^) $(filter %.o,$^) + -T $(filter %.lds,$^) $(filter %.o,$^); \ + $(cmd_vdso_check) -VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined +VDSO_LDFLAGS = -shared --hash-style=both --build-id=sha1 -Bsymbolic --no-undefined -z noexecstack diff --git a/arch/sparc/vdso/vclock_gettime.c b/arch/sparc/vdso/vclock_gettime.c index 79607804ea1b..1d9859392e4c 100644 --- a/arch/sparc/vdso/vclock_gettime.c +++ b/arch/sparc/vdso/vclock_gettime.c @@ -12,382 +12,48 @@ * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved. */ -#include <linux/kernel.h> -#include <linux/time.h> -#include <linux/string.h> -#include <asm/io.h> -#include <asm/unistd.h> -#include <asm/timex.h> -#include <asm/clocksource.h> -#include <asm/vvar.h> +#include <linux/compiler.h> +#include <linux/types.h> -#ifdef CONFIG_SPARC64 -#define SYSCALL_STRING \ - "ta 0x6d;" \ - "bcs,a 1f;" \ - " sub %%g0, %%o0, %%o0;" \ - "1:" -#else -#define SYSCALL_STRING \ - "ta 0x10;" \ - "bcs,a 1f;" \ - " sub %%g0, %%o0, %%o0;" \ - "1:" -#endif - -#define SYSCALL_CLOBBERS \ - "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \ - "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \ - "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \ - "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \ - "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \ - "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \ - "cc", "memory" - -/* - * Compute the vvar page's address in the process address space, and return it - * as a pointer to the vvar_data. - */ -notrace static __always_inline struct vvar_data *get_vvar_data(void) -{ - unsigned long ret; - - /* - * vdso data page is the first vDSO page so grab the PC - * and move up a page to get to the data page. - */ - __asm__("rd %%pc, %0" : "=r" (ret)); - ret &= ~(8192 - 1); - ret -= 8192; - - return (struct vvar_data *) ret; -} +#include <vdso/gettime.h> -notrace static long vdso_fallback_gettime(long clock, struct __kernel_old_timespec *ts) -{ - register long num __asm__("g1") = __NR_clock_gettime; - register long o0 __asm__("o0") = clock; - register long o1 __asm__("o1") = (long) ts; +#include <asm/vdso/gettimeofday.h> - __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), - "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); - return o0; -} +#include "../../../../lib/vdso/gettimeofday.c" -notrace static long vdso_fallback_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) { - register long num __asm__("g1") = __NR_gettimeofday; - register long o0 __asm__("o0") = (long) tv; - register long o1 __asm__("o1") = (long) tz; - - __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num), - "0" (o0), "r" (o1) : SYSCALL_CLOBBERS); - return o0; + return __cvdso_gettimeofday(tv, tz); } -#ifdef CONFIG_SPARC64 -notrace static __always_inline u64 __shr64(u64 val, int amt) -{ - return val >> amt; -} +int gettimeofday(struct __kernel_old_timeval *, struct timezone *) + __weak __alias(__vdso_gettimeofday); -notrace static __always_inline u64 vread_tick(void) +#if defined(CONFIG_SPARC64) +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) { - u64 ret; - - __asm__ __volatile__("rd %%tick, %0" : "=r" (ret)); - return ret; + return __cvdso_clock_gettime(clock, ts); } -notrace static __always_inline u64 vread_tick_stick(void) -{ - u64 ret; +int clock_gettime(clockid_t, struct __kernel_timespec *) + __weak __alias(__vdso_clock_gettime); - __asm__ __volatile__("rd %%asr24, %0" : "=r" (ret)); - return ret; -} #else -notrace static __always_inline u64 __shr64(u64 val, int amt) -{ - u64 ret; - - __asm__ __volatile__("sllx %H1, 32, %%g1\n\t" - "srl %L1, 0, %L1\n\t" - "or %%g1, %L1, %%g1\n\t" - "srlx %%g1, %2, %L0\n\t" - "srlx %L0, 32, %H0" - : "=r" (ret) - : "r" (val), "r" (amt) - : "g1"); - return ret; -} - -notrace static __always_inline u64 vread_tick(void) -{ - register unsigned long long ret asm("o4"); - - __asm__ __volatile__("rd %%tick, %L0\n\t" - "srlx %L0, 32, %H0" - : "=r" (ret)); - return ret; -} - -notrace static __always_inline u64 vread_tick_stick(void) -{ - register unsigned long long ret asm("o4"); - - __asm__ __volatile__("rd %%asr24, %L0\n\t" - "srlx %L0, 32, %H0" - : "=r" (ret)); - return ret; -} -#endif -notrace static __always_inline u64 vgetsns(struct vvar_data *vvar) +int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts) { - u64 v; - u64 cycles; - - cycles = vread_tick(); - v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask; - return v * vvar->clock.mult; + return __cvdso_clock_gettime32(clock, ts); } -notrace static __always_inline u64 vgetsns_stick(struct vvar_data *vvar) -{ - u64 v; - u64 cycles; +int clock_gettime(clockid_t, struct old_timespec32 *) + __weak __alias(__vdso_clock_gettime); - cycles = vread_tick_stick(); - v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask; - return v * vvar->clock.mult; -} - -notrace static __always_inline int do_realtime(struct vvar_data *vvar, - struct __kernel_old_timespec *ts) +int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts) { - unsigned long seq; - u64 ns; - - do { - seq = vvar_read_begin(vvar); - ts->tv_sec = vvar->wall_time_sec; - ns = vvar->wall_time_snsec; - ns += vgetsns(vvar); - ns = __shr64(ns, vvar->clock.shift); - } while (unlikely(vvar_read_retry(vvar, seq))); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; - - return 0; + return __cvdso_clock_gettime(clock, ts); } -notrace static __always_inline int do_realtime_stick(struct vvar_data *vvar, - struct __kernel_old_timespec *ts) -{ - unsigned long seq; - u64 ns; - - do { - seq = vvar_read_begin(vvar); - ts->tv_sec = vvar->wall_time_sec; - ns = vvar->wall_time_snsec; - ns += vgetsns_stick(vvar); - ns = __shr64(ns, vvar->clock.shift); - } while (unlikely(vvar_read_retry(vvar, seq))); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; +int clock_gettime64(clockid_t, struct __kernel_timespec *) + __weak __alias(__vdso_clock_gettime64); - return 0; -} - -notrace static __always_inline int do_monotonic(struct vvar_data *vvar, - struct __kernel_old_timespec *ts) -{ - unsigned long seq; - u64 ns; - - do { - seq = vvar_read_begin(vvar); - ts->tv_sec = vvar->monotonic_time_sec; - ns = vvar->monotonic_time_snsec; - ns += vgetsns(vvar); - ns = __shr64(ns, vvar->clock.shift); - } while (unlikely(vvar_read_retry(vvar, seq))); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; - - return 0; -} - -notrace static __always_inline int do_monotonic_stick(struct vvar_data *vvar, - struct __kernel_old_timespec *ts) -{ - unsigned long seq; - u64 ns; - - do { - seq = vvar_read_begin(vvar); - ts->tv_sec = vvar->monotonic_time_sec; - ns = vvar->monotonic_time_snsec; - ns += vgetsns_stick(vvar); - ns = __shr64(ns, vvar->clock.shift); - } while (unlikely(vvar_read_retry(vvar, seq))); - - ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns); - ts->tv_nsec = ns; - - return 0; -} - -notrace static int do_realtime_coarse(struct vvar_data *vvar, - struct __kernel_old_timespec *ts) -{ - unsigned long seq; - - do { - seq = vvar_read_begin(vvar); - ts->tv_sec = vvar->wall_time_coarse_sec; - ts->tv_nsec = vvar->wall_time_coarse_nsec; - } while (unlikely(vvar_read_retry(vvar, seq))); - return 0; -} - -notrace static int do_monotonic_coarse(struct vvar_data *vvar, - struct __kernel_old_timespec *ts) -{ - unsigned long seq; - - do { - seq = vvar_read_begin(vvar); - ts->tv_sec = vvar->monotonic_time_coarse_sec; - ts->tv_nsec = vvar->monotonic_time_coarse_nsec; - } while (unlikely(vvar_read_retry(vvar, seq))); - - return 0; -} - -notrace int -__vdso_clock_gettime(clockid_t clock, struct __kernel_old_timespec *ts) -{ - struct vvar_data *vvd = get_vvar_data(); - - switch (clock) { - case CLOCK_REALTIME: - if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) - break; - return do_realtime(vvd, ts); - case CLOCK_MONOTONIC: - if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) - break; - return do_monotonic(vvd, ts); - case CLOCK_REALTIME_COARSE: - return do_realtime_coarse(vvd, ts); - case CLOCK_MONOTONIC_COARSE: - return do_monotonic_coarse(vvd, ts); - } - /* - * Unknown clock ID ? Fall back to the syscall. - */ - return vdso_fallback_gettime(clock, ts); -} -int -clock_gettime(clockid_t, struct __kernel_old_timespec *) - __attribute__((weak, alias("__vdso_clock_gettime"))); - -notrace int -__vdso_clock_gettime_stick(clockid_t clock, struct __kernel_old_timespec *ts) -{ - struct vvar_data *vvd = get_vvar_data(); - - switch (clock) { - case CLOCK_REALTIME: - if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) - break; - return do_realtime_stick(vvd, ts); - case CLOCK_MONOTONIC: - if (unlikely(vvd->vclock_mode == VCLOCK_NONE)) - break; - return do_monotonic_stick(vvd, ts); - case CLOCK_REALTIME_COARSE: - return do_realtime_coarse(vvd, ts); - case CLOCK_MONOTONIC_COARSE: - return do_monotonic_coarse(vvd, ts); - } - /* - * Unknown clock ID ? Fall back to the syscall. - */ - return vdso_fallback_gettime(clock, ts); -} - -notrace int -__vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) -{ - struct vvar_data *vvd = get_vvar_data(); - - if (likely(vvd->vclock_mode != VCLOCK_NONE)) { - if (likely(tv != NULL)) { - union tstv_t { - struct __kernel_old_timespec ts; - struct __kernel_old_timeval tv; - } *tstv = (union tstv_t *) tv; - do_realtime(vvd, &tstv->ts); - /* - * Assign before dividing to ensure that the division is - * done in the type of tv_usec, not tv_nsec. - * - * There cannot be > 1 billion usec in a second: - * do_realtime() has already distributed such overflow - * into tv_sec. So we can assign it to an int safely. - */ - tstv->tv.tv_usec = tstv->ts.tv_nsec; - tstv->tv.tv_usec /= 1000; - } - if (unlikely(tz != NULL)) { - /* Avoid memcpy. Some old compilers fail to inline it */ - tz->tz_minuteswest = vvd->tz_minuteswest; - tz->tz_dsttime = vvd->tz_dsttime; - } - return 0; - } - return vdso_fallback_gettimeofday(tv, tz); -} -int -gettimeofday(struct __kernel_old_timeval *, struct timezone *) - __attribute__((weak, alias("__vdso_gettimeofday"))); - -notrace int -__vdso_gettimeofday_stick(struct __kernel_old_timeval *tv, struct timezone *tz) -{ - struct vvar_data *vvd = get_vvar_data(); - - if (likely(vvd->vclock_mode != VCLOCK_NONE)) { - if (likely(tv != NULL)) { - union tstv_t { - struct __kernel_old_timespec ts; - struct __kernel_old_timeval tv; - } *tstv = (union tstv_t *) tv; - do_realtime_stick(vvd, &tstv->ts); - /* - * Assign before dividing to ensure that the division is - * done in the type of tv_usec, not tv_nsec. - * - * There cannot be > 1 billion usec in a second: - * do_realtime() has already distributed such overflow - * into tv_sec. So we can assign it to an int safely. - */ - tstv->tv.tv_usec = tstv->ts.tv_nsec; - tstv->tv.tv_usec /= 1000; - } - if (unlikely(tz != NULL)) { - /* Avoid memcpy. Some old compilers fail to inline it */ - tz->tz_minuteswest = vvd->tz_minuteswest; - tz->tz_dsttime = vvd->tz_dsttime; - } - return 0; - } - return vdso_fallback_gettimeofday(tv, tz); -} +#endif diff --git a/arch/sparc/vdso/vdso-layout.lds.S b/arch/sparc/vdso/vdso-layout.lds.S index d31e57e8a3bb..180e5d0ee071 100644 --- a/arch/sparc/vdso/vdso-layout.lds.S +++ b/arch/sparc/vdso/vdso-layout.lds.S @@ -4,15 +4,9 @@ * This script controls its layout. */ -#if defined(BUILD_VDSO64) -# define SHDR_SIZE 64 -#elif defined(BUILD_VDSO32) -# define SHDR_SIZE 40 -#else -# error unknown VDSO target -#endif - -#define NUM_FAKE_SHDRS 7 +#include <vdso/datapage.h> +#include <vdso/page.h> +#include <asm/vdso/vsyscall.h> SECTIONS { @@ -23,8 +17,7 @@ SECTIONS * segment. Page size is 8192 for both 64-bit and 32-bit vdso binaries */ - vvar_start = . -8192; - vvar_data = vvar_start; + VDSO_VVAR_SYMS . = SIZEOF_HEADERS; @@ -47,19 +40,8 @@ SECTIONS *(.bss*) *(.dynbss*) *(.gnu.linkonce.b.*) - - /* - * Ideally this would live in a C file: kept in here for - * compatibility with x86-64. - */ - VDSO_FAKE_SECTION_TABLE_START = .; - . = . + NUM_FAKE_SHDRS * SHDR_SIZE; - VDSO_FAKE_SECTION_TABLE_END = .; } :text - .fake_shstrtab : { *(.fake_shstrtab) } :text - - .note : { *(.note.*) } :text :note .eh_frame_hdr : { *(.eh_frame_hdr) } :text :eh_frame_hdr diff --git a/arch/sparc/vdso/vdso.lds.S b/arch/sparc/vdso/vdso.lds.S index 629ab6900df7..f3caa29a331c 100644 --- a/arch/sparc/vdso/vdso.lds.S +++ b/arch/sparc/vdso/vdso.lds.S @@ -18,10 +18,8 @@ VERSION { global: clock_gettime; __vdso_clock_gettime; - __vdso_clock_gettime_stick; gettimeofday; __vdso_gettimeofday; - __vdso_gettimeofday_stick; local: *; }; } diff --git a/arch/sparc/vdso/vdso2c.c b/arch/sparc/vdso/vdso2c.c index dc81240aab6f..e5c61214a0e2 100644 --- a/arch/sparc/vdso/vdso2c.c +++ b/arch/sparc/vdso/vdso2c.c @@ -58,28 +58,6 @@ const char *outfilename; -/* Symbols that we need in vdso2c. */ -enum { - sym_vvar_start, - sym_VDSO_FAKE_SECTION_TABLE_START, - sym_VDSO_FAKE_SECTION_TABLE_END, -}; - -struct vdso_sym { - const char *name; - int export; -}; - -struct vdso_sym required_syms[] = { - [sym_vvar_start] = {"vvar_start", 1}, - [sym_VDSO_FAKE_SECTION_TABLE_START] = { - "VDSO_FAKE_SECTION_TABLE_START", 0 - }, - [sym_VDSO_FAKE_SECTION_TABLE_END] = { - "VDSO_FAKE_SECTION_TABLE_END", 0 - }, -}; - __attribute__((format(printf, 1, 2))) __attribute__((noreturn)) static void fail(const char *format, ...) { @@ -119,8 +97,6 @@ static void fail(const char *format, ...) #define PUT_BE(x, val) \ PBE(x, val, 64, PBE(x, val, 32, PBE(x, val, 16, LAST_PBE(x, val)))) -#define NSYMS ARRAY_SIZE(required_syms) - #define BITSFUNC3(name, bits, suffix) name##bits##suffix #define BITSFUNC2(name, bits, suffix) BITSFUNC3(name, bits, suffix) #define BITSFUNC(name) BITSFUNC2(name, ELF_BITS, ) diff --git a/arch/sparc/vdso/vdso2c.h b/arch/sparc/vdso/vdso2c.h index 60d69acc748f..bad6a0593f4c 100644 --- a/arch/sparc/vdso/vdso2c.h +++ b/arch/sparc/vdso/vdso2c.h @@ -17,11 +17,9 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, unsigned long mapping_size; int i; unsigned long j; - ELF(Shdr) *symtab_hdr = NULL, *strtab_hdr; + ELF(Shdr) *symtab_hdr = NULL; ELF(Ehdr) *hdr = (ELF(Ehdr) *)raw_addr; ELF(Dyn) *dyn = 0, *dyn_end = 0; - INT_BITS syms[NSYMS] = {}; - ELF(Phdr) *pt = (ELF(Phdr) *)(raw_addr + GET_BE(&hdr->e_phoff)); /* Walk the segment table. */ @@ -72,42 +70,6 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, if (!symtab_hdr) fail("no symbol table\n"); - strtab_hdr = raw_addr + GET_BE(&hdr->e_shoff) + - GET_BE(&hdr->e_shentsize) * GET_BE(&symtab_hdr->sh_link); - - /* Walk the symbol table */ - for (i = 0; - i < GET_BE(&symtab_hdr->sh_size) / GET_BE(&symtab_hdr->sh_entsize); - i++) { - int k; - - ELF(Sym) *sym = raw_addr + GET_BE(&symtab_hdr->sh_offset) + - GET_BE(&symtab_hdr->sh_entsize) * i; - const char *name = raw_addr + GET_BE(&strtab_hdr->sh_offset) + - GET_BE(&sym->st_name); - - for (k = 0; k < NSYMS; k++) { - if (!strcmp(name, required_syms[k].name)) { - if (syms[k]) { - fail("duplicate symbol %s\n", - required_syms[k].name); - } - - /* - * Careful: we use negative addresses, but - * st_value is unsigned, so we rely - * on syms[k] being a signed type of the - * correct width. - */ - syms[k] = GET_BE(&sym->st_value); - } - } - } - - /* Validate mapping addresses. */ - if (syms[sym_vvar_start] % 8192) - fail("vvar_begin must be a multiple of 8192\n"); - if (!name) { fwrite(stripped_addr, stripped_len, 1, outfile); return; @@ -133,10 +95,5 @@ static void BITSFUNC(go)(void *raw_addr, size_t raw_len, fprintf(outfile, "const struct vdso_image %s_builtin = {\n", name); fprintf(outfile, "\t.data = raw_data,\n"); fprintf(outfile, "\t.size = %lu,\n", mapping_size); - for (i = 0; i < NSYMS; i++) { - if (required_syms[i].export && syms[i]) - fprintf(outfile, "\t.sym_%s = %" PRIi64 ",\n", - required_syms[i].name, (int64_t)syms[i]); - } fprintf(outfile, "};\n"); } diff --git a/arch/sparc/vdso/vdso32/vdso32.lds.S b/arch/sparc/vdso/vdso32/vdso32.lds.S index 218930fdff03..a14e4f77e6f2 100644 --- a/arch/sparc/vdso/vdso32/vdso32.lds.S +++ b/arch/sparc/vdso/vdso32/vdso32.lds.S @@ -17,10 +17,10 @@ VERSION { global: clock_gettime; __vdso_clock_gettime; - __vdso_clock_gettime_stick; + clock_gettime64; + __vdso_clock_gettime64; gettimeofday; __vdso_gettimeofday; - __vdso_gettimeofday_stick; local: *; }; } diff --git a/arch/sparc/vdso/vma.c b/arch/sparc/vdso/vma.c index c454689ce5fa..60029d60f4d3 100644 --- a/arch/sparc/vdso/vma.c +++ b/arch/sparc/vdso/vma.c @@ -16,17 +16,16 @@ #include <linux/linkage.h> #include <linux/random.h> #include <linux/elf.h> +#include <linux/vdso_datastore.h> #include <asm/cacheflush.h> #include <asm/spitfire.h> #include <asm/vdso.h> -#include <asm/vvar.h> #include <asm/page.h> -unsigned int __read_mostly vdso_enabled = 1; +#include <vdso/datapage.h> +#include <asm/vdso/vsyscall.h> -static struct vm_special_mapping vvar_mapping = { - .name = "[vvar]" -}; +unsigned int __read_mostly vdso_enabled = 1; #ifdef CONFIG_SPARC64 static struct vm_special_mapping vdso_mapping64 = { @@ -40,207 +39,8 @@ static struct vm_special_mapping vdso_mapping32 = { }; #endif -struct vvar_data *vvar_data; - -struct vdso_elfinfo32 { - Elf32_Ehdr *hdr; - Elf32_Sym *dynsym; - unsigned long dynsymsize; - const char *dynstr; - unsigned long text; -}; - -struct vdso_elfinfo64 { - Elf64_Ehdr *hdr; - Elf64_Sym *dynsym; - unsigned long dynsymsize; - const char *dynstr; - unsigned long text; -}; - -struct vdso_elfinfo { - union { - struct vdso_elfinfo32 elf32; - struct vdso_elfinfo64 elf64; - } u; -}; - -static void *one_section64(struct vdso_elfinfo64 *e, const char *name, - unsigned long *size) -{ - const char *snames; - Elf64_Shdr *shdrs; - unsigned int i; - - shdrs = (void *)e->hdr + e->hdr->e_shoff; - snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset; - for (i = 1; i < e->hdr->e_shnum; i++) { - if (!strcmp(snames+shdrs[i].sh_name, name)) { - if (size) - *size = shdrs[i].sh_size; - return (void *)e->hdr + shdrs[i].sh_offset; - } - } - return NULL; -} - -static int find_sections64(const struct vdso_image *image, struct vdso_elfinfo *_e) -{ - struct vdso_elfinfo64 *e = &_e->u.elf64; - - e->hdr = image->data; - e->dynsym = one_section64(e, ".dynsym", &e->dynsymsize); - e->dynstr = one_section64(e, ".dynstr", NULL); - - if (!e->dynsym || !e->dynstr) { - pr_err("VDSO64: Missing symbol sections.\n"); - return -ENODEV; - } - return 0; -} - -static Elf64_Sym *find_sym64(const struct vdso_elfinfo64 *e, const char *name) -{ - unsigned int i; - - for (i = 0; i < (e->dynsymsize / sizeof(Elf64_Sym)); i++) { - Elf64_Sym *s = &e->dynsym[i]; - if (s->st_name == 0) - continue; - if (!strcmp(e->dynstr + s->st_name, name)) - return s; - } - return NULL; -} - -static int patchsym64(struct vdso_elfinfo *_e, const char *orig, - const char *new) -{ - struct vdso_elfinfo64 *e = &_e->u.elf64; - Elf64_Sym *osym = find_sym64(e, orig); - Elf64_Sym *nsym = find_sym64(e, new); - - if (!nsym || !osym) { - pr_err("VDSO64: Missing symbols.\n"); - return -ENODEV; - } - osym->st_value = nsym->st_value; - osym->st_size = nsym->st_size; - osym->st_info = nsym->st_info; - osym->st_other = nsym->st_other; - osym->st_shndx = nsym->st_shndx; - - return 0; -} - -static void *one_section32(struct vdso_elfinfo32 *e, const char *name, - unsigned long *size) -{ - const char *snames; - Elf32_Shdr *shdrs; - unsigned int i; - - shdrs = (void *)e->hdr + e->hdr->e_shoff; - snames = (void *)e->hdr + shdrs[e->hdr->e_shstrndx].sh_offset; - for (i = 1; i < e->hdr->e_shnum; i++) { - if (!strcmp(snames+shdrs[i].sh_name, name)) { - if (size) - *size = shdrs[i].sh_size; - return (void *)e->hdr + shdrs[i].sh_offset; - } - } - return NULL; -} - -static int find_sections32(const struct vdso_image *image, struct vdso_elfinfo *_e) -{ - struct vdso_elfinfo32 *e = &_e->u.elf32; - - e->hdr = image->data; - e->dynsym = one_section32(e, ".dynsym", &e->dynsymsize); - e->dynstr = one_section32(e, ".dynstr", NULL); - - if (!e->dynsym || !e->dynstr) { - pr_err("VDSO32: Missing symbol sections.\n"); - return -ENODEV; - } - return 0; -} - -static Elf32_Sym *find_sym32(const struct vdso_elfinfo32 *e, const char *name) -{ - unsigned int i; - - for (i = 0; i < (e->dynsymsize / sizeof(Elf32_Sym)); i++) { - Elf32_Sym *s = &e->dynsym[i]; - if (s->st_name == 0) - continue; - if (!strcmp(e->dynstr + s->st_name, name)) - return s; - } - return NULL; -} - -static int patchsym32(struct vdso_elfinfo *_e, const char *orig, - const char *new) -{ - struct vdso_elfinfo32 *e = &_e->u.elf32; - Elf32_Sym *osym = find_sym32(e, orig); - Elf32_Sym *nsym = find_sym32(e, new); - - if (!nsym || !osym) { - pr_err("VDSO32: Missing symbols.\n"); - return -ENODEV; - } - osym->st_value = nsym->st_value; - osym->st_size = nsym->st_size; - osym->st_info = nsym->st_info; - osym->st_other = nsym->st_other; - osym->st_shndx = nsym->st_shndx; - - return 0; -} - -static int find_sections(const struct vdso_image *image, struct vdso_elfinfo *e, - bool elf64) -{ - if (elf64) - return find_sections64(image, e); - else - return find_sections32(image, e); -} - -static int patch_one_symbol(struct vdso_elfinfo *e, const char *orig, - const char *new_target, bool elf64) -{ - if (elf64) - return patchsym64(e, orig, new_target); - else - return patchsym32(e, orig, new_target); -} - -static int stick_patch(const struct vdso_image *image, struct vdso_elfinfo *e, bool elf64) -{ - int err; - - err = find_sections(image, e, elf64); - if (err) - return err; - - err = patch_one_symbol(e, - "__vdso_gettimeofday", - "__vdso_gettimeofday_stick", elf64); - if (err) - return err; - - return patch_one_symbol(e, - "__vdso_clock_gettime", - "__vdso_clock_gettime_stick", elf64); - return 0; -} - /* - * Allocate pages for the vdso and vvar, and copy in the vdso text from the + * Allocate pages for the vdso and copy in the vdso text from the * kernel image. */ static int __init init_vdso_image(const struct vdso_image *image, @@ -248,16 +48,8 @@ static int __init init_vdso_image(const struct vdso_image *image, bool elf64) { int cnpages = (image->size) / PAGE_SIZE; - struct page *dp, **dpp = NULL; struct page *cp, **cpp = NULL; - struct vdso_elfinfo ei; - int i, dnpages = 0; - - if (tlb_type != spitfire) { - int err = stick_patch(image, &ei, elf64); - if (err) - return err; - } + int i; /* * First, the vdso text. This is initialied data, an integral number of @@ -280,31 +72,6 @@ static int __init init_vdso_image(const struct vdso_image *image, copy_page(page_address(cp), image->data + i * PAGE_SIZE); } - /* - * Now the vvar page. This is uninitialized data. - */ - - if (vvar_data == NULL) { - dnpages = (sizeof(struct vvar_data) / PAGE_SIZE) + 1; - if (WARN_ON(dnpages != 1)) - goto oom; - dpp = kzalloc_objs(struct page *, dnpages); - vvar_mapping.pages = dpp; - - if (!dpp) - goto oom; - - dp = alloc_page(GFP_KERNEL); - if (!dp) - goto oom; - - dpp[0] = dp; - vvar_data = page_address(dp); - memset(vvar_data, 0, PAGE_SIZE); - - vvar_data->seq = 0; - } - return 0; oom: if (cpp != NULL) { @@ -316,15 +83,6 @@ static int __init init_vdso_image(const struct vdso_image *image, vdso_mapping->pages = NULL; } - if (dpp != NULL) { - for (i = 0; i < dnpages; i++) { - if (dpp[i] != NULL) - __free_page(dpp[i]); - } - kfree(dpp); - vvar_mapping.pages = NULL; - } - pr_warn("Cannot allocate vdso\n"); vdso_enabled = 0; return -ENOMEM; @@ -359,9 +117,12 @@ static unsigned long vdso_addr(unsigned long start, unsigned int len) return start + (offset << PAGE_SHIFT); } +static_assert(VDSO_NR_PAGES == __VDSO_PAGES); + static int map_vdso(const struct vdso_image *image, struct vm_special_mapping *vdso_mapping) { + const size_t area_size = image->size + VDSO_NR_PAGES * PAGE_SIZE; struct mm_struct *mm = current->mm; struct vm_area_struct *vma; unsigned long text_start, addr = 0; @@ -374,23 +135,20 @@ static int map_vdso(const struct vdso_image *image, * region is free. */ if (current->flags & PF_RANDOMIZE) { - addr = get_unmapped_area(NULL, 0, - image->size - image->sym_vvar_start, - 0, 0); + addr = get_unmapped_area(NULL, 0, area_size, 0, 0); if (IS_ERR_VALUE(addr)) { ret = addr; goto up_fail; } - addr = vdso_addr(addr, image->size - image->sym_vvar_start); + addr = vdso_addr(addr, area_size); } - addr = get_unmapped_area(NULL, addr, - image->size - image->sym_vvar_start, 0, 0); + addr = get_unmapped_area(NULL, addr, area_size, 0, 0); if (IS_ERR_VALUE(addr)) { ret = addr; goto up_fail; } - text_start = addr - image->sym_vvar_start; + text_start = addr + VDSO_NR_PAGES * PAGE_SIZE; current->mm->context.vdso = (void __user *)text_start; /* @@ -408,11 +166,7 @@ static int map_vdso(const struct vdso_image *image, goto up_fail; } - vma = _install_special_mapping(mm, - addr, - -image->sym_vvar_start, - VM_READ|VM_MAYREAD, - &vvar_mapping); + vma = vdso_install_vvar_mapping(mm, addr); if (IS_ERR(vma)) { ret = PTR_ERR(vma); |
