diff options
Diffstat (limited to 'include/vdso')
| -rw-r--r-- | include/vdso/datapage.h | 27 | ||||
| -rw-r--r-- | include/vdso/helpers.h | 31 |
2 files changed, 34 insertions, 24 deletions
diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index 23c39b96190f..5977723fb3b5 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -4,24 +4,16 @@ #ifndef __ASSEMBLY__ -#include <linux/compiler.h> +#include <linux/types.h> + #include <uapi/linux/bits.h> #include <uapi/linux/time.h> -#include <uapi/linux/types.h> -#include <uapi/asm-generic/errno-base.h> #include <vdso/align.h> #include <vdso/bits.h> #include <vdso/cache.h> -#include <vdso/clocksource.h> -#include <vdso/ktime.h> -#include <vdso/limits.h> -#include <vdso/math64.h> #include <vdso/page.h> -#include <vdso/processor.h> #include <vdso/time.h> -#include <vdso/time32.h> -#include <vdso/time64.h> #ifdef CONFIG_ARCH_HAS_VDSO_TIME_DATA #include <asm/vdso/time_data.h> @@ -80,8 +72,8 @@ struct vdso_timestamp { * @mask: clocksource mask * @mult: clocksource multiplier * @shift: clocksource shift - * @basetime[clock_id]: basetime per clock_id - * @offset[clock_id]: time namespace offset per clock_id + * @basetime: basetime per clock_id + * @offset: time namespace offset per clock_id * * See also struct vdso_time_data for basic access and ordering information as * struct vdso_clock is used there. @@ -184,17 +176,6 @@ enum vdso_pages { VDSO_NR_PAGES }; -/* - * The generic vDSO implementation requires that gettimeofday.h - * provides: - * - __arch_get_hw_counter(): to get the hw counter based on the - * clock_mode. - * - gettimeofday_fallback(): fallback for gettimeofday. - * - clock_gettime_fallback(): fallback for clock_gettime. - * - clock_getres_fallback(): fallback for clock_getres. - */ -#include <asm/vdso/gettimeofday.h> - #else /* !__ASSEMBLY__ */ #ifdef CONFIG_VDSO_GETRANDOM diff --git a/include/vdso/helpers.h b/include/vdso/helpers.h index 1a5ee9d9052c..a3bf4f1c0d37 100644 --- a/include/vdso/helpers.h +++ b/include/vdso/helpers.h @@ -6,6 +6,13 @@ #include <asm/barrier.h> #include <vdso/datapage.h> +#include <vdso/processor.h> +#include <vdso/clocksource.h> + +static __always_inline bool vdso_is_timens_clock(const struct vdso_clock *vc) +{ + return IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS; +} static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc) { @@ -18,6 +25,28 @@ static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc) return seq; } +/* + * Variant of vdso_read_begin() to handle VDSO_CLOCKMODE_TIMENS. + * + * Time namespace enabled tasks have a special VVAR page installed which has + * vc->seq set to 1 and vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non + * time namespace affected tasks this does not affect performance because if + * vc->seq is odd, i.e. a concurrent update is in progress the extra check for + * vc->clock_mode is just a few extra instructions while spin waiting for + * vc->seq to become even again. + */ +static __always_inline bool vdso_read_begin_timens(const struct vdso_clock *vc, u32 *seq) +{ + while (unlikely((*seq = READ_ONCE(vc->seq)) & 1)) { + if (vdso_is_timens_clock(vc)) + return true; + cpu_relax(); + } + smp_rmb(); + + return false; +} + static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc, u32 start) { @@ -25,7 +54,7 @@ static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc, smp_rmb(); seq = READ_ONCE(vc->seq); - return seq != start; + return unlikely(seq != start); } static __always_inline void vdso_write_seq_begin(struct vdso_clock *vc) |