diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/asm-generic/bitsperlong.h | 9 | ||||
| -rw-r--r-- | include/linux/clocksource.h | 6 | ||||
| -rw-r--r-- | include/linux/time_namespace.h | 39 | ||||
| -rw-r--r-- | include/linux/vdso_datastore.h | 6 | ||||
| -rw-r--r-- | include/vdso/datapage.h | 27 | ||||
| -rw-r--r-- | include/vdso/helpers.h | 31 |
6 files changed, 69 insertions, 49 deletions
diff --git a/include/asm-generic/bitsperlong.h b/include/asm-generic/bitsperlong.h index 1023e2a4bd37..90e8aeebfd2f 100644 --- a/include/asm-generic/bitsperlong.h +++ b/include/asm-generic/bitsperlong.h @@ -19,6 +19,15 @@ #error Inconsistent word size. Check asm/bitsperlong.h #endif +#if __CHAR_BIT__ * __SIZEOF_LONG__ != __BITS_PER_LONG +#error Inconsistent word size. Check asm/bitsperlong.h +#endif + +#ifndef __ASSEMBLER__ +_Static_assert(sizeof(long) * 8 == __BITS_PER_LONG, + "Inconsistent word size. Check asm/bitsperlong.h"); +#endif + #ifndef BITS_PER_LONG_LONG #define BITS_PER_LONG_LONG 64 #endif diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index ccf5c0ca26b7..7c38190b10bf 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -25,8 +25,7 @@ struct clocksource_base; struct clocksource; struct module; -#if defined(CONFIG_ARCH_CLOCKSOURCE_DATA) || \ - defined(CONFIG_GENERIC_GETTIMEOFDAY) +#if defined(CONFIG_GENERIC_GETTIMEOFDAY) #include <asm/clocksource.h> #endif @@ -103,9 +102,6 @@ struct clocksource { u32 shift; u64 max_idle_ns; u32 maxadj; -#ifdef CONFIG_ARCH_CLOCKSOURCE_DATA - struct arch_clocksource_data archdata; -#endif u64 max_cycles; u64 max_raw_delta; const char *name; diff --git a/include/linux/time_namespace.h b/include/linux/time_namespace.h index c514d0e5a45c..58bd9728df58 100644 --- a/include/linux/time_namespace.h +++ b/include/linux/time_namespace.h @@ -8,6 +8,7 @@ #include <linux/ns_common.h> #include <linux/err.h> #include <linux/time64.h> +#include <linux/cleanup.h> struct user_namespace; extern struct user_namespace init_user_ns; @@ -25,7 +26,9 @@ struct time_namespace { struct ucounts *ucounts; struct ns_common ns; struct timens_offsets offsets; +#ifdef CONFIG_TIME_NS_VDSO struct page *vvar_page; +#endif /* If set prevents changing offsets after any task joined namespace. */ bool frozen_offsets; } __randomize_layout; @@ -38,9 +41,6 @@ static inline struct time_namespace *to_time_ns(struct ns_common *ns) return container_of(ns, struct time_namespace, ns); } void __init time_ns_init(void); -extern int vdso_join_timens(struct task_struct *task, - struct time_namespace *ns); -extern void timens_commit(struct task_struct *tsk, struct time_namespace *ns); static inline struct time_namespace *get_time_ns(struct time_namespace *ns) { @@ -53,7 +53,6 @@ struct time_namespace *copy_time_ns(u64 flags, struct time_namespace *old_ns); void free_time_ns(struct time_namespace *ns); void timens_on_fork(struct nsproxy *nsproxy, struct task_struct *tsk); -struct page *find_timens_vvar_page(struct vm_area_struct *vma); static inline void put_time_ns(struct time_namespace *ns) { @@ -117,17 +116,6 @@ static inline void __init time_ns_init(void) { } -static inline int vdso_join_timens(struct task_struct *task, - struct time_namespace *ns) -{ - return 0; -} - -static inline void timens_commit(struct task_struct *tsk, - struct time_namespace *ns) -{ -} - static inline struct time_namespace *get_time_ns(struct time_namespace *ns) { return NULL; @@ -154,11 +142,6 @@ static inline void timens_on_fork(struct nsproxy *nsproxy, return; } -static inline struct page *find_timens_vvar_page(struct vm_area_struct *vma) -{ - return NULL; -} - static inline void timens_add_monotonic(struct timespec64 *ts) { } static inline void timens_add_boottime(struct timespec64 *ts) { } @@ -175,4 +158,20 @@ static inline ktime_t timens_ktime_to_host(clockid_t clockid, ktime_t tim) } #endif +#ifdef CONFIG_TIME_NS_VDSO +extern void timens_commit(struct task_struct *tsk, struct time_namespace *ns); +struct page *find_timens_vvar_page(struct vm_area_struct *vma); +#else /* !CONFIG_TIME_NS_VDSO */ +static inline void timens_commit(struct task_struct *tsk, struct time_namespace *ns) +{ +} + +static inline struct page *find_timens_vvar_page(struct vm_area_struct *vma) +{ + return NULL; +} +#endif /* CONFIG_TIME_NS_VDSO */ + +DEFINE_FREE(time_ns, struct time_namespace *, if (_T) put_time_ns(_T)) + #endif /* _LINUX_TIMENS_H */ diff --git a/include/linux/vdso_datastore.h b/include/linux/vdso_datastore.h index a91fa24b06e0..0b530428db71 100644 --- a/include/linux/vdso_datastore.h +++ b/include/linux/vdso_datastore.h @@ -2,9 +2,15 @@ #ifndef _LINUX_VDSO_DATASTORE_H #define _LINUX_VDSO_DATASTORE_H +#ifdef CONFIG_HAVE_GENERIC_VDSO #include <linux/mm_types.h> extern const struct vm_special_mapping vdso_vvar_mapping; struct vm_area_struct *vdso_install_vvar_mapping(struct mm_struct *mm, unsigned long addr); +void __init vdso_setup_data_pages(void); +#else /* !CONFIG_HAVE_GENERIC_VDSO */ +static inline void vdso_setup_data_pages(void) { } +#endif /* CONFIG_HAVE_GENERIC_VDSO */ + #endif /* _LINUX_VDSO_DATASTORE_H */ diff --git a/include/vdso/datapage.h b/include/vdso/datapage.h index 23c39b96190f..5977723fb3b5 100644 --- a/include/vdso/datapage.h +++ b/include/vdso/datapage.h @@ -4,24 +4,16 @@ #ifndef __ASSEMBLY__ -#include <linux/compiler.h> +#include <linux/types.h> + #include <uapi/linux/bits.h> #include <uapi/linux/time.h> -#include <uapi/linux/types.h> -#include <uapi/asm-generic/errno-base.h> #include <vdso/align.h> #include <vdso/bits.h> #include <vdso/cache.h> -#include <vdso/clocksource.h> -#include <vdso/ktime.h> -#include <vdso/limits.h> -#include <vdso/math64.h> #include <vdso/page.h> -#include <vdso/processor.h> #include <vdso/time.h> -#include <vdso/time32.h> -#include <vdso/time64.h> #ifdef CONFIG_ARCH_HAS_VDSO_TIME_DATA #include <asm/vdso/time_data.h> @@ -80,8 +72,8 @@ struct vdso_timestamp { * @mask: clocksource mask * @mult: clocksource multiplier * @shift: clocksource shift - * @basetime[clock_id]: basetime per clock_id - * @offset[clock_id]: time namespace offset per clock_id + * @basetime: basetime per clock_id + * @offset: time namespace offset per clock_id * * See also struct vdso_time_data for basic access and ordering information as * struct vdso_clock is used there. @@ -184,17 +176,6 @@ enum vdso_pages { VDSO_NR_PAGES }; -/* - * The generic vDSO implementation requires that gettimeofday.h - * provides: - * - __arch_get_hw_counter(): to get the hw counter based on the - * clock_mode. - * - gettimeofday_fallback(): fallback for gettimeofday. - * - clock_gettime_fallback(): fallback for clock_gettime. - * - clock_getres_fallback(): fallback for clock_getres. - */ -#include <asm/vdso/gettimeofday.h> - #else /* !__ASSEMBLY__ */ #ifdef CONFIG_VDSO_GETRANDOM diff --git a/include/vdso/helpers.h b/include/vdso/helpers.h index 1a5ee9d9052c..a3bf4f1c0d37 100644 --- a/include/vdso/helpers.h +++ b/include/vdso/helpers.h @@ -6,6 +6,13 @@ #include <asm/barrier.h> #include <vdso/datapage.h> +#include <vdso/processor.h> +#include <vdso/clocksource.h> + +static __always_inline bool vdso_is_timens_clock(const struct vdso_clock *vc) +{ + return IS_ENABLED(CONFIG_TIME_NS) && vc->clock_mode == VDSO_CLOCKMODE_TIMENS; +} static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc) { @@ -18,6 +25,28 @@ static __always_inline u32 vdso_read_begin(const struct vdso_clock *vc) return seq; } +/* + * Variant of vdso_read_begin() to handle VDSO_CLOCKMODE_TIMENS. + * + * Time namespace enabled tasks have a special VVAR page installed which has + * vc->seq set to 1 and vc->clock_mode set to VDSO_CLOCKMODE_TIMENS. For non + * time namespace affected tasks this does not affect performance because if + * vc->seq is odd, i.e. a concurrent update is in progress the extra check for + * vc->clock_mode is just a few extra instructions while spin waiting for + * vc->seq to become even again. + */ +static __always_inline bool vdso_read_begin_timens(const struct vdso_clock *vc, u32 *seq) +{ + while (unlikely((*seq = READ_ONCE(vc->seq)) & 1)) { + if (vdso_is_timens_clock(vc)) + return true; + cpu_relax(); + } + smp_rmb(); + + return false; +} + static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc, u32 start) { @@ -25,7 +54,7 @@ static __always_inline u32 vdso_read_retry(const struct vdso_clock *vc, smp_rmb(); seq = READ_ONCE(vc->seq); - return seq != start; + return unlikely(seq != start); } static __always_inline void vdso_write_seq_begin(struct vdso_clock *vc) |