summaryrefslogtreecommitdiff
path: root/arch/x86/entry/vdso
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/entry/vdso')
-rw-r--r--arch/x86/entry/vdso/common/vclock_gettime.c20
-rw-r--r--arch/x86/entry/vdso/common/vfutex.c71
-rw-r--r--arch/x86/entry/vdso/vdso32/Makefile5
-rw-r--r--arch/x86/entry/vdso/vdso32/sigreturn.S10
-rw-r--r--arch/x86/entry/vdso/vdso32/vdso32.lds.S5
-rw-r--r--arch/x86/entry/vdso/vdso32/vfutex.c1
-rw-r--r--arch/x86/entry/vdso/vdso64/Makefile7
-rw-r--r--arch/x86/entry/vdso/vdso64/vdso64.lds.S7
-rw-r--r--arch/x86/entry/vdso/vdso64/vdsox32.lds.S7
-rw-r--r--arch/x86/entry/vdso/vdso64/vfutex.c1
-rw-r--r--arch/x86/entry/vdso/vma.c31
11 files changed, 141 insertions, 24 deletions
diff --git a/arch/x86/entry/vdso/common/vclock_gettime.c b/arch/x86/entry/vdso/common/vclock_gettime.c
index 57066f346b3f..304dbd1f9db4 100644
--- a/arch/x86/entry/vdso/common/vclock_gettime.c
+++ b/arch/x86/entry/vdso/common/vclock_gettime.c
@@ -15,6 +15,7 @@
#include "lib/vdso/gettimeofday.c"
+#if defined(__x86_64__) || defined(CONFIG_COMPAT_32BIT_TIME)
int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
{
return __cvdso_gettimeofday(tv, tz);
@@ -29,6 +30,7 @@ __kernel_old_time_t __vdso_time(__kernel_old_time_t *t)
}
__kernel_old_time_t time(__kernel_old_time_t *t) __attribute__((weak, alias("__vdso_time")));
+#endif /* CONFIG_COMPAT_32BIT_TIME */
#if defined(CONFIG_X86_64) && !defined(BUILD_VDSO32_64)
@@ -51,6 +53,7 @@ int clock_getres(clockid_t, struct __kernel_timespec *)
#else
/* i386 only */
+#ifdef CONFIG_COMPAT_32BIT_TIME
int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts)
{
return __cvdso_clock_gettime32(clock, ts);
@@ -59,14 +62,6 @@ int __vdso_clock_gettime(clockid_t clock, struct old_timespec32 *ts)
int clock_gettime(clockid_t, struct old_timespec32 *)
__attribute__((weak, alias("__vdso_clock_gettime")));
-int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts)
-{
- return __cvdso_clock_gettime(clock, ts);
-}
-
-int clock_gettime64(clockid_t, struct __kernel_timespec *)
- __attribute__((weak, alias("__vdso_clock_gettime64")));
-
int __vdso_clock_getres(clockid_t clock, struct old_timespec32 *res)
{
return __cvdso_clock_getres_time32(clock, res);
@@ -74,6 +69,15 @@ int __vdso_clock_getres(clockid_t clock, struct old_timespec32 *res)
int clock_getres(clockid_t, struct old_timespec32 *)
__attribute__((weak, alias("__vdso_clock_getres")));
+#endif /* CONFIG_COMPAT_32BIT_TIME */
+
+int __vdso_clock_gettime64(clockid_t clock, struct __kernel_timespec *ts)
+{
+ return __cvdso_clock_gettime(clock, ts);
+}
+
+int clock_gettime64(clockid_t, struct __kernel_timespec *)
+ __attribute__((weak, alias("__vdso_clock_gettime64")));
int __vdso_clock_getres_time64(clockid_t clock, struct __kernel_timespec *ts)
{
diff --git a/arch/x86/entry/vdso/common/vfutex.c b/arch/x86/entry/vdso/common/vfutex.c
new file mode 100644
index 000000000000..454f059278e4
--- /dev/null
+++ b/arch/x86/entry/vdso/common/vfutex.c
@@ -0,0 +1,71 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <vdso/futex.h>
+
+/*
+ * Assembly template for the try unlock functions. The basic functionality is:
+ *
+ * mov esi, %eax Move the TID into EAX
+ * xor %ecx, %ecx Clear ECX
+ * lock_cmpxchgl %ecx, (%rdi) Attempt the TID -> 0 transition
+ * .Lcs_start: Start of the critical section
+ * jnz .Lcs_end If cmpxchl failed jump to the end
+ * .Lcs_success: Start of the success section
+ * movq %rcx, (%rdx) Set the pending op pointer to 0
+ * .Lcs_end: End of the critical section
+ *
+ * .Lcs_start and .Lcs_end establish the critical section range. .Lcs_success is
+ * technically not required, but there for illustration, debugging and testing.
+ *
+ * When CONFIG_COMPAT is enabled then the 64-bit VDSO provides two functions.
+ * One for the regular 64-bit sized pending operation pointer and one for a
+ * 32-bit sized pointer to support gaming emulators.
+ *
+ * The 32-bit VDSO provides only the one for 32-bit sized pointers.
+ */
+#define __stringify_1(x...) #x
+#define __stringify(x...) __stringify_1(x)
+
+#define LABEL(prefix, which) __stringify(prefix##_try_unlock_cs_##which:)
+
+#define JNZ_END(prefix) "jnz " __stringify(prefix) "_try_unlock_cs_end\n"
+
+#define CLEAR_POPQ "movq %[zero], %a[pop]\n"
+#define CLEAR_POPL "movl %k[zero], %a[pop]\n"
+
+#define futex_robust_try_unlock(prefix, clear_pop, __lock, __tid, __pop)\
+({ \
+ asm volatile ( \
+ " \n" \
+ " lock cmpxchgl %k[zero], %a[lock] \n" \
+ " \n" \
+ LABEL(prefix, start) \
+ " \n" \
+ JNZ_END(prefix) \
+ " \n" \
+ LABEL(prefix, success) \
+ " \n" \
+ clear_pop \
+ " \n" \
+ LABEL(prefix, end) \
+ : [tid] "+&a" (__tid) \
+ : [lock] "D" (__lock), \
+ [pop] "d" (__pop), \
+ [zero] "r" (0UL) \
+ : "memory" \
+ ); \
+ __tid; \
+})
+
+#ifdef __x86_64__
+__u32 __vdso_futex_robust_list64_try_unlock(__u32 *lock, __u32 tid, __u64 *pop)
+{
+ return futex_robust_try_unlock(__futex_list64, CLEAR_POPQ, lock, tid, pop);
+}
+#endif /* __x86_64__ */
+
+#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT)
+__u32 __vdso_futex_robust_list32_try_unlock(__u32 *lock, __u32 tid, __u32 *pop)
+{
+ return futex_robust_try_unlock(__futex_list32, CLEAR_POPL, lock, tid, pop);
+}
+#endif /* CONFIG_X86_32 || CONFIG_COMPAT */
diff --git a/arch/x86/entry/vdso/vdso32/Makefile b/arch/x86/entry/vdso/vdso32/Makefile
index ded4fc6a48cd..ab4b1f635f66 100644
--- a/arch/x86/entry/vdso/vdso32/Makefile
+++ b/arch/x86/entry/vdso/vdso32/Makefile
@@ -7,8 +7,9 @@
vdsos-y := 32
# Files to link into the vDSO:
-vobjs-y := note.o vclock_gettime.o vgetcpu.o
-vobjs-y += system_call.o sigreturn.o
+vobjs-y := note.o vclock_gettime.o vgetcpu.o
+vobjs-y += system_call.o sigreturn.o
+vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o
# Compilation flags
flags-y := -DBUILD_VDSO32 -m32 -mregparm=0
diff --git a/arch/x86/entry/vdso/vdso32/sigreturn.S b/arch/x86/entry/vdso/vdso32/sigreturn.S
index b33fcc501ba3..328bd3a4ef51 100644
--- a/arch/x86/entry/vdso/vdso32/sigreturn.S
+++ b/arch/x86/entry/vdso/vdso32/sigreturn.S
@@ -22,17 +22,7 @@
CFI_OFFSET cs, IA32_SIGCONTEXT_cs
CFI_OFFSET ss, IA32_SIGCONTEXT_ss
CFI_OFFSET ds, IA32_SIGCONTEXT_ds
-/*
- * .cfi_offset eflags requires LLVM 16 or newer:
- *
- * https://github.com/llvm/llvm-project/commit/67bd3c58c0c7389e39c5a2f4d3b1a30459ccf5b7
- *
- * Check for 16.0.1 to ensure the support is present, as 16.0.0 may be a
- * prerelease version.
- */
-#if defined(CONFIG_AS_IS_GNU) || (defined(CONFIG_AS_IS_LLVM) && CONFIG_AS_VERSION >= 160001)
CFI_OFFSET eflags, IA32_SIGCONTEXT_flags
-#endif
.endm
/*
diff --git a/arch/x86/entry/vdso/vdso32/vdso32.lds.S b/arch/x86/entry/vdso/vdso32/vdso32.lds.S
index 55554f80d930..00629192db56 100644
--- a/arch/x86/entry/vdso/vdso32/vdso32.lds.S
+++ b/arch/x86/entry/vdso/vdso32/vdso32.lds.S
@@ -23,13 +23,18 @@ VERSION
{
LINUX_2.6 {
global:
+#ifdef CONFIG_COMPAT_32BIT_TIME
__vdso_clock_gettime;
__vdso_gettimeofday;
__vdso_time;
__vdso_clock_getres;
+#endif /* CONFIG_COMPAT_32BIT_TIME */
__vdso_clock_gettime64;
__vdso_clock_getres_time64;
__vdso_getcpu;
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+ __vdso_futex_robust_list32_try_unlock;
+#endif
};
LINUX_2.5 {
diff --git a/arch/x86/entry/vdso/vdso32/vfutex.c b/arch/x86/entry/vdso/vdso32/vfutex.c
new file mode 100644
index 000000000000..940a6ee30026
--- /dev/null
+++ b/arch/x86/entry/vdso/vdso32/vfutex.c
@@ -0,0 +1 @@
+#include "common/vfutex.c"
diff --git a/arch/x86/entry/vdso/vdso64/Makefile b/arch/x86/entry/vdso/vdso64/Makefile
index bfffaf1aeecc..7c0790065b5e 100644
--- a/arch/x86/entry/vdso/vdso64/Makefile
+++ b/arch/x86/entry/vdso/vdso64/Makefile
@@ -8,9 +8,10 @@ vdsos-y := 64
vdsos-$(CONFIG_X86_X32_ABI) += x32
# Files to link into the vDSO:
-vobjs-y := note.o vclock_gettime.o vgetcpu.o
-vobjs-y += vgetrandom.o vgetrandom-chacha.o
-vobjs-$(CONFIG_X86_SGX) += vsgx.o
+vobjs-y := note.o vclock_gettime.o vgetcpu.o
+vobjs-y += vgetrandom.o vgetrandom-chacha.o
+vobjs-$(CONFIG_X86_SGX) += vsgx.o
+vobjs-$(CONFIG_FUTEX_ROBUST_UNLOCK) += vfutex.o
# Compilation flags
flags-y := -DBUILD_VDSO64 -m64 -mcmodel=small
diff --git a/arch/x86/entry/vdso/vdso64/vdso64.lds.S b/arch/x86/entry/vdso/vdso64/vdso64.lds.S
index 5ce3f2b6373a..4a72122da81b 100644
--- a/arch/x86/entry/vdso/vdso64/vdso64.lds.S
+++ b/arch/x86/entry/vdso/vdso64/vdso64.lds.S
@@ -32,6 +32,13 @@ VERSION {
#endif
getrandom;
__vdso_getrandom;
+
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+ __vdso_futex_robust_list64_try_unlock;
+#ifdef CONFIG_COMPAT
+ __vdso_futex_robust_list32_try_unlock;
+#endif
+#endif
local: *;
};
}
diff --git a/arch/x86/entry/vdso/vdso64/vdsox32.lds.S b/arch/x86/entry/vdso/vdso64/vdsox32.lds.S
index 3dbd20c8dacc..b917dc69f62f 100644
--- a/arch/x86/entry/vdso/vdso64/vdsox32.lds.S
+++ b/arch/x86/entry/vdso/vdso64/vdsox32.lds.S
@@ -22,6 +22,13 @@ VERSION {
__vdso_getcpu;
__vdso_time;
__vdso_clock_getres;
+
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+ __vdso_futex_robust_list64_try_unlock;
+#ifdef CONFIG_COMPAT
+ __vdso_futex_robust_list32_try_unlock;
+#endif
+#endif
local: *;
};
}
diff --git a/arch/x86/entry/vdso/vdso64/vfutex.c b/arch/x86/entry/vdso/vdso64/vfutex.c
new file mode 100644
index 000000000000..940a6ee30026
--- /dev/null
+++ b/arch/x86/entry/vdso/vdso64/vfutex.c
@@ -0,0 +1 @@
+#include "common/vfutex.c"
diff --git a/arch/x86/entry/vdso/vma.c b/arch/x86/entry/vdso/vma.c
index a6bfcc8243cd..18dfd80a81ef 100644
--- a/arch/x86/entry/vdso/vma.c
+++ b/arch/x86/entry/vdso/vma.c
@@ -6,6 +6,7 @@
*/
#include <linux/mm.h>
#include <linux/err.h>
+#include <linux/futex.h>
#include <linux/sched.h>
#include <linux/sched/task_stack.h>
#include <linux/slab.h>
@@ -73,6 +74,31 @@ static void vdso_fix_landing(const struct vdso_image *image,
regs->ip = new_vma->vm_start + ipoffset;
}
+#ifdef CONFIG_FUTEX_ROBUST_UNLOCK
+static void vdso_futex_robust_unlock_update_ips(void)
+{
+ const struct vdso_image *image = current->mm->context.vdso_image;
+ unsigned long vdso = (unsigned long) current->mm->context.vdso;
+ struct futex_mm_data *fd = &current->mm->futex;
+ unsigned int idx = 0;
+
+ futex_reset_cs_ranges(fd);
+
+#ifdef CONFIG_X86_64
+ futex_set_vdso_cs_range(fd, idx, vdso + image->sym___futex_list64_try_unlock_cs_start,
+ vdso + image->sym___futex_list64_try_unlock_cs_end, false);
+ idx++;
+#endif /* CONFIG_X86_64 */
+
+#if defined(CONFIG_X86_32) || defined(CONFIG_COMPAT)
+ futex_set_vdso_cs_range(fd, idx, vdso + image->sym___futex_list32_try_unlock_cs_start,
+ vdso + image->sym___futex_list32_try_unlock_cs_end, true);
+#endif /* CONFIG_X86_32 || CONFIG_COMPAT */
+}
+#else
+static inline void vdso_futex_robust_unlock_update_ips(void) { }
+#endif
+
static int vdso_mremap(const struct vm_special_mapping *sm,
struct vm_area_struct *new_vma)
{
@@ -80,6 +106,7 @@ static int vdso_mremap(const struct vm_special_mapping *sm,
vdso_fix_landing(image, new_vma);
current->mm->context.vdso = (void __user *)new_vma->vm_start;
+ vdso_futex_robust_unlock_update_ips();
return 0;
}
@@ -178,13 +205,15 @@ static int map_vdso(const struct vdso_image *image, unsigned long addr)
if (IS_ERR(vma)) {
ret = PTR_ERR(vma);
do_munmap(mm, text_start, image->size, NULL);
- do_munmap(mm, addr, image->size, NULL);
+ do_munmap(mm, addr, VDSO_NR_PAGES * PAGE_SIZE, NULL);
goto up_fail;
}
current->mm->context.vdso = (void __user *)text_start;
current->mm->context.vdso_image = image;
+ vdso_futex_robust_unlock_update_ips();
+
up_fail:
mmap_write_unlock(mm);
return ret;
generated by cgit v1.2.3 (git 2.0.4) at 2026-09-26 19:30:04 +0000