diff options
| author | Peter Zijlstra <peterz@infradead.org> | 2011-07-28 10:43:51 +0200 |
|---|---|---|
| committer | Clark Williams <williams@redhat.com> | 2012-04-13 11:01:51 -0500 |
| commit | 8f2a48d21a612df271a4217c6be0467b277c3906 (patch) | |
| tree | f9c1971fccca2a00c9c5f18414bc1af8a80d990b | |
| parent | d9d1b9404178e97a4cad32926041f32af3096945 (diff) | |
mm, rt: kmap_atomic scheduling
In fact, with migrate_disable() existing one could play games with
kmap_atomic. You could save/restore the kmap_atomic slots on context
switch (if there are any in use of course), this should be esp easy now
that we have a kmap_atomic stack.
Something like the below.. it wants replacing all the preempt_disable()
stuff with pagefault_disable() && migrate_disable() of course, but then
you can flip kmaps around like below.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
[dvhart@linux.intel.com: build fix]
Link: http://lkml.kernel.org/r/1311842631.5890.208.camel@twins
| -rw-r--r-- | arch/x86/kernel/process_32.c | 36 | ||||
| -rw-r--r-- | include/linux/sched.h | 5 | ||||
| -rw-r--r-- | mm/memory.c | 2 |
3 files changed, 43 insertions, 0 deletions
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c index ae6847303e26..2b0882add416 100644 --- a/arch/x86/kernel/process_32.c +++ b/arch/x86/kernel/process_32.c @@ -36,6 +36,7 @@ #include <linux/uaccess.h> #include <linux/io.h> #include <linux/kdebug.h> +#include <linux/highmem.h> #include <asm/pgtable.h> #include <asm/ldt.h> @@ -285,6 +286,41 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p) task_thread_info(next_p)->flags & _TIF_WORK_CTXSW_NEXT)) __switch_to_xtra(prev_p, next_p, tss); +#if defined CONFIG_PREEMPT_RT_FULL && defined CONFIG_HIGHMEM + /* + * Save @prev's kmap_atomic stack + */ + prev_p->kmap_idx = __this_cpu_read(__kmap_atomic_idx); + if (unlikely(prev_p->kmap_idx)) { + int i; + + for (i = 0; i < prev_p->kmap_idx; i++) { + int idx = i + KM_TYPE_NR * smp_processor_id(); + + pte_t *ptep = kmap_pte - idx; + prev_p->kmap_pte[i] = *ptep; + kpte_clear_flush(ptep, __fix_to_virt(FIX_KMAP_BEGIN + idx)); + } + + __this_cpu_write(__kmap_atomic_idx, 0); + } + + /* + * Restore @next_p's kmap_atomic stack + */ + if (unlikely(next_p->kmap_idx)) { + int i; + + __this_cpu_write(__kmap_atomic_idx, next_p->kmap_idx); + + for (i = 0; i < next_p->kmap_idx; i++) { + int idx = i + KM_TYPE_NR * smp_processor_id(); + + set_pte(kmap_pte - idx, next_p->kmap_pte[i]); + } + } +#endif + /* * Leave lazy mode, flushing any hypercalls made here. * This must be done before restoring TLS segments so diff --git a/include/linux/sched.h b/include/linux/sched.h index 372c15df2ebc..7fc83210e8fb 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -63,6 +63,7 @@ struct sched_param { #include <linux/nodemask.h> #include <linux/mm_types.h> +#include <asm/kmap_types.h> #include <asm/page.h> #include <asm/ptrace.h> #include <asm/cputime.h> @@ -1647,6 +1648,10 @@ struct task_struct { struct rcu_head put_rcu; int softirq_nestcnt; #endif +#if defined CONFIG_PREEMPT_RT_FULL && defined CONFIG_HIGHMEM + int kmap_idx; + pte_t kmap_pte[KM_TYPE_NR]; +#endif }; #ifdef CONFIG_PREEMPT_RT_FULL diff --git a/mm/memory.c b/mm/memory.c index 68676fdccd14..c63016be154f 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -3470,6 +3470,7 @@ unlock: #ifdef CONFIG_PREEMPT_RT_FULL void pagefault_disable(void) { + migrate_disable(); current->pagefault_disabled++; /* * make sure to have issued the store before a pagefault @@ -3487,6 +3488,7 @@ void pagefault_enable(void) */ barrier(); current->pagefault_disabled--; + migrate_enable(); } EXPORT_SYMBOL_GPL(pagefault_enable); #endif |