summaryrefslogtreecommitdiff
path: root/include
diff options
context:
space:
mode:
authorAvi Kivity <avi@redhat.com>2011-05-11 05:56:53 -0400
committerAvi Kivity <avi@redhat.com>2011-05-22 08:48:07 -0400
commite3aa52d665ec1a962d1cf025a2e5ee84b3b33406 (patch)
tree8e4e9516fe5cccf2b73c7537327f3dd21634c6d7 /include
parentd2f62766d5778bbaf80d4feb90a23c7edc371a54 (diff)
parent29ce831000081dd757d3116bf774aafffc4b6b20 (diff)
Merge commit '29ce831000081dd757d3116bf774aafffc4b6b20' into next
* commit '29ce831000081dd757d3116bf774aafffc4b6b20': (34 commits) rcu: provide rcu_virt_note_context_switch() function. rcu: get rid of signed overflow in check_cpu_stall() rcu: optimize rcutiny rcu: prevent call_rcu() from diving into rcu core if irqs disabled rcu: further lower priority in rcu_yield() rcu: introduce kfree_rcu() rcu: fix spelling rcu: call __rcu_read_unlock() in exit_rcu for tree RCU rcu: Converge TINY_RCU expedited and normal boosting rcu: remove useless ->boosted_this_gp field rcu: code cleanups in TINY_RCU priority boosting. rcu: Switch to this_cpu() primitives rcu: Use WARN_ON_ONCE for DEBUG_OBJECTS_RCU_HEAD warnings rcu: mark rcutorture boosting callback as being on-stack rcu: add DEBUG_OBJECTS_RCU_HEAD check for alignment rcu: Enable DEBUG_OBJECTS_RCU_HEAD from !PREEMPT rcu: Add forward-progress diagnostic for per-CPU kthreads rcu: add grace-period age and more kthread state to tracing rcu: fix tracing bug thinko on boost-balk attribution rcu: update tracing documentation for new rcutorture and rcuboost ... Pulling in rcu_virt_note_context_switch(). Signed-off-by: Avi Kivity <avi@redhat.com> * commit '29ce831000081dd757d3116bf774aafffc4b6b20': (34 commits) rcu: provide rcu_virt_note_context_switch() function. rcu: get rid of signed overflow in check_cpu_stall() rcu: optimize rcutiny rcu: prevent call_rcu() from diving into rcu core if irqs disabled rcu: further lower priority in rcu_yield() rcu: introduce kfree_rcu() rcu: fix spelling rcu: call __rcu_read_unlock() in exit_rcu for tree RCU rcu: Converge TINY_RCU expedited and normal boosting rcu: remove useless ->boosted_this_gp field rcu: code cleanups in TINY_RCU priority boosting. rcu: Switch to this_cpu() primitives rcu: Use WARN_ON_ONCE for DEBUG_OBJECTS_RCU_HEAD warnings rcu: mark rcutorture boosting callback as being on-stack rcu: add DEBUG_OBJECTS_RCU_HEAD check for alignment rcu: Enable DEBUG_OBJECTS_RCU_HEAD from !PREEMPT rcu: Add forward-progress diagnostic for per-CPU kthreads rcu: add grace-period age and more kthread state to tracing rcu: fix tracing bug thinko on boost-balk attribution rcu: update tracing documentation for new rcutorture and rcuboost ...
Diffstat (limited to 'include')
-rw-r--r--include/linux/interrupt.h1
-rw-r--r--include/linux/rcupdate.h70
-rw-r--r--include/linux/rcutiny.h8
-rw-r--r--include/linux/rcutree.h13
-rw-r--r--include/trace/events/irq.h3
5 files changed, 91 insertions, 4 deletions
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index bea0ac750712..6c12989839d9 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -414,7 +414,6 @@ enum
TASKLET_SOFTIRQ,
SCHED_SOFTIRQ,
HRTIMER_SOFTIRQ,
- RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */
NR_SOFTIRQS
};
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index ff422d2b7f90..99f9aa7c2804 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -47,6 +47,18 @@
extern int rcutorture_runnable; /* for sysctl */
#endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
+#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
+extern void rcutorture_record_test_transition(void);
+extern void rcutorture_record_progress(unsigned long vernum);
+#else
+static inline void rcutorture_record_test_transition(void)
+{
+}
+static inline void rcutorture_record_progress(unsigned long vernum)
+{
+}
+#endif
+
#define UINT_CMP_GE(a, b) (UINT_MAX / 2 >= (a) - (b))
#define UINT_CMP_LT(a, b) (UINT_MAX / 2 < (a) - (b))
#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
@@ -68,7 +80,6 @@ extern void call_rcu_sched(struct rcu_head *head,
extern void synchronize_sched(void);
extern void rcu_barrier_bh(void);
extern void rcu_barrier_sched(void);
-extern int sched_expedited_torture_stats(char *page);
static inline void __rcu_read_lock_bh(void)
{
@@ -774,6 +785,7 @@ extern struct debug_obj_descr rcuhead_debug_descr;
static inline void debug_rcu_head_queue(struct rcu_head *head)
{
+ WARN_ON_ONCE((unsigned long)head & 0x3);
debug_object_activate(head, &rcuhead_debug_descr);
debug_object_active_state(head, &rcuhead_debug_descr,
STATE_RCU_HEAD_READY,
@@ -797,4 +809,60 @@ static inline void debug_rcu_head_unqueue(struct rcu_head *head)
}
#endif /* #else !CONFIG_DEBUG_OBJECTS_RCU_HEAD */
+static __always_inline bool __is_kfree_rcu_offset(unsigned long offset)
+{
+ return offset < 4096;
+}
+
+static __always_inline
+void __kfree_rcu(struct rcu_head *head, unsigned long offset)
+{
+ typedef void (*rcu_callback)(struct rcu_head *);
+
+ BUILD_BUG_ON(!__builtin_constant_p(offset));
+
+ /* See the kfree_rcu() header comment. */
+ BUILD_BUG_ON(!__is_kfree_rcu_offset(offset));
+
+ call_rcu(head, (rcu_callback)offset);
+}
+
+extern void kfree(const void *);
+
+static inline void __rcu_reclaim(struct rcu_head *head)
+{
+ unsigned long offset = (unsigned long)head->func;
+
+ if (__is_kfree_rcu_offset(offset))
+ kfree((void *)head - offset);
+ else
+ head->func(head);
+}
+
+/**
+ * kfree_rcu() - kfree an object after a grace period.
+ * @ptr: pointer to kfree
+ * @rcu_head: the name of the struct rcu_head within the type of @ptr.
+ *
+ * Many rcu callbacks functions just call kfree() on the base structure.
+ * These functions are trivial, but their size adds up, and furthermore
+ * when they are used in a kernel module, that module must invoke the
+ * high-latency rcu_barrier() function at module-unload time.
+ *
+ * The kfree_rcu() function handles this issue. Rather than encoding a
+ * function address in the embedded rcu_head structure, kfree_rcu() instead
+ * encodes the offset of the rcu_head structure within the base structure.
+ * Because the functions are not allowed in the low-order 4096 bytes of
+ * kernel virtual memory, offsets up to 4095 bytes can be accommodated.
+ * If the offset is larger than 4095 bytes, a compile-time error will
+ * be generated in __kfree_rcu(). If this error is triggered, you can
+ * either fall back to use of call_rcu() or rearrange the structure to
+ * position the rcu_head structure into the first 4096 bytes.
+ *
+ * Note that the allowable offset might decrease in the future, for example,
+ * to allow something like kmem_cache_free_rcu().
+ */
+#define kfree_rcu(ptr, rcu_head) \
+ __kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
+
#endif /* __LINUX_RCUPDATE_H */
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 30ebd7c8d874..52b3e0281fd0 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -100,6 +100,14 @@ static inline void rcu_note_context_switch(int cpu)
}
/*
+ * Take advantage of the fact that there is only one CPU, which
+ * allows us to ignore virtualization-based context switches.
+ */
+static inline void rcu_virt_note_context_switch(int cpu)
+{
+}
+
+/*
* Return the number of grace periods.
*/
static inline long rcu_batches_completed(void)
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 3a933482734a..e65d06634dd8 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -35,6 +35,16 @@ extern void rcu_note_context_switch(int cpu);
extern int rcu_needs_cpu(int cpu);
extern void rcu_cpu_stall_reset(void);
+/*
+ * Note a virtualization-based context switch. This is simply a
+ * wrapper around rcu_note_context_switch(), which allows TINY_RCU
+ * to save a few bytes.
+ */
+static inline void rcu_virt_note_context_switch(int cpu)
+{
+ rcu_note_context_switch(cpu);
+}
+
#ifdef CONFIG_TREE_PREEMPT_RCU
extern void exit_rcu(void);
@@ -58,9 +68,12 @@ static inline void synchronize_rcu_bh_expedited(void)
extern void rcu_barrier(void);
+extern unsigned long rcutorture_testseq;
+extern unsigned long rcutorture_vernum;
extern long rcu_batches_completed(void);
extern long rcu_batches_completed_bh(void);
extern long rcu_batches_completed_sched(void);
+
extern void rcu_force_quiescent_state(void);
extern void rcu_bh_force_quiescent_state(void);
extern void rcu_sched_force_quiescent_state(void);
diff --git a/include/trace/events/irq.h b/include/trace/events/irq.h
index 1c09820df585..ae045ca7d356 100644
--- a/include/trace/events/irq.h
+++ b/include/trace/events/irq.h
@@ -20,8 +20,7 @@ struct softirq_action;
softirq_name(BLOCK_IOPOLL), \
softirq_name(TASKLET), \
softirq_name(SCHED), \
- softirq_name(HRTIMER), \
- softirq_name(RCU))
+ softirq_name(HRTIMER))
/**
* irq_handler_entry - called immediately before the irq action handler