diff options
| -rw-r--r-- | kernel/time/clockevents.c | 33 | ||||
| -rw-r--r-- | kernel/time/hrtimer.c | 15 | ||||
| -rw-r--r-- | kernel/time/tick-broadcast.c | 36 | ||||
| -rw-r--r-- | kernel/time/tick-internal.h | 2 |
4 files changed, 55 insertions, 31 deletions
diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 0014d163f989..62ad7c000386 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -615,34 +615,41 @@ void clockevents_handle_noop(struct clock_event_device *dev) { } -/** - * clockevents_exchange_device - release and request clock devices - * @old: device to release (can be NULL) - * @new: device to request (can be NULL) - * - * Called from various tick functions with clockevents_lock held and - * interrupts disabled. - */ -void clockevents_exchange_device(struct clock_event_device *old, - struct clock_event_device *new) +void __clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new) { /* * Caller releases a clock event device. We queue it into the * released list and do a notify add later. */ if (old) { - module_put(old->owner); clockevents_switch_state(old, CLOCK_EVT_STATE_DETACHED); list_move(&old->list, &clockevents_released); } if (new) { - BUG_ON(!clockevent_state_detached(new)); + WARN_ON(!clockevent_state_detached(new)); clockevents_shutdown(new); } } /** + * clockevents_exchange_device - release and request clock devices + * @old: device to release (can be NULL) + * @new: device to request (can be NULL) + * + * Called from various tick functions with clockevents_lock held and + * interrupts disabled. + */ +void clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new) +{ + __clockevents_exchange_device(old, new); + if (old) + module_put(old->owner); +} + +/** * clockevents_suspend - suspend clock devices */ void clockevents_suspend(void) @@ -699,7 +706,7 @@ void tick_offline_cpu(unsigned int cpu) if (cpumask_test_cpu(cpu, dev->cpumask) && cpumask_weight(dev->cpumask) == 1 && !tick_is_broadcast_device(dev)) { - BUG_ON(!clockevent_state_detached(dev)); + WARN_ON(!clockevent_state_detached(dev)); list_del(&dev->list); } } diff --git a/kernel/time/hrtimer.c b/kernel/time/hrtimer.c index 530d61257b9a..cbf1693c86b3 100644 --- a/kernel/time/hrtimer.c +++ b/kernel/time/hrtimer.c @@ -1263,13 +1263,23 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b { bool was_first = false; + /* + * Updating the sort key while @timer is queued can temporarily + * make the tree inconsistent. This is safe under cpu_base->lock: + * no other queue operation can observe that state. + * hrtimer_can_update_in_place() either confirms that the new expiry + * fits between the neighbours or timerqueue_linked_del() removes the + * timer without consulting the expiry. + */ + hrtimer_set_expires_range_ns(timer, expires, delta_ns); + expires = hrtimer_get_expires(timer); + /* Remove it from the timer queue if active */ if (timer->is_queued) { was_first = !timerqueue_linked_prev(&timer->node); /* Try to update in place to avoid the de/enqueue dance */ if (hrtimer_can_update_in_place(timer, base, expires)) { - hrtimer_set_expires_range_ns(timer, expires, delta_ns); trace_hrtimer_start(timer, mode, true); if (was_first) base->expires_next = expires; @@ -1280,9 +1290,6 @@ remove_and_enqueue_same_base(struct hrtimer *timer, struct hrtimer_clock_base *b timerqueue_linked_del(&base->active, &timer->node); } - /* Set the new expiry time */ - hrtimer_set_expires_range_ns(timer, expires, delta_ns); - debug_activate(timer, mode, timer->is_queued); base->cpu_base->active_bases |= 1 << base->index; diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 115e0bf01276..bda3d2391a60 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -165,23 +165,31 @@ static bool tick_set_oneshot_wakeup_device(struct clock_event_device *newdev, */ void tick_install_broadcast_device(struct clock_event_device *dev, int cpu) { - struct clock_event_device *cur = tick_broadcast_device.evtdev; + struct clock_event_device *cur; - if (tick_set_oneshot_wakeup_device(dev, cpu)) - return; + scoped_guard(raw_spinlock_irqsave, &tick_broadcast_lock) { - if (!tick_check_broadcast_device(cur, dev)) - return; + if (tick_set_oneshot_wakeup_device(dev, cpu)) + return; - if (!try_module_get(dev->owner)) - return; + cur = tick_broadcast_device.evtdev; + if (!tick_check_broadcast_device(cur, dev)) + return; - clockevents_exchange_device(cur, dev); + if (!try_module_get(dev->owner)) + return; + + __clockevents_exchange_device(cur, dev); + if (cur) + cur->event_handler = clockevents_handle_noop; + WRITE_ONCE(tick_broadcast_device.evtdev, dev); + if (!cpumask_empty(tick_broadcast_mask)) + tick_broadcast_start_periodic(dev); + } + + /* Module release must be outside of the lock */ if (cur) - cur->event_handler = clockevents_handle_noop; - tick_broadcast_device.evtdev = dev; - if (!cpumask_empty(tick_broadcast_mask)) - tick_broadcast_start_periodic(dev); + module_put(cur->owner); if (!(dev->features & CLOCK_EVT_FEAT_ONESHOT)) return; @@ -1218,7 +1226,7 @@ int tick_broadcast_oneshot_active(void) */ bool tick_broadcast_oneshot_available(void) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; + struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev); return bc ? bc->features & CLOCK_EVT_FEAT_ONESHOT : false; } @@ -1226,7 +1234,7 @@ bool tick_broadcast_oneshot_available(void) #else int __tick_broadcast_oneshot_control(enum tick_broadcast_state state) { - struct clock_event_device *bc = tick_broadcast_device.evtdev; + struct clock_event_device *bc = READ_ONCE(tick_broadcast_device.evtdev); if (!bc || (bc->features & CLOCK_EVT_FEAT_HRTIMER)) return -EBUSY; diff --git a/kernel/time/tick-internal.h b/kernel/time/tick-internal.h index 182974c4f21b..65680db95053 100644 --- a/kernel/time/tick-internal.h +++ b/kernel/time/tick-internal.h @@ -55,6 +55,8 @@ static inline void clockevent_set_state(struct clock_event_device *dev, } extern void clockevents_shutdown(struct clock_event_device *dev); +extern void __clockevents_exchange_device(struct clock_event_device *old, + struct clock_event_device *new); extern void clockevents_exchange_device(struct clock_event_device *old, struct clock_event_device *new); extern void clockevents_switch_state(struct clock_event_device *dev, |