diff options
Diffstat (limited to 'block')
-rw-r--r-- | block/blk-core.c | 22 | ||||
-rw-r--r-- | block/blk-iopoll.c | 3 | ||||
-rw-r--r-- | block/blk-softirq.c | 3 |
3 files changed, 15 insertions, 13 deletions
diff --git a/block/blk-core.c b/block/blk-core.c index 1f61b74867e4..f06832826bff 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -237,7 +237,7 @@ EXPORT_SYMBOL(blk_delay_queue); **/ void blk_start_queue(struct request_queue *q) { - WARN_ON(!irqs_disabled()); + WARN_ON_NONRT(!irqs_disabled()); queue_flag_clear(QUEUE_FLAG_STOPPED, q); __blk_run_queue(q); @@ -302,7 +302,11 @@ void __blk_run_queue(struct request_queue *q) { if (unlikely(blk_queue_stopped(q))) return; - + /* + * q->request_fn() can drop q->queue_lock and reenable + * interrupts, but must return with q->queue_lock held and + * interrupts disabled. + */ q->request_fn(q); } EXPORT_SYMBOL(__blk_run_queue); @@ -2779,11 +2783,11 @@ static void queue_unplugged(struct request_queue *q, unsigned int depth, * this lock). */ if (from_schedule) { - spin_unlock(q->queue_lock); + spin_unlock_irq(q->queue_lock); blk_run_queue_async(q); } else { __blk_run_queue(q); - spin_unlock(q->queue_lock); + spin_unlock_irq(q->queue_lock); } } @@ -2809,7 +2813,6 @@ static void flush_plug_callbacks(struct blk_plug *plug) void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) { struct request_queue *q; - unsigned long flags; struct request *rq; LIST_HEAD(list); unsigned int depth; @@ -2830,11 +2833,6 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) q = NULL; depth = 0; - /* - * Save and disable interrupts here, to avoid doing it for every - * queue lock we have to take. - */ - local_irq_save(flags); while (!list_empty(&list)) { rq = list_entry_rq(list.next); list_del_init(&rq->queuelist); @@ -2847,7 +2845,7 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) queue_unplugged(q, depth, from_schedule); q = rq->q; depth = 0; - spin_lock(q->queue_lock); + spin_lock_irq(q->queue_lock); } /* @@ -2874,8 +2872,6 @@ void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule) */ if (q) queue_unplugged(q, depth, from_schedule); - - local_irq_restore(flags); } void blk_finish_plug(struct blk_plug *plug) diff --git a/block/blk-iopoll.c b/block/blk-iopoll.c index 58916afbbda5..f7ca9b411d2a 100644 --- a/block/blk-iopoll.c +++ b/block/blk-iopoll.c @@ -38,6 +38,7 @@ void blk_iopoll_sched(struct blk_iopoll *iop) list_add_tail(&iop->list, &__get_cpu_var(blk_cpu_iopoll)); __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); local_irq_restore(flags); + preempt_check_resched_rt(); } EXPORT_SYMBOL(blk_iopoll_sched); @@ -135,6 +136,7 @@ static void blk_iopoll_softirq(struct softirq_action *h) __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); local_irq_enable(); + preempt_check_resched_rt(); } /** @@ -204,6 +206,7 @@ static int __cpuinit blk_iopoll_cpu_notify(struct notifier_block *self, &__get_cpu_var(blk_cpu_iopoll)); __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); local_irq_enable(); + preempt_check_resched_rt(); } return NOTIFY_OK; diff --git a/block/blk-softirq.c b/block/blk-softirq.c index 467c8de88642..3fe236829aeb 100644 --- a/block/blk-softirq.c +++ b/block/blk-softirq.c @@ -51,6 +51,7 @@ static void trigger_softirq(void *data) raise_softirq_irqoff(BLOCK_SOFTIRQ); local_irq_restore(flags); + preempt_check_resched_rt(); } /* @@ -93,6 +94,7 @@ static int __cpuinit blk_cpu_notify(struct notifier_block *self, &__get_cpu_var(blk_cpu_done)); raise_softirq_irqoff(BLOCK_SOFTIRQ); local_irq_enable(); + preempt_check_resched_rt(); } return NOTIFY_OK; @@ -150,6 +152,7 @@ do_local: goto do_local; local_irq_restore(flags); + preempt_check_resched_rt(); } /** |