From 4ec591790356f0e5a95f8d278b0cfd04aea2ae52 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Fri, 22 Jan 2021 10:33:05 +0800 Subject: scsi: sbitmap: Remove sbitmap_clear_bit_unlock No one uses this helper any more, so kill it. Link: https://lore.kernel.org/r/20210122023317.687987-2-ming.lei@redhat.com Cc: Omar Sandoval Cc: Kashyap Desai Cc: Sumanesh Samanta Cc: Ewan D. Milne Cc: Hannes Reinecke Tested-by: Sumanesh Samanta Reviewed-by: Hannes Reinecke Reviewed-by: Johannes Thumshirn Signed-off-by: Ming Lei Signed-off-by: Martin K. Petersen --- include/linux/sbitmap.h | 6 ------ 1 file changed, 6 deletions(-) (limited to 'include/linux/sbitmap.h') diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index 74cc6384715e..16353fbee765 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -315,12 +315,6 @@ static inline void sbitmap_deferred_clear_bit(struct sbitmap *sb, unsigned int b set_bit(SB_NR_TO_BIT(sb, bitnr), addr); } -static inline void sbitmap_clear_bit_unlock(struct sbitmap *sb, - unsigned int bitnr) -{ - clear_bit_unlock(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr)); -} - static inline int sbitmap_test_bit(struct sbitmap *sb, unsigned int bitnr) { return test_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr)); -- cgit v1.2.3 From efe1f3a1d5833c0ddd61ee50dbef8908f65a0a5e Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Fri, 22 Jan 2021 10:33:06 +0800 Subject: scsi: sbitmap: Maintain allocation round_robin in sbitmap Currently the allocation round_robin info is maintained by sbitmap_queue. However, bit allocation really belongs to sbitmap. Move it there. Link: https://lore.kernel.org/r/20210122023317.687987-3-ming.lei@redhat.com Cc: Omar Sandoval Cc: Kashyap Desai Cc: Sumanesh Samanta Cc: Ewan D. Milne Cc: Hannes Reinecke Cc: virtualization@lists.linux-foundation.org Tested-by: Sumanesh Samanta Reviewed-by: Hannes Reinecke Signed-off-by: Ming Lei Signed-off-by: Martin K. Petersen --- include/linux/sbitmap.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'include/linux/sbitmap.h') diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index 16353fbee765..734ee6214cd6 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -56,6 +56,11 @@ struct sbitmap { */ unsigned int map_nr; + /** + * @round_robin: Allocate bits in strict round-robin order. + */ + bool round_robin; + /** * @map: Allocated bitmap. */ @@ -124,11 +129,6 @@ struct sbitmap_queue { */ atomic_t ws_active; - /** - * @round_robin: Allocate bits in strict round-robin order. - */ - bool round_robin; - /** * @min_shallow_depth: The minimum shallow depth which may be passed to * sbitmap_queue_get_shallow() or __sbitmap_queue_get_shallow(). @@ -144,11 +144,14 @@ struct sbitmap_queue { * given, a good default is chosen. * @flags: Allocation flags. * @node: Memory node to allocate on. + * @round_robin: If true, be stricter about allocation order; always allocate + * starting from the last allocated bit. This is less efficient + * than the default behavior (false). * * Return: Zero on success or negative errno on failure. */ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, - gfp_t flags, int node); + gfp_t flags, int node, bool round_robin); /** * sbitmap_free() - Free memory used by a &struct sbitmap. @@ -174,15 +177,12 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth); * sbitmap_get() - Try to allocate a free bit from a &struct sbitmap. * @sb: Bitmap to allocate from. * @alloc_hint: Hint for where to start searching for a free bit. - * @round_robin: If true, be stricter about allocation order; always allocate - * starting from the last allocated bit. This is less efficient - * than the default behavior (false). * * This operation provides acquire barrier semantics if it succeeds. * * Return: Non-negative allocated bit number if successful, -1 otherwise. */ -int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint, bool round_robin); +int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint); /** * sbitmap_get_shallow() - Try to allocate a free bit from a &struct sbitmap, -- cgit v1.2.3 From c548e62bcf6adc7066ff201e9ecc88e536dd8890 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Fri, 22 Jan 2021 10:33:08 +0800 Subject: scsi: sbitmap: Move allocation hint into sbitmap Allocation hint should have belonged to sbitmap. Also, when sbitmap's depth is high and there is no need to use mulitple wakeup queues, user can benefit from percpu allocation hint too. Move allocation hint into sbitmap, then SCSI device queue can benefit from allocation hint when converting to plain sbitmap. Convert vhost/scsi.c to use sbitmap allocation with percpu alloc hint. This is more efficient than the previous approach. Link: https://lore.kernel.org/r/20210122023317.687987-5-ming.lei@redhat.com Cc: Omar Sandoval Cc: Kashyap Desai Cc: Sumanesh Samanta Cc: Ewan D. Milne Cc: Mike Christie Cc: virtualization@lists.linux-foundation.org Tested-by: Sumanesh Samanta Reviewed-by: Hannes Reinecke Signed-off-by: Ming Lei Signed-off-by: Martin K. Petersen --- include/linux/sbitmap.h | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) (limited to 'include/linux/sbitmap.h') diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index 734ee6214cd6..247776fcc02c 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -65,6 +65,14 @@ struct sbitmap { * @map: Allocated bitmap. */ struct sbitmap_word *map; + + /* + * @alloc_hint: Cache of last successfully allocated or freed bit. + * + * This is per-cpu, which allows multiple users to stick to different + * cachelines until the map is exhausted. + */ + unsigned int __percpu *alloc_hint; }; #define SBQ_WAIT_QUEUES 8 @@ -100,14 +108,6 @@ struct sbitmap_queue { */ struct sbitmap sb; - /* - * @alloc_hint: Cache of last successfully allocated or freed bit. - * - * This is per-cpu, which allows multiple users to stick to different - * cachelines until the map is exhausted. - */ - unsigned int __percpu *alloc_hint; - /** * @wake_batch: Number of bits which must be freed before we wake up any * waiters. @@ -147,11 +147,13 @@ struct sbitmap_queue { * @round_robin: If true, be stricter about allocation order; always allocate * starting from the last allocated bit. This is less efficient * than the default behavior (false). + * @alloc_hint: If true, apply percpu hint for where to start searching for + * a free bit. * * Return: Zero on success or negative errno on failure. */ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, - gfp_t flags, int node, bool round_robin); + gfp_t flags, int node, bool round_robin, bool alloc_hint); /** * sbitmap_free() - Free memory used by a &struct sbitmap. @@ -159,6 +161,7 @@ int sbitmap_init_node(struct sbitmap *sb, unsigned int depth, int shift, */ static inline void sbitmap_free(struct sbitmap *sb) { + free_percpu(sb->alloc_hint); kfree(sb->map); sb->map = NULL; } @@ -176,19 +179,17 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth); /** * sbitmap_get() - Try to allocate a free bit from a &struct sbitmap. * @sb: Bitmap to allocate from. - * @alloc_hint: Hint for where to start searching for a free bit. * * This operation provides acquire barrier semantics if it succeeds. * * Return: Non-negative allocated bit number if successful, -1 otherwise. */ -int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint); +int sbitmap_get(struct sbitmap *sb); /** * sbitmap_get_shallow() - Try to allocate a free bit from a &struct sbitmap, * limiting the depth used from each word. * @sb: Bitmap to allocate from. - * @alloc_hint: Hint for where to start searching for a free bit. * @shallow_depth: The maximum number of bits to allocate from a single word. * * This rather specific operation allows for having multiple users with @@ -200,8 +201,7 @@ int sbitmap_get(struct sbitmap *sb, unsigned int alloc_hint); * * Return: Non-negative allocated bit number if successful, -1 otherwise. */ -int sbitmap_get_shallow(struct sbitmap *sb, unsigned int alloc_hint, - unsigned long shallow_depth); +int sbitmap_get_shallow(struct sbitmap *sb, unsigned long shallow_depth); /** * sbitmap_any_bit_set() - Check for a set bit in a &struct sbitmap. @@ -315,6 +315,18 @@ static inline void sbitmap_deferred_clear_bit(struct sbitmap *sb, unsigned int b set_bit(SB_NR_TO_BIT(sb, bitnr), addr); } +/* + * Pair of sbitmap_get, and this one applies both cleared bit and + * allocation hint. + */ +static inline void sbitmap_put(struct sbitmap *sb, unsigned int bitnr) +{ + sbitmap_deferred_clear_bit(sb, bitnr); + + if (likely(sb->alloc_hint && !sb->round_robin && bitnr < sb->depth)) + *this_cpu_ptr(sb->alloc_hint) = bitnr; +} + static inline int sbitmap_test_bit(struct sbitmap *sb, unsigned int bitnr) { return test_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr)); @@ -363,7 +375,6 @@ int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, static inline void sbitmap_queue_free(struct sbitmap_queue *sbq) { kfree(sbq->ws); - free_percpu(sbq->alloc_hint); sbitmap_free(&sbq->sb); } -- cgit v1.2.3 From cbb9950b41dd9dfb7c2be3429ba09f83b8b1ff98 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Fri, 22 Jan 2021 10:33:09 +0800 Subject: scsi: sbitmap: Export sbitmap_weight SCSI's .device_busy will be converted to sbitmap and sbitmap_weight is needed. Export the helper. The only existing user of sbitmap_weight() uses it to find out how many bits are set and not cleared. Align sbitmap_weight() meaning with this usage model. Link: https://lore.kernel.org/r/20210122023317.687987-6-ming.lei@redhat.com Cc: Omar Sandoval Cc: Kashyap Desai Cc: Sumanesh Samanta Cc: Ewan D. Milne Tested-by: Sumanesh Samanta Reviewed-by: Hannes Reinecke Signed-off-by: Ming Lei Signed-off-by: Martin K. Petersen --- include/linux/sbitmap.h | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'include/linux/sbitmap.h') diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index 247776fcc02c..c65ba887dcc3 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -341,6 +341,16 @@ static inline int sbitmap_test_bit(struct sbitmap *sb, unsigned int bitnr) */ void sbitmap_show(struct sbitmap *sb, struct seq_file *m); + +/** + * sbitmap_weight() - Return how many set and not cleared bits in a &struct + * sbitmap. + * @sb: Bitmap to check. + * + * Return: How many set and not cleared bits set + */ +unsigned int sbitmap_weight(const struct sbitmap *sb); + /** * sbitmap_bitmap_show() - Write a hex dump of a &struct sbitmap to a &struct * seq_file. -- cgit v1.2.3 From 2d13b1ea9f4affdaa7af0e0e4a1358d28f80c54f Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Fri, 22 Jan 2021 10:33:10 +0800 Subject: scsi: sbitmap: Add sbitmap_calculate_shift() helper Move code for calculating default shift into a public helper which can be used by SCSI. Link: https://lore.kernel.org/r/20210122023317.687987-7-ming.lei@redhat.com Cc: Omar Sandoval Cc: Kashyap Desai Cc: Sumanesh Samanta Cc: Ewan D. Milne Tested-by: Sumanesh Samanta Reviewed-by: Hannes Reinecke Signed-off-by: Ming Lei Signed-off-by: Martin K. Petersen --- include/linux/sbitmap.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'include/linux/sbitmap.h') diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index c65ba887dcc3..3087e1f15fdd 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -332,6 +332,24 @@ static inline int sbitmap_test_bit(struct sbitmap *sb, unsigned int bitnr) return test_bit(SB_NR_TO_BIT(sb, bitnr), __sbitmap_word(sb, bitnr)); } +static inline int sbitmap_calculate_shift(unsigned int depth) +{ + int shift = ilog2(BITS_PER_LONG); + + /* + * If the bitmap is small, shrink the number of bits per word so + * we spread over a few cachelines, at least. If less than 4 + * bits, just forget about it, it's not going to work optimally + * anyway. + */ + if (depth >= 4) { + while ((4U << shift) > depth) + shift--; + } + + return shift; +} + /** * sbitmap_show() - Dump &struct sbitmap information to a &struct seq_file. * @sb: Bitmap to show. -- cgit v1.2.3 From 035e9f471691a16c32b389c8b2f236043a2a50d7 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Tue, 16 Mar 2021 20:26:48 -0700 Subject: scsi: sbitmap: Silence a debug kernel warning triggered by sbitmap_put() All sbitmap code uses implied preemption protection to update sb->alloc_hint except sbitmap_put(). Using implied preemption protection is safe since the value of sb->alloc_hint only affects performance of sbitmap allocations but not their correctness. Change this_cpu_ptr() in sbitmap_put() into raw_cpu_ptr() to suppress the following kernel warning that appears with preemption debugging enabled (CONFIG_DEBUG_PREEMPT): BUG: using smp_processor_id() in preemptible [00000000] code: scsi_eh_0/152 caller is debug_smp_processor_id+0x17/0x20 CPU: 1 PID: 152 Comm: scsi_eh_0 Tainted: G W 5.12.0-rc1-dbg+ #6 Call Trace: show_stack+0x52/0x58 dump_stack+0xaf/0xf3 check_preemption_disabled+0xce/0xd0 debug_smp_processor_id+0x17/0x20 scsi_device_unbusy+0x13a/0x1c0 [scsi_mod] scsi_finish_command+0x4d/0x290 [scsi_mod] scsi_eh_flush_done_q+0x1e7/0x280 [scsi_mod] ata_scsi_port_error_handler+0x592/0x750 [libata] ata_scsi_error+0x1a0/0x1f0 [libata] scsi_error_handler+0x19e/0x330 [scsi_mod] kthread+0x222/0x250 ret_from_fork+0x1f/0x30 Link: https://lore.kernel.org/r/20210317032648.9080-1-bvanassche@acm.org Fixes: c548e62bcf6a ("scsi: sbitmap: Move allocation hint into sbitmap") Cc: Hannes Reinecke Cc: Omar Sandoval Reviewed-by: Ming Lei Reviewed-by: Christoph Hellwig Signed-off-by: Bart Van Assche Signed-off-by: Martin K. Petersen --- include/linux/sbitmap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include/linux/sbitmap.h') diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index 3087e1f15fdd..2713e689ad66 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -324,7 +324,7 @@ static inline void sbitmap_put(struct sbitmap *sb, unsigned int bitnr) sbitmap_deferred_clear_bit(sb, bitnr); if (likely(sb->alloc_hint && !sb->round_robin && bitnr < sb->depth)) - *this_cpu_ptr(sb->alloc_hint) = bitnr; + *raw_cpu_ptr(sb->alloc_hint) = bitnr; } static inline int sbitmap_test_bit(struct sbitmap *sb, unsigned int bitnr) -- cgit v1.2.3