From 10b88a4b17d31a7409494b179dcb76e7ab2fcaea Mon Sep 17 00:00:00 2001 From: Sowmini Varadhan Date: Thu, 12 Mar 2015 20:02:35 -0400 Subject: sparc: Break up monolithic iommu table/lock into finer graularity pools and lock Investigation of multithreaded iperf experiments on an ethernet interface show the iommu->lock as the hottest lock identified by lockstat, with something of the order of 21M contentions out of 27M acquisitions, and an average wait time of 26 us for the lock. This is not efficient. A more scalable design is to follow the ppc model, where the iommu_table has multiple pools, each stretching over a segment of the map, and with a separate lock for each pool. This model allows for better parallelization of the iommu map search. This patch adds the iommu range alloc/free function infrastructure. Signed-off-by: Sowmini Varadhan Signed-off-by: David S. Miller --- lib/iommu-common.c | 220 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 lib/iommu-common.c (limited to 'lib/iommu-common.c') diff --git a/lib/iommu-common.c b/lib/iommu-common.c new file mode 100644 index 000000000000..7583f9b7846b --- /dev/null +++ b/lib/iommu-common.c @@ -0,0 +1,220 @@ +/* + * IOMMU mmap management and range allocation functions. + * Based almost entirely upon the powerpc iommu allocator. + */ + +#include +#include +#include +#include +#include +#include + +#define IOMMU_LARGE_ALLOC 15 + +/* + * Initialize iommu_pool entries for the iommu_table. `num_entries' + * is the number of table entries. If `large_pool' is set to true, + * the top 1/4 of the table will be set aside for pool allocations + * of more than IOMMU_LARGE_ALLOC pages. + */ +extern void iommu_tbl_pool_init(struct iommu_table *iommu, + unsigned long num_entries, + u32 page_table_shift, + const struct iommu_tbl_ops *iommu_tbl_ops, + bool large_pool, u32 npools) +{ + unsigned int start, i; + struct iommu_pool *p = &(iommu->large_pool); + + if (npools == 0) + iommu->nr_pools = IOMMU_NR_POOLS; + else + iommu->nr_pools = npools; + BUG_ON(npools > IOMMU_NR_POOLS); + + iommu->page_table_shift = page_table_shift; + iommu->iommu_tbl_ops = iommu_tbl_ops; + start = 0; + if (large_pool) + iommu->flags |= IOMMU_HAS_LARGE_POOL; + + if (!large_pool) + iommu->poolsize = num_entries/iommu->nr_pools; + else + iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools; + for (i = 0; i < iommu->nr_pools; i++) { + spin_lock_init(&(iommu->arena_pool[i].lock)); + iommu->arena_pool[i].start = start; + iommu->arena_pool[i].hint = start; + start += iommu->poolsize; /* start for next pool */ + iommu->arena_pool[i].end = start - 1; + } + if (!large_pool) + return; + /* initialize large_pool */ + spin_lock_init(&(p->lock)); + p->start = start; + p->hint = p->start; + p->end = num_entries; +} +EXPORT_SYMBOL(iommu_tbl_pool_init); + +unsigned long iommu_tbl_range_alloc(struct device *dev, + struct iommu_table *iommu, + unsigned long npages, + unsigned long *handle, + unsigned int pool_hash) +{ + unsigned long n, end, start, limit, boundary_size; + struct iommu_pool *arena; + int pass = 0; + unsigned int pool_nr; + unsigned int npools = iommu->nr_pools; + unsigned long flags; + bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0); + bool largealloc = (large_pool && npages > IOMMU_LARGE_ALLOC); + unsigned long shift; + + /* Sanity check */ + if (unlikely(npages == 0)) { + printk_ratelimited("npages == 0\n"); + return DMA_ERROR_CODE; + } + + if (largealloc) { + arena = &(iommu->large_pool); + spin_lock_irqsave(&arena->lock, flags); + pool_nr = 0; /* to keep compiler happy */ + } else { + /* pick out pool_nr */ + pool_nr = pool_hash & (npools - 1); + arena = &(iommu->arena_pool[pool_nr]); + + /* find first available unlocked pool */ + while (!spin_trylock_irqsave(&(arena->lock), flags)) { + pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1); + arena = &(iommu->arena_pool[pool_nr]); + } + } + + again: + if (pass == 0 && handle && *handle && + (*handle >= arena->start) && (*handle < arena->end)) + start = *handle; + else + start = arena->hint; + + limit = arena->end; + + /* The case below can happen if we have a small segment appended + * to a large, or when the previous alloc was at the very end of + * the available space. If so, go back to the beginning and flush. + */ + if (start >= limit) { + start = arena->start; + if (iommu->iommu_tbl_ops->reset != NULL) + iommu->iommu_tbl_ops->reset(iommu); + } + + if (dev) + boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, + 1 << iommu->page_table_shift); + else + boundary_size = ALIGN(1UL << 32, 1 << iommu->page_table_shift); + + shift = iommu->page_table_map_base >> iommu->page_table_shift; + boundary_size = boundary_size >> iommu->page_table_shift; + /* + * if the iommu has a non-trivial cookie <-> index mapping, we set + * things up so that iommu_is_span_boundary() merely checks if the + * (index + npages) < num_tsb_entries + */ + if (iommu->iommu_tbl_ops->cookie_to_index != NULL) { + shift = 0; + boundary_size = iommu->poolsize * iommu->nr_pools; + } + n = iommu_area_alloc(iommu->map, limit, start, npages, shift, + boundary_size, 0); + if (n == -1) { + if (likely(pass == 0)) { + /* First failure, rescan from the beginning. */ + arena->hint = arena->start; + if (iommu->iommu_tbl_ops->reset != NULL) + iommu->iommu_tbl_ops->reset(iommu); + pass++; + goto again; + } else if (!largealloc && pass <= iommu->nr_pools) { + spin_unlock(&(arena->lock)); + pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1); + arena = &(iommu->arena_pool[pool_nr]); + while (!spin_trylock(&(arena->lock))) { + pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1); + arena = &(iommu->arena_pool[pool_nr]); + } + arena->hint = arena->start; + pass++; + goto again; + } else { + /* give up */ + spin_unlock_irqrestore(&(arena->lock), flags); + return DMA_ERROR_CODE; + } + } + + end = n + npages; + + arena->hint = end; + + /* Update handle for SG allocations */ + if (handle) + *handle = end; + spin_unlock_irqrestore(&(arena->lock), flags); + + return n; +} +EXPORT_SYMBOL(iommu_tbl_range_alloc); + +static struct iommu_pool *get_pool(struct iommu_table *tbl, + unsigned long entry) +{ + struct iommu_pool *p; + unsigned long largepool_start = tbl->large_pool.start; + bool large_pool = ((tbl->flags & IOMMU_HAS_LARGE_POOL) != 0); + + /* The large pool is the last pool at the top of the table */ + if (large_pool && entry >= largepool_start) { + p = &tbl->large_pool; + } else { + unsigned int pool_nr = entry / tbl->poolsize; + + BUG_ON(pool_nr >= tbl->nr_pools); + p = &tbl->arena_pool[pool_nr]; + } + return p; +} + +void iommu_tbl_range_free(struct iommu_table *iommu, u64 dma_addr, + unsigned long npages, bool do_demap, void *demap_arg) +{ + unsigned long entry; + struct iommu_pool *pool; + unsigned long flags; + unsigned long shift = iommu->page_table_shift; + + if (iommu->iommu_tbl_ops->cookie_to_index != NULL) { + entry = (*iommu->iommu_tbl_ops->cookie_to_index)(dma_addr, + demap_arg); + } else { + entry = (dma_addr - iommu->page_table_map_base) >> shift; + } + pool = get_pool(iommu, entry); + + spin_lock_irqsave(&(pool->lock), flags); + if (do_demap && iommu->iommu_tbl_ops->demap != NULL) + (*iommu->iommu_tbl_ops->demap)(demap_arg, entry, npages); + + bitmap_clear(iommu->map, entry, npages); + spin_unlock_irqrestore(&(pool->lock), flags); +} +EXPORT_SYMBOL(iommu_tbl_range_free); -- cgit v1.2.3 From cb97201cb060d13da0b87fd1bf68208c7389c5b1 Mon Sep 17 00:00:00 2001 From: Sowmini Varadhan Date: Thu, 16 Apr 2015 22:28:04 -0400 Subject: iommu-common: Fix PARISC compile-time warnings Fixes warnings due to - no DMA_ERROR_CODE on PARISC, - sizeof (unsigned long) == 4 bytes on PARISC. Signed-off-by: Sowmini Varadhan Signed-off-by: David S. Miller --- lib/iommu-common.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib/iommu-common.c') diff --git a/lib/iommu-common.c b/lib/iommu-common.c index 7583f9b7846b..fac4f35250c9 100644 --- a/lib/iommu-common.c +++ b/lib/iommu-common.c @@ -10,6 +10,10 @@ #include #include +#ifndef DMA_ERROR_CODE +#define DMA_ERROR_CODE (~(dma_addr_t)0x0) +#endif + #define IOMMU_LARGE_ALLOC 15 /* @@ -121,7 +125,7 @@ unsigned long iommu_tbl_range_alloc(struct device *dev, boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 1 << iommu->page_table_shift); else - boundary_size = ALIGN(1UL << 32, 1 << iommu->page_table_shift); + boundary_size = ALIGN(1ULL << 32, 1 << iommu->page_table_shift); shift = iommu->page_table_map_base >> iommu->page_table_shift; boundary_size = boundary_size >> iommu->page_table_shift; -- cgit v1.2.3 From c12f048ffdf3a5802239426dc290290929268dc9 Mon Sep 17 00:00:00 2001 From: "David S. Miller" Date: Sat, 18 Apr 2015 12:31:25 -0700 Subject: sparc: Revert generic IOMMU allocator. I applied the wrong version of this patch series, V4 instead of V10, due to a patchwork bundling snafu. Signed-off-by: David S. Miller --- lib/iommu-common.c | 224 ----------------------------------------------------- 1 file changed, 224 deletions(-) delete mode 100644 lib/iommu-common.c (limited to 'lib/iommu-common.c') diff --git a/lib/iommu-common.c b/lib/iommu-common.c deleted file mode 100644 index fac4f35250c9..000000000000 --- a/lib/iommu-common.c +++ /dev/null @@ -1,224 +0,0 @@ -/* - * IOMMU mmap management and range allocation functions. - * Based almost entirely upon the powerpc iommu allocator. - */ - -#include -#include -#include -#include -#include -#include - -#ifndef DMA_ERROR_CODE -#define DMA_ERROR_CODE (~(dma_addr_t)0x0) -#endif - -#define IOMMU_LARGE_ALLOC 15 - -/* - * Initialize iommu_pool entries for the iommu_table. `num_entries' - * is the number of table entries. If `large_pool' is set to true, - * the top 1/4 of the table will be set aside for pool allocations - * of more than IOMMU_LARGE_ALLOC pages. - */ -extern void iommu_tbl_pool_init(struct iommu_table *iommu, - unsigned long num_entries, - u32 page_table_shift, - const struct iommu_tbl_ops *iommu_tbl_ops, - bool large_pool, u32 npools) -{ - unsigned int start, i; - struct iommu_pool *p = &(iommu->large_pool); - - if (npools == 0) - iommu->nr_pools = IOMMU_NR_POOLS; - else - iommu->nr_pools = npools; - BUG_ON(npools > IOMMU_NR_POOLS); - - iommu->page_table_shift = page_table_shift; - iommu->iommu_tbl_ops = iommu_tbl_ops; - start = 0; - if (large_pool) - iommu->flags |= IOMMU_HAS_LARGE_POOL; - - if (!large_pool) - iommu->poolsize = num_entries/iommu->nr_pools; - else - iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools; - for (i = 0; i < iommu->nr_pools; i++) { - spin_lock_init(&(iommu->arena_pool[i].lock)); - iommu->arena_pool[i].start = start; - iommu->arena_pool[i].hint = start; - start += iommu->poolsize; /* start for next pool */ - iommu->arena_pool[i].end = start - 1; - } - if (!large_pool) - return; - /* initialize large_pool */ - spin_lock_init(&(p->lock)); - p->start = start; - p->hint = p->start; - p->end = num_entries; -} -EXPORT_SYMBOL(iommu_tbl_pool_init); - -unsigned long iommu_tbl_range_alloc(struct device *dev, - struct iommu_table *iommu, - unsigned long npages, - unsigned long *handle, - unsigned int pool_hash) -{ - unsigned long n, end, start, limit, boundary_size; - struct iommu_pool *arena; - int pass = 0; - unsigned int pool_nr; - unsigned int npools = iommu->nr_pools; - unsigned long flags; - bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0); - bool largealloc = (large_pool && npages > IOMMU_LARGE_ALLOC); - unsigned long shift; - - /* Sanity check */ - if (unlikely(npages == 0)) { - printk_ratelimited("npages == 0\n"); - return DMA_ERROR_CODE; - } - - if (largealloc) { - arena = &(iommu->large_pool); - spin_lock_irqsave(&arena->lock, flags); - pool_nr = 0; /* to keep compiler happy */ - } else { - /* pick out pool_nr */ - pool_nr = pool_hash & (npools - 1); - arena = &(iommu->arena_pool[pool_nr]); - - /* find first available unlocked pool */ - while (!spin_trylock_irqsave(&(arena->lock), flags)) { - pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1); - arena = &(iommu->arena_pool[pool_nr]); - } - } - - again: - if (pass == 0 && handle && *handle && - (*handle >= arena->start) && (*handle < arena->end)) - start = *handle; - else - start = arena->hint; - - limit = arena->end; - - /* The case below can happen if we have a small segment appended - * to a large, or when the previous alloc was at the very end of - * the available space. If so, go back to the beginning and flush. - */ - if (start >= limit) { - start = arena->start; - if (iommu->iommu_tbl_ops->reset != NULL) - iommu->iommu_tbl_ops->reset(iommu); - } - - if (dev) - boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, - 1 << iommu->page_table_shift); - else - boundary_size = ALIGN(1ULL << 32, 1 << iommu->page_table_shift); - - shift = iommu->page_table_map_base >> iommu->page_table_shift; - boundary_size = boundary_size >> iommu->page_table_shift; - /* - * if the iommu has a non-trivial cookie <-> index mapping, we set - * things up so that iommu_is_span_boundary() merely checks if the - * (index + npages) < num_tsb_entries - */ - if (iommu->iommu_tbl_ops->cookie_to_index != NULL) { - shift = 0; - boundary_size = iommu->poolsize * iommu->nr_pools; - } - n = iommu_area_alloc(iommu->map, limit, start, npages, shift, - boundary_size, 0); - if (n == -1) { - if (likely(pass == 0)) { - /* First failure, rescan from the beginning. */ - arena->hint = arena->start; - if (iommu->iommu_tbl_ops->reset != NULL) - iommu->iommu_tbl_ops->reset(iommu); - pass++; - goto again; - } else if (!largealloc && pass <= iommu->nr_pools) { - spin_unlock(&(arena->lock)); - pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1); - arena = &(iommu->arena_pool[pool_nr]); - while (!spin_trylock(&(arena->lock))) { - pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1); - arena = &(iommu->arena_pool[pool_nr]); - } - arena->hint = arena->start; - pass++; - goto again; - } else { - /* give up */ - spin_unlock_irqrestore(&(arena->lock), flags); - return DMA_ERROR_CODE; - } - } - - end = n + npages; - - arena->hint = end; - - /* Update handle for SG allocations */ - if (handle) - *handle = end; - spin_unlock_irqrestore(&(arena->lock), flags); - - return n; -} -EXPORT_SYMBOL(iommu_tbl_range_alloc); - -static struct iommu_pool *get_pool(struct iommu_table *tbl, - unsigned long entry) -{ - struct iommu_pool *p; - unsigned long largepool_start = tbl->large_pool.start; - bool large_pool = ((tbl->flags & IOMMU_HAS_LARGE_POOL) != 0); - - /* The large pool is the last pool at the top of the table */ - if (large_pool && entry >= largepool_start) { - p = &tbl->large_pool; - } else { - unsigned int pool_nr = entry / tbl->poolsize; - - BUG_ON(pool_nr >= tbl->nr_pools); - p = &tbl->arena_pool[pool_nr]; - } - return p; -} - -void iommu_tbl_range_free(struct iommu_table *iommu, u64 dma_addr, - unsigned long npages, bool do_demap, void *demap_arg) -{ - unsigned long entry; - struct iommu_pool *pool; - unsigned long flags; - unsigned long shift = iommu->page_table_shift; - - if (iommu->iommu_tbl_ops->cookie_to_index != NULL) { - entry = (*iommu->iommu_tbl_ops->cookie_to_index)(dma_addr, - demap_arg); - } else { - entry = (dma_addr - iommu->page_table_map_base) >> shift; - } - pool = get_pool(iommu, entry); - - spin_lock_irqsave(&(pool->lock), flags); - if (do_demap && iommu->iommu_tbl_ops->demap != NULL) - (*iommu->iommu_tbl_ops->demap)(demap_arg, entry, npages); - - bitmap_clear(iommu->map, entry, npages); - spin_unlock_irqrestore(&(pool->lock), flags); -} -EXPORT_SYMBOL(iommu_tbl_range_free); -- cgit v1.2.3 From ff7d37a502022149655c18035b99a53391be0383 Mon Sep 17 00:00:00 2001 From: Sowmini Varadhan Date: Thu, 9 Apr 2015 15:33:30 -0400 Subject: Break up monolithic iommu table/lock into finer graularity pools and lock Investigation of multithreaded iperf experiments on an ethernet interface show the iommu->lock as the hottest lock identified by lockstat, with something of the order of 21M contentions out of 27M acquisitions, and an average wait time of 26 us for the lock. This is not efficient. A more scalable design is to follow the ppc model, where the iommu_map_table has multiple pools, each stretching over a segment of the map, and with a separate lock for each pool. This model allows for better parallelization of the iommu map search. This patch adds the iommu range alloc/free function infrastructure. Signed-off-by: Sowmini Varadhan Acked-by: Benjamin Herrenschmidt Signed-off-by: David S. Miller --- lib/iommu-common.c | 266 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 lib/iommu-common.c (limited to 'lib/iommu-common.c') diff --git a/lib/iommu-common.c b/lib/iommu-common.c new file mode 100644 index 000000000000..b99f1d744a8d --- /dev/null +++ b/lib/iommu-common.c @@ -0,0 +1,266 @@ +/* + * IOMMU mmap management and range allocation functions. + * Based almost entirely upon the powerpc iommu allocator. + */ + +#include +#include +#include +#include +#include +#include +#include + +unsigned long iommu_large_alloc = 15; + +static DEFINE_PER_CPU(unsigned int, iommu_pool_hash); + +static inline bool need_flush(struct iommu_map_table *iommu) +{ + return (iommu->lazy_flush != NULL && + (iommu->flags & IOMMU_NEED_FLUSH) != 0); +} + +static inline void set_flush(struct iommu_map_table *iommu) +{ + iommu->flags |= IOMMU_NEED_FLUSH; +} + +static inline void clear_flush(struct iommu_map_table *iommu) +{ + iommu->flags &= ~IOMMU_NEED_FLUSH; +} + +static void setup_iommu_pool_hash(void) +{ + unsigned int i; + static bool do_once; + + if (do_once) + return; + do_once = true; + for_each_possible_cpu(i) + per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS); +} + +/* + * Initialize iommu_pool entries for the iommu_map_table. `num_entries' + * is the number of table entries. If `large_pool' is set to true, + * the top 1/4 of the table will be set aside for pool allocations + * of more than iommu_large_alloc pages. + */ +extern void iommu_tbl_pool_init(struct iommu_map_table *iommu, + unsigned long num_entries, + u32 table_shift, + void (*lazy_flush)(struct iommu_map_table *), + bool large_pool, u32 npools, + bool skip_span_boundary_check) +{ + unsigned int start, i; + struct iommu_pool *p = &(iommu->large_pool); + + setup_iommu_pool_hash(); + if (npools == 0) + iommu->nr_pools = IOMMU_NR_POOLS; + else + iommu->nr_pools = npools; + BUG_ON(npools > IOMMU_NR_POOLS); + + iommu->table_shift = table_shift; + iommu->lazy_flush = lazy_flush; + start = 0; + if (skip_span_boundary_check) + iommu->flags |= IOMMU_NO_SPAN_BOUND; + if (large_pool) + iommu->flags |= IOMMU_HAS_LARGE_POOL; + + if (!large_pool) + iommu->poolsize = num_entries/iommu->nr_pools; + else + iommu->poolsize = (num_entries * 3 / 4)/iommu->nr_pools; + for (i = 0; i < iommu->nr_pools; i++) { + spin_lock_init(&(iommu->pools[i].lock)); + iommu->pools[i].start = start; + iommu->pools[i].hint = start; + start += iommu->poolsize; /* start for next pool */ + iommu->pools[i].end = start - 1; + } + if (!large_pool) + return; + /* initialize large_pool */ + spin_lock_init(&(p->lock)); + p->start = start; + p->hint = p->start; + p->end = num_entries; +} +EXPORT_SYMBOL(iommu_tbl_pool_init); + +unsigned long iommu_tbl_range_alloc(struct device *dev, + struct iommu_map_table *iommu, + unsigned long npages, + unsigned long *handle, + unsigned long mask, + unsigned int align_order) +{ + unsigned int pool_hash = __this_cpu_read(iommu_pool_hash); + unsigned long n, end, start, limit, boundary_size; + struct iommu_pool *pool; + int pass = 0; + unsigned int pool_nr; + unsigned int npools = iommu->nr_pools; + unsigned long flags; + bool large_pool = ((iommu->flags & IOMMU_HAS_LARGE_POOL) != 0); + bool largealloc = (large_pool && npages > iommu_large_alloc); + unsigned long shift; + unsigned long align_mask = 0; + + if (align_order > 0) + align_mask = 0xffffffffffffffffl >> (64 - align_order); + + /* Sanity check */ + if (unlikely(npages == 0)) { + WARN_ON_ONCE(1); + return DMA_ERROR_CODE; + } + + if (largealloc) { + pool = &(iommu->large_pool); + pool_nr = 0; /* to keep compiler happy */ + } else { + /* pick out pool_nr */ + pool_nr = pool_hash & (npools - 1); + pool = &(iommu->pools[pool_nr]); + } + spin_lock_irqsave(&pool->lock, flags); + + again: + if (pass == 0 && handle && *handle && + (*handle >= pool->start) && (*handle < pool->end)) + start = *handle; + else + start = pool->hint; + + limit = pool->end; + + /* The case below can happen if we have a small segment appended + * to a large, or when the previous alloc was at the very end of + * the available space. If so, go back to the beginning. If a + * flush is needed, it will get done based on the return value + * from iommu_area_alloc() below. + */ + if (start >= limit) + start = pool->start; + shift = iommu->table_map_base >> iommu->table_shift; + if (limit + shift > mask) { + limit = mask - shift + 1; + /* If we're constrained on address range, first try + * at the masked hint to avoid O(n) search complexity, + * but on second pass, start at 0 in pool 0. + */ + if ((start & mask) >= limit || pass > 0) { + spin_unlock(&(pool->lock)); + pool = &(iommu->pools[0]); + spin_lock(&(pool->lock)); + start = pool->start; + } else { + start &= mask; + } + } + + if (dev) + boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, + 1 << iommu->table_shift); + else + boundary_size = ALIGN(1UL << 32, 1 << iommu->table_shift); + + boundary_size = boundary_size >> iommu->table_shift; + /* + * if the skip_span_boundary_check had been set during init, we set + * things up so that iommu_is_span_boundary() merely checks if the + * (index + npages) < num_tsb_entries + */ + if ((iommu->flags & IOMMU_NO_SPAN_BOUND) != 0) { + shift = 0; + boundary_size = iommu->poolsize * iommu->nr_pools; + } + n = iommu_area_alloc(iommu->map, limit, start, npages, shift, + boundary_size, align_mask); + if (n == -1) { + if (likely(pass == 0)) { + /* First failure, rescan from the beginning. */ + pool->hint = pool->start; + set_flush(iommu); + pass++; + goto again; + } else if (!largealloc && pass <= iommu->nr_pools) { + spin_unlock(&(pool->lock)); + pool_nr = (pool_nr + 1) & (iommu->nr_pools - 1); + pool = &(iommu->pools[pool_nr]); + spin_lock(&(pool->lock)); + pool->hint = pool->start; + set_flush(iommu); + pass++; + goto again; + } else { + /* give up */ + n = DMA_ERROR_CODE; + goto bail; + } + } + if (n < pool->hint || need_flush(iommu)) { + clear_flush(iommu); + iommu->lazy_flush(iommu); + } + + end = n + npages; + pool->hint = end; + + /* Update handle for SG allocations */ + if (handle) + *handle = end; +bail: + spin_unlock_irqrestore(&(pool->lock), flags); + + return n; +} +EXPORT_SYMBOL(iommu_tbl_range_alloc); + +static struct iommu_pool *get_pool(struct iommu_map_table *tbl, + unsigned long entry) +{ + struct iommu_pool *p; + unsigned long largepool_start = tbl->large_pool.start; + bool large_pool = ((tbl->flags & IOMMU_HAS_LARGE_POOL) != 0); + + /* The large pool is the last pool at the top of the table */ + if (large_pool && entry >= largepool_start) { + p = &tbl->large_pool; + } else { + unsigned int pool_nr = entry / tbl->poolsize; + + BUG_ON(pool_nr >= tbl->nr_pools); + p = &tbl->pools[pool_nr]; + } + return p; +} + +/* Caller supplies the index of the entry into the iommu map table + * itself when the mapping from dma_addr to the entry is not the + * default addr->entry mapping below. + */ +void iommu_tbl_range_free(struct iommu_map_table *iommu, u64 dma_addr, + unsigned long npages, unsigned long entry) +{ + struct iommu_pool *pool; + unsigned long flags; + unsigned long shift = iommu->table_shift; + + if (entry == DMA_ERROR_CODE) /* use default addr->entry mapping */ + entry = (dma_addr - iommu->table_map_base) >> shift; + pool = get_pool(iommu, entry); + + spin_lock_irqsave(&(pool->lock), flags); + bitmap_clear(iommu->map, entry, npages); + spin_unlock_irqrestore(&(pool->lock), flags); +} +EXPORT_SYMBOL(iommu_tbl_range_free); -- cgit v1.2.3 From 2f0c0fdc085c0d415457a1c52344f72e12c4cec6 Mon Sep 17 00:00:00 2001 From: Sowmini Varadhan Date: Sat, 18 Apr 2015 12:33:55 -0700 Subject: iommu-common: Fix PARISC compile-time warnings Fixes warnings due to - no DMA_ERROR_CODE on PARISC, - sizeof (unsigned long) == 4 bytes on PARISC. Signed-off-by: Sowmini Varadhan Signed-off-by: David S. Miller --- lib/iommu-common.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib/iommu-common.c') diff --git a/lib/iommu-common.c b/lib/iommu-common.c index b99f1d744a8d..a1a517cba7ec 100644 --- a/lib/iommu-common.c +++ b/lib/iommu-common.c @@ -11,6 +11,10 @@ #include #include +#ifndef DMA_ERROR_CODE +#define DMA_ERROR_CODE (~(dma_addr_t)0x0) +#endif + unsigned long iommu_large_alloc = 15; static DEFINE_PER_CPU(unsigned int, iommu_pool_hash); @@ -171,7 +175,7 @@ unsigned long iommu_tbl_range_alloc(struct device *dev, boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 1 << iommu->table_shift); else - boundary_size = ALIGN(1UL << 32, 1 << iommu->table_shift); + boundary_size = ALIGN(1ULL << 32, 1 << iommu->table_shift); boundary_size = boundary_size >> iommu->table_shift; /* -- cgit v1.2.3 From b0cc836d306c12462a60e72aae8f6d2318f10817 Mon Sep 17 00:00:00 2001 From: Sowmini Varadhan Date: Sun, 19 Apr 2015 13:13:30 -0400 Subject: iommu-common: fix x86_64 compiler warnings Declare iommu_large_alloc as static. Remove extern definition for iommu_tbl_pool_init(). Signed-off-by: Sowmini Varadhan Tested-by: Guenter Roeck Reviewed-by: Guenter Roeck Signed-off-by: David S. Miller --- lib/iommu-common.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib/iommu-common.c') diff --git a/lib/iommu-common.c b/lib/iommu-common.c index a1a517cba7ec..a9a53f566237 100644 --- a/lib/iommu-common.c +++ b/lib/iommu-common.c @@ -15,7 +15,7 @@ #define DMA_ERROR_CODE (~(dma_addr_t)0x0) #endif -unsigned long iommu_large_alloc = 15; +static unsigned long iommu_large_alloc = 15; static DEFINE_PER_CPU(unsigned int, iommu_pool_hash); @@ -53,12 +53,12 @@ static void setup_iommu_pool_hash(void) * the top 1/4 of the table will be set aside for pool allocations * of more than iommu_large_alloc pages. */ -extern void iommu_tbl_pool_init(struct iommu_map_table *iommu, - unsigned long num_entries, - u32 table_shift, - void (*lazy_flush)(struct iommu_map_table *), - bool large_pool, u32 npools, - bool skip_span_boundary_check) +void iommu_tbl_pool_init(struct iommu_map_table *iommu, + unsigned long num_entries, + u32 table_shift, + void (*lazy_flush)(struct iommu_map_table *), + bool large_pool, u32 npools, + bool skip_span_boundary_check) { unsigned int start, i; struct iommu_pool *p = &(iommu->large_pool); -- cgit v1.2.3 From 7b3372d4c2bced80598771aab8fea87c40ebb52a Mon Sep 17 00:00:00 2001 From: Sowmini Varadhan Date: Sun, 19 Apr 2015 13:13:31 -0400 Subject: iommu-common: rename iommu_pool_hash to iommu_hash_common When CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set, the DEFINE_PER_CPU_SECTION macro will define an extern __pcpu_unique_##name variable that could conflict with the same definition in powerpc at this time. Avoid that conflict by renaming iommu_pool_hash in iommu-common.c Thanks to Guenter Roeck for catching this, and helping to test the fix. Signed-off-by: Sowmini Varadhan Tested-by: Guenter Roeck Reviewed-by: Guenter Roeck Signed-off-by: David S. Miller --- lib/iommu-common.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/iommu-common.c') diff --git a/lib/iommu-common.c b/lib/iommu-common.c index a9a53f566237..df30632f0bef 100644 --- a/lib/iommu-common.c +++ b/lib/iommu-common.c @@ -17,7 +17,7 @@ static unsigned long iommu_large_alloc = 15; -static DEFINE_PER_CPU(unsigned int, iommu_pool_hash); +static DEFINE_PER_CPU(unsigned int, iommu_hash_common); static inline bool need_flush(struct iommu_map_table *iommu) { @@ -44,7 +44,7 @@ static void setup_iommu_pool_hash(void) return; do_once = true; for_each_possible_cpu(i) - per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS); + per_cpu(iommu_hash_common, i) = hash_32(i, IOMMU_POOL_HASHBITS); } /* @@ -106,7 +106,7 @@ unsigned long iommu_tbl_range_alloc(struct device *dev, unsigned long mask, unsigned int align_order) { - unsigned int pool_hash = __this_cpu_read(iommu_pool_hash); + unsigned int pool_hash = __this_cpu_read(iommu_hash_common); unsigned long n, end, start, limit, boundary_size; struct iommu_pool *pool; int pass = 0; -- cgit v1.2.3