diff options
Diffstat (limited to 'arch/powerpc/kernel/iommu.c')
-rw-r--r-- | arch/powerpc/kernel/iommu.c | 116 |
1 files changed, 54 insertions, 62 deletions
diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index 2d0c9ef555e9..8f1f4e539c4b 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -31,6 +31,7 @@ #include <linux/string.h> #include <linux/dma-mapping.h> #include <linux/bitops.h> +#include <linux/iommu-helper.h> #include <asm/io.h> #include <asm/prom.h> #include <asm/iommu.h> @@ -81,17 +82,19 @@ static int __init setup_iommu(char *str) __setup("protect4gb=", setup_protect4gb); __setup("iommu=", setup_iommu); -static unsigned long iommu_range_alloc(struct iommu_table *tbl, +static unsigned long iommu_range_alloc(struct device *dev, + struct iommu_table *tbl, unsigned long npages, unsigned long *handle, unsigned long mask, unsigned int align_order) { - unsigned long n, end, i, start; + unsigned long n, end, start; unsigned long limit; int largealloc = npages > 15; int pass = 0; unsigned long align_mask; + unsigned long boundary_size; align_mask = 0xffffffffffffffffl >> (64 - align_order); @@ -136,14 +139,17 @@ static unsigned long iommu_range_alloc(struct iommu_table *tbl, start &= mask; } - n = find_next_zero_bit(tbl->it_map, limit, start); - - /* Align allocation */ - n = (n + align_mask) & ~align_mask; - - end = n + npages; + if (dev) + boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, + 1 << IOMMU_PAGE_SHIFT); + else + boundary_size = ALIGN(1UL << 32, 1 << IOMMU_PAGE_SHIFT); + /* 4GB boundary for iseries_hv_alloc and iseries_hv_map */ - if (unlikely(end >= limit)) { + n = iommu_area_alloc(tbl->it_map, limit, start, npages, + tbl->it_offset, boundary_size >> IOMMU_PAGE_SHIFT, + align_mask); + if (n == -1) { if (likely(pass < 2)) { /* First failure, just rescan the half of the table. * Second failure, rescan the other half of the table. @@ -158,14 +164,7 @@ static unsigned long iommu_range_alloc(struct iommu_table *tbl, } } - for (i = n; i < end; i++) - if (test_bit(i, tbl->it_map)) { - start = i+1; - goto again; - } - - for (i = n; i < end; i++) - __set_bit(i, tbl->it_map); + end = n + npages; /* Bump the hint to a new block for small allocs. */ if (largealloc) { @@ -184,16 +183,17 @@ static unsigned long iommu_range_alloc(struct iommu_table *tbl, return n; } -static dma_addr_t iommu_alloc(struct iommu_table *tbl, void *page, - unsigned int npages, enum dma_data_direction direction, - unsigned long mask, unsigned int align_order) +static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl, + void *page, unsigned int npages, + enum dma_data_direction direction, + unsigned long mask, unsigned int align_order) { unsigned long entry, flags; dma_addr_t ret = DMA_ERROR_CODE; spin_lock_irqsave(&(tbl->it_lock), flags); - entry = iommu_range_alloc(tbl, npages, NULL, mask, align_order); + entry = iommu_range_alloc(dev, tbl, npages, NULL, mask, align_order); if (unlikely(entry == DMA_ERROR_CODE)) { spin_unlock_irqrestore(&(tbl->it_lock), flags); @@ -224,7 +224,6 @@ static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, unsigned int npages) { unsigned long entry, free_entry; - unsigned long i; entry = dma_addr >> IOMMU_PAGE_SHIFT; free_entry = entry - tbl->it_offset; @@ -246,9 +245,7 @@ static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, } ppc_md.tce_free(tbl, entry, npages); - - for (i = 0; i < npages; i++) - __clear_bit(free_entry+i, tbl->it_map); + iommu_area_free(tbl->it_map, free_entry, npages); } static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, @@ -270,15 +267,18 @@ static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, spin_unlock_irqrestore(&(tbl->it_lock), flags); } -int iommu_map_sg(struct iommu_table *tbl, struct scatterlist *sglist, +int iommu_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, unsigned long mask, enum dma_data_direction direction) { + struct iommu_table *tbl = dev->archdata.dma_data; dma_addr_t dma_next = 0, dma_addr; unsigned long flags; struct scatterlist *s, *outs, *segstart; int outcount, incount, i; + unsigned int align; unsigned long handle; + unsigned int max_seg_size; BUG_ON(direction == DMA_NONE); @@ -297,6 +297,7 @@ int iommu_map_sg(struct iommu_table *tbl, struct scatterlist *sglist, spin_lock_irqsave(&(tbl->it_lock), flags); + max_seg_size = dma_get_max_seg_size(dev); for_each_sg(sglist, s, nelems, i) { unsigned long vaddr, npages, entry, slen; @@ -309,7 +310,12 @@ int iommu_map_sg(struct iommu_table *tbl, struct scatterlist *sglist, /* Allocate iommu entries for that segment */ vaddr = (unsigned long) sg_virt(s); npages = iommu_num_pages(vaddr, slen); - entry = iommu_range_alloc(tbl, npages, &handle, mask >> IOMMU_PAGE_SHIFT, 0); + align = 0; + if (IOMMU_PAGE_SHIFT < PAGE_SHIFT && slen >= PAGE_SIZE && + (vaddr & ~PAGE_MASK) == 0) + align = PAGE_SHIFT - IOMMU_PAGE_SHIFT; + entry = iommu_range_alloc(dev, tbl, npages, &handle, + mask >> IOMMU_PAGE_SHIFT, align); DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen); @@ -338,7 +344,8 @@ int iommu_map_sg(struct iommu_table *tbl, struct scatterlist *sglist, /* We cannot merge if: * - allocated dma_addr isn't contiguous to previous allocation */ - if (novmerge || (dma_addr != dma_next)) { + if (novmerge || (dma_addr != dma_next) || + (outs->dma_length + s->length > max_seg_size)) { /* Can't merge: create a new segment */ segstart = s; outcount++; @@ -446,9 +453,6 @@ void iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist, struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid) { unsigned long sz; - unsigned long start_index, end_index; - unsigned long entries_per_4g; - unsigned long index; static int welcomed = 0; struct page *page; @@ -470,6 +474,7 @@ struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid) #ifdef CONFIG_CRASH_DUMP if (ppc_md.tce_get) { + unsigned long index; unsigned long tceval; unsigned long tcecount = 0; @@ -500,23 +505,6 @@ struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid) ppc_md.tce_free(tbl, tbl->it_offset, tbl->it_size); #endif - /* - * DMA cannot cross 4 GB boundary. Mark last entry of each 4 - * GB chunk as reserved. - */ - if (protect4gb) { - entries_per_4g = 0x100000000l >> IOMMU_PAGE_SHIFT; - - /* Mark the last bit before a 4GB boundary as used */ - start_index = tbl->it_offset | (entries_per_4g - 1); - start_index -= tbl->it_offset; - - end_index = tbl->it_size; - - for (index = start_index; index < end_index - 1; index += entries_per_4g) - __set_bit(index, tbl->it_map); - } - if (!welcomed) { printk(KERN_INFO "IOMMU table initialized, virtual merging %s\n", novmerge ? "disabled" : "enabled"); @@ -526,16 +514,14 @@ struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid) return tbl; } -void iommu_free_table(struct device_node *dn) +void iommu_free_table(struct iommu_table *tbl, const char *node_name) { - struct pci_dn *pdn = dn->data; - struct iommu_table *tbl = pdn->iommu_table; unsigned long bitmap_sz, i; unsigned int order; if (!tbl || !tbl->it_map) { printk(KERN_ERR "%s: expected TCE map for %s\n", __FUNCTION__, - dn->full_name); + node_name); return; } @@ -544,7 +530,7 @@ void iommu_free_table(struct device_node *dn) for (i = 0; i < (tbl->it_size/64); i++) { if (tbl->it_map[i] != 0) { printk(KERN_WARNING "%s: Unexpected TCEs for %s\n", - __FUNCTION__, dn->full_name); + __FUNCTION__, node_name); break; } } @@ -566,13 +552,13 @@ void iommu_free_table(struct device_node *dn) * need not be page aligned, the dma_addr_t returned will point to the same * byte within the page as vaddr. */ -dma_addr_t iommu_map_single(struct iommu_table *tbl, void *vaddr, - size_t size, unsigned long mask, - enum dma_data_direction direction) +dma_addr_t iommu_map_single(struct device *dev, struct iommu_table *tbl, + void *vaddr, size_t size, unsigned long mask, + enum dma_data_direction direction) { dma_addr_t dma_handle = DMA_ERROR_CODE; unsigned long uaddr; - unsigned int npages; + unsigned int npages, align; BUG_ON(direction == DMA_NONE); @@ -580,8 +566,13 @@ dma_addr_t iommu_map_single(struct iommu_table *tbl, void *vaddr, npages = iommu_num_pages(uaddr, size); if (tbl) { - dma_handle = iommu_alloc(tbl, vaddr, npages, direction, - mask >> IOMMU_PAGE_SHIFT, 0); + align = 0; + if (IOMMU_PAGE_SHIFT < PAGE_SHIFT && size >= PAGE_SIZE && + ((unsigned long)vaddr & ~PAGE_MASK) == 0) + align = PAGE_SHIFT - IOMMU_PAGE_SHIFT; + + dma_handle = iommu_alloc(dev, tbl, vaddr, npages, direction, + mask >> IOMMU_PAGE_SHIFT, align); if (dma_handle == DMA_ERROR_CODE) { if (printk_ratelimit()) { printk(KERN_INFO "iommu_alloc failed, " @@ -612,8 +603,9 @@ void iommu_unmap_single(struct iommu_table *tbl, dma_addr_t dma_handle, * Returns the virtual address of the buffer and sets dma_handle * to the dma address (mapping) of the first page. */ -void *iommu_alloc_coherent(struct iommu_table *tbl, size_t size, - dma_addr_t *dma_handle, unsigned long mask, gfp_t flag, int node) +void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl, + size_t size, dma_addr_t *dma_handle, + unsigned long mask, gfp_t flag, int node) { void *ret = NULL; dma_addr_t mapping; @@ -647,7 +639,7 @@ void *iommu_alloc_coherent(struct iommu_table *tbl, size_t size, /* Set up tces to cover the allocated range */ nio_pages = size >> IOMMU_PAGE_SHIFT; io_order = get_iommu_order(size); - mapping = iommu_alloc(tbl, ret, nio_pages, DMA_BIDIRECTIONAL, + mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL, mask >> IOMMU_PAGE_SHIFT, io_order); if (mapping == DMA_ERROR_CODE) { free_pages((unsigned long)ret, order); |