summaryrefslogtreecommitdiff
path: root/drivers/iommu
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/iommu')
-rw-r--r--drivers/iommu/Kconfig75
-rw-r--r--drivers/iommu/Makefile2
-rw-r--r--drivers/iommu/amd_iommu.c396
-rw-r--r--drivers/iommu/amd_iommu_types.h40
-rw-r--r--drivers/iommu/amd_iommu_v2.c38
-rw-r--r--drivers/iommu/dma-iommu.c11
-rw-r--r--drivers/iommu/dmar.c12
-rw-r--r--drivers/iommu/ipmmu-vmsa.c2
-rw-r--r--drivers/iommu/msm_iommu_dev.c25
-rw-r--r--drivers/iommu/s390-iommu.c4
-rw-r--r--drivers/iommu/shmobile-iommu.c402
-rw-r--r--drivers/iommu/shmobile-ipmmu.c129
-rw-r--r--drivers/iommu/shmobile-ipmmu.h34
13 files changed, 291 insertions, 879 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig
index b9094e9da537..a1e75cba18e0 100644
--- a/drivers/iommu/Kconfig
+++ b/drivers/iommu/Kconfig
@@ -263,81 +263,6 @@ config EXYNOS_IOMMU_DEBUG
Say N unless you need kernel log message for IOMMU debugging.
-config SHMOBILE_IPMMU
- bool
-
-config SHMOBILE_IPMMU_TLB
- bool
-
-config SHMOBILE_IOMMU
- bool "IOMMU for Renesas IPMMU/IPMMUI"
- default n
- depends on ARM && MMU
- depends on ARCH_SHMOBILE || COMPILE_TEST
- select IOMMU_API
- select ARM_DMA_USE_IOMMU
- select SHMOBILE_IPMMU
- select SHMOBILE_IPMMU_TLB
- help
- Support for Renesas IPMMU/IPMMUI. This option enables
- remapping of DMA memory accesses from all of the IP blocks
- on the ICB.
-
- Warning: Drivers (including userspace drivers of UIO
- devices) of the IP blocks on the ICB *must* use addresses
- allocated from the IPMMU (iova) for DMA with this option
- enabled.
-
- If unsure, say N.
-
-choice
- prompt "IPMMU/IPMMUI address space size"
- default SHMOBILE_IOMMU_ADDRSIZE_2048MB
- depends on SHMOBILE_IOMMU
- help
- This option sets IPMMU/IPMMUI address space size by
- adjusting the 1st level page table size. The page table size
- is calculated as follows:
-
- page table size = number of page table entries * 4 bytes
- number of page table entries = address space size / 1 MiB
-
- For example, when the address space size is 2048 MiB, the
- 1st level page table size is 8192 bytes.
-
- config SHMOBILE_IOMMU_ADDRSIZE_2048MB
- bool "2 GiB"
-
- config SHMOBILE_IOMMU_ADDRSIZE_1024MB
- bool "1 GiB"
-
- config SHMOBILE_IOMMU_ADDRSIZE_512MB
- bool "512 MiB"
-
- config SHMOBILE_IOMMU_ADDRSIZE_256MB
- bool "256 MiB"
-
- config SHMOBILE_IOMMU_ADDRSIZE_128MB
- bool "128 MiB"
-
- config SHMOBILE_IOMMU_ADDRSIZE_64MB
- bool "64 MiB"
-
- config SHMOBILE_IOMMU_ADDRSIZE_32MB
- bool "32 MiB"
-
-endchoice
-
-config SHMOBILE_IOMMU_L1SIZE
- int
- default 8192 if SHMOBILE_IOMMU_ADDRSIZE_2048MB
- default 4096 if SHMOBILE_IOMMU_ADDRSIZE_1024MB
- default 2048 if SHMOBILE_IOMMU_ADDRSIZE_512MB
- default 1024 if SHMOBILE_IOMMU_ADDRSIZE_256MB
- default 512 if SHMOBILE_IOMMU_ADDRSIZE_128MB
- default 256 if SHMOBILE_IOMMU_ADDRSIZE_64MB
- default 128 if SHMOBILE_IOMMU_ADDRSIZE_32MB
-
config IPMMU_VMSA
bool "Renesas VMSA-compatible IPMMU"
depends on ARM_LPAE
diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile
index 68faca02225d..42fc0c25cf1a 100644
--- a/drivers/iommu/Makefile
+++ b/drivers/iommu/Makefile
@@ -22,7 +22,5 @@ obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o
obj-$(CONFIG_TEGRA_IOMMU_GART) += tegra-gart.o
obj-$(CONFIG_TEGRA_IOMMU_SMMU) += tegra-smmu.o
obj-$(CONFIG_EXYNOS_IOMMU) += exynos-iommu.o
-obj-$(CONFIG_SHMOBILE_IOMMU) += shmobile-iommu.o
-obj-$(CONFIG_SHMOBILE_IPMMU) += shmobile-ipmmu.o
obj-$(CONFIG_FSL_PAMU) += fsl_pamu.o fsl_pamu_domain.o
obj-$(CONFIG_S390_IOMMU) += s390-iommu.o
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 8b2be1e7714f..539b0dea8034 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -35,6 +35,7 @@
#include <linux/msi.h>
#include <linux/dma-contiguous.h>
#include <linux/irqdomain.h>
+#include <linux/percpu.h>
#include <asm/irq_remapping.h>
#include <asm/io_apic.h>
#include <asm/apic.h>
@@ -114,6 +115,45 @@ struct kmem_cache *amd_iommu_irq_cache;
static void update_domain(struct protection_domain *domain);
static int protection_domain_init(struct protection_domain *domain);
+/*
+ * For dynamic growth the aperture size is split into ranges of 128MB of
+ * DMA address space each. This struct represents one such range.
+ */
+struct aperture_range {
+
+ spinlock_t bitmap_lock;
+
+ /* address allocation bitmap */
+ unsigned long *bitmap;
+ unsigned long offset;
+ unsigned long next_bit;
+
+ /*
+ * Array of PTE pages for the aperture. In this array we save all the
+ * leaf pages of the domain page table used for the aperture. This way
+ * we don't need to walk the page table to find a specific PTE. We can
+ * just calculate its address in constant time.
+ */
+ u64 *pte_pages[64];
+};
+
+/*
+ * Data container for a dma_ops specific protection domain
+ */
+struct dma_ops_domain {
+ /* generic protection domain information */
+ struct protection_domain domain;
+
+ /* size of the aperture for the mappings */
+ unsigned long aperture_size;
+
+ /* aperture index we start searching for free addresses */
+ u32 __percpu *next_index;
+
+ /* address space relevant data */
+ struct aperture_range *aperture[APERTURE_MAX_RANGES];
+};
+
/****************************************************************************
*
* Helper functions
@@ -1167,11 +1207,21 @@ static u64 *alloc_pte(struct protection_domain *domain,
end_lvl = PAGE_SIZE_LEVEL(page_size);
while (level > end_lvl) {
- if (!IOMMU_PTE_PRESENT(*pte)) {
+ u64 __pte, __npte;
+
+ __pte = *pte;
+
+ if (!IOMMU_PTE_PRESENT(__pte)) {
page = (u64 *)get_zeroed_page(gfp);
if (!page)
return NULL;
- *pte = PM_LEVEL_PDE(level, virt_to_phys(page));
+
+ __npte = PM_LEVEL_PDE(level, virt_to_phys(page));
+
+ if (cmpxchg64(pte, __pte, __npte)) {
+ free_page((unsigned long)page);
+ continue;
+ }
}
/* No level skipping support yet */
@@ -1376,8 +1426,10 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
bool populate, gfp_t gfp)
{
int index = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
- struct amd_iommu *iommu;
unsigned long i, old_size, pte_pgsize;
+ struct aperture_range *range;
+ struct amd_iommu *iommu;
+ unsigned long flags;
#ifdef CONFIG_IOMMU_STRESS
populate = false;
@@ -1386,15 +1438,17 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
if (index >= APERTURE_MAX_RANGES)
return -ENOMEM;
- dma_dom->aperture[index] = kzalloc(sizeof(struct aperture_range), gfp);
- if (!dma_dom->aperture[index])
+ range = kzalloc(sizeof(struct aperture_range), gfp);
+ if (!range)
return -ENOMEM;
- dma_dom->aperture[index]->bitmap = (void *)get_zeroed_page(gfp);
- if (!dma_dom->aperture[index]->bitmap)
+ range->bitmap = (void *)get_zeroed_page(gfp);
+ if (!range->bitmap)
goto out_free;
- dma_dom->aperture[index]->offset = dma_dom->aperture_size;
+ range->offset = dma_dom->aperture_size;
+
+ spin_lock_init(&range->bitmap_lock);
if (populate) {
unsigned long address = dma_dom->aperture_size;
@@ -1407,14 +1461,20 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
if (!pte)
goto out_free;
- dma_dom->aperture[index]->pte_pages[i] = pte_page;
+ range->pte_pages[i] = pte_page;
address += APERTURE_RANGE_SIZE / 64;
}
}
- old_size = dma_dom->aperture_size;
- dma_dom->aperture_size += APERTURE_RANGE_SIZE;
+ spin_lock_irqsave(&dma_dom->domain.lock, flags);
+
+ /* First take the bitmap_lock and then publish the range */
+ spin_lock(&range->bitmap_lock);
+
+ old_size = dma_dom->aperture_size;
+ dma_dom->aperture[index] = range;
+ dma_dom->aperture_size += APERTURE_RANGE_SIZE;
/* Reserve address range used for MSI messages */
if (old_size < MSI_ADDR_BASE_LO &&
@@ -1461,62 +1521,123 @@ static int alloc_new_range(struct dma_ops_domain *dma_dom,
update_domain(&dma_dom->domain);
+ spin_unlock(&range->bitmap_lock);
+
+ spin_unlock_irqrestore(&dma_dom->domain.lock, flags);
+
return 0;
out_free:
update_domain(&dma_dom->domain);
- free_page((unsigned long)dma_dom->aperture[index]->bitmap);
+ free_page((unsigned long)range->bitmap);
- kfree(dma_dom->aperture[index]);
- dma_dom->aperture[index] = NULL;
+ kfree(range);
return -ENOMEM;
}
+static dma_addr_t dma_ops_aperture_alloc(struct dma_ops_domain *dom,
+ struct aperture_range *range,
+ unsigned long pages,
+ unsigned long dma_mask,
+ unsigned long boundary_size,
+ unsigned long align_mask,
+ bool trylock)
+{
+ unsigned long offset, limit, flags;
+ dma_addr_t address;
+ bool flush = false;
+
+ offset = range->offset >> PAGE_SHIFT;
+ limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
+ dma_mask >> PAGE_SHIFT);
+
+ if (trylock) {
+ if (!spin_trylock_irqsave(&range->bitmap_lock, flags))
+ return -1;
+ } else {
+ spin_lock_irqsave(&range->bitmap_lock, flags);
+ }
+
+ address = iommu_area_alloc(range->bitmap, limit, range->next_bit,
+ pages, offset, boundary_size, align_mask);
+ if (address == -1) {
+ /* Nothing found, retry one time */
+ address = iommu_area_alloc(range->bitmap, limit,
+ 0, pages, offset, boundary_size,
+ align_mask);
+ flush = true;
+ }
+
+ if (address != -1)
+ range->next_bit = address + pages;
+
+ spin_unlock_irqrestore(&range->bitmap_lock, flags);
+
+ if (flush) {
+ domain_flush_tlb(&dom->domain);
+ domain_flush_complete(&dom->domain);
+ }
+
+ return address;
+}
+
static unsigned long dma_ops_area_alloc(struct device *dev,
struct dma_ops_domain *dom,
unsigned int pages,
unsigned long align_mask,
- u64 dma_mask,
- unsigned long start)
+ u64 dma_mask)
{
- unsigned long next_bit = dom->next_address % APERTURE_RANGE_SIZE;
- int max_index = dom->aperture_size >> APERTURE_RANGE_SHIFT;
- int i = start >> APERTURE_RANGE_SHIFT;
unsigned long boundary_size, mask;
unsigned long address = -1;
- unsigned long limit;
+ bool first = true;
+ u32 start, i;
- next_bit >>= PAGE_SHIFT;
+ preempt_disable();
mask = dma_get_seg_boundary(dev);
+again:
+ start = this_cpu_read(*dom->next_index);
+
+ /* Sanity check - is it really necessary? */
+ if (unlikely(start > APERTURE_MAX_RANGES)) {
+ start = 0;
+ this_cpu_write(*dom->next_index, 0);
+ }
+
boundary_size = mask + 1 ? ALIGN(mask + 1, PAGE_SIZE) >> PAGE_SHIFT :
1UL << (BITS_PER_LONG - PAGE_SHIFT);
- for (;i < max_index; ++i) {
- unsigned long offset = dom->aperture[i]->offset >> PAGE_SHIFT;
+ for (i = 0; i < APERTURE_MAX_RANGES; ++i) {
+ struct aperture_range *range;
+ int index;
- if (dom->aperture[i]->offset >= dma_mask)
- break;
+ index = (start + i) % APERTURE_MAX_RANGES;
- limit = iommu_device_max_index(APERTURE_RANGE_PAGES, offset,
- dma_mask >> PAGE_SHIFT);
+ range = dom->aperture[index];
- address = iommu_area_alloc(dom->aperture[i]->bitmap,
- limit, next_bit, pages, 0,
- boundary_size, align_mask);
+ if (!range || range->offset >= dma_mask)
+ continue;
+
+ address = dma_ops_aperture_alloc(dom, range, pages,
+ dma_mask, boundary_size,
+ align_mask, first);
if (address != -1) {
- address = dom->aperture[i]->offset +
- (address << PAGE_SHIFT);
- dom->next_address = address + (pages << PAGE_SHIFT);
+ address = range->offset + (address << PAGE_SHIFT);
+ this_cpu_write(*dom->next_index, index);
break;
}
+ }
- next_bit = 0;
+ if (address == -1 && first) {
+ first = false;
+ goto again;
}
+ preempt_enable();
+
return address;
}
@@ -1526,21 +1647,14 @@ static unsigned long dma_ops_alloc_addresses(struct device *dev,
unsigned long align_mask,
u64 dma_mask)
{
- unsigned long address;
-
-#ifdef CONFIG_IOMMU_STRESS
- dom->next_address = 0;
- dom->need_flush = true;
-#endif
+ unsigned long address = -1;
- address = dma_ops_area_alloc(dev, dom, pages, align_mask,
- dma_mask, dom->next_address);
+ while (address == -1) {
+ address = dma_ops_area_alloc(dev, dom, pages,
+ align_mask, dma_mask);
- if (address == -1) {
- dom->next_address = 0;
- address = dma_ops_area_alloc(dev, dom, pages, align_mask,
- dma_mask, 0);
- dom->need_flush = true;
+ if (address == -1 && alloc_new_range(dom, false, GFP_ATOMIC))
+ break;
}
if (unlikely(address == -1))
@@ -1562,6 +1676,7 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom,
{
unsigned i = address >> APERTURE_RANGE_SHIFT;
struct aperture_range *range = dom->aperture[i];
+ unsigned long flags;
BUG_ON(i >= APERTURE_MAX_RANGES || range == NULL);
@@ -1570,12 +1685,18 @@ static void dma_ops_free_addresses(struct dma_ops_domain *dom,
return;
#endif
- if (address >= dom->next_address)
- dom->need_flush = true;
+ if (amd_iommu_unmap_flush) {
+ domain_flush_tlb(&dom->domain);
+ domain_flush_complete(&dom->domain);
+ }
address = (address % APERTURE_RANGE_SIZE) >> PAGE_SHIFT;
+ spin_lock_irqsave(&range->bitmap_lock, flags);
+ if (address + pages > range->next_bit)
+ range->next_bit = address + pages;
bitmap_clear(range->bitmap, address, pages);
+ spin_unlock_irqrestore(&range->bitmap_lock, flags);
}
@@ -1755,6 +1876,8 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
if (!dom)
return;
+ free_percpu(dom->next_index);
+
del_domain_from_list(&dom->domain);
free_pagetable(&dom->domain);
@@ -1769,6 +1892,23 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
kfree(dom);
}
+static int dma_ops_domain_alloc_apertures(struct dma_ops_domain *dma_dom,
+ int max_apertures)
+{
+ int ret, i, apertures;
+
+ apertures = dma_dom->aperture_size >> APERTURE_RANGE_SHIFT;
+ ret = 0;
+
+ for (i = apertures; i < max_apertures; ++i) {
+ ret = alloc_new_range(dma_dom, false, GFP_KERNEL);
+ if (ret)
+ break;
+ }
+
+ return ret;
+}
+
/*
* Allocates a new protection domain usable for the dma_ops functions.
* It also initializes the page table and the address allocator data
@@ -1777,6 +1917,7 @@ static void dma_ops_domain_free(struct dma_ops_domain *dom)
static struct dma_ops_domain *dma_ops_domain_alloc(void)
{
struct dma_ops_domain *dma_dom;
+ int cpu;
dma_dom = kzalloc(sizeof(struct dma_ops_domain), GFP_KERNEL);
if (!dma_dom)
@@ -1785,6 +1926,10 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
if (protection_domain_init(&dma_dom->domain))
goto free_dma_dom;
+ dma_dom->next_index = alloc_percpu(u32);
+ if (!dma_dom->next_index)
+ goto free_dma_dom;
+
dma_dom->domain.mode = PAGE_MODE_2_LEVEL;
dma_dom->domain.pt_root = (void *)get_zeroed_page(GFP_KERNEL);
dma_dom->domain.flags = PD_DMA_OPS_MASK;
@@ -1792,8 +1937,6 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
if (!dma_dom->domain.pt_root)
goto free_dma_dom;
- dma_dom->need_flush = false;
-
add_domain_to_list(&dma_dom->domain);
if (alloc_new_range(dma_dom, true, GFP_KERNEL))
@@ -1804,8 +1947,9 @@ static struct dma_ops_domain *dma_ops_domain_alloc(void)
* a valid dma-address. So we can use 0 as error value
*/
dma_dom->aperture[0]->bitmap[0] = 1;
- dma_dom->next_address = 0;
+ for_each_possible_cpu(cpu)
+ *per_cpu_ptr(dma_dom->next_index, cpu) = 0;
return dma_dom;
@@ -2328,7 +2472,7 @@ static dma_addr_t dma_ops_domain_map(struct dma_ops_domain *dom,
else if (direction == DMA_BIDIRECTIONAL)
__pte |= IOMMU_PTE_IR | IOMMU_PTE_IW;
- WARN_ON(*pte);
+ WARN_ON_ONCE(*pte);
*pte = __pte;
@@ -2357,7 +2501,7 @@ static void dma_ops_domain_unmap(struct dma_ops_domain *dom,
pte += PM_LEVEL_INDEX(0, address);
- WARN_ON(!*pte);
+ WARN_ON_ONCE(!*pte);
*pte = 0ULL;
}
@@ -2393,26 +2537,11 @@ static dma_addr_t __map_single(struct device *dev,
if (align)
align_mask = (1UL << get_order(size)) - 1;
-retry:
address = dma_ops_alloc_addresses(dev, dma_dom, pages, align_mask,
dma_mask);
- if (unlikely(address == DMA_ERROR_CODE)) {
- /*
- * setting next_address here will let the address
- * allocator only scan the new allocated range in the
- * first run. This is a small optimization.
- */
- dma_dom->next_address = dma_dom->aperture_size;
- if (alloc_new_range(dma_dom, false, GFP_ATOMIC))
- goto out;
-
- /*
- * aperture was successfully enlarged by 128 MB, try
- * allocation again
- */
- goto retry;
- }
+ if (address == DMA_ERROR_CODE)
+ goto out;
start = address;
for (i = 0; i < pages; ++i) {
@@ -2427,11 +2556,10 @@ retry:
ADD_STATS_COUNTER(alloced_io_mem, size);
- if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) {
- domain_flush_tlb(&dma_dom->domain);
- dma_dom->need_flush = false;
- } else if (unlikely(amd_iommu_np_cache))
+ if (unlikely(amd_iommu_np_cache)) {
domain_flush_pages(&dma_dom->domain, address, size);
+ domain_flush_complete(&dma_dom->domain);
+ }
out:
return address;
@@ -2478,11 +2606,6 @@ static void __unmap_single(struct dma_ops_domain *dma_dom,
SUB_STATS_COUNTER(alloced_io_mem, size);
dma_ops_free_addresses(dma_dom, dma_addr, pages);
-
- if (amd_iommu_unmap_flush || dma_dom->need_flush) {
- domain_flush_pages(&dma_dom->domain, flush_addr, size);
- dma_dom->need_flush = false;
- }
}
/*
@@ -2493,11 +2616,9 @@ static dma_addr_t map_page(struct device *dev, struct page *page,
enum dma_data_direction dir,
struct dma_attrs *attrs)
{
- unsigned long flags;
+ phys_addr_t paddr = page_to_phys(page) + offset;
struct protection_domain *domain;
- dma_addr_t addr;
u64 dma_mask;
- phys_addr_t paddr = page_to_phys(page) + offset;
INC_STATS_COUNTER(cnt_map_single);
@@ -2509,19 +2630,8 @@ static dma_addr_t map_page(struct device *dev, struct page *page,
dma_mask = *dev->dma_mask;
- spin_lock_irqsave(&domain->lock, flags);
-
- addr = __map_single(dev, domain->priv, paddr, size, dir, false,
+ return __map_single(dev, domain->priv, paddr, size, dir, false,
dma_mask);
- if (addr == DMA_ERROR_CODE)
- goto out;
-
- domain_flush_complete(domain);
-
-out:
- spin_unlock_irqrestore(&domain->lock, flags);
-
- return addr;
}
/*
@@ -2530,7 +2640,6 @@ out:
static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
enum dma_data_direction dir, struct dma_attrs *attrs)
{
- unsigned long flags;
struct protection_domain *domain;
INC_STATS_COUNTER(cnt_unmap_single);
@@ -2539,13 +2648,7 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size,
if (IS_ERR(domain))
return;
- spin_lock_irqsave(&domain->lock, flags);
-
__unmap_single(domain->priv, dma_addr, size, dir);
-
- domain_flush_complete(domain);
-
- spin_unlock_irqrestore(&domain->lock, flags);
}
/*
@@ -2556,7 +2659,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
- unsigned long flags;
struct protection_domain *domain;
int i;
struct scatterlist *s;
@@ -2572,8 +2674,6 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
dma_mask = *dev->dma_mask;
- spin_lock_irqsave(&domain->lock, flags);
-
for_each_sg(sglist, s, nelems, i) {
paddr = sg_phys(s);
@@ -2588,12 +2688,8 @@ static int map_sg(struct device *dev, struct scatterlist *sglist,
goto unmap;
}
- domain_flush_complete(domain);
-
-out:
- spin_unlock_irqrestore(&domain->lock, flags);
-
return mapped_elems;
+
unmap:
for_each_sg(sglist, s, mapped_elems, i) {
if (s->dma_address)
@@ -2602,9 +2698,7 @@ unmap:
s->dma_address = s->dma_length = 0;
}
- mapped_elems = 0;
-
- goto out;
+ return 0;
}
/*
@@ -2615,7 +2709,6 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir,
struct dma_attrs *attrs)
{
- unsigned long flags;
struct protection_domain *domain;
struct scatterlist *s;
int i;
@@ -2626,17 +2719,11 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist,
if (IS_ERR(domain))
return;
- spin_lock_irqsave(&domain->lock, flags);
-
for_each_sg(sglist, s, nelems, i) {
__unmap_single(domain->priv, s->dma_address,
s->dma_length, dir);
s->dma_address = s->dma_length = 0;
}
-
- domain_flush_complete(domain);
-
- spin_unlock_irqrestore(&domain->lock, flags);
}
/*
@@ -2648,7 +2735,6 @@ static void *alloc_coherent(struct device *dev, size_t size,
{
u64 dma_mask = dev->coherent_dma_mask;
struct protection_domain *domain;
- unsigned long flags;
struct page *page;
INC_STATS_COUNTER(cnt_alloc_coherent);
@@ -2680,19 +2766,11 @@ static void *alloc_coherent(struct device *dev, size_t size,
if (!dma_mask)
dma_mask = *dev->dma_mask;
- spin_lock_irqsave(&domain->lock, flags);
-
*dma_addr = __map_single(dev, domain->priv, page_to_phys(page),
size, DMA_BIDIRECTIONAL, true, dma_mask);
- if (*dma_addr == DMA_ERROR_CODE) {
- spin_unlock_irqrestore(&domain->lock, flags);
+ if (*dma_addr == DMA_ERROR_CODE)
goto out_free;
- }
-
- domain_flush_complete(domain);
-
- spin_unlock_irqrestore(&domain->lock, flags);
return page_address(page);
@@ -2712,7 +2790,6 @@ static void free_coherent(struct device *dev, size_t size,
struct dma_attrs *attrs)
{
struct protection_domain *domain;
- unsigned long flags;
struct page *page;
INC_STATS_COUNTER(cnt_free_coherent);
@@ -2724,14 +2801,8 @@ static void free_coherent(struct device *dev, size_t size,
if (IS_ERR(domain))
goto free_mem;
- spin_lock_irqsave(&domain->lock, flags);
-
__unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL);
- domain_flush_complete(domain);
-
- spin_unlock_irqrestore(&domain->lock, flags);
-
free_mem:
if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
__free_pages(page, get_order(size));
@@ -2746,14 +2817,43 @@ static int amd_iommu_dma_supported(struct device *dev, u64 mask)
return check_device(dev);
}
+static int set_dma_mask(struct device *dev, u64 mask)
+{
+ struct protection_domain *domain;
+ int max_apertures = 1;
+
+ domain = get_domain(dev);
+ if (IS_ERR(domain))
+ return PTR_ERR(domain);
+
+ if (mask == DMA_BIT_MASK(64))
+ max_apertures = 8;
+ else if (mask > DMA_BIT_MASK(32))
+ max_apertures = 4;
+
+ /*
+ * To prevent lock contention it doesn't make sense to allocate more
+ * apertures than online cpus
+ */
+ if (max_apertures > num_online_cpus())
+ max_apertures = num_online_cpus();
+
+ if (dma_ops_domain_alloc_apertures(domain->priv, max_apertures))
+ dev_err(dev, "Can't allocate %d iommu apertures\n",
+ max_apertures);
+
+ return 0;
+}
+
static struct dma_map_ops amd_iommu_dma_ops = {
- .alloc = alloc_coherent,
- .free = free_coherent,
- .map_page = map_page,
- .unmap_page = unmap_page,
- .map_sg = map_sg,
- .unmap_sg = unmap_sg,
- .dma_supported = amd_iommu_dma_supported,
+ .alloc = alloc_coherent,
+ .free = free_coherent,
+ .map_page = map_page,
+ .unmap_page = unmap_page,
+ .map_sg = map_sg,
+ .unmap_sg = unmap_sg,
+ .dma_supported = amd_iommu_dma_supported,
+ .set_dma_mask = set_dma_mask,
};
int __init amd_iommu_init_api(void)
@@ -3757,11 +3857,9 @@ static struct irq_domain *get_irq_domain(struct irq_alloc_info *info)
case X86_IRQ_ALLOC_TYPE_MSI:
case X86_IRQ_ALLOC_TYPE_MSIX:
devid = get_device_id(&info->msi_dev->dev);
- if (devid >= 0) {
- iommu = amd_iommu_rlookup_table[devid];
- if (iommu)
- return iommu->msi_domain;
- }
+ iommu = amd_iommu_rlookup_table[devid];
+ if (iommu)
+ return iommu->msi_domain;
break;
default:
break;
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index b08cf57bf455..9d32b20a5e9a 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -425,46 +425,6 @@ struct protection_domain {
};
/*
- * For dynamic growth the aperture size is split into ranges of 128MB of
- * DMA address space each. This struct represents one such range.
- */
-struct aperture_range {
-
- /* address allocation bitmap */
- unsigned long *bitmap;
-
- /*
- * Array of PTE pages for the aperture. In this array we save all the
- * leaf pages of the domain page table used for the aperture. This way
- * we don't need to walk the page table to find a specific PTE. We can
- * just calculate its address in constant time.
- */
- u64 *pte_pages[64];
-
- unsigned long offset;
-};
-
-/*
- * Data container for a dma_ops specific protection domain
- */
-struct dma_ops_domain {
- /* generic protection domain information */
- struct protection_domain domain;
-
- /* size of the aperture for the mappings */
- unsigned long aperture_size;
-
- /* address we start to search for free addresses */
- unsigned long next_address;
-
- /* address space relevant data */
- struct aperture_range *aperture[APERTURE_MAX_RANGES];
-
- /* This will be set to true when TLB needs to be flushed */
- bool need_flush;
-};
-
-/*
* Structure where we save information about one hardware AMD IOMMU in the
* system.
*/
diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd_iommu_v2.c
index 7caf2fa237f2..c865737326e1 100644
--- a/drivers/iommu/amd_iommu_v2.c
+++ b/drivers/iommu/amd_iommu_v2.c
@@ -432,7 +432,7 @@ static void mn_release(struct mmu_notifier *mn, struct mm_struct *mm)
unbind_pasid(pasid_state);
}
-static struct mmu_notifier_ops iommu_mn = {
+static const struct mmu_notifier_ops iommu_mn = {
.release = mn_release,
.clear_flush_young = mn_clear_flush_young,
.invalidate_page = mn_invalidate_page,
@@ -513,43 +513,39 @@ static bool access_error(struct vm_area_struct *vma, struct fault *fault)
static void do_fault(struct work_struct *work)
{
struct fault *fault = container_of(work, struct fault, work);
- struct mm_struct *mm;
struct vm_area_struct *vma;
+ int ret = VM_FAULT_ERROR;
+ unsigned int flags = 0;
+ struct mm_struct *mm;
u64 address;
- int ret, write;
-
- write = !!(fault->flags & PPR_FAULT_WRITE);
mm = fault->state->mm;
address = fault->address;
+ if (fault->flags & PPR_FAULT_USER)
+ flags |= FAULT_FLAG_USER;
+ if (fault->flags & PPR_FAULT_WRITE)
+ flags |= FAULT_FLAG_WRITE;
+
down_read(&mm->mmap_sem);
vma = find_extend_vma(mm, address);
- if (!vma || address < vma->vm_start) {
+ if (!vma || address < vma->vm_start)
/* failed to get a vma in the right range */
- up_read(&mm->mmap_sem);
- handle_fault_error(fault);
goto out;
- }
/* Check if we have the right permissions on the vma */
- if (access_error(vma, fault)) {
- up_read(&mm->mmap_sem);
- handle_fault_error(fault);
+ if (access_error(vma, fault))
goto out;
- }
- ret = handle_mm_fault(mm, vma, address, write);
- if (ret & VM_FAULT_ERROR) {
- /* failed to service fault */
- up_read(&mm->mmap_sem);
- handle_fault_error(fault);
- goto out;
- }
+ ret = handle_mm_fault(mm, vma, address, flags);
+out:
up_read(&mm->mmap_sem);
-out:
+ if (ret & VM_FAULT_ERROR)
+ /* failed to service fault */
+ handle_fault_error(fault);
+
finish_pri_tag(fault->dev_state, fault->state, fault->tag);
put_pasid_state(fault->state);
diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 3a20db4f8604..72d6182666cb 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -21,10 +21,13 @@
#include <linux/device.h>
#include <linux/dma-iommu.h>
+#include <linux/gfp.h>
#include <linux/huge_mm.h>
#include <linux/iommu.h>
#include <linux/iova.h>
#include <linux/mm.h>
+#include <linux/scatterlist.h>
+#include <linux/vmalloc.h>
int iommu_dma_init(void)
{
@@ -191,6 +194,7 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count, gfp_t gfp)
{
struct page **pages;
unsigned int i = 0, array_size = count * sizeof(*pages);
+ unsigned int order = MAX_ORDER;
if (array_size <= PAGE_SIZE)
pages = kzalloc(array_size, GFP_KERNEL);
@@ -204,14 +208,15 @@ static struct page **__iommu_dma_alloc_pages(unsigned int count, gfp_t gfp)
while (count) {
struct page *page = NULL;
- int j, order = __fls(count);
+ int j;
/*
* Higher-order allocations are a convenience rather
* than a necessity, hence using __GFP_NORETRY until
* falling back to single-page allocations.
*/
- for (order = min(order, MAX_ORDER); order > 0; order--) {
+ for (order = min_t(unsigned int, order, __fls(count));
+ order > 0; order--) {
page = alloc_pages(gfp | __GFP_NORETRY, order);
if (!page)
continue;
@@ -453,7 +458,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
size_t s_offset = iova_offset(iovad, s->offset);
size_t s_length = s->length;
- sg_dma_address(s) = s->offset;
+ sg_dma_address(s) = s_offset;
sg_dma_len(s) = s_length;
s->offset -= s_offset;
s_length = iova_align(iovad, s_length + s_offset);
diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c
index 80e3c176008e..62a400c5ba06 100644
--- a/drivers/iommu/dmar.c
+++ b/drivers/iommu/dmar.c
@@ -1063,13 +1063,19 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
raw_spin_lock_init(&iommu->register_lock);
- drhd->iommu = iommu;
-
- if (intel_iommu_enabled)
+ if (intel_iommu_enabled) {
iommu->iommu_dev = iommu_device_create(NULL, iommu,
intel_iommu_groups,
"%s", iommu->name);
+ if (IS_ERR(iommu->iommu_dev)) {
+ err = PTR_ERR(iommu->iommu_dev);
+ goto err_unmap;
+ }
+ }
+
+ drhd->iommu = iommu;
+
return 0;
err_unmap:
diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 5b1166d407c4..2fdbac67a77f 100644
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -295,7 +295,7 @@ static struct iommu_gather_ops ipmmu_gather_ops = {
static int ipmmu_domain_init_context(struct ipmmu_vmsa_domain *domain)
{
- phys_addr_t ttbr;
+ u64 ttbr;
/*
* Allocate the page table operations.
diff --git a/drivers/iommu/msm_iommu_dev.c b/drivers/iommu/msm_iommu_dev.c
index b6d01f97e537..4b09e815accf 100644
--- a/drivers/iommu/msm_iommu_dev.c
+++ b/drivers/iommu/msm_iommu_dev.c
@@ -359,30 +359,19 @@ static struct platform_driver msm_iommu_ctx_driver = {
.remove = msm_iommu_ctx_remove,
};
+static struct platform_driver * const drivers[] = {
+ &msm_iommu_driver,
+ &msm_iommu_ctx_driver,
+};
+
static int __init msm_iommu_driver_init(void)
{
- int ret;
- ret = platform_driver_register(&msm_iommu_driver);
- if (ret != 0) {
- pr_err("Failed to register IOMMU driver\n");
- goto error;
- }
-
- ret = platform_driver_register(&msm_iommu_ctx_driver);
- if (ret != 0) {
- platform_driver_unregister(&msm_iommu_driver);
- pr_err("Failed to register IOMMU context driver\n");
- goto error;
- }
-
-error:
- return ret;
+ return platform_register_drivers(drivers, ARRAY_SIZE(drivers));
}
static void __exit msm_iommu_driver_exit(void)
{
- platform_driver_unregister(&msm_iommu_ctx_driver);
- platform_driver_unregister(&msm_iommu_driver);
+ platform_unregister_drivers(drivers, ARRAY_SIZE(drivers));
}
subsys_initcall(msm_iommu_driver_init);
diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 471ee36b9c6e..a04d491cf431 100644
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -49,7 +49,7 @@ static bool s390_iommu_capable(enum iommu_cap cap)
}
}
-struct iommu_domain *s390_domain_alloc(unsigned domain_type)
+static struct iommu_domain *s390_domain_alloc(unsigned domain_type)
{
struct s390_domain *s390_domain;
@@ -73,7 +73,7 @@ struct iommu_domain *s390_domain_alloc(unsigned domain_type)
return &s390_domain->domain;
}
-void s390_domain_free(struct iommu_domain *domain)
+static void s390_domain_free(struct iommu_domain *domain)
{
struct s390_domain *s390_domain = to_s390_domain(domain);
diff --git a/drivers/iommu/shmobile-iommu.c b/drivers/iommu/shmobile-iommu.c
deleted file mode 100644
index a0287519a1d4..000000000000
--- a/drivers/iommu/shmobile-iommu.c
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * IOMMU for IPMMU/IPMMUI
- * Copyright (C) 2012 Hideki EIRAKU
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- */
-
-#include <linux/dma-mapping.h>
-#include <linux/io.h>
-#include <linux/iommu.h>
-#include <linux/platform_device.h>
-#include <linux/sizes.h>
-#include <linux/slab.h>
-#include <asm/dma-iommu.h>
-#include "shmobile-ipmmu.h"
-
-#define L1_SIZE CONFIG_SHMOBILE_IOMMU_L1SIZE
-#define L1_LEN (L1_SIZE / 4)
-#define L1_ALIGN L1_SIZE
-#define L2_SIZE SZ_1K
-#define L2_LEN (L2_SIZE / 4)
-#define L2_ALIGN L2_SIZE
-
-struct shmobile_iommu_domain_pgtable {
- uint32_t *pgtable;
- dma_addr_t handle;
-};
-
-struct shmobile_iommu_archdata {
- struct list_head attached_list;
- struct dma_iommu_mapping *iommu_mapping;
- spinlock_t attach_lock;
- struct shmobile_iommu_domain *attached;
- int num_attached_devices;
- struct shmobile_ipmmu *ipmmu;
-};
-
-struct shmobile_iommu_domain {
- struct shmobile_iommu_domain_pgtable l1, l2[L1_LEN];
- spinlock_t map_lock;
- spinlock_t attached_list_lock;
- struct list_head attached_list;
- struct iommu_domain domain;
-};
-
-static struct shmobile_iommu_archdata *ipmmu_archdata;
-static struct kmem_cache *l1cache, *l2cache;
-
-static struct shmobile_iommu_domain *to_sh_domain(struct iommu_domain *dom)
-{
- return container_of(dom, struct shmobile_iommu_domain, domain);
-}
-
-static int pgtable_alloc(struct shmobile_iommu_domain_pgtable *pgtable,
- struct kmem_cache *cache, size_t size)
-{
- pgtable->pgtable = kmem_cache_zalloc(cache, GFP_ATOMIC);
- if (!pgtable->pgtable)
- return -ENOMEM;
- pgtable->handle = dma_map_single(NULL, pgtable->pgtable, size,
- DMA_TO_DEVICE);
- return 0;
-}
-
-static void pgtable_free(struct shmobile_iommu_domain_pgtable *pgtable,
- struct kmem_cache *cache, size_t size)
-{
- dma_unmap_single(NULL, pgtable->handle, size, DMA_TO_DEVICE);
- kmem_cache_free(cache, pgtable->pgtable);
-}
-
-static uint32_t pgtable_read(struct shmobile_iommu_domain_pgtable *pgtable,
- unsigned int index)
-{
- return pgtable->pgtable[index];
-}
-
-static void pgtable_write(struct shmobile_iommu_domain_pgtable *pgtable,
- unsigned int index, unsigned int count, uint32_t val)
-{
- unsigned int i;
-
- for (i = 0; i < count; i++)
- pgtable->pgtable[index + i] = val;
- dma_sync_single_for_device(NULL, pgtable->handle + index * sizeof(val),
- sizeof(val) * count, DMA_TO_DEVICE);
-}
-
-static struct iommu_domain *shmobile_iommu_domain_alloc(unsigned type)
-{
- struct shmobile_iommu_domain *sh_domain;
- int i, ret;
-
- if (type != IOMMU_DOMAIN_UNMANAGED)
- return NULL;
-
- sh_domain = kzalloc(sizeof(*sh_domain), GFP_KERNEL);
- if (!sh_domain)
- return NULL;
- ret = pgtable_alloc(&sh_domain->l1, l1cache, L1_SIZE);
- if (ret < 0) {
- kfree(sh_domain);
- return NULL;
- }
- for (i = 0; i < L1_LEN; i++)
- sh_domain->l2[i].pgtable = NULL;
- spin_lock_init(&sh_domain->map_lock);
- spin_lock_init(&sh_domain->attached_list_lock);
- INIT_LIST_HEAD(&sh_domain->attached_list);
- return &sh_domain->domain;
-}
-
-static void shmobile_iommu_domain_free(struct iommu_domain *domain)
-{
- struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
- int i;
-
- for (i = 0; i < L1_LEN; i++) {
- if (sh_domain->l2[i].pgtable)
- pgtable_free(&sh_domain->l2[i], l2cache, L2_SIZE);
- }
- pgtable_free(&sh_domain->l1, l1cache, L1_SIZE);
- kfree(sh_domain);
-}
-
-static int shmobile_iommu_attach_device(struct iommu_domain *domain,
- struct device *dev)
-{
- struct shmobile_iommu_archdata *archdata = dev->archdata.iommu;
- struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
- int ret = -EBUSY;
-
- if (!archdata)
- return -ENODEV;
- spin_lock(&sh_domain->attached_list_lock);
- spin_lock(&archdata->attach_lock);
- if (archdata->attached != sh_domain) {
- if (archdata->attached)
- goto err;
- ipmmu_tlb_set(archdata->ipmmu, sh_domain->l1.handle, L1_SIZE,
- 0);
- ipmmu_tlb_flush(archdata->ipmmu);
- archdata->attached = sh_domain;
- archdata->num_attached_devices = 0;
- list_add(&archdata->attached_list, &sh_domain->attached_list);
- }
- archdata->num_attached_devices++;
- ret = 0;
-err:
- spin_unlock(&archdata->attach_lock);
- spin_unlock(&sh_domain->attached_list_lock);
- return ret;
-}
-
-static void shmobile_iommu_detach_device(struct iommu_domain *domain,
- struct device *dev)
-{
- struct shmobile_iommu_archdata *archdata = dev->archdata.iommu;
- struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
-
- if (!archdata)
- return;
- spin_lock(&sh_domain->attached_list_lock);
- spin_lock(&archdata->attach_lock);
- archdata->num_attached_devices--;
- if (!archdata->num_attached_devices) {
- ipmmu_tlb_set(archdata->ipmmu, 0, 0, 0);
- ipmmu_tlb_flush(archdata->ipmmu);
- archdata->attached = NULL;
- list_del(&archdata->attached_list);
- }
- spin_unlock(&archdata->attach_lock);
- spin_unlock(&sh_domain->attached_list_lock);
-}
-
-static void domain_tlb_flush(struct shmobile_iommu_domain *sh_domain)
-{
- struct shmobile_iommu_archdata *archdata;
-
- spin_lock(&sh_domain->attached_list_lock);
- list_for_each_entry(archdata, &sh_domain->attached_list, attached_list)
- ipmmu_tlb_flush(archdata->ipmmu);
- spin_unlock(&sh_domain->attached_list_lock);
-}
-
-static int l2alloc(struct shmobile_iommu_domain *sh_domain,
- unsigned int l1index)
-{
- int ret;
-
- if (!sh_domain->l2[l1index].pgtable) {
- ret = pgtable_alloc(&sh_domain->l2[l1index], l2cache, L2_SIZE);
- if (ret < 0)
- return ret;
- }
- pgtable_write(&sh_domain->l1, l1index, 1,
- sh_domain->l2[l1index].handle | 0x1);
- return 0;
-}
-
-static void l2realfree(struct shmobile_iommu_domain_pgtable *l2)
-{
- if (l2->pgtable)
- pgtable_free(l2, l2cache, L2_SIZE);
-}
-
-static void l2free(struct shmobile_iommu_domain *sh_domain,
- unsigned int l1index,
- struct shmobile_iommu_domain_pgtable *l2)
-{
- pgtable_write(&sh_domain->l1, l1index, 1, 0);
- if (sh_domain->l2[l1index].pgtable) {
- *l2 = sh_domain->l2[l1index];
- sh_domain->l2[l1index].pgtable = NULL;
- }
-}
-
-static int shmobile_iommu_map(struct iommu_domain *domain, unsigned long iova,
- phys_addr_t paddr, size_t size, int prot)
-{
- struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL };
- struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
- unsigned int l1index, l2index;
- int ret;
-
- l1index = iova >> 20;
- switch (size) {
- case SZ_4K:
- l2index = (iova >> 12) & 0xff;
- spin_lock(&sh_domain->map_lock);
- ret = l2alloc(sh_domain, l1index);
- if (!ret)
- pgtable_write(&sh_domain->l2[l1index], l2index, 1,
- paddr | 0xff2);
- spin_unlock(&sh_domain->map_lock);
- break;
- case SZ_64K:
- l2index = (iova >> 12) & 0xf0;
- spin_lock(&sh_domain->map_lock);
- ret = l2alloc(sh_domain, l1index);
- if (!ret)
- pgtable_write(&sh_domain->l2[l1index], l2index, 0x10,
- paddr | 0xff1);
- spin_unlock(&sh_domain->map_lock);
- break;
- case SZ_1M:
- spin_lock(&sh_domain->map_lock);
- l2free(sh_domain, l1index, &l2);
- pgtable_write(&sh_domain->l1, l1index, 1, paddr | 0xc02);
- spin_unlock(&sh_domain->map_lock);
- ret = 0;
- break;
- default:
- ret = -EINVAL;
- }
- if (!ret)
- domain_tlb_flush(sh_domain);
- l2realfree(&l2);
- return ret;
-}
-
-static size_t shmobile_iommu_unmap(struct iommu_domain *domain,
- unsigned long iova, size_t size)
-{
- struct shmobile_iommu_domain_pgtable l2 = { .pgtable = NULL };
- struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
- unsigned int l1index, l2index;
- uint32_t l2entry = 0;
- size_t ret = 0;
-
- l1index = iova >> 20;
- if (!(iova & 0xfffff) && size >= SZ_1M) {
- spin_lock(&sh_domain->map_lock);
- l2free(sh_domain, l1index, &l2);
- spin_unlock(&sh_domain->map_lock);
- ret = SZ_1M;
- goto done;
- }
- l2index = (iova >> 12) & 0xff;
- spin_lock(&sh_domain->map_lock);
- if (sh_domain->l2[l1index].pgtable)
- l2entry = pgtable_read(&sh_domain->l2[l1index], l2index);
- switch (l2entry & 3) {
- case 1:
- if (l2index & 0xf)
- break;
- pgtable_write(&sh_domain->l2[l1index], l2index, 0x10, 0);
- ret = SZ_64K;
- break;
- case 2:
- pgtable_write(&sh_domain->l2[l1index], l2index, 1, 0);
- ret = SZ_4K;
- break;
- }
- spin_unlock(&sh_domain->map_lock);
-done:
- if (ret)
- domain_tlb_flush(sh_domain);
- l2realfree(&l2);
- return ret;
-}
-
-static phys_addr_t shmobile_iommu_iova_to_phys(struct iommu_domain *domain,
- dma_addr_t iova)
-{
- struct shmobile_iommu_domain *sh_domain = to_sh_domain(domain);
- uint32_t l1entry = 0, l2entry = 0;
- unsigned int l1index, l2index;
-
- l1index = iova >> 20;
- l2index = (iova >> 12) & 0xff;
- spin_lock(&sh_domain->map_lock);
- if (sh_domain->l2[l1index].pgtable)
- l2entry = pgtable_read(&sh_domain->l2[l1index], l2index);
- else
- l1entry = pgtable_read(&sh_domain->l1, l1index);
- spin_unlock(&sh_domain->map_lock);
- switch (l2entry & 3) {
- case 1:
- return (l2entry & ~0xffff) | (iova & 0xffff);
- case 2:
- return (l2entry & ~0xfff) | (iova & 0xfff);
- default:
- if ((l1entry & 3) == 2)
- return (l1entry & ~0xfffff) | (iova & 0xfffff);
- return 0;
- }
-}
-
-static int find_dev_name(struct shmobile_ipmmu *ipmmu, const char *dev_name)
-{
- unsigned int i, n = ipmmu->num_dev_names;
-
- for (i = 0; i < n; i++) {
- if (strcmp(ipmmu->dev_names[i], dev_name) == 0)
- return 1;
- }
- return 0;
-}
-
-static int shmobile_iommu_add_device(struct device *dev)
-{
- struct shmobile_iommu_archdata *archdata = ipmmu_archdata;
- struct dma_iommu_mapping *mapping;
-
- if (!find_dev_name(archdata->ipmmu, dev_name(dev)))
- return 0;
- mapping = archdata->iommu_mapping;
- if (!mapping) {
- mapping = arm_iommu_create_mapping(&platform_bus_type, 0,
- L1_LEN << 20);
- if (IS_ERR(mapping))
- return PTR_ERR(mapping);
- archdata->iommu_mapping = mapping;
- }
- dev->archdata.iommu = archdata;
- if (arm_iommu_attach_device(dev, mapping))
- pr_err("arm_iommu_attach_device failed\n");
- return 0;
-}
-
-static const struct iommu_ops shmobile_iommu_ops = {
- .domain_alloc = shmobile_iommu_domain_alloc,
- .domain_free = shmobile_iommu_domain_free,
- .attach_dev = shmobile_iommu_attach_device,
- .detach_dev = shmobile_iommu_detach_device,
- .map = shmobile_iommu_map,
- .unmap = shmobile_iommu_unmap,
- .map_sg = default_iommu_map_sg,
- .iova_to_phys = shmobile_iommu_iova_to_phys,
- .add_device = shmobile_iommu_add_device,
- .pgsize_bitmap = SZ_1M | SZ_64K | SZ_4K,
-};
-
-int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu)
-{
- static struct shmobile_iommu_archdata *archdata;
-
- l1cache = kmem_cache_create("shmobile-iommu-pgtable1", L1_SIZE,
- L1_ALIGN, SLAB_HWCACHE_ALIGN, NULL);
- if (!l1cache)
- return -ENOMEM;
- l2cache = kmem_cache_create("shmobile-iommu-pgtable2", L2_SIZE,
- L2_ALIGN, SLAB_HWCACHE_ALIGN, NULL);
- if (!l2cache) {
- kmem_cache_destroy(l1cache);
- return -ENOMEM;
- }
- archdata = kzalloc(sizeof(*archdata), GFP_KERNEL);
- if (!archdata) {
- kmem_cache_destroy(l1cache);
- kmem_cache_destroy(l2cache);
- return -ENOMEM;
- }
- spin_lock_init(&archdata->attach_lock);
- archdata->ipmmu = ipmmu;
- ipmmu_archdata = archdata;
- bus_set_iommu(&platform_bus_type, &shmobile_iommu_ops);
- return 0;
-}
diff --git a/drivers/iommu/shmobile-ipmmu.c b/drivers/iommu/shmobile-ipmmu.c
deleted file mode 100644
index 951651a9746b..000000000000
--- a/drivers/iommu/shmobile-ipmmu.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * IPMMU/IPMMUI
- * Copyright (C) 2012 Hideki EIRAKU
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- */
-
-#include <linux/err.h>
-#include <linux/export.h>
-#include <linux/io.h>
-#include <linux/platform_device.h>
-#include <linux/slab.h>
-#include <linux/platform_data/sh_ipmmu.h>
-#include "shmobile-ipmmu.h"
-
-#define IMCTR1 0x000
-#define IMCTR2 0x004
-#define IMASID 0x010
-#define IMTTBR 0x014
-#define IMTTBCR 0x018
-
-#define IMCTR1_TLBEN (1 << 0)
-#define IMCTR1_FLUSH (1 << 1)
-
-static void ipmmu_reg_write(struct shmobile_ipmmu *ipmmu, unsigned long reg_off,
- unsigned long data)
-{
- iowrite32(data, ipmmu->ipmmu_base + reg_off);
-}
-
-void ipmmu_tlb_flush(struct shmobile_ipmmu *ipmmu)
-{
- if (!ipmmu)
- return;
-
- spin_lock(&ipmmu->flush_lock);
- if (ipmmu->tlb_enabled)
- ipmmu_reg_write(ipmmu, IMCTR1, IMCTR1_FLUSH | IMCTR1_TLBEN);
- else
- ipmmu_reg_write(ipmmu, IMCTR1, IMCTR1_FLUSH);
- spin_unlock(&ipmmu->flush_lock);
-}
-
-void ipmmu_tlb_set(struct shmobile_ipmmu *ipmmu, unsigned long phys, int size,
- int asid)
-{
- if (!ipmmu)
- return;
-
- spin_lock(&ipmmu->flush_lock);
- switch (size) {
- default:
- ipmmu->tlb_enabled = 0;
- break;
- case 0x2000:
- ipmmu_reg_write(ipmmu, IMTTBCR, 1);
- ipmmu->tlb_enabled = 1;
- break;
- case 0x1000:
- ipmmu_reg_write(ipmmu, IMTTBCR, 2);
- ipmmu->tlb_enabled = 1;
- break;
- case 0x800:
- ipmmu_reg_write(ipmmu, IMTTBCR, 3);
- ipmmu->tlb_enabled = 1;
- break;
- case 0x400:
- ipmmu_reg_write(ipmmu, IMTTBCR, 4);
- ipmmu->tlb_enabled = 1;
- break;
- case 0x200:
- ipmmu_reg_write(ipmmu, IMTTBCR, 5);
- ipmmu->tlb_enabled = 1;
- break;
- case 0x100:
- ipmmu_reg_write(ipmmu, IMTTBCR, 6);
- ipmmu->tlb_enabled = 1;
- break;
- case 0x80:
- ipmmu_reg_write(ipmmu, IMTTBCR, 7);
- ipmmu->tlb_enabled = 1;
- break;
- }
- ipmmu_reg_write(ipmmu, IMTTBR, phys);
- ipmmu_reg_write(ipmmu, IMASID, asid);
- spin_unlock(&ipmmu->flush_lock);
-}
-
-static int ipmmu_probe(struct platform_device *pdev)
-{
- struct shmobile_ipmmu *ipmmu;
- struct resource *res;
- struct shmobile_ipmmu_platform_data *pdata = pdev->dev.platform_data;
-
- ipmmu = devm_kzalloc(&pdev->dev, sizeof(*ipmmu), GFP_KERNEL);
- if (!ipmmu) {
- dev_err(&pdev->dev, "cannot allocate device data\n");
- return -ENOMEM;
- }
- spin_lock_init(&ipmmu->flush_lock);
- ipmmu->dev = &pdev->dev;
-
- res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
- ipmmu->ipmmu_base = devm_ioremap_resource(&pdev->dev, res);
- if (IS_ERR(ipmmu->ipmmu_base))
- return PTR_ERR(ipmmu->ipmmu_base);
-
- ipmmu->dev_names = pdata->dev_names;
- ipmmu->num_dev_names = pdata->num_dev_names;
- platform_set_drvdata(pdev, ipmmu);
- ipmmu_reg_write(ipmmu, IMCTR1, 0x0); /* disable TLB */
- ipmmu_reg_write(ipmmu, IMCTR2, 0x0); /* disable PMB */
- return ipmmu_iommu_init(ipmmu);
-}
-
-static struct platform_driver ipmmu_driver = {
- .probe = ipmmu_probe,
- .driver = {
- .name = "ipmmu",
- },
-};
-
-static int __init ipmmu_init(void)
-{
- return platform_driver_register(&ipmmu_driver);
-}
-subsys_initcall(ipmmu_init);
diff --git a/drivers/iommu/shmobile-ipmmu.h b/drivers/iommu/shmobile-ipmmu.h
deleted file mode 100644
index 9524743ca1fb..000000000000
--- a/drivers/iommu/shmobile-ipmmu.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* shmobile-ipmmu.h
- *
- * Copyright (C) 2012 Hideki EIRAKU
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- */
-
-#ifndef __SHMOBILE_IPMMU_H__
-#define __SHMOBILE_IPMMU_H__
-
-struct shmobile_ipmmu {
- struct device *dev;
- void __iomem *ipmmu_base;
- int tlb_enabled;
- spinlock_t flush_lock;
- const char * const *dev_names;
- unsigned int num_dev_names;
-};
-
-#ifdef CONFIG_SHMOBILE_IPMMU_TLB
-void ipmmu_tlb_flush(struct shmobile_ipmmu *ipmmu);
-void ipmmu_tlb_set(struct shmobile_ipmmu *ipmmu, unsigned long phys, int size,
- int asid);
-int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu);
-#else
-static inline int ipmmu_iommu_init(struct shmobile_ipmmu *ipmmu)
-{
- return -EINVAL;
-}
-#endif
-
-#endif /* __SHMOBILE_IPMMU_H__ */
generated by cgit v1.2.3 (git 2.0.4) at 2024-11-18 23:44:17 +0000