diff options
Diffstat (limited to 'kernel/dma/mapping.c')
| -rw-r--r-- | kernel/dma/mapping.c | 93 |
1 files changed, 62 insertions, 31 deletions
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 37163eb49f9f..bf2651a70b7c 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -126,11 +126,9 @@ static bool dma_go_direct(struct device *dev, dma_addr_t mask, if (likely(!ops)) return true; -#ifdef CONFIG_DMA_OPS_BYPASS - if (dev->dma_ops_bypass) + if (IS_ENABLED(CONFIG_DMA_OPS_BYPASS) && dev_dma_ops_bypass(dev)) return min_not_zero(mask, dev->bus_dma_limit) >= dma_direct_get_required_mask(dev); -#endif return false; } @@ -157,6 +155,7 @@ dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, { const struct dma_map_ops *ops = get_dma_ops(dev); bool is_mmio = attrs & DMA_ATTR_MMIO; + bool is_cc_shared = attrs & DMA_ATTR_CC_SHARED; dma_addr_t addr = DMA_MAPPING_ERROR; BUG_ON(!valid_dma_direction(dir)); @@ -164,9 +163,15 @@ dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, if (WARN_ON_ONCE(!dev->dma_mask)) return DMA_MAPPING_ERROR; + if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT)) + return DMA_MAPPING_ERROR; + if (dma_map_direct(dev, ops) || - (!is_mmio && arch_dma_map_phys_direct(dev, phys + size))) - addr = dma_direct_map_phys(dev, phys, size, dir, attrs); + (!is_mmio && !is_cc_shared && + arch_dma_map_phys_direct(dev, phys + size))) + addr = dma_direct_map_phys(dev, phys, size, dir, attrs, true); + else if (is_cc_shared) + return DMA_MAPPING_ERROR; else if (use_dma_iommu(dev)) addr = iommu_dma_map_phys(dev, phys, size, dir, attrs); else if (ops->map_phys) @@ -203,17 +208,22 @@ void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, { const struct dma_map_ops *ops = get_dma_ops(dev); bool is_mmio = attrs & DMA_ATTR_MMIO; + bool is_cc_shared = attrs & DMA_ATTR_CC_SHARED; BUG_ON(!valid_dma_direction(dir)); + if (dma_map_direct(dev, ops) || - (!is_mmio && arch_dma_unmap_phys_direct(dev, addr + size))) - dma_direct_unmap_phys(dev, addr, size, dir, attrs); + (!is_mmio && !is_cc_shared && + arch_dma_unmap_phys_direct(dev, addr + size))) + dma_direct_unmap_phys(dev, addr, size, dir, attrs, true); + else if (is_cc_shared) + return; else if (use_dma_iommu(dev)) iommu_dma_unmap_phys(dev, addr, size, dir, attrs); else if (ops->unmap_phys) ops->unmap_phys(dev, addr, size, dir, attrs); trace_dma_unmap_phys(dev, addr, size, dir, attrs); - debug_dma_unmap_phys(dev, addr, size, dir); + debug_dma_unmap_phys(dev, addr, size, dir, attrs); } EXPORT_SYMBOL_GPL(dma_unmap_phys); @@ -235,6 +245,9 @@ static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, BUG_ON(!valid_dma_direction(dir)); + if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT)) + return -EOPNOTSUPP; + if (WARN_ON_ONCE(!dev->dma_mask)) return 0; @@ -336,7 +349,7 @@ void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, BUG_ON(!valid_dma_direction(dir)); trace_dma_unmap_sg(dev, sg, nents, dir, attrs); - debug_dma_unmap_sg(dev, sg, nents, dir); + debug_dma_unmap_sg(dev, sg, nents, dir, attrs); if (dma_map_direct(dev, ops) || arch_dma_unmap_sg_direct(dev, sg, nents)) dma_direct_unmap_sg(dev, sg, nents, dir, attrs); @@ -350,10 +363,6 @@ EXPORT_SYMBOL(dma_unmap_sg_attrs); dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { - if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && - WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr)))) - return DMA_MAPPING_ERROR; - return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO); } EXPORT_SYMBOL(dma_map_resource); @@ -373,7 +382,7 @@ void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, BUG_ON(!valid_dma_direction(dir)); if (dma_map_direct(dev, ops)) - dma_direct_sync_single_for_cpu(dev, addr, size, dir); + dma_direct_sync_single_for_cpu(dev, addr, size, dir, true); else if (use_dma_iommu(dev)) iommu_dma_sync_single_for_cpu(dev, addr, size, dir); else if (ops->sync_single_for_cpu) @@ -461,7 +470,7 @@ bool dma_need_unmap(struct device *dev) { if (!dma_map_direct(dev, get_dma_ops(dev))) return true; - if (!dev->dma_skip_sync) + if (!dev_dma_skip_sync(dev)) return true; return IS_ENABLED(CONFIG_DMA_API_DEBUG); } @@ -477,16 +486,16 @@ static void dma_setup_need_sync(struct device *dev) * mapping, if any. During the device initialization, it's * enough to check only for the DMA coherence. */ - dev->dma_skip_sync = dev_is_dma_coherent(dev); + dev_assign_dma_skip_sync(dev, dev_is_dma_coherent(dev)); else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu && !ops->sync_sg_for_device && !ops->sync_sg_for_cpu) /* * Synchronization is not possible when none of DMA sync ops * is set. */ - dev->dma_skip_sync = true; + dev_set_dma_skip_sync(dev); else - dev->dma_skip_sync = false; + dev_clear_dma_skip_sync(dev); } #else /* !CONFIG_DMA_NEED_SYNC */ static inline void dma_setup_need_sync(struct device *dev) { } @@ -528,13 +537,21 @@ EXPORT_SYMBOL(dma_get_sgtable_attrs); */ pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) { + pgprot_t dma_prot; + if (dev_is_dma_coherent(dev)) - return prot; + dma_prot = prot; #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE - if (attrs & DMA_ATTR_WRITE_COMBINE) - return pgprot_writecombine(prot); + else if (attrs & DMA_ATTR_WRITE_COMBINE) + dma_prot = pgprot_writecombine(prot); #endif - return pgprot_dmacoherent(prot); + else + dma_prot = pgprot_dmacoherent(prot); + + if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED)) + return pgprot_decrypted(dma_prot); + else + return pgprot_encrypted(dma_prot); } #endif /* CONFIG_MMU */ @@ -629,6 +646,15 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, if (WARN_ON_ONCE(flag & __GFP_COMP)) return NULL; + if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED)) { + trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag, + attrs); + return NULL; + } + + if (force_dma_unencrypted(dev)) + attrs |= __DMA_ATTR_ALLOC_CC_SHARED; + if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) { trace_dma_alloc(dev, cpu_addr, *dma_handle, size, DMA_BIDIRECTIONAL, flag, attrs); @@ -638,7 +664,7 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, /* let the implementation decide on the zone to allocate from: */ flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); - if (dma_alloc_direct(dev, ops)) { + if (dma_alloc_direct(dev, ops) || arch_dma_alloc_direct(dev)) { cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs); } else if (use_dma_iommu(dev)) { cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs); @@ -678,8 +704,8 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, if (!cpu_addr) return; - debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - if (dma_alloc_direct(dev, ops)) + debug_dma_free_coherent(dev, size, cpu_addr, dma_handle, attrs); + if (dma_alloc_direct(dev, ops) || arch_dma_free_direct(dev, dma_handle)) dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); else if (use_dma_iommu(dev)) iommu_dma_free(dev, size, cpu_addr, dma_handle, attrs); @@ -718,7 +744,7 @@ struct page *dma_alloc_pages(struct device *dev, size_t size, if (page) { trace_dma_alloc_pages(dev, page_to_virt(page), *dma_handle, size, dir, gfp, 0); - debug_dma_alloc_pages(dev, page, size, dir, *dma_handle, 0); + debug_dma_alloc_pages(dev, page, size, dir, *dma_handle); } else { trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0); } @@ -752,12 +778,14 @@ EXPORT_SYMBOL_GPL(dma_free_pages); int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma, size_t size, struct page *page) { - unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; + const pgoff_t pgoff_start = vma_start_pgoff(vma); + const pgoff_t pgoff_end = vma_end_pgoff(vma); + const unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; - if (vma->vm_pgoff >= count || vma_pages(vma) > count - vma->vm_pgoff) + if (pgoff_start >= count || pgoff_end > count) return -ENXIO; return remap_pfn_range(vma, vma->vm_start, - page_to_pfn(page) + vma->vm_pgoff, + page_to_pfn(page) + pgoff_start, vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot); } EXPORT_SYMBOL_GPL(dma_mmap_pages); @@ -768,7 +796,7 @@ static struct sg_table *alloc_single_sgt(struct device *dev, size_t size, struct sg_table *sgt; struct page *page; - sgt = kmalloc(sizeof(*sgt), gfp); + sgt = kmalloc_obj(*sgt, gfp); if (!sgt) return NULL; if (sg_alloc_table(sgt, 1, gfp)) @@ -825,7 +853,7 @@ void dma_free_noncontiguous(struct device *dev, size_t size, struct sg_table *sgt, enum dma_data_direction dir) { trace_dma_free_sgt(dev, sgt, size, dir); - debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); + debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir, 0); if (use_dma_iommu(dev)) iommu_dma_free_noncontiguous(dev, size, sgt, dir); @@ -970,6 +998,9 @@ size_t dma_max_mapping_size(struct device *dev) const struct dma_map_ops *ops = get_dma_ops(dev); size_t size = SIZE_MAX; + if (!dev->dma_mask) + return 0; + if (dma_map_direct(dev, ops)) size = dma_direct_max_mapping_size(dev); else if (use_dma_iommu(dev)) |