summaryrefslogtreecommitdiff
path: root/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'kernel')
-rw-r--r--kernel/cgroup/dmem.c4
-rw-r--r--kernel/dma/contiguous.c6
-rw-r--r--kernel/dma/debug.c7
-rw-r--r--kernel/dma/direct.c30
-rw-r--r--kernel/dma/mapping.c10
-rw-r--r--kernel/dma/ops_helpers.c11
-rw-r--r--kernel/dma/swiotlb.c54
7 files changed, 71 insertions, 51 deletions
diff --git a/kernel/cgroup/dmem.c b/kernel/cgroup/dmem.c
index 10e148d64e85..d49ed5d4a923 100644
--- a/kernel/cgroup/dmem.c
+++ b/kernel/cgroup/dmem.c
@@ -28,7 +28,7 @@ DEFINE_STATIC_SRCU(dmemcg_srcu);
struct dmem_cgroup_region {
/**
* @ref: References keeping the region alive.
- * Keeps the region reference alive after a succesful RCU lookup.
+ * Keeps the region reference alive after a successful RCU lookup.
*/
struct kref ref;
@@ -696,7 +696,7 @@ EXPORT_SYMBOL_GPL(dmem_cgroup_uncharge);
* dmem_cgroup_try_charge() - Try charging a new allocation to a region.
* @region: dmem region to charge
* @size: Size (in bytes) to charge.
- * @ret_pool: On succesfull allocation, the pool that is charged.
+ * @ret_pool: On successful allocation, the pool that is charged.
* @ret_limit_pool: On a failed allocation, the limiting pool.
*
* This function charges the @region region for a size of @size bytes.
diff --git a/kernel/dma/contiguous.c b/kernel/dma/contiguous.c
index 66093460584e..90937dc7c627 100644
--- a/kernel/dma/contiguous.c
+++ b/kernel/dma/contiguous.c
@@ -451,8 +451,8 @@ struct page *dma_alloc_contiguous(struct device *dev, size_t size, gfp_t gfp)
*
* This function releases memory allocated by dma_alloc_contiguous(). As the
* cma_release returns false when provided pages do not belong to contiguous
- * area and true otherwise, this function then does a fallback __free_pages()
- * upon a false-return.
+ * area and true otherwise, this function then does a fallback
+ * free_pages_exact() upon a false-return.
*/
void dma_free_contiguous(struct device *dev, struct page *page, size_t size)
{
@@ -476,7 +476,7 @@ void dma_free_contiguous(struct device *dev, struct page *page, size_t size)
}
/* not in any cma, free from buddy */
- __free_pages(page, get_order(size));
+ free_pages_exact(page_address(page), size);
}
/*
diff --git a/kernel/dma/debug.c b/kernel/dma/debug.c
index 026d50de340e..36c1f7cbb2ae 100644
--- a/kernel/dma/debug.c
+++ b/kernel/dma/debug.c
@@ -489,8 +489,10 @@ static int active_cacheline_insert(struct dma_debug_entry *entry,
/* If the device is not writing memory then we don't have any
* concerns about the cpu consuming stale data. This mitigates
* legitimate usages of overlapping mappings.
+ * MMIO mappings are never CPU-cacheable and do not need tracking either.
*/
- if (entry->direction == DMA_TO_DEVICE)
+ if (entry->direction == DMA_TO_DEVICE ||
+ entry->attrs & DMA_ATTR_MMIO)
return 0;
spin_lock_irqsave(&radix_lock, flags);
@@ -519,7 +521,8 @@ static void active_cacheline_remove(struct dma_debug_entry *entry)
unsigned long flags;
/* ...mirror the insert case */
- if (entry->direction == DMA_TO_DEVICE)
+ if (entry->direction == DMA_TO_DEVICE ||
+ entry->attrs & DMA_ATTR_MMIO)
return;
spin_lock_irqsave(&radix_lock, flags);
diff --git a/kernel/dma/direct.c b/kernel/dma/direct.c
index da665ca22d5c..25e9a86835fd 100644
--- a/kernel/dma/direct.c
+++ b/kernel/dma/direct.c
@@ -120,6 +120,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
int node = dev_to_node(dev);
struct page *page;
u64 phys_limit;
+ void *va;
WARN_ON_ONCE(!PAGE_ALIGNED(size));
@@ -133,9 +134,9 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
dma_free_contiguous(dev, page, size);
}
- while ((page = alloc_pages_node(node, gfp, get_order(size)))
- && !dma_coherent_ok(dev, page_to_phys(page), size)) {
- __free_pages(page, get_order(size));
+ while ((va = alloc_pages_exact_nid(node, size, gfp)) &&
+ !dma_coherent_ok(dev, virt_to_phys(va), size)) {
+ free_pages_exact(va, size);
if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
phys_limit < DMA_BIT_MASK(64) &&
@@ -146,6 +147,7 @@ static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,
else
return NULL;
}
+ page = va ? virt_to_page(va) : NULL;
return page;
}
@@ -179,6 +181,22 @@ static struct page *dma_direct_alloc_from_pool(struct device *dev, size_t size,
return page;
}
+/*
+ * Check if the allocation can skip the kernel mapping and return the page
+ * pointer as the cookie.
+ */
+static bool dma_direct_use_no_mapping(struct device *dev, unsigned long attrs)
+{
+ if ((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) !=
+ DMA_ATTR_NO_KERNEL_MAPPING)
+ return false;
+ if (is_swiotlb_for_alloc(dev))
+ return false;
+ if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_ALLOC) && !dev_is_dma_coherent(dev))
+ return false;
+ return true;
+}
+
static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp)
{
@@ -226,8 +244,7 @@ void *dma_direct_alloc(struct device *dev, size_t size,
if (attrs & DMA_ATTR_NO_WARN)
gfp |= __GFP_NOWARN;
- if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==
- DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev))
+ if (dma_direct_use_no_mapping(dev, attrs))
return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);
if (!dev_is_dma_coherent(dev)) {
@@ -374,8 +391,7 @@ void dma_direct_free(struct device *dev, size_t size,
if (attrs & __DMA_ATTR_ALLOC_CC_SHARED)
mark_mem_encrypted = true;
- if (((attrs & (DMA_ATTR_NO_KERNEL_MAPPING | __DMA_ATTR_ALLOC_CC_SHARED)) ==
- DMA_ATTR_NO_KERNEL_MAPPING) && !is_swiotlb_for_alloc(dev)) {
+ if (dma_direct_use_no_mapping(dev, attrs)) {
/* cpu_addr is a struct page cookie, not a kernel address */
dma_free_contiguous(dev, cpu_addr, size);
return;
diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c
index bf2651a70b7c..82e701648d60 100644
--- a/kernel/dma/mapping.c
+++ b/kernel/dma/mapping.c
@@ -1012,19 +1012,19 @@ size_t dma_max_mapping_size(struct device *dev)
}
EXPORT_SYMBOL_GPL(dma_max_mapping_size);
-size_t dma_opt_mapping_size(struct device *dev)
+size_t dma_max_opt_mapping_size(struct device *dev)
{
const struct dma_map_ops *ops = get_dma_ops(dev);
size_t size = SIZE_MAX;
if (use_dma_iommu(dev))
- size = iommu_dma_opt_mapping_size();
- else if (ops && ops->opt_mapping_size)
- size = ops->opt_mapping_size();
+ size = iommu_dma_max_opt_mapping_size();
+ else if (ops && ops->max_opt_mapping_size)
+ size = ops->max_opt_mapping_size();
return min(dma_max_mapping_size(dev), size);
}
-EXPORT_SYMBOL_GPL(dma_opt_mapping_size);
+EXPORT_SYMBOL_GPL(dma_max_opt_mapping_size);
unsigned long dma_get_merge_boundary(struct device *dev)
{
diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c
index 6b5f9208d31c..5cdf3f60cf47 100644
--- a/kernel/dma/ops_helpers.c
+++ b/kernel/dma/ops_helpers.c
@@ -67,10 +67,13 @@ struct page *dma_common_alloc_pages(struct device *dev, size_t size,
phys_addr_t phys;
page = dma_alloc_contiguous(dev, size, gfp);
- if (!page)
- page = alloc_pages_node(dev_to_node(dev), gfp, get_order(size));
- if (!page)
- return NULL;
+ if (!page) {
+ void *va = alloc_pages_exact_nid(dev_to_node(dev), size, gfp);
+
+ if (!va)
+ return NULL;
+ page = virt_to_page(va);
+ }
phys = page_to_phys(page);
if (use_dma_iommu(dev))
diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c
index 5edea93ec635..2c57b49ac7c0 100644
--- a/kernel/dma/swiotlb.c
+++ b/kernel/dma/swiotlb.c
@@ -100,6 +100,7 @@ static struct io_tlb_mem io_tlb_default_mem;
static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
static unsigned long default_nareas;
+static bool swiotlb_cmdline_size_set __initdata;
/**
* struct io_tlb_area - IO TLB memory area descriptor
@@ -117,27 +118,20 @@ struct io_tlb_area {
spinlock_t lock;
};
-/*
- * Round up number of slabs to the next power of 2. The last area is going
- * be smaller than the rest if default_nslabs is not power of two.
- * The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
- * otherwise a segment may span two or more areas. It conflicts with free
- * contiguous slots tracking: free slots are treated contiguous no matter
- * whether they cross an area boundary.
- *
- * Return true if default_nslabs is rounded up.
- */
-static bool round_up_default_nslabs(void)
+/* Return a power-of-two number of slabs that can be split between areas. */
+static unsigned long swiotlb_calc_nslabs(unsigned long size,
+ unsigned long nareas)
{
- if (!default_nareas)
- return false;
+ unsigned long nslabs;
- if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
- default_nslabs = IO_TLB_SEGSIZE * default_nareas;
- else if (is_power_of_2(default_nslabs))
- return false;
- default_nslabs = roundup_pow_of_two(default_nslabs);
- return true;
+ nslabs = ALIGN(DIV_ROUND_UP(size, IO_TLB_SIZE), IO_TLB_SEGSIZE);
+ if (nareas && nslabs < IO_TLB_SEGSIZE * nareas)
+ nslabs = IO_TLB_SEGSIZE * nareas;
+
+ if (!is_power_of_2(nslabs))
+ nslabs = roundup_pow_of_two(nslabs);
+
+ return nslabs;
}
/**
@@ -150,6 +144,8 @@ static bool round_up_default_nslabs(void)
*/
static void swiotlb_adjust_nareas(unsigned int nareas)
{
+ unsigned long nslabs;
+
if (!nareas)
nareas = 1;
else if (!is_power_of_2(nareas))
@@ -158,9 +154,12 @@ static void swiotlb_adjust_nareas(unsigned int nareas)
default_nareas = nareas;
pr_info("area num %d.\n", nareas);
- if (round_up_default_nslabs())
+ nslabs = swiotlb_calc_nslabs(default_nslabs << IO_TLB_SHIFT, nareas);
+ if (nslabs != default_nslabs) {
+ default_nslabs = nslabs;
pr_info("SWIOTLB bounce buffer size roundup to %luMB",
(default_nslabs << IO_TLB_SHIFT) >> 20);
+ }
}
/**
@@ -255,6 +254,7 @@ setup_io_tlb_npages(char *str)
/* avoid tail segment of size < IO_TLB_SEGSIZE */
default_nslabs =
ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
+ swiotlb_cmdline_size_set = true;
}
if (*str == ',')
++str;
@@ -285,7 +285,7 @@ setup_io_tlb_npages(char *str)
}
early_param("swiotlb", setup_io_tlb_npages);
-unsigned long swiotlb_size_or_default(void)
+unsigned long swiotlb_default_pool_size(void)
{
return default_nslabs << IO_TLB_SHIFT;
}
@@ -297,13 +297,11 @@ void __init swiotlb_adjust_size(unsigned long size)
* architectures such as those supporting memory encryption to
* adjust/expand SWIOTLB size for their use.
*/
- if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
+ if (swiotlb_cmdline_size_set)
return;
- size = ALIGN(size, IO_TLB_SIZE);
- default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
- if (round_up_default_nslabs())
- size = default_nslabs << IO_TLB_SHIFT;
+ default_nslabs = swiotlb_calc_nslabs(size, default_nareas);
+ size = default_nslabs << IO_TLB_SHIFT;
pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
}
@@ -2083,8 +2081,8 @@ static int __init rmem_swiotlb_setup(unsigned long node,
of_get_flat_dt_prop(node, "no-map", NULL))
return -EINVAL;
- pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
- &rmem->base, (unsigned long)rmem->size / SZ_1M);
+ pr_info("Reserved memory: created restricted DMA pool at %pa, size %llu KiB\n",
+ &rmem->base, (unsigned long long)(rmem->size / SZ_1K));
return 0;
}
generated by cgit v1.2.3 (git 2.0.4) at 2026-10-24 22:49:35 +0000