summaryrefslogtreecommitdiff
path: root/mm
diff options
context:
space:
mode:
Diffstat (limited to 'mm')
-rw-r--r--mm/Kconfig11
-rw-r--r--mm/damon/core.c55
-rw-r--r--mm/filemap.c5
-rw-r--r--mm/gup_test.c12
-rw-r--r--mm/huge_memory.c50
-rw-r--r--mm/khugepaged.c734
-rw-r--r--mm/memory.c4
-rw-r--r--mm/page_alloc.c18
-rw-r--r--mm/page_isolation.c19
-rw-r--r--mm/readahead.c6
-rw-r--r--mm/shrinker_debug.c4
-rw-r--r--mm/truncate.c8
-rw-r--r--mm/vmscan.c69
-rw-r--r--mm/zsmalloc.c3
14 files changed, 681 insertions, 317 deletions
diff --git a/mm/Kconfig b/mm/Kconfig
index 210770ba2b26..9e0ca4824905 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -989,17 +989,6 @@ config THP_SWAP
For selection by architectures with reasonable THP sizes.
-config READ_ONLY_THP_FOR_FS
- bool "Read-only THP for filesystems (EXPERIMENTAL)"
- depends on TRANSPARENT_HUGEPAGE
-
- help
- Allow khugepaged to put read-only file-backed pages in THP.
-
- This is marked experimental because it is a new feature. Write
- support of file THPs will be developed in the next few release
- cycles.
-
config NO_PAGE_MAPCOUNT
bool "No per-page mapcount (EXPERIMENTAL)"
help
diff --git a/mm/damon/core.c b/mm/damon/core.c
index 265d51ade25b..7e4b9affc5b0 100644
--- a/mm/damon/core.c
+++ b/mm/damon/core.c
@@ -1387,10 +1387,36 @@ static int damon_commit_target(
return 0;
}
+/*
+ * damon_revert_target_commits() - revert unsuccessful target commits.
+ * @dst: Commit destination context
+ * @failed: Commit failed destination target
+ * @src: Commit source context
+ *
+ * Revert target states that changed by damon_commit_target(), and cannot be
+ * cleaned up by the destination context's ops.cleanup_target().
+ */
+static void damon_revert_target_commits(struct damon_ctx *dst,
+ struct damon_target *failed, struct damon_ctx *src)
+{
+ struct damon_target *target;
+
+ if (!damon_target_has_pid(src))
+ return;
+ if (dst->ops.cleanup_target)
+ return;
+ damon_for_each_target(target, dst) {
+ if (target == failed)
+ return;
+ put_pid(target->pid);
+ }
+}
+
static int damon_commit_targets(
struct damon_ctx *dst, struct damon_ctx *src)
{
struct damon_target *dst_target, *next, *src_target, *new_target;
+ struct damon_target *failed;
int i = 0, j = 0, err;
damon_for_each_target_safe(dst_target, next, dst) {
@@ -1404,8 +1430,10 @@ static int damon_commit_targets(
dst_target, damon_target_has_pid(dst),
src_target, damon_target_has_pid(src),
src->min_region_sz);
- if (err)
- return err;
+ if (err) {
+ failed = dst_target;
+ goto out;
+ }
} else {
struct damos *s;
@@ -1419,25 +1447,34 @@ static int damon_commit_targets(
}
}
+ failed = NULL;
damon_for_each_target_safe(src_target, next, src) {
if (j++ < i)
continue;
/* target to remove has no matching dst */
- if (src_target->obsolete)
- return -EINVAL;
+ if (src_target->obsolete) {
+ err = -EINVAL;
+ goto out;
+ }
new_target = damon_new_target();
- if (!new_target)
- return -ENOMEM;
+ if (!new_target) {
+ err = -ENOMEM;
+ goto out;
+ }
err = damon_commit_target(new_target, false,
src_target, damon_target_has_pid(src),
src->min_region_sz);
if (err) {
damon_destroy_target(new_target, NULL);
- return err;
+ goto out;
}
damon_add_target(dst, new_target);
}
return 0;
+
+out:
+ damon_revert_target_commits(dst, failed, src);
+ return err;
}
static void damon_commit_filter(struct damon_filter *dst,
@@ -1571,8 +1608,10 @@ int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
*/
if (!damon_attrs_equals(&dst->attrs, &src->attrs)) {
err = damon_set_attrs(dst, &src->attrs);
- if (err)
+ if (err) {
+ damon_revert_target_commits(dst, NULL, src);
return err;
+ }
}
dst->pause = src->pause;
dst->ops = src->ops;
diff --git a/mm/filemap.c b/mm/filemap.c
index 7e467c81d213..5af62e6abca5 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -189,7 +189,6 @@ static void filemap_unaccount_folio(struct address_space *mapping,
lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, -nr);
} else if (folio_test_pmd_mappable(folio)) {
lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr);
- filemap_nr_thps_dec(mapping);
}
if (test_bit(AS_KERNEL_FILE, &folio->mapping->flags))
mod_node_page_state(folio_pgdat(folio),
@@ -3324,8 +3323,6 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf)
unsigned int thp_order = 0;
unsigned short mmap_miss;
- ractl._max_index = vmf->vma->vm_pgoff + vma_pages(vmf->vma) - 1;
-
/* Use the readahead code, even if readahead is disabled */
if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && (vm_flags & VM_HUGEPAGE)) {
/*
@@ -3421,7 +3418,6 @@ static struct file *do_sync_mmap_readahead(struct vm_fault *vmf)
* mmap read-around
*/
ra->start = max_t(long, 0, vmf->pgoff - ra->ra_pages / 2);
- ra->start = max(ra->start, vmf->vma->vm_pgoff);
ra->size = ra->ra_pages;
ra->async_size = ra->ra_pages / 4;
ra->order = 0;
@@ -3469,7 +3465,6 @@ static struct file *do_async_mmap_readahead(struct vm_fault *vmf,
}
if (folio_test_readahead(folio)) {
- ractl._max_index = vmf->vma->vm_pgoff + vma_pages(vmf->vma) - 1;
fpin = maybe_unlock_mmap_for_io(vmf, fpin);
page_cache_async_ra(&ractl, folio, ra->ra_pages);
}
diff --git a/mm/gup_test.c b/mm/gup_test.c
index 9dd48db897b9..eb4c9cda16ed 100644
--- a/mm/gup_test.c
+++ b/mm/gup_test.c
@@ -105,11 +105,15 @@ static int __gup_test_ioctl(unsigned int cmd,
unsigned long i, nr_pages, addr, next;
long nr;
struct page **pages;
+ unsigned long end;
int ret = 0;
bool needs_mmap_lock =
cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
- if (gup->size > ULONG_MAX)
+ if (gup->addr > ULONG_MAX || gup->size > ULONG_MAX)
+ return -EINVAL;
+ if (check_add_overflow((unsigned long)gup->addr,
+ (unsigned long)gup->size, &end))
return -EINVAL;
nr_pages = gup->size / PAGE_SIZE;
@@ -125,13 +129,13 @@ static int __gup_test_ioctl(unsigned int cmd,
i = 0;
nr = gup->nr_pages_per_call;
start_time = ktime_get();
- for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
+ for (addr = gup->addr; addr < end; addr = next) {
if (nr != gup->nr_pages_per_call)
break;
next = addr + nr * PAGE_SIZE;
- if (next > gup->addr + gup->size) {
- next = gup->addr + gup->size;
+ if (next > end) {
+ next = end;
nr = (next - addr) / PAGE_SIZE;
}
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 64492dcb9d1e..2bccb0a53a0a 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -89,9 +89,6 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
{
struct inode *inode;
- if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS))
- return false;
-
if (!vma->vm_file)
return false;
@@ -100,7 +97,10 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma)
if (IS_ANON_FILE(inode))
return false;
- return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode);
+ if (!mapping_pmd_folio_support(vma->vm_file->f_mapping))
+ return false;
+
+ return S_ISREG(inode->i_mode);
}
/* If returns true, we are unable to access the VMA's folios. */
@@ -702,6 +702,8 @@ static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC);
DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK);
DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE);
+DEFINE_MTHP_STAT_ATTR(collapse_alloc, MTHP_STAT_COLLAPSE_ALLOC);
+DEFINE_MTHP_STAT_ATTR(collapse_alloc_failed, MTHP_STAT_COLLAPSE_ALLOC_FAILED);
DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT);
DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN);
DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK);
@@ -718,6 +720,10 @@ DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED);
DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED);
DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON);
DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED);
+DEFINE_MTHP_STAT_ATTR(collapse_exceed_swap_pte, MTHP_STAT_COLLAPSE_EXCEED_SWAP);
+DEFINE_MTHP_STAT_ATTR(collapse_exceed_none_pte, MTHP_STAT_COLLAPSE_EXCEED_NONE);
+DEFINE_MTHP_STAT_ATTR(collapse_exceed_shared_pte, MTHP_STAT_COLLAPSE_EXCEED_SHARED);
+
static struct attribute *anon_stats_attrs[] = {
&anon_fault_alloc_attr.attr,
@@ -734,6 +740,9 @@ static struct attribute *anon_stats_attrs[] = {
&split_deferred_attr.attr,
&nr_anon_attr.attr,
&nr_anon_partially_mapped_attr.attr,
+ &collapse_exceed_swap_pte_attr.attr,
+ &collapse_exceed_none_pte_attr.attr,
+ &collapse_exceed_shared_pte_attr.attr,
NULL,
};
@@ -767,6 +776,8 @@ static struct attribute *any_stats_attrs[] = {
#endif
&split_attr.attr,
&split_failed_attr.attr,
+ &collapse_alloc_attr.attr,
+ &collapse_alloc_failed_attr.attr,
NULL,
};
@@ -3753,33 +3764,9 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order,
if (!folio->mapping && !folio_test_anon(folio))
return -EBUSY;
- if (folio_test_anon(folio)) {
- /* order-1 is not supported for anonymous THP. */
- if (new_order == 1)
- return -EINVAL;
- } else if (split_type == SPLIT_TYPE_NON_UNIFORM || new_order) {
- if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
- !mapping_large_folio_support(folio->mapping)) {
- /*
- * We can always split a folio down to a single page
- * (new_order == 0) uniformly.
- *
- * For any other scenario
- * a) uniform split targeting a large folio
- * (new_order > 0)
- * b) any non-uniform split
- * we must confirm that the file system supports large
- * folios.
- *
- * Note that we might still have THPs in such
- * mappings, which is created from khugepaged when
- * CONFIG_READ_ONLY_THP_FOR_FS is enabled. But in that
- * case, the mapping does not actually support large
- * folios properly.
- */
- return -EINVAL;
- }
- }
+ /* order-1 is not supported for anonymous THP. */
+ if (folio_test_anon(folio) && new_order == 1)
+ return -EINVAL;
/*
* swapcache folio could only be split to order 0
@@ -3867,7 +3854,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n
} else {
lruvec_stat_mod_folio(folio,
NR_FILE_THPS, -nr);
- filemap_nr_thps_dec(mapping);
}
}
}
diff --git a/mm/khugepaged.c b/mm/khugepaged.c
index 73e262cb30dd..617bca76db49 100644
--- a/mm/khugepaged.c
+++ b/mm/khugepaged.c
@@ -99,6 +99,8 @@ static DEFINE_READ_MOSTLY_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
static struct kmem_cache *mm_slot_cache __ro_after_init;
+#define KHUGEPAGED_MIN_MTHP_ORDER 2
+
struct collapse_control {
bool is_khugepaged;
@@ -110,6 +112,9 @@ struct collapse_control {
/* nodemask for allocation fallback */
nodemask_t alloc_nmask;
+
+ /* Each bit represents a single occupied (!none/zero) page. */
+ DECLARE_BITMAP(mthp_present_ptes, MAX_PTRS_PER_PTE);
};
/**
@@ -348,6 +353,98 @@ static bool pte_none_or_zero(pte_t pte)
return pte_present(pte) && is_zero_pfn(pte_pfn(pte));
}
+/**
+ * collapse_max_ptes_none - Calculate maximum allowed empty PTEs or PTEs mapping
+ * the shared zeropage for the given collapse operation.
+ * @cc: The collapse control struct
+ * @vma: The vma to check for userfaultfd
+ * @order: The folio order being collapsed to
+ *
+ * Return: Maximum number of empty/shared zeropage PTEs for the collapse operation
+ */
+static unsigned int collapse_max_ptes_none(struct collapse_control *cc,
+ struct vm_area_struct *vma, unsigned int order)
+{
+ const unsigned int max_ptes_none = khugepaged_max_ptes_none;
+
+ if (vma && userfaultfd_armed(vma))
+ return 0;
+ /* for MADV_COLLAPSE, allow any empty/shared zeropage PTEs */
+ if (!cc->is_khugepaged)
+ return HPAGE_PMD_NR;
+ /* for PMD collapse, respect the user defined maximum */
+ if (is_pmd_order(order))
+ return max_ptes_none;
+ /*
+ * for mTHP collapse with the sysctl value set to KHUGEPAGED_MAX_PTES_LIMIT,
+ * scale the maximum number of PTEs to the order of the collapse.
+ */
+ if (max_ptes_none == KHUGEPAGED_MAX_PTES_LIMIT)
+ return (1 << order) - 1;
+ /*
+ * For mTHP collapse of values other than 0 or KHUGEPAGED_MAX_PTES_LIMIT,
+ * emit a warning and return 0.
+ */
+ if (max_ptes_none)
+ pr_warn_once("mTHP collapse does not support max_ptes_none"
+ " values other than 0 or %u, defaulting to 0.\n",
+ KHUGEPAGED_MAX_PTES_LIMIT);
+ return 0;
+}
+
+/**
+ * collapse_max_ptes_shared - Calculate maximum allowed PTEs that map shared
+ * anonymous pages for the given collapse operation.
+ * @cc: The collapse control struct
+ * @order: The folio order being collapsed to
+ *
+ * Return: Maximum number of PTEs that map shared anonymous pages for the
+ * collapse operation
+ */
+static unsigned int collapse_max_ptes_shared(struct collapse_control *cc,
+ unsigned int order)
+{
+ /*
+ * For MADV_COLLAPSE, do not restrict the number of PTEs that map shared
+ * anonymous pages.
+ */
+ if (!cc->is_khugepaged)
+ return HPAGE_PMD_NR;
+ /*
+ * for mTHP collapse do not allow collapsing anonymous memory pages that
+ * are shared between processes.
+ */
+ if (!is_pmd_order(order))
+ return 0;
+ /* for PMD collapse, respect the user defined maximum */
+ return khugepaged_max_ptes_shared;
+}
+
+/**
+ * collapse_max_ptes_swap - Calculate the maximum allowed non-present PTEs or the
+ * maximum allowed non-present pagecache entries for the given collapse operation.
+ * @cc: The collapse control struct
+ * @order: The folio order being collapsed to
+ *
+ * Return: Maximum number of non-present PTEs or the maximum allowed non-present
+ * pagecache entries for the collapse operation.
+ */
+static unsigned int collapse_max_ptes_swap(struct collapse_control *cc,
+ unsigned int order)
+{
+ /*
+ * For MADV_COLLAPSE, do not restrict the number PTEs entries or
+ * pagecache entries that are non-present.
+ */
+ if (!cc->is_khugepaged)
+ return HPAGE_PMD_NR;
+ /* for mTHP collapse do not allow any non-present PTEs or pagecache entries */
+ if (!is_pmd_order(order))
+ return 0;
+ /* for PMD collapse, respect the user defined maximum */
+ return khugepaged_max_ptes_swap;
+}
+
int hugepage_madvise(struct vm_area_struct *vma,
vm_flags_t *vm_flags, int advice)
{
@@ -406,26 +503,32 @@ static inline int collapse_test_exit_or_disable(struct mm_struct *mm)
mm_flags_test(MMF_DISABLE_THP_COMPLETELY, mm);
}
-static bool hugepage_pmd_enabled(void)
+static inline bool anon_hpage_enabled(void)
+{
+ if (READ_ONCE(huge_anon_orders_always))
+ return true;
+ if (READ_ONCE(huge_anon_orders_madvise))
+ return true;
+ if (READ_ONCE(huge_anon_orders_inherit) &&
+ hugepage_global_enabled())
+ return true;
+ return false;
+}
+
+static bool hugepage_enabled(void)
{
/*
* We cover the anon, shmem and the file-backed case here; file-backed
- * hugepages, when configured in, are determined by the global control.
- * Anon pmd-sized hugepages are determined by the pmd-size control.
+ * hugepages are determined by the global control.
+ * Anon hugepages are determined by its per-size mTHP control.
* Shmem pmd-sized hugepages are also determined by its pmd-size control,
* except when the global shmem_huge is set to SHMEM_HUGE_DENY.
*/
- if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) &&
- hugepage_global_enabled())
+ if (hugepage_global_enabled())
return true;
- if (test_bit(PMD_ORDER, &huge_anon_orders_always))
- return true;
- if (test_bit(PMD_ORDER, &huge_anon_orders_madvise))
- return true;
- if (test_bit(PMD_ORDER, &huge_anon_orders_inherit) &&
- hugepage_global_enabled())
+ if (anon_hpage_enabled())
return true;
- if (IS_ENABLED(CONFIG_SHMEM) && shmem_hpage_pmd_enabled())
+ if (shmem_hpage_pmd_enabled())
return true;
return false;
}
@@ -462,14 +565,36 @@ void __khugepaged_enter(struct mm_struct *mm)
wake_up_interruptible(&khugepaged_wait);
}
+/*
+ * Check what orders are possible based on the vma and collapse type.
+ * This is used to determine if mTHP collapse is a viable option.
+ */
+static unsigned long collapse_possible_orders(struct vm_area_struct *vma,
+ vm_flags_t vm_flags, enum tva_type tva_flags)
+{
+ unsigned long orders;
+
+ /* If khugepaged is scanning an anonymous vma, allow mTHP collapse */
+ if ((tva_flags == TVA_KHUGEPAGED) && vma_is_anonymous(vma))
+ orders = THP_ORDERS_ALL_ANON;
+ else
+ orders = BIT(HPAGE_PMD_ORDER);
+
+ return thp_vma_allowable_orders(vma, vm_flags, tva_flags, orders);
+}
+
+static bool collapse_possible(struct vm_area_struct *vma,
+ vm_flags_t vm_flags, enum tva_type tva_flags)
+{
+ return collapse_possible_orders(vma, vm_flags, tva_flags);
+}
+
void khugepaged_enter_vma(struct vm_area_struct *vma,
vm_flags_t vm_flags)
{
- if (!mm_flags_test(MMF_VM_HUGEPAGE, vma->vm_mm) &&
- hugepage_pmd_enabled()) {
- if (thp_vma_allowable_order(vma, vm_flags, TVA_KHUGEPAGED, PMD_ORDER))
- __khugepaged_enter(vma->vm_mm);
- }
+ if (!mm_flags_test(MMF_VM_HUGEPAGE, vma->vm_mm) && hugepage_enabled()
+ && collapse_possible(vma, vm_flags, TVA_KHUGEPAGED))
+ __khugepaged_enter(vma->vm_mm);
}
void __khugepaged_exit(struct mm_struct *mm)
@@ -541,8 +666,11 @@ static void release_pte_pages(pte_t *pte, pte_t *_pte,
static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma,
unsigned long start_addr, pte_t *pte, struct collapse_control *cc,
- struct list_head *compound_pagelist)
+ unsigned int order, struct list_head *compound_pagelist)
{
+ const unsigned int max_ptes_none = collapse_max_ptes_none(cc, vma, order);
+ const unsigned int max_ptes_shared = collapse_max_ptes_shared(cc, order);
+ const unsigned long nr_pages = 1UL << order;
struct page *page = NULL;
struct folio *folio = NULL;
unsigned long addr = start_addr;
@@ -550,20 +678,18 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma,
int none_or_zero = 0, shared = 0, referenced = 0;
enum scan_result result = SCAN_FAIL;
- for (_pte = pte; _pte < pte + HPAGE_PMD_NR;
+ for (_pte = pte; _pte < pte + nr_pages;
_pte++, addr += PAGE_SIZE) {
pte_t pteval = ptep_get(_pte);
if (pte_none_or_zero(pteval)) {
- ++none_or_zero;
- if (!userfaultfd_armed(vma) &&
- (!cc->is_khugepaged ||
- none_or_zero <= khugepaged_max_ptes_none)) {
- continue;
- } else {
+ if (++none_or_zero > max_ptes_none) {
result = SCAN_EXCEED_NONE_PTE;
- count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
+ if (is_pmd_order(order))
+ count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
+ count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_NONE);
goto out;
}
+ continue;
}
if (!pte_present(pteval)) {
result = SCAN_PTE_NON_PRESENT;
@@ -594,14 +720,28 @@ static enum scan_result __collapse_huge_page_isolate(struct vm_area_struct *vma,
/* See collapse_scan_pmd(). */
if (folio_maybe_mapped_shared(folio)) {
- ++shared;
- if (cc->is_khugepaged &&
- shared > khugepaged_max_ptes_shared) {
+ /*
+ * TODO: Support shared pages without leading to further
+ * mTHP collapses. Currently bringing in new pages via
+ * shared may cause a future higher order collapse on a
+ * rescan of the same range.
+ */
+ if (++shared > max_ptes_shared) {
result = SCAN_EXCEED_SHARED_PTE;
- count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
+ if (is_pmd_order(order))
+ count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
+ count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_SHARED);
goto out;
}
}
+ /*
+ * TODO: In some cases of partially-mapped folios, we'd actually
+ * want to collapse.
+ */
+ if (!is_pmd_order(order) && folio_order(folio) >= order) {
+ result = SCAN_PTE_MAPPED_HUGEPAGE;
+ goto out;
+ }
if (folio_test_large(folio)) {
struct folio *f;
@@ -678,29 +818,29 @@ next:
} else {
result = SCAN_SUCCEED;
trace_mm_collapse_huge_page_isolate(folio, none_or_zero,
- referenced, result);
+ referenced, result, order);
return result;
}
out:
release_pte_pages(pte, _pte, compound_pagelist);
trace_mm_collapse_huge_page_isolate(folio, none_or_zero,
- referenced, result);
+ referenced, result, order);
return result;
}
static void __collapse_huge_page_copy_succeeded(pte_t *pte,
- struct vm_area_struct *vma,
- unsigned long address,
- spinlock_t *ptl,
- struct list_head *compound_pagelist)
+ struct vm_area_struct *vma, unsigned long address,
+ spinlock_t *ptl, unsigned int order,
+ struct list_head *compound_pagelist)
{
- unsigned long end = address + HPAGE_PMD_SIZE;
+ const unsigned long nr_pages = 1UL << order;
+ unsigned long end = address + (PAGE_SIZE * nr_pages);
struct folio *src, *tmp;
pte_t pteval;
pte_t *_pte;
unsigned int nr_ptes;
- for (_pte = pte; _pte < pte + HPAGE_PMD_NR; _pte += nr_ptes,
+ for (_pte = pte; _pte < pte + nr_pages; _pte += nr_ptes,
address += nr_ptes * PAGE_SIZE) {
nr_ptes = 1;
pteval = ptep_get(_pte);
@@ -753,11 +893,10 @@ static void __collapse_huge_page_copy_succeeded(pte_t *pte,
}
static void __collapse_huge_page_copy_failed(pte_t *pte,
- pmd_t *pmd,
- pmd_t orig_pmd,
- struct vm_area_struct *vma,
- struct list_head *compound_pagelist)
+ pmd_t *pmd, pmd_t orig_pmd, struct vm_area_struct *vma,
+ unsigned int order, struct list_head *compound_pagelist)
{
+ const unsigned long nr_pages = 1UL << order;
spinlock_t *pmd_ptl;
/*
@@ -773,7 +912,7 @@ static void __collapse_huge_page_copy_failed(pte_t *pte,
* Release both raw and compound pages isolated
* in __collapse_huge_page_isolate.
*/
- release_pte_pages(pte, pte + HPAGE_PMD_NR, compound_pagelist);
+ release_pte_pages(pte, pte + nr_pages, compound_pagelist);
}
/*
@@ -793,16 +932,17 @@ static void __collapse_huge_page_copy_failed(pte_t *pte,
*/
static enum scan_result __collapse_huge_page_copy(pte_t *pte, struct folio *folio,
pmd_t *pmd, pmd_t orig_pmd, struct vm_area_struct *vma,
- unsigned long address, spinlock_t *ptl,
+ unsigned long address, spinlock_t *ptl, unsigned int order,
struct list_head *compound_pagelist)
{
+ const unsigned long nr_pages = 1UL << order;
unsigned int i;
enum scan_result result = SCAN_SUCCEED;
/*
* Copying pages' contents is subject to memory poison at any iteration.
*/
- for (i = 0; i < HPAGE_PMD_NR; i++) {
+ for (i = 0; i < nr_pages; i++) {
pte_t pteval = ptep_get(pte + i);
struct page *page = folio_page(folio, i);
unsigned long src_addr = address + i * PAGE_SIZE;
@@ -821,10 +961,10 @@ static enum scan_result __collapse_huge_page_copy(pte_t *pte, struct folio *foli
if (likely(result == SCAN_SUCCEED))
__collapse_huge_page_copy_succeeded(pte, vma, address, ptl,
- compound_pagelist);
+ order, compound_pagelist);
else
__collapse_huge_page_copy_failed(pte, pmd, orig_pmd, vma,
- compound_pagelist);
+ order, compound_pagelist);
return result;
}
@@ -905,12 +1045,13 @@ static int collapse_find_target_node(struct collapse_control *cc)
/*
* If mmap_lock temporarily dropped, revalidate vma
- * before taking mmap_lock.
+ * after taking the mmap_lock again.
* Returns enum scan_result value.
*/
static enum scan_result hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address,
- bool expect_anon, struct vm_area_struct **vmap, struct collapse_control *cc)
+ bool expect_anon, struct vm_area_struct **vmap,
+ struct collapse_control *cc, unsigned int order)
{
struct vm_area_struct *vma;
enum tva_type type = cc->is_khugepaged ? TVA_KHUGEPAGED :
@@ -923,15 +1064,22 @@ static enum scan_result hugepage_vma_revalidate(struct mm_struct *mm, unsigned l
if (!vma)
return SCAN_VMA_NULL;
+ /*
+ * We cannot collapse VMA regions that do not span the full PMD. This is
+ * due to the potential of the PMD being shared by another VMA leaving
+ * us vulnerable to a race condition. Always check the PMD order here to
+ * ensure its not shared by another VMA. We'd need to lock all VMAs in
+ * the PMD range to support this.
+ */
if (!thp_vma_suitable_order(vma, address, PMD_ORDER))
return SCAN_ADDRESS_RANGE;
- if (!thp_vma_allowable_order(vma, vma->vm_flags, type, PMD_ORDER))
+ if (!thp_vma_allowable_orders(vma, vma->vm_flags, type, BIT(order)))
return SCAN_VMA_CHECK;
/*
* Anon VMA expected, the address may be unmapped then
* remapped to file after khugepaged reaquired the mmap_lock.
*
- * thp_vma_allowable_order may return true for qualified file
+ * thp_vma_allowable_orders may return true for qualified file
* vmas.
*/
if (expect_anon && (!(*vmap)->anon_vma || !vma_is_anonymous(*vmap)))
@@ -989,16 +1137,20 @@ static enum scan_result check_pmd_still_valid(struct mm_struct *mm,
* Bring missing pages in from swap, to complete THP collapse.
* Only done if khugepaged_scan_pmd believes it is worthwhile.
*
+ * For mTHP orders the function bails on the first swap entry, because
+ * faulting pages back in during collapse could re-populate PTEs that
+ * push a later scan over the threshold for a higher-order collapse.
+ *
* Called and returns without pte mapped or spinlocks held.
* Returns result: if not SCAN_SUCCEED, mmap_lock has been released.
*/
static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm,
- struct vm_area_struct *vma, unsigned long start_addr, pmd_t *pmd,
- int referenced)
+ struct vm_area_struct *vma, unsigned long start_addr,
+ pmd_t *pmd, int referenced, unsigned int order)
{
int swapped_in = 0;
vm_fault_t ret = 0;
- unsigned long addr, end = start_addr + (HPAGE_PMD_NR * PAGE_SIZE);
+ unsigned long addr, end = start_addr + (PAGE_SIZE << order);
enum scan_result result;
pte_t *pte = NULL;
spinlock_t *ptl;
@@ -1030,6 +1182,20 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm,
pte_present(vmf.orig_pte))
continue;
+ /*
+ * TODO: Support swapin without leading to further mTHP
+ * collapses. Currently bringing in new pages via swapin may
+ * cause a future higher order collapse on a rescan of the same
+ * range.
+ */
+ if (!is_pmd_order(order)) {
+ count_mthp_stat(order, MTHP_STAT_COLLAPSE_EXCEED_SWAP);
+ pte_unmap(pte);
+ mmap_read_unlock(mm);
+ result = SCAN_EXCEED_SWAP_PTE;
+ goto out;
+ }
+
vmf.pte = pte;
vmf.ptl = ptl;
ret = do_swap_page(&vmf);
@@ -1064,62 +1230,69 @@ static enum scan_result __collapse_huge_page_swapin(struct mm_struct *mm,
result = SCAN_SUCCEED;
out:
- trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, result);
+ trace_mm_collapse_huge_page_swapin(mm, swapped_in, referenced, result,
+ order);
return result;
}
static enum scan_result alloc_charge_folio(struct folio **foliop, struct mm_struct *mm,
- struct collapse_control *cc)
+ struct collapse_control *cc, unsigned int order)
{
gfp_t gfp = (cc->is_khugepaged ? alloc_hugepage_khugepaged_gfpmask() :
GFP_TRANSHUGE);
int node = collapse_find_target_node(cc);
struct folio *folio;
- folio = __folio_alloc(gfp, HPAGE_PMD_ORDER, node, &cc->alloc_nmask);
+ folio = __folio_alloc(gfp, order, node, &cc->alloc_nmask);
if (!folio) {
*foliop = NULL;
- count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
+ if (is_pmd_order(order))
+ count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
+ count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC_FAILED);
return SCAN_ALLOC_HUGE_PAGE_FAIL;
}
- count_vm_event(THP_COLLAPSE_ALLOC);
+ if (is_pmd_order(order))
+ count_vm_event(THP_COLLAPSE_ALLOC);
+ count_mthp_stat(order, MTHP_STAT_COLLAPSE_ALLOC);
+
if (unlikely(mem_cgroup_charge(folio, mm, gfp))) {
folio_put(folio);
*foliop = NULL;
return SCAN_CGROUP_CHARGE_FAIL;
}
- count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1);
+ if (is_pmd_order(order))
+ count_memcg_folio_events(folio, THP_COLLAPSE_ALLOC, 1);
*foliop = folio;
return SCAN_SUCCEED;
}
-static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long address,
- int referenced, int unmapped, struct collapse_control *cc)
+/*
+ * collapse_huge_page expects the mmap_lock to be unlocked before entering and
+ * will always return with the lock unlocked, to avoid holding the mmap_lock
+ * while allocating a THP, as that could trigger direct reclaim/compaction.
+ * Note that the VMA must be rechecked after grabbing the mmap_lock again.
+ */
+static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long start_addr,
+ int referenced, int unmapped, struct collapse_control *cc,
+ unsigned int order)
{
+ const unsigned long pmd_addr = start_addr & HPAGE_PMD_MASK;
+ const unsigned long end_addr = start_addr + (PAGE_SIZE << order);
LIST_HEAD(compound_pagelist);
pmd_t *pmd, _pmd;
- pte_t *pte;
+ pte_t *pte = NULL;
pgtable_t pgtable;
struct folio *folio;
spinlock_t *pmd_ptl, *pte_ptl;
enum scan_result result = SCAN_FAIL;
struct vm_area_struct *vma;
struct mmu_notifier_range range;
+ bool anon_vma_locked = false;
- VM_BUG_ON(address & ~HPAGE_PMD_MASK);
-
- /*
- * Before allocating the hugepage, release the mmap_lock read lock.
- * The allocation can take potentially a long time if it involves
- * sync compaction, and we do not need to hold the mmap_lock during
- * that. We will recheck the vma after taking it again in write mode.
- */
- mmap_read_unlock(mm);
-
- result = alloc_charge_folio(&folio, mm, cc);
+ result = alloc_charge_folio(&folio, mm, cc, order);
if (result != SCAN_SUCCEED)
goto out_nolock;
@@ -1129,13 +1302,14 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
}
mmap_read_lock(mm);
- result = hugepage_vma_revalidate(mm, address, true, &vma, cc);
+ result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true,
+ &vma, cc, order);
if (result != SCAN_SUCCEED) {
mmap_read_unlock(mm);
goto out_nolock;
}
- result = find_pmd_or_thp_or_none(mm, address, &pmd);
+ result = find_pmd_or_thp_or_none(mm, pmd_addr, &pmd);
if (result != SCAN_SUCCEED) {
mmap_read_unlock(mm);
goto out_nolock;
@@ -1147,8 +1321,8 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
* released when it fails. So we jump out_nolock directly in
* that case. Continuing to collapse causes inconsistency.
*/
- result = __collapse_huge_page_swapin(mm, vma, address, pmd,
- referenced);
+ result = __collapse_huge_page_swapin(mm, vma, start_addr, pmd,
+ referenced, order);
if (result != SCAN_SUCCEED)
goto out_nolock;
}
@@ -1163,19 +1337,28 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
* mmap_lock.
*/
mmap_write_lock(mm);
- result = hugepage_vma_revalidate(mm, address, true, &vma, cc);
+ result = hugepage_vma_revalidate(mm, pmd_addr, /*expect_anon=*/ true,
+ &vma, cc, order);
if (result != SCAN_SUCCEED)
goto out_up_write;
/* check if the pmd is still valid */
vma_start_write(vma);
- result = check_pmd_still_valid(mm, address, pmd);
+ result = check_pmd_still_valid(mm, pmd_addr, pmd);
if (result != SCAN_SUCCEED)
goto out_up_write;
anon_vma_lock_write(vma->anon_vma);
+ anon_vma_locked = true;
- mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, address,
- address + HPAGE_PMD_SIZE);
+ /*
+ * Only notify about the PTE range we will actually modify. While we
+ * temporary unmap the whole PTE table for mTHP collapse, we'll remap
+ * it later, leaving other PTEs effectively unmodified. The locks we
+ * hold prevent anybody from stumbling over such temporarily unmapped
+ * PTE tables.
+ */
+ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm, start_addr,
+ end_addr);
mmu_notifier_invalidate_range_start(&range);
pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
@@ -1187,25 +1370,23 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
* Parallel GUP-fast is fine since GUP-fast will back off when
* it detects PMD is changed.
*/
- _pmd = pmdp_collapse_flush(vma, address, pmd);
+ _pmd = pmdp_collapse_flush(vma, pmd_addr, pmd);
spin_unlock(pmd_ptl);
mmu_notifier_invalidate_range_end(&range);
tlb_remove_table_sync_one();
- pte = pte_offset_map_lock(mm, &_pmd, address, &pte_ptl);
+ pte = pte_offset_map_lock(mm, &_pmd, start_addr, &pte_ptl);
if (pte) {
- result = __collapse_huge_page_isolate(vma, address, pte, cc,
- &compound_pagelist);
+ result = __collapse_huge_page_isolate(vma, start_addr, pte, cc,
+ order, &compound_pagelist);
spin_unlock(pte_ptl);
} else {
result = SCAN_NO_PTE_TABLE;
}
if (unlikely(result != SCAN_SUCCEED)) {
- if (pte)
- pte_unmap(pte);
spin_lock(pmd_ptl);
- BUG_ON(!pmd_none(*pmd));
+ VM_WARN_ON_ONCE(!pmd_none(*pmd));
/*
* We can only use set_pmd_at when establishing
* hugepmds and never for establishing regular pmds that
@@ -1213,20 +1394,24 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
*/
pmd_populate(mm, pmd, pmd_pgtable(_pmd));
spin_unlock(pmd_ptl);
- anon_vma_unlock_write(vma->anon_vma);
goto out_up_write;
}
/*
- * All pages are isolated and locked so anon_vma rmap
- * can't run anymore.
+ * For PMD collapse all pages are isolated and locked so anon_vma
+ * rmap can't run anymore. For mTHP collapse the PMD entry has been
+ * removed and not all pages are isolated and locked, so we must hold
+ * the lock to prevent neighboring folios from attempting to access
+ * this PMD until its reinstalled.
*/
- anon_vma_unlock_write(vma->anon_vma);
+ if (is_pmd_order(order)) {
+ anon_vma_unlock_write(vma->anon_vma);
+ anon_vma_locked = false;
+ }
result = __collapse_huge_page_copy(pte, folio, pmd, _pmd,
- vma, address, pte_ptl,
- &compound_pagelist);
- pte_unmap(pte);
+ vma, start_addr, pte_ptl,
+ order, &compound_pagelist);
if (unlikely(result != SCAN_SUCCEED))
goto out_up_write;
@@ -1236,37 +1421,190 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long a
* write.
*/
__folio_mark_uptodate(folio);
- pgtable = pmd_pgtable(_pmd);
-
spin_lock(pmd_ptl);
- BUG_ON(!pmd_none(*pmd));
- pgtable_trans_huge_deposit(mm, pmd, pgtable);
- map_anon_folio_pmd_nopf(folio, pmd, vma, address);
+ VM_WARN_ON_ONCE(!pmd_none(*pmd));
+ if (is_pmd_order(order)) {
+ pgtable = pmd_pgtable(_pmd);
+ pgtable_trans_huge_deposit(mm, pmd, pgtable);
+ map_anon_folio_pmd_nopf(folio, pmd, vma, pmd_addr);
+ } else {
+ /*
+ * Some architectures (e.g. MIPS) walk the live page table in
+ * their implementation. update_mmu_cache_range() must be called
+ * with a valid page table hierarchy and the PTE lock held.
+ * Acquire it nested inside pmd_ptl when they are distinct locks.
+ */
+ if (pte_ptl != pmd_ptl)
+ spin_lock_nested(pte_ptl, SINGLE_DEPTH_NESTING);
+ pmd_populate(mm, pmd, pmd_pgtable(_pmd));
+ map_anon_folio_pte_nopf(folio, pte, vma, start_addr,
+ /*uffd_wp=*/ false);
+ if (pte_ptl != pmd_ptl)
+ spin_unlock(pte_ptl);
+ }
spin_unlock(pmd_ptl);
folio = NULL;
result = SCAN_SUCCEED;
out_up_write:
+ if (anon_vma_locked)
+ anon_vma_unlock_write(vma->anon_vma);
+ if (pte)
+ pte_unmap(pte);
mmap_write_unlock(mm);
out_nolock:
if (folio)
folio_put(folio);
- trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result);
+ trace_mm_collapse_huge_page(mm, result == SCAN_SUCCEED, result, order);
return result;
}
+/* Return the highest naturally aligned order that fits at @offset within a PMD. */
+static unsigned int max_order_from_offset(unsigned int offset)
+{
+ if (offset == 0)
+ return HPAGE_PMD_ORDER;
+
+ return min_t(unsigned int, __ffs(offset), HPAGE_PMD_ORDER);
+}
+
+/*
+ * mthp_collapse() consumes the bitmap that is generated during
+ * collapse_scan_pmd() to determine what regions and mTHP orders fit best.
+ *
+ * Each bit in cc->mthp_present_ptes represents a single occupied (!none/zero)
+ * page. We start at the PMD order and check if it is eligible for collapse;
+ * if not, we check the left and right halves of the PTE page table we are
+ * examining at a lower order.
+ *
+ * For each of these, we determine how many PTE entries are occupied in the
+ * range of PTE entries we propose to collapse, then we compare this to a
+ * threshold number of PTE entries which would need to be occupied for a
+ * collapse to be permitted at that order (accounting for max_ptes_none).
+ *
+ * If a collapse is permitted, we attempt to collapse the PTE range into a
+ * mTHP.
+ */
+static enum scan_result mthp_collapse(struct mm_struct *mm,
+ unsigned long address, int referenced, int unmapped,
+ struct collapse_control *cc, unsigned long enabled_orders)
+{
+ unsigned int nr_occupied_ptes, nr_ptes, max_ptes_none;
+ enum scan_result last_result = SCAN_FAIL;
+ int collapsed = 0;
+ bool alloc_failed = false;
+ unsigned long collapse_address;
+ unsigned int offset = 0;
+ unsigned int order = HPAGE_PMD_ORDER;
+
+ while (offset < HPAGE_PMD_NR) {
+ nr_ptes = 1UL << order;
+
+ if (!test_bit(order, &enabled_orders))
+ goto next_order;
+
+ max_ptes_none = collapse_max_ptes_none(cc, NULL, order);
+ nr_occupied_ptes = bitmap_weight_from(cc->mthp_present_ptes, offset,
+ offset + nr_ptes);
+
+ /*
+ * Swap PTEs accepted during the scan are counted in @unmapped,
+ * not in the present-PTE bitmap. Account them for the PMD-order
+ * candidate.
+ */
+ if (is_pmd_order(order))
+ nr_occupied_ptes += unmapped;
+
+ if (nr_occupied_ptes >= nr_ptes - max_ptes_none) {
+ enum scan_result ret;
+
+ collapse_address = address + offset * PAGE_SIZE;
+ ret = collapse_huge_page(mm, collapse_address, referenced,
+ unmapped, cc, order);
+
+ switch (ret) {
+ /* Cases where we continue to next collapse candidate */
+ case SCAN_SUCCEED:
+ collapsed += nr_ptes;
+ fallthrough;
+ case SCAN_PTE_MAPPED_HUGEPAGE:
+ goto next_offset;
+ /* Cases where lower orders might still succeed */
+ case SCAN_ALLOC_HUGE_PAGE_FAIL:
+ alloc_failed = true;
+ fallthrough;
+ case SCAN_LACK_REFERENCED_PAGE:
+ case SCAN_EXCEED_NONE_PTE:
+ case SCAN_EXCEED_SWAP_PTE:
+ case SCAN_EXCEED_SHARED_PTE:
+ case SCAN_PAGE_LOCK:
+ case SCAN_PAGE_COUNT:
+ case SCAN_PAGE_NULL:
+ case SCAN_DEL_PAGE_LRU:
+ case SCAN_PTE_NON_PRESENT:
+ case SCAN_PTE_UFFD_WP:
+ case SCAN_PAGE_LAZYFREE:
+ last_result = ret;
+ goto next_order;
+ /* Cases where no further collapse is possible */
+ case SCAN_PMD_MAPPED:
+ fallthrough;
+ default:
+ last_result = ret;
+ goto done;
+ }
+ }
+
+next_order:
+ /*
+ * Continue with the next smaller order if there is still
+ * any smaller order enabled. When at the smallest order
+ * we must always move to the next offset.
+ */
+ if (order > KHUGEPAGED_MIN_MTHP_ORDER &&
+ (enabled_orders & GENMASK(order - 1, 0))) {
+ order--;
+ continue;
+ }
+next_offset:
+ /*
+ * Advance past the region we just processed and determine the
+ * highest order we can attempt next. Since huge pages must be
+ * naturally aligned, the max order we can attempt next is
+ * limited by the alignment of the new offset.
+ * E.g. if we collapsed a order-2 mTHP at offset 0, offset
+ * becomes 4 and __ffs(4) == 2, so the next attempt starts at
+ * order 2.
+ */
+ offset += nr_ptes;
+ order = max_order_from_offset(offset);
+ }
+done:
+ if (collapsed)
+ return SCAN_SUCCEED;
+ if (alloc_failed)
+ return SCAN_ALLOC_HUGE_PAGE_FAIL;
+ return last_result;
+}
+
static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
struct vm_area_struct *vma, unsigned long start_addr,
bool *lock_dropped, struct collapse_control *cc)
{
+ const unsigned int max_ptes_shared = collapse_max_ptes_shared(cc, HPAGE_PMD_ORDER);
+ const unsigned int max_ptes_swap = collapse_max_ptes_swap(cc, HPAGE_PMD_ORDER);
+ unsigned int max_ptes_none = collapse_max_ptes_none(cc, vma, HPAGE_PMD_ORDER);
+ enum tva_type tva_flags = cc->is_khugepaged ? TVA_KHUGEPAGED : TVA_FORCED_COLLAPSE;
pmd_t *pmd;
- pte_t *pte, *_pte;
+ pte_t *pte, *_pte, pteval;
+ int i;
int none_or_zero = 0, shared = 0, referenced = 0;
enum scan_result result = SCAN_FAIL;
struct page *page = NULL;
struct folio *folio = NULL;
unsigned long addr;
+ unsigned long enabled_orders;
spinlock_t *ptl;
int node = NUMA_NO_NODE, unmapped = 0;
@@ -1278,8 +1616,19 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
goto out;
}
+ bitmap_zero(cc->mthp_present_ptes, MAX_PTRS_PER_PTE);
memset(cc->node_load, 0, sizeof(cc->node_load));
nodes_clear(cc->alloc_nmask);
+
+ enabled_orders = collapse_possible_orders(vma, vma->vm_flags, tva_flags);
+
+ /*
+ * If PMD is the only enabled order, enforce max_ptes_none, otherwise
+ * scan all pages to populate the bitmap for mTHP collapse.
+ */
+ if (enabled_orders != BIT(HPAGE_PMD_ORDER))
+ max_ptes_none = KHUGEPAGED_MAX_PTES_LIMIT;
+
pte = pte_offset_map_lock(mm, pmd, start_addr, &ptl);
if (!pte) {
cc->progress++;
@@ -1287,42 +1636,41 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
goto out;
}
- for (addr = start_addr, _pte = pte; _pte < pte + HPAGE_PMD_NR;
- _pte++, addr += PAGE_SIZE) {
+ for (i = 0; i < HPAGE_PMD_NR; i++) {
+ _pte = pte + i;
+ addr = start_addr + i * PAGE_SIZE;
+ pteval = ptep_get(_pte);
+
cc->progress++;
- pte_t pteval = ptep_get(_pte);
if (pte_none_or_zero(pteval)) {
- ++none_or_zero;
- if (!userfaultfd_armed(vma) &&
- (!cc->is_khugepaged ||
- none_or_zero <= khugepaged_max_ptes_none)) {
- continue;
- } else {
+ if (++none_or_zero > max_ptes_none) {
result = SCAN_EXCEED_NONE_PTE;
count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
+ count_mthp_stat(HPAGE_PMD_ORDER,
+ MTHP_STAT_COLLAPSE_EXCEED_NONE);
goto out_unmap;
}
+ continue;
}
if (!pte_present(pteval)) {
- ++unmapped;
- if (!cc->is_khugepaged ||
- unmapped <= khugepaged_max_ptes_swap) {
- /*
- * Always be strict with uffd-wp
- * enabled swap entries. Please see
- * comment below for pte_uffd_wp().
- */
- if (pte_swp_uffd_wp_any(pteval)) {
- result = SCAN_PTE_UFFD_WP;
- goto out_unmap;
- }
- continue;
- } else {
+ if (++unmapped > max_ptes_swap) {
result = SCAN_EXCEED_SWAP_PTE;
count_vm_event(THP_SCAN_EXCEED_SWAP_PTE);
+ count_mthp_stat(HPAGE_PMD_ORDER,
+ MTHP_STAT_COLLAPSE_EXCEED_SWAP);
+ goto out_unmap;
+ }
+ /*
+ * Always be strict with uffd-wp
+ * enabled swap entries. Please see
+ * comment below for pte_uffd_wp().
+ */
+ if (pte_swp_uffd_wp_any(pteval)) {
+ result = SCAN_PTE_UFFD_WP;
goto out_unmap;
}
+ continue;
}
if (pte_uffd_wp(pteval)) {
/*
@@ -1365,15 +1713,17 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
* is shared.
*/
if (folio_maybe_mapped_shared(folio)) {
- ++shared;
- if (cc->is_khugepaged &&
- shared > khugepaged_max_ptes_shared) {
+ if (++shared > max_ptes_shared) {
result = SCAN_EXCEED_SHARED_PTE;
count_vm_event(THP_SCAN_EXCEED_SHARED_PTE);
+ count_mthp_stat(HPAGE_PMD_ORDER,
+ MTHP_STAT_COLLAPSE_EXCEED_SHARED);
goto out_unmap;
}
}
+ /* Set bit for occupied pages */
+ __set_bit(i, cc->mthp_present_ptes);
/*
* Record which node the original page is from and save this
* information to cc->node_load[].
@@ -1430,9 +1780,11 @@ static enum scan_result collapse_scan_pmd(struct mm_struct *mm,
out_unmap:
pte_unmap_unlock(pte, ptl);
if (result == SCAN_SUCCEED) {
- result = collapse_huge_page(mm, start_addr, referenced,
- unmapped, cc);
- /* collapse_huge_page will return with the mmap_lock released */
+ /* collapse_huge_page expects the lock to be dropped before calling */
+ mmap_read_unlock(mm);
+ result = mthp_collapse(mm, start_addr, referenced,
+ unmapped, cc, enabled_orders);
+ /* mmap_lock was released above, set lock_dropped */
*lock_dropped = true;
}
out:
@@ -1900,10 +2252,16 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
int nr_none = 0;
bool is_shmem = shmem_file(file);
- VM_BUG_ON(!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && !is_shmem);
- VM_BUG_ON(start & (HPAGE_PMD_NR - 1));
+ /*
+ * MADV_COLLAPSE ignores shmem huge config, so do not check shmem
+ *
+ * TODO: once shmem always calls mapping_set_large_folios() on its
+ * mapping, the shmem check can be removed.
+ */
+ VM_WARN_ON_ONCE(!is_shmem && !mapping_pmd_folio_support(mapping));
+ VM_WARN_ON_ONCE(start & (HPAGE_PMD_NR - 1));
- result = alloc_charge_folio(&new_folio, mm, cc);
+ result = alloc_charge_folio(&new_folio, mm, cc, HPAGE_PMD_ORDER);
if (result != SCAN_SUCCEED)
goto out;
@@ -1986,20 +2344,22 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
}
} else if (folio_test_dirty(folio)) {
/*
- * khugepaged only works on read-only fd,
- * so this page is dirty because it hasn't
- * been flushed since first write. There
- * won't be new dirty pages.
+ * This page is dirty because it hasn't
+ * been flushed since first write.
*
- * Trigger async flush here and hope the
- * writeback is done when khugepaged
- * revisits this page.
+ * Trigger async flush for read-only files and
+ * hope the writeback is done when khugepaged
+ * revisits this page. Writable files can have
+ * their folios dirty at any time; blindly
+ * flushing them would cause undesirable
+ * system-wide writeback.
*
* This is a one-off situation. We are not
* forcing writeback in loop.
*/
xas_unlock_irq(&xas);
- filemap_flush(mapping);
+ if (!inode_is_open_for_write(mapping->host))
+ filemap_flush(mapping);
result = SCAN_PAGE_DIRTY_OR_WRITEBACK;
goto xa_unlocked;
} else if (folio_test_writeback(folio)) {
@@ -2045,8 +2405,8 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
if (!is_shmem && (folio_test_dirty(folio) ||
folio_test_writeback(folio))) {
/*
- * khugepaged only works on read-only fd, so this
- * folio is dirty because it hasn't been flushed
+ * khugepaged only works on clean file-backed folios,
+ * so this folio is dirty because it hasn't been flushed
* since first write.
*/
result = SCAN_PAGE_DIRTY_OR_WRITEBACK;
@@ -2091,6 +2451,27 @@ static enum scan_result collapse_file(struct mm_struct *mm, unsigned long addr,
}
/*
+ * At this point, the folio is locked and unmapped. If the PTE
+ * was dirty, try_to_unmap() has transferred the dirty bit to
+ * the folio and we must not collapse it into a clean
+ * file-backed folio.
+ *
+ * If the folio is clean here, no one can write it until we
+ * drop the folio lock. A write through a stale TLB entry came
+ * from a clean PTE and must fault because the PTE has been
+ * cleared; the fault path has to take the folio lock before
+ * installing a writable mapping. Buffered write paths also
+ * have to take the folio lock before modifying file contents
+ * without a mapping, typically via write_begin_get_folio().
+ */
+ if (!is_shmem && folio_test_dirty(folio)) {
+ result = SCAN_PAGE_DIRTY_OR_WRITEBACK;
+ xas_unlock_irq(&xas);
+ folio_putback_lru(folio);
+ goto out_unlock;
+ }
+
+ /*
* Accumulate the folios that are being collapsed.
*/
list_add_tail(&folio->lru, &pagelist);
@@ -2102,21 +2483,6 @@ out_unlock:
goto xa_unlocked;
}
- if (!is_shmem) {
- filemap_nr_thps_inc(mapping);
- /*
- * Paired with the fence in do_dentry_open() -> get_write_access()
- * to ensure i_writecount is up to date and the update to nr_thps
- * is visible. Ensures the page cache will be truncated if the
- * file is opened writable.
- */
- smp_mb();
- if (inode_is_open_for_write(mapping->host)) {
- result = SCAN_FAIL;
- filemap_nr_thps_dec(mapping);
- }
- }
-
xa_locked:
xas_unlock_irq(&xas);
xa_unlocked:
@@ -2294,19 +2660,6 @@ rollback:
folio_putback_lru(folio);
folio_put(folio);
}
- /*
- * Undo the updates of filemap_nr_thps_inc for non-SHMEM
- * file only. This undo is not needed unless failure is
- * due to SCAN_COPY_MC.
- */
- if (!is_shmem && result == SCAN_COPY_MC) {
- filemap_nr_thps_dec(mapping);
- /*
- * Paired with the fence in do_dentry_open() -> get_write_access()
- * to ensure the update to nr_thps is visible.
- */
- smp_mb();
- }
new_folio->mapping = NULL;
@@ -2322,6 +2675,8 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
unsigned long addr, struct file *file, pgoff_t start,
struct collapse_control *cc)
{
+ const unsigned int max_ptes_none = collapse_max_ptes_none(cc, NULL, HPAGE_PMD_ORDER);
+ const unsigned int max_ptes_swap = collapse_max_ptes_swap(cc, HPAGE_PMD_ORDER);
struct folio *folio = NULL;
struct address_space *mapping = file->f_mapping;
XA_STATE(xas, &mapping->i_pages, start);
@@ -2340,8 +2695,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
if (xa_is_value(folio)) {
swap += 1 << xas_get_order(&xas);
- if (cc->is_khugepaged &&
- swap > khugepaged_max_ptes_swap) {
+ if (swap > max_ptes_swap) {
result = SCAN_EXCEED_SWAP_PTE;
count_vm_event(THP_SCAN_EXCEED_SWAP_PTE);
break;
@@ -2412,8 +2766,7 @@ static enum scan_result collapse_scan_file(struct mm_struct *mm,
cc->progress += HPAGE_PMD_NR;
if (result == SCAN_SUCCEED) {
- if (cc->is_khugepaged &&
- present < HPAGE_PMD_NR - khugepaged_max_ptes_none) {
+ if (present < HPAGE_PMD_NR - max_ptes_none) {
result = SCAN_EXCEED_NONE_PTE;
count_vm_event(THP_SCAN_EXCEED_NONE_PTE);
} else {
@@ -2532,7 +2885,7 @@ static void collapse_scan_mm_slot(unsigned int progress_max,
cc->progress++;
break;
}
- if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_KHUGEPAGED, PMD_ORDER)) {
+ if (!collapse_possible(vma, vma->vm_flags, TVA_KHUGEPAGED)) {
cc->progress++;
continue;
}
@@ -2607,7 +2960,7 @@ breakouterloop_mmap_lock:
static int khugepaged_has_work(void)
{
- return !list_empty(&khugepaged_scan.mm_head) && hugepage_pmd_enabled();
+ return !list_empty(&khugepaged_scan.mm_head) && hugepage_enabled();
}
static int khugepaged_wait_event(void)
@@ -2680,7 +3033,7 @@ static void khugepaged_wait_work(void)
return;
}
- if (hugepage_pmd_enabled())
+ if (hugepage_enabled())
wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
}
@@ -2711,7 +3064,7 @@ void set_recommended_min_free_kbytes(void)
int nr_zones = 0;
unsigned long recommended_min;
- if (!hugepage_pmd_enabled()) {
+ if (!hugepage_enabled()) {
calculate_min_free_kbytes();
goto update_wmarks;
}
@@ -2761,7 +3114,7 @@ int start_stop_khugepaged(void)
int err = 0;
mutex_lock(&khugepaged_mutex);
- if (hugepage_pmd_enabled()) {
+ if (hugepage_enabled()) {
if (!khugepaged_thread)
khugepaged_thread = kthread_run(khugepaged, NULL,
"khugepaged");
@@ -2787,7 +3140,7 @@ fail:
void khugepaged_min_free_kbytes_update(void)
{
mutex_lock(&khugepaged_mutex);
- if (hugepage_pmd_enabled() && khugepaged_thread)
+ if (hugepage_enabled() && khugepaged_thread)
set_recommended_min_free_kbytes();
mutex_unlock(&khugepaged_mutex);
}
@@ -2842,9 +3195,15 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
BUG_ON(vma->vm_start > start);
BUG_ON(vma->vm_end < end);
- if (!thp_vma_allowable_order(vma, vma->vm_flags, TVA_FORCED_COLLAPSE, PMD_ORDER))
+ if (!collapse_possible(vma, vma->vm_flags, TVA_FORCED_COLLAPSE))
return -EINVAL;
+ hstart = ALIGN(start, HPAGE_PMD_SIZE);
+ hend = ALIGN_DOWN(end, HPAGE_PMD_SIZE);
+
+ if (hstart >= hend)
+ return 0;
+
cc = kmalloc_obj(*cc);
if (!cc)
return -ENOMEM;
@@ -2854,9 +3213,6 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
mmgrab(mm);
lru_add_drain_all();
- hstart = ALIGN(start, HPAGE_PMD_SIZE);
- hend = ALIGN_DOWN(end, HPAGE_PMD_SIZE);
-
for (addr = hstart; addr < hend; addr += HPAGE_PMD_SIZE) {
enum scan_result result = SCAN_FAIL;
@@ -2866,8 +3222,8 @@ int madvise_collapse(struct vm_area_struct *vma, unsigned long start,
mmap_unlocked = false;
*lock_dropped = true;
result = hugepage_vma_revalidate(mm, addr, false, &vma,
- cc);
- if (result != SCAN_SUCCEED) {
+ cc, HPAGE_PMD_ORDER);
+ if (result != SCAN_SUCCEED) {
last_fail = result;
goto out_nolock;
}
diff --git a/mm/memory.c b/mm/memory.c
index 56be920c56d7..ff338c2abe92 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4386,7 +4386,7 @@ void unmap_mapping_folio(struct folio *folio)
details.zap_flags = ZAP_FLAG_DROP_MARKER;
i_mmap_lock_read(mapping);
- if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
+ if (unlikely(mapping_mapped(mapping)))
unmap_mapping_range_tree(&mapping->i_mmap, first_index,
last_index, &details);
i_mmap_unlock_read(mapping);
@@ -4416,7 +4416,7 @@ void unmap_mapping_pages(struct address_space *mapping, pgoff_t start,
last_index = ULONG_MAX;
i_mmap_lock_read(mapping);
- if (unlikely(!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root)))
+ if (unlikely(mapping_mapped(mapping)))
unmap_mapping_range_tree(&mapping->i_mmap, first_index,
last_index, &details);
i_mmap_unlock_read(mapping);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index f7db8f049bd2..ee902a468c2f 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1246,7 +1246,7 @@ void __clear_page_tag_ref(struct page *page)
/* Should be called only if mem_alloc_profiling_enabled() */
static noinline
void __pgalloc_tag_add(struct page *page, struct task_struct *task,
- unsigned int nr)
+ unsigned int nr, gfp_t gfp_flags)
{
union pgtag_ref_handle handle;
union codetag_ref ref;
@@ -1260,17 +1260,17 @@ void __pgalloc_tag_add(struct page *page, struct task_struct *task,
* page_ext is not available yet, record the pfn so we can
* clear the tag ref later when page_ext is initialized.
*/
- alloc_tag_add_early_pfn(page_to_pfn(page));
+ alloc_tag_add_early_pfn(page_to_pfn(page), gfp_flags);
if (task->alloc_tag)
alloc_tag_set_inaccurate(task->alloc_tag);
}
}
static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
- unsigned int nr)
+ unsigned int nr, gfp_t gfp_flags)
{
if (mem_alloc_profiling_enabled())
- __pgalloc_tag_add(page, task, nr);
+ __pgalloc_tag_add(page, task, nr, gfp_flags);
}
/* Should be called only if mem_alloc_profiling_enabled() */
@@ -1303,7 +1303,7 @@ static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr)
#else /* CONFIG_MEM_ALLOC_PROFILING */
static inline void pgalloc_tag_add(struct page *page, struct task_struct *task,
- unsigned int nr) {}
+ unsigned int nr, gfp_t gfp_flags) {}
static inline void pgalloc_tag_sub(struct page *page, unsigned int nr) {}
static inline void pgalloc_tag_sub_pages(struct alloc_tag *tag, unsigned int nr) {}
@@ -1858,7 +1858,7 @@ inline void post_alloc_hook(struct page *page, unsigned int order,
set_page_owner(page, order, gfp_flags);
page_table_check_alloc(page, order);
- pgalloc_tag_add(page, current, 1 << order);
+ pgalloc_tag_add(page, current, 1 << order, gfp_flags);
}
static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
@@ -6653,7 +6653,8 @@ static int sysctl_min_unmapped_ratio_sysctl_handler(const struct ctl_table *tabl
if (rc)
return rc;
- setup_min_unmapped_ratio();
+ if (write)
+ setup_min_unmapped_ratio();
return 0;
}
@@ -6680,7 +6681,8 @@ static int sysctl_min_slab_ratio_sysctl_handler(const struct ctl_table *table, i
if (rc)
return rc;
- setup_min_slab_ratio();
+ if (write)
+ setup_min_slab_ratio();
return 0;
}
diff --git a/mm/page_isolation.c b/mm/page_isolation.c
index 7a9d631945a3..32ce8a7d9df3 100644
--- a/mm/page_isolation.c
+++ b/mm/page_isolation.c
@@ -41,8 +41,14 @@ bool page_is_unmovable(struct zone *zone, struct page *page,
* We need not scan over tail pages because we don't
* handle each tail page individually in migration.
*/
- if (PageHuge(page) || PageCompound(page)) {
+ if (PageCompound(page)) {
struct folio *folio = page_folio(page);
+ unsigned long nr_pages, pfn;
+ unsigned int order;
+
+ order = compound_order(&folio->page);
+ if (order > MAX_FOLIO_ORDER)
+ return true;
if (folio_test_hugetlb(folio)) {
struct hstate *h;
@@ -54,15 +60,16 @@ bool page_is_unmovable(struct zone *zone, struct page *page,
* The huge page may be freed so can not
* use folio_hstate() directly.
*/
- h = size_to_hstate(folio_size(folio));
- if (h && !hugepage_migration_supported(h))
+ h = size_to_hstate(PAGE_SIZE << order);
+ if (!h || !hugepage_migration_supported(h))
return true;
-
- } else if (!folio_test_lru(folio)) {
+ } else if (!PageLRU(page)) {
return true;
}
- *step = folio_nr_pages(folio) - folio_page_idx(folio, page);
+ nr_pages = 1UL << order;
+ pfn = page_to_pfn(page);
+ *step = (pfn | (nr_pages - 1)) + 1 - pfn;
return false;
}
diff --git a/mm/readahead.c b/mm/readahead.c
index 38ce16e3fcbd..558c92957518 100644
--- a/mm/readahead.c
+++ b/mm/readahead.c
@@ -335,8 +335,6 @@ static void do_page_cache_ra(struct readahead_control *ractl,
return;
end_index = (isize - 1) >> PAGE_SHIFT;
- if (end_index > ractl->_max_index)
- end_index = ractl->_max_index;
if (index > end_index)
return;
/* Don't read past the page containing the last byte of the file */
@@ -487,7 +485,7 @@ void page_cache_ra_order(struct readahead_control *ractl,
pgoff_t start = readahead_index(ractl);
pgoff_t index = start;
unsigned int min_order = mapping_min_folio_order(mapping);
- pgoff_t limit;
+ pgoff_t limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT;
pgoff_t mark;
unsigned int nofs;
int err = 0;
@@ -500,8 +498,6 @@ void page_cache_ra_order(struct readahead_control *ractl,
goto fallback;
}
- limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT;
- limit = min(limit, ractl->_max_index);
if (limit > index + ra->size - 1) {
limit = index + ra->size - 1;
mark = index + ra->size - ra->async_size;
diff --git a/mm/shrinker_debug.c b/mm/shrinker_debug.c
index affa64437302..cda4e86428c8 100644
--- a/mm/shrinker_debug.c
+++ b/mm/shrinker_debug.c
@@ -57,8 +57,6 @@ static int shrinker_debugfs_count_show(struct seq_file *m, void *v)
if (!count_per_node)
return -ENOMEM;
- rcu_read_lock();
-
memcg_aware = shrinker->flags & SHRINKER_MEMCG_AWARE;
memcg = mem_cgroup_iter(NULL, NULL, NULL);
@@ -88,8 +86,6 @@ static int shrinker_debugfs_count_show(struct seq_file *m, void *v)
}
} while ((memcg = mem_cgroup_iter(NULL, memcg, NULL)) != NULL);
- rcu_read_unlock();
-
kfree(count_per_node);
return ret;
}
diff --git a/mm/truncate.c b/mm/truncate.c
index 12cc89f89afc..b58ba940be47 100644
--- a/mm/truncate.c
+++ b/mm/truncate.c
@@ -177,7 +177,7 @@ int truncate_inode_folio(struct address_space *mapping, struct folio *folio)
return 0;
}
-static int try_folio_split_or_unmap(struct folio *folio, struct page *split_at,
+static int folio_split_or_unmap(struct folio *folio, struct page *split_at,
unsigned long min_order)
{
enum ttu_flags ttu_flags =
@@ -186,7 +186,7 @@ static int try_folio_split_or_unmap(struct folio *folio, struct page *split_at,
TTU_IGNORE_MLOCK;
int ret;
- ret = try_folio_split_to_order(folio, split_at, min_order);
+ ret = folio_split(folio, min_order, split_at, NULL);
/*
* If the split fails, unmap the folio, so it will be refaulted
@@ -252,7 +252,7 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)
min_order = mapping_min_folio_order(folio->mapping);
split_at = folio_page(folio, PAGE_ALIGN_DOWN(offset) / PAGE_SIZE);
- if (!try_folio_split_or_unmap(folio, split_at, min_order)) {
+ if (!folio_split_or_unmap(folio, split_at, min_order)) {
/*
* try to split at offset + length to make sure folios within
* the range can be dropped, especially to avoid memory waste
@@ -279,7 +279,7 @@ bool truncate_inode_partial_folio(struct folio *folio, loff_t start, loff_t end)
/* make sure folio2 is large and does not change its mapping */
if (folio_test_large(folio2) &&
folio2->mapping == folio->mapping)
- try_folio_split_or_unmap(folio2, split_at2, min_order);
+ folio_split_or_unmap(folio2, split_at2, min_order);
folio_unlock(folio2);
out:
diff --git a/mm/vmscan.c b/mm/vmscan.c
index b21a15f36cce..35c3bb15ae96 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -612,45 +612,14 @@ typedef enum {
PAGE_CLEAN,
} pageout_t;
-static pageout_t writeout(struct folio *folio, struct address_space *mapping,
- struct swap_iocb **plug, struct list_head *folio_list)
-{
- int res;
-
- folio_set_reclaim(folio);
-
- /*
- * The large shmem folio can be split if CONFIG_THP_SWAP is not enabled
- * or we failed to allocate contiguous swap entries, in which case
- * the split out folios get added back to folio_list.
- */
- if (shmem_mapping(mapping))
- res = shmem_writeout(folio, plug, folio_list);
- else
- res = swap_writeout(folio, plug);
-
- if (res < 0)
- handle_write_error(mapping, folio, res);
- if (res == AOP_WRITEPAGE_ACTIVATE) {
- folio_clear_reclaim(folio);
- return PAGE_ACTIVATE;
- }
-
- /* synchronous write? */
- if (!folio_test_writeback(folio))
- folio_clear_reclaim(folio);
-
- trace_mm_vmscan_write_folio(folio);
- node_stat_add_folio(folio, NR_VMSCAN_WRITE);
- return PAGE_SUCCESS;
-}
-
/*
* pageout is called by shrink_folio_list() for each dirty folio.
*/
static pageout_t pageout(struct folio *folio, struct address_space *mapping,
struct swap_iocb **plug, struct list_head *folio_list)
{
+ int res;
+
/*
* We no longer attempt to writeback filesystem folios here, other
* than tmpfs/shmem. That's taken care of in page-writeback.
@@ -674,7 +643,33 @@ static pageout_t pageout(struct folio *folio, struct address_space *mapping,
return PAGE_ACTIVATE;
if (!folio_clear_dirty_for_io(folio))
return PAGE_CLEAN;
- return writeout(folio, mapping, plug, folio_list);
+
+ folio_set_reclaim(folio);
+
+ /*
+ * The large shmem folio can be split if CONFIG_THP_SWAP is not enabled
+ * or we failed to allocate contiguous swap entries, in which case
+ * the split out folios get added back to folio_list.
+ */
+ if (shmem_mapping(mapping))
+ res = shmem_writeout(folio, plug, folio_list);
+ else
+ res = swap_writeout(folio, plug);
+
+ if (res < 0)
+ handle_write_error(mapping, folio, res);
+ if (res == AOP_WRITEPAGE_ACTIVATE) {
+ folio_clear_reclaim(folio);
+ return PAGE_ACTIVATE;
+ }
+
+ /* synchronous write? */
+ if (!folio_test_writeback(folio))
+ folio_clear_reclaim(folio);
+
+ trace_mm_vmscan_write_folio(folio);
+ node_stat_add_folio(folio, NR_VMSCAN_WRITE);
+ return PAGE_SUCCESS;
}
/*
@@ -6711,11 +6706,11 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
return 1;
set_task_reclaim_state(current, &sc.reclaim_state);
- trace_mm_vmscan_direct_reclaim_begin(sc.gfp_mask, order, 0);
+ trace_mm_vmscan_direct_reclaim_begin(sc.gfp_mask, order, NULL);
nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
- trace_mm_vmscan_direct_reclaim_end(nr_reclaimed, 0);
+ trace_mm_vmscan_direct_reclaim_end(nr_reclaimed, NULL);
set_task_reclaim_state(current, NULL);
return nr_reclaimed;
@@ -7781,7 +7776,7 @@ static unsigned long __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask,
delayacct_freepages_end();
psi_memstall_leave(&pflags);
- trace_mm_vmscan_node_reclaim_end(sc->nr_reclaimed, 0);
+ trace_mm_vmscan_node_reclaim_end(sc->nr_reclaimed, NULL);
return sc->nr_reclaimed;
}
diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 63128ddb7959..83f5820c45f9 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -565,8 +565,7 @@ static int zs_stats_size_show(struct seq_file *s, void *v)
total_freeable += freeable;
}
- seq_puts(s, "\n");
- seq_printf(s, " %5s %5s ", "Total", "");
+ seq_printf(s, "\n %5s %5s ", "Total", "");
for (fg = ZS_INUSE_RATIO_10; fg < NR_FULLNESS_GROUPS; fg++)
seq_printf(s, "%9lu ", inuse_totals[fg]);
generated by cgit v1.2.3 (git 2.0.4) at 2026-10-13 18:57:30 +0000