diff options
Diffstat (limited to 'mm/huge_memory.c')
| -rw-r--r-- | mm/huge_memory.c | 1484 |
1 files changed, 857 insertions, 627 deletions
diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 8e2746ea74ad..1e5d68acf62a 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -14,6 +14,7 @@ #include <linux/mmu_notifier.h> #include <linux/rmap.h> #include <linux/swap.h> +#include <linux/list_lru.h> #include <linux/shrinker.h> #include <linux/mm_inline.h> #include <linux/swapops.h> @@ -40,6 +41,7 @@ #include <linux/pgalloc.h> #include <linux/pgalloc_tag.h> #include <linux/pagewalk.h> +#include <linux/cleanup.h> #include <asm/tlb.h> #include "internal.h" @@ -67,6 +69,8 @@ unsigned long transparent_hugepage_flags __read_mostly = (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)| (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG); +static struct lock_class_key deferred_split_key; +static struct list_lru deferred_split_lru; static struct shrinker *deferred_split_shrinker; static unsigned long deferred_split_count(struct shrinker *shrink, struct shrink_control *sc); @@ -74,21 +78,24 @@ static unsigned long deferred_split_scan(struct shrinker *shrink, struct shrink_control *sc); static bool split_underused_thp = true; -static atomic_t huge_zero_refcount; +#define HUGE_ZERO_UNSET_PFN (~0UL) struct folio *huge_zero_folio __read_mostly; -unsigned long huge_zero_pfn __read_mostly = ~0UL; +unsigned long huge_zero_pfn __read_mostly = HUGE_ZERO_UNSET_PFN; +#ifndef CONFIG_PERSISTENT_HUGE_ZERO_FOLIO +static atomic_t huge_zero_refcount; +static DEFINE_SPINLOCK(huge_zero_lock); +static struct shrinker *huge_zero_folio_shrinker; +#endif + unsigned long huge_anon_orders_always __read_mostly; unsigned long huge_anon_orders_madvise __read_mostly; unsigned long huge_anon_orders_inherit __read_mostly; static bool anon_orders_configured __initdata; -static inline bool file_thp_enabled(struct vm_area_struct *vma) +static inline bool file_thp_enabled(const struct vm_area_struct *vma) { struct inode *inode; - if (!IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS)) - return false; - if (!vma->vm_file) return false; @@ -97,7 +104,79 @@ static inline bool file_thp_enabled(struct vm_area_struct *vma) if (IS_ANON_FILE(inode)) return false; - return !inode_is_open_for_write(inode) && S_ISREG(inode->i_mode); + if (!mapping_pmd_folio_support(vma->vm_file->f_mapping)) + return false; + + return S_ISREG(inode->i_mode); +} + +/* If returns true, we are unable to access the VMA's folios. */ +static bool vma_is_special_huge(const struct vm_area_struct *vma) +{ + if (vma_is_dax(vma)) + return false; + return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT); +} + +static bool vma_file_bypass_thp_tuneables(const struct vm_area_struct *vma, + enum tva_type type) +{ + const bool has_huge_fault = vma->vm_ops->huge_fault; + + /* MADV_COLLAPSE ignores tuneables. */ + if (type == TVA_FORCED_COLLAPSE) + return true; + /* Huge PFN mappings are uncompactable so the policy doesn't apply. */ + if (vma_test(vma, VMA_PFNMAP_BIT) && has_huge_fault) + return true; + return false; +} + +static bool vma_file_allow_thp_tuneables(vm_flags_t vm_flags) +{ + /* THP=always? */ + if (hugepage_global_always()) + return true; + /* THP=madvise and marked MADV_HUGEPAGE? */ + if (hugepage_global_enabled() && (vm_flags & VM_HUGEPAGE)) + return true; + return false; +} + +static bool vma_file_check_thp_tuneables(const struct vm_area_struct *vma, + vm_flags_t vm_flags, enum tva_type type) +{ + return vma_file_bypass_thp_tuneables(vma, type) || + vma_file_allow_thp_tuneables(vm_flags); +} + +static bool vma_can_map_huge_file(const struct vm_area_struct *vma, + vm_flags_t vm_flags, enum tva_type type) +{ + const bool has_huge_fault = vma->vm_ops->huge_fault; + + /* + * Enforce THP collapse requirements as necessary. Anonymous vmas + * were already handled in thp_vma_allowable_orders(). + */ + if (!vma_file_check_thp_tuneables(vma, vm_flags, type)) + return false; + + switch (type) { + case TVA_PAGEFAULT: + /* + * Trust that ->huge_fault() handlers know what they are doing + * in fault path. + */ + return has_huge_fault; + case TVA_SMAPS: + if (has_huge_fault) + return true; + fallthrough; + default: + /* Only regular file is valid in collapse path. */ + return file_thp_enabled(vma); + } } unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, @@ -113,8 +192,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, /* Check the intersection of requested and supported orders. */ if (vma_is_anonymous(vma)) supported_orders = THP_ORDERS_ALL_ANON; - else if (vma_is_special_huge(vma)) - supported_orders = THP_ORDERS_ALL_SPECIAL; + else if (vma_is_dax(vma) || vma_is_special_huge(vma)) + supported_orders = THP_ORDERS_ALL_SPECIAL_DAX; else supported_orders = THP_ORDERS_ALL_FILE_DEFAULT; @@ -169,30 +248,11 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, */ if (!in_pf && shmem_file(vma->vm_file)) return orders & shmem_allowable_huge_orders(file_inode(vma->vm_file), - vma, vma->vm_pgoff, 0, + vma, vma_start_pgoff(vma), 0, forced_collapse); - if (!vma_is_anonymous(vma)) { - /* - * Enforce THP collapse requirements as necessary. Anonymous vmas - * were already handled in thp_vma_allowable_orders(). - */ - if (!forced_collapse && - (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && - !hugepage_global_always()))) - return 0; - - /* - * Trust that ->huge_fault() handlers know what they are doing - * in fault path. - */ - if (((in_pf || smaps)) && vma->vm_ops->huge_fault) - return orders; - /* Only regular file is valid in collapse path */ - if (((!in_pf || smaps)) && file_thp_enabled(vma)) - return orders; - return 0; - } + if (!vma_is_anonymous(vma)) + return vma_can_map_huge_file(vma, vm_flags, type) ? orders : 0; if (vma_is_temporary_stack(vma)) return 0; @@ -210,33 +270,74 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, return orders; } -static bool get_huge_zero_folio(void) +static struct folio *alloc_huge_zero_folio(void) { struct folio *zero_folio; -retry: - if (likely(atomic_inc_not_zero(&huge_zero_refcount))) - return true; zero_folio = folio_alloc((GFP_TRANSHUGE | __GFP_ZERO | __GFP_ZEROTAGS) & ~__GFP_MOVABLE, HPAGE_PMD_ORDER); if (!zero_folio) { count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED); - return false; + return NULL; + } + folio_clear_large_rmappable(zero_folio); /* Explicitly not rmappable. */ + return zero_folio; +} + +#ifdef CONFIG_PERSISTENT_HUGE_ZERO_FOLIO +static int __init huge_zero_init(void) +{ + huge_zero_folio = alloc_huge_zero_folio(); + if (!huge_zero_folio) { + pr_warn("Allocating persistent huge zero folio failed\n"); + } else { + huge_zero_pfn = folio_pfn(huge_zero_folio); + count_vm_event(THP_ZERO_PAGE_ALLOC); } - /* Ensure zero folio won't have large_rmappable flag set. */ - folio_clear_large_rmappable(zero_folio); - preempt_disable(); - if (cmpxchg(&huge_zero_folio, NULL, zero_folio)) { - preempt_enable(); + return 0; +} + +static void __init huge_zero_shrinker_exit(void) +{ +} + +struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) +{ + return huge_zero_folio; +} + +void mm_put_huge_zero_folio(struct mm_struct *mm) +{ +} +#else +static bool get_huge_zero_folio(void) +{ + struct folio *zero_folio; + + /* Paired with atomic_set_release(). */ + if (likely(atomic_inc_not_zero(&huge_zero_refcount))) + return true; + + zero_folio = alloc_huge_zero_folio(); + if (unlikely(!zero_folio)) + return false; + + /* Paired with critical section in shrink_huge_zero_folio_scan(). */ + spin_lock(&huge_zero_lock); + if (huge_zero_folio) { + /* Somebody else already installed it. */ + atomic_inc(&huge_zero_refcount); + spin_unlock(&huge_zero_lock); folio_put(zero_folio); - goto retry; + return true; } + WRITE_ONCE(huge_zero_folio, zero_folio); WRITE_ONCE(huge_zero_pfn, folio_pfn(zero_folio)); + /* Paired with atomic_inc_not_zero(). +1 for shrinker pin. */ + atomic_set_release(&huge_zero_refcount, 2); + spin_unlock(&huge_zero_lock); - /* We take additional reference here. It will be put back by shrinker */ - atomic_set(&huge_zero_refcount, 2); - preempt_enable(); count_vm_event(THP_ZERO_PAGE_ALLOC); return true; } @@ -247,14 +348,59 @@ static void put_huge_zero_folio(void) * Counter should never go to zero here. Only shrinker can put * last reference. */ - BUG_ON(atomic_dec_and_test(&huge_zero_refcount)); + WARN_ON_ONCE(atomic_dec_and_test(&huge_zero_refcount)); } -struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) +static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink, + struct shrink_control *sc) { - if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) - return huge_zero_folio; + /* we can free zero page only if last reference remains */ + return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; +} +static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink, + struct shrink_control *sc) +{ + struct folio *zero_folio; + + /* Paired with critical section in get_huge_zero_folio(). */ + scoped_guard(spinlock, &huge_zero_lock) { + /* Paired with atomic_inc_not_zero() in get_huge_zero_folio(). */ + if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) != 1) + return 0; + + zero_folio = huge_zero_folio; + VM_WARN_ON_ONCE(!zero_folio); + WRITE_ONCE(huge_zero_folio, NULL); + WRITE_ONCE(huge_zero_pfn, HUGE_ZERO_UNSET_PFN); + } + + folio_put(zero_folio); + return HPAGE_PMD_NR; +} + +static int __init huge_zero_init(void) +{ + huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero"); + if (!huge_zero_folio_shrinker) { + shrinker_free(deferred_split_shrinker); + list_lru_destroy(&deferred_split_lru); + return -ENOMEM; + } + + huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count; + huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan; + shrinker_register(huge_zero_folio_shrinker); + return 0; +} + +static void __init huge_zero_shrinker_exit(void) +{ + shrinker_free(huge_zero_folio_shrinker); +} + +struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) +{ if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) return READ_ONCE(huge_zero_folio); @@ -269,35 +415,10 @@ struct folio *mm_get_huge_zero_folio(struct mm_struct *mm) void mm_put_huge_zero_folio(struct mm_struct *mm) { - if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) - return; - if (mm_flags_test(MMF_HUGE_ZERO_FOLIO, mm)) put_huge_zero_folio(); } - -static unsigned long shrink_huge_zero_folio_count(struct shrinker *shrink, - struct shrink_control *sc) -{ - /* we can free zero page only if last reference remains */ - return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0; -} - -static unsigned long shrink_huge_zero_folio_scan(struct shrinker *shrink, - struct shrink_control *sc) -{ - if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) { - struct folio *zero_folio = xchg(&huge_zero_folio, NULL); - BUG_ON(zero_folio == NULL); - WRITE_ONCE(huge_zero_pfn, ~0UL); - folio_put(zero_folio); - return HPAGE_PMD_NR; - } - - return 0; -} - -static struct shrinker *huge_zero_folio_shrinker; +#endif /* CONFIG_PERSISTENT_HUGE_ZERO_FOLIO */ #ifdef CONFIG_SYSFS static ssize_t enabled_show(struct kobject *kobj, @@ -316,30 +437,77 @@ static ssize_t enabled_show(struct kobject *kobj, return sysfs_emit(buf, "%s\n", output); } +enum anon_enabled_mode { + ANON_ENABLED_ALWAYS = 0, + ANON_ENABLED_INHERIT = 1, + ANON_ENABLED_MADVISE = 2, + ANON_ENABLED_NEVER = 3, +}; + +static const char * const anon_enabled_mode_strings[] = { + [ANON_ENABLED_ALWAYS] = "always", + [ANON_ENABLED_INHERIT] = "inherit", + [ANON_ENABLED_MADVISE] = "madvise", + [ANON_ENABLED_NEVER] = "never", +}; + +enum global_enabled_mode { + GLOBAL_ENABLED_ALWAYS = 0, + GLOBAL_ENABLED_MADVISE = 1, + GLOBAL_ENABLED_NEVER = 2, +}; + +static const char * const global_enabled_mode_strings[] = { + [GLOBAL_ENABLED_ALWAYS] = "always", + [GLOBAL_ENABLED_MADVISE] = "madvise", + [GLOBAL_ENABLED_NEVER] = "never", +}; + +static bool set_global_enabled_mode(enum global_enabled_mode mode) +{ + static const unsigned long thp_flags[] = { + TRANSPARENT_HUGEPAGE_FLAG, + TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, + }; + enum global_enabled_mode m; + bool changed = false; + + for (m = 0; m < ARRAY_SIZE(thp_flags); m++) { + if (m == mode) + changed |= !test_and_set_bit(thp_flags[m], + &transparent_hugepage_flags); + else + changed |= test_and_clear_bit(thp_flags[m], + &transparent_hugepage_flags); + } + + return changed; +} + static ssize_t enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - ssize_t ret = count; - - if (sysfs_streq(buf, "always")) { - clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); - set_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); - } else if (sysfs_streq(buf, "madvise")) { - clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); - set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); - } else if (sysfs_streq(buf, "never")) { - clear_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags); - } else - ret = -EINVAL; + int mode; - if (ret > 0) { + mode = sysfs_match_string(global_enabled_mode_strings, buf); + if (mode < 0) + return -EINVAL; + + if (set_global_enabled_mode(mode)) { int err = start_stop_khugepaged(); + if (err) - ret = err; + return err; + } else { + /* + * Recalculate watermarks even when the mode didn't + * change, as the previous code always called + * start_stop_khugepaged() which does this internally. + */ + set_recommended_min_free_kbytes(); } - return ret; + return count; } static struct kobj_attribute enabled_attr = __ATTR_RW(enabled); @@ -374,61 +542,75 @@ ssize_t single_hugepage_flag_store(struct kobject *kobj, return count; } +enum defrag_mode { + DEFRAG_ALWAYS = 0, + DEFRAG_DEFER, + DEFRAG_DEFER_MADVISE, + DEFRAG_MADVISE, + DEFRAG_NEVER, +}; + +static const char * const defrag_mode_strings[] = { + [DEFRAG_ALWAYS] = "always", + [DEFRAG_DEFER] = "defer", + [DEFRAG_DEFER_MADVISE] = "defer+madvise", + [DEFRAG_MADVISE] = "madvise", + [DEFRAG_NEVER] = "never", +}; + +static const enum transparent_hugepage_flag defrag_flags[] = { + [DEFRAG_ALWAYS] = TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, + [DEFRAG_DEFER] = TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, + [DEFRAG_DEFER_MADVISE] = TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, + [DEFRAG_MADVISE] = TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, +}; + static ssize_t defrag_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - const char *output; + int active = DEFRAG_NEVER; + int len = 0; + int i; - if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, - &transparent_hugepage_flags)) - output = "[always] defer defer+madvise madvise never"; - else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, - &transparent_hugepage_flags)) - output = "always [defer] defer+madvise madvise never"; - else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, - &transparent_hugepage_flags)) - output = "always defer [defer+madvise] madvise never"; - else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, - &transparent_hugepage_flags)) - output = "always defer defer+madvise [madvise] never"; - else - output = "always defer defer+madvise madvise [never]"; + for (i = 0; i < ARRAY_SIZE(defrag_flags); i++) { + if (test_bit(defrag_flags[i], &transparent_hugepage_flags)) { + active = i; + break; + } + } - return sysfs_emit(buf, "%s\n", output); + for (i = 0; i < ARRAY_SIZE(defrag_mode_strings); i++) { + if (i == active) + len += sysfs_emit_at(buf, len, "[%s] ", + defrag_mode_strings[i]); + else + len += sysfs_emit_at(buf, len, "%s ", + defrag_mode_strings[i]); + } + + /* Replace trailing space with newline */ + buf[len - 1] = '\n'; + + return len; } static ssize_t defrag_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { - if (sysfs_streq(buf, "always")) { - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); - set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); - } else if (sysfs_streq(buf, "defer+madvise")) { - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); - set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); - } else if (sysfs_streq(buf, "defer")) { - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); - set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); - } else if (sysfs_streq(buf, "madvise")) { - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); - set_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); - } else if (sysfs_streq(buf, "never")) { - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags); - clear_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags); - } else + int mode, m; + + mode = sysfs_match_string(defrag_mode_strings, buf); + if (mode < 0) return -EINVAL; + for (m = 0; m < ARRAY_SIZE(defrag_flags); m++) { + if (m == mode) + set_bit(defrag_flags[m], &transparent_hugepage_flags); + else + clear_bit(defrag_flags[m], &transparent_hugepage_flags); + } + return count; } static struct kobj_attribute defrag_attr = __ATTR_RW(defrag); @@ -515,48 +697,54 @@ static ssize_t anon_enabled_show(struct kobject *kobj, return sysfs_emit(buf, "%s\n", output); } +static bool set_anon_enabled_mode(int order, enum anon_enabled_mode mode) +{ + static unsigned long *enabled_orders[] = { + &huge_anon_orders_always, + &huge_anon_orders_inherit, + &huge_anon_orders_madvise, + }; + enum anon_enabled_mode m; + bool changed = false; + + spin_lock(&huge_anon_orders_lock); + for (m = 0; m < ARRAY_SIZE(enabled_orders); m++) { + if (m == mode) + changed |= !__test_and_set_bit(order, enabled_orders[m]); + else + changed |= __test_and_clear_bit(order, enabled_orders[m]); + } + spin_unlock(&huge_anon_orders_lock); + + return changed; +} + static ssize_t anon_enabled_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { int order = to_thpsize(kobj)->order; - ssize_t ret = count; - - if (sysfs_streq(buf, "always")) { - spin_lock(&huge_anon_orders_lock); - clear_bit(order, &huge_anon_orders_inherit); - clear_bit(order, &huge_anon_orders_madvise); - set_bit(order, &huge_anon_orders_always); - spin_unlock(&huge_anon_orders_lock); - } else if (sysfs_streq(buf, "inherit")) { - spin_lock(&huge_anon_orders_lock); - clear_bit(order, &huge_anon_orders_always); - clear_bit(order, &huge_anon_orders_madvise); - set_bit(order, &huge_anon_orders_inherit); - spin_unlock(&huge_anon_orders_lock); - } else if (sysfs_streq(buf, "madvise")) { - spin_lock(&huge_anon_orders_lock); - clear_bit(order, &huge_anon_orders_always); - clear_bit(order, &huge_anon_orders_inherit); - set_bit(order, &huge_anon_orders_madvise); - spin_unlock(&huge_anon_orders_lock); - } else if (sysfs_streq(buf, "never")) { - spin_lock(&huge_anon_orders_lock); - clear_bit(order, &huge_anon_orders_always); - clear_bit(order, &huge_anon_orders_inherit); - clear_bit(order, &huge_anon_orders_madvise); - spin_unlock(&huge_anon_orders_lock); - } else - ret = -EINVAL; + int mode; + + mode = sysfs_match_string(anon_enabled_mode_strings, buf); + if (mode < 0) + return -EINVAL; - if (ret > 0) { - int err; + if (set_anon_enabled_mode(order, mode)) { + int err = start_stop_khugepaged(); - err = start_stop_khugepaged(); if (err) - ret = err; + return err; + } else { + /* + * Recalculate watermarks even when the mode didn't + * change, as the previous code always called + * start_stop_khugepaged() which does this internally. + */ + set_recommended_min_free_kbytes(); } - return ret; + + return count; } static struct kobj_attribute anon_enabled_attr = @@ -624,6 +812,8 @@ static struct kobj_attribute _name##_attr = __ATTR_RO(_name) DEFINE_MTHP_STAT_ATTR(anon_fault_alloc, MTHP_STAT_ANON_FAULT_ALLOC); DEFINE_MTHP_STAT_ATTR(anon_fault_fallback, MTHP_STAT_ANON_FAULT_FALLBACK); DEFINE_MTHP_STAT_ATTR(anon_fault_fallback_charge, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); +DEFINE_MTHP_STAT_ATTR(collapse_alloc, MTHP_STAT_COLLAPSE_ALLOC); +DEFINE_MTHP_STAT_ATTR(collapse_alloc_failed, MTHP_STAT_COLLAPSE_ALLOC_FAILED); DEFINE_MTHP_STAT_ATTR(zswpout, MTHP_STAT_ZSWPOUT); DEFINE_MTHP_STAT_ATTR(swpin, MTHP_STAT_SWPIN); DEFINE_MTHP_STAT_ATTR(swpin_fallback, MTHP_STAT_SWPIN_FALLBACK); @@ -640,6 +830,10 @@ DEFINE_MTHP_STAT_ATTR(split_failed, MTHP_STAT_SPLIT_FAILED); DEFINE_MTHP_STAT_ATTR(split_deferred, MTHP_STAT_SPLIT_DEFERRED); DEFINE_MTHP_STAT_ATTR(nr_anon, MTHP_STAT_NR_ANON); DEFINE_MTHP_STAT_ATTR(nr_anon_partially_mapped, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED); +DEFINE_MTHP_STAT_ATTR(collapse_exceed_swap_pte, MTHP_STAT_COLLAPSE_EXCEED_SWAP); +DEFINE_MTHP_STAT_ATTR(collapse_exceed_none_pte, MTHP_STAT_COLLAPSE_EXCEED_NONE); +DEFINE_MTHP_STAT_ATTR(collapse_exceed_shared_pte, MTHP_STAT_COLLAPSE_EXCEED_SHARED); + static struct attribute *anon_stats_attrs[] = { &anon_fault_alloc_attr.attr, @@ -656,6 +850,9 @@ static struct attribute *anon_stats_attrs[] = { &split_deferred_attr.attr, &nr_anon_attr.attr, &nr_anon_partially_mapped_attr.attr, + &collapse_exceed_swap_pte_attr.attr, + &collapse_exceed_none_pte_attr.attr, + &collapse_exceed_shared_pte_attr.attr, NULL, }; @@ -689,6 +886,8 @@ static struct attribute *any_stats_attrs[] = { #endif &split_attr.attr, &split_failed_attr.attr, + &collapse_alloc_attr.attr, + &collapse_alloc_failed_attr.attr, NULL, }; @@ -729,10 +928,8 @@ static struct thpsize *thpsize_create(int order, struct kobject *parent) ret = kobject_init_and_add(&thpsize->kobj, &thpsize_ktype, parent, "hugepages-%lukB", size); - if (ret) { - kfree(thpsize); - goto err; - } + if (ret) + goto err_put; ret = sysfs_add_group(&thpsize->kobj, &any_ctrl_attr_grp); @@ -857,50 +1054,40 @@ static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj) } #endif /* CONFIG_SYSFS */ +int folio_memcg_alloc_deferred(struct folio *folio) +{ + if (mem_cgroup_disabled()) + return 0; + return folio_memcg_list_lru_alloc(folio, &deferred_split_lru, GFP_KERNEL); +} + static int __init thp_shrinker_init(void) { deferred_split_shrinker = shrinker_alloc(SHRINKER_NUMA_AWARE | - SHRINKER_MEMCG_AWARE | - SHRINKER_NONSLAB, + SHRINKER_MEMCG_AWARE, "thp-deferred_split"); if (!deferred_split_shrinker) return -ENOMEM; - deferred_split_shrinker->count_objects = deferred_split_count; - deferred_split_shrinker->scan_objects = deferred_split_scan; - shrinker_register(deferred_split_shrinker); - - if (IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) { - /* - * Bump the reference of the huge_zero_folio and do not - * initialize the shrinker. - * - * huge_zero_folio will always be NULL on failure. We assume - * that get_huge_zero_folio() will most likely not fail as - * thp_shrinker_init() is invoked early on during boot. - */ - if (!get_huge_zero_folio()) - pr_warn("Allocating persistent huge zero folio failed\n"); - return 0; - } - - huge_zero_folio_shrinker = shrinker_alloc(0, "thp-zero"); - if (!huge_zero_folio_shrinker) { + if (list_lru_init_memcg_key(&deferred_split_lru, + deferred_split_shrinker, + &deferred_split_key)) { shrinker_free(deferred_split_shrinker); return -ENOMEM; } - huge_zero_folio_shrinker->count_objects = shrink_huge_zero_folio_count; - huge_zero_folio_shrinker->scan_objects = shrink_huge_zero_folio_scan; - shrinker_register(huge_zero_folio_shrinker); + deferred_split_shrinker->count_objects = deferred_split_count; + deferred_split_shrinker->scan_objects = deferred_split_scan; + shrinker_register(deferred_split_shrinker); - return 0; + return huge_zero_init(); } static void __init thp_shrinker_exit(void) { - shrinker_free(huge_zero_folio_shrinker); shrinker_free(deferred_split_shrinker); + list_lru_destroy(&deferred_split_lru); + huge_zero_shrinker_exit(); } static int __init hugepage_init(void) @@ -1080,103 +1267,6 @@ pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) return pmd; } -static struct deferred_split *split_queue_node(int nid) -{ - struct pglist_data *pgdata = NODE_DATA(nid); - - return &pgdata->deferred_split_queue; -} - -#ifdef CONFIG_MEMCG -static inline -struct mem_cgroup *folio_split_queue_memcg(struct folio *folio, - struct deferred_split *queue) -{ - if (mem_cgroup_disabled()) - return NULL; - if (split_queue_node(folio_nid(folio)) == queue) - return NULL; - return container_of(queue, struct mem_cgroup, deferred_split_queue); -} - -static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg) -{ - return memcg ? &memcg->deferred_split_queue : split_queue_node(nid); -} -#else -static inline -struct mem_cgroup *folio_split_queue_memcg(struct folio *folio, - struct deferred_split *queue) -{ - return NULL; -} - -static struct deferred_split *memcg_split_queue(int nid, struct mem_cgroup *memcg) -{ - return split_queue_node(nid); -} -#endif - -static struct deferred_split *split_queue_lock(int nid, struct mem_cgroup *memcg) -{ - struct deferred_split *queue; - -retry: - queue = memcg_split_queue(nid, memcg); - spin_lock(&queue->split_queue_lock); - /* - * There is a period between setting memcg to dying and reparenting - * deferred split queue, and during this period the THPs in the deferred - * split queue will be hidden from the shrinker side. - */ - if (unlikely(memcg_is_dying(memcg))) { - spin_unlock(&queue->split_queue_lock); - memcg = parent_mem_cgroup(memcg); - goto retry; - } - - return queue; -} - -static struct deferred_split * -split_queue_lock_irqsave(int nid, struct mem_cgroup *memcg, unsigned long *flags) -{ - struct deferred_split *queue; - -retry: - queue = memcg_split_queue(nid, memcg); - spin_lock_irqsave(&queue->split_queue_lock, *flags); - if (unlikely(memcg_is_dying(memcg))) { - spin_unlock_irqrestore(&queue->split_queue_lock, *flags); - memcg = parent_mem_cgroup(memcg); - goto retry; - } - - return queue; -} - -static struct deferred_split *folio_split_queue_lock(struct folio *folio) -{ - return split_queue_lock(folio_nid(folio), folio_memcg(folio)); -} - -static struct deferred_split * -folio_split_queue_lock_irqsave(struct folio *folio, unsigned long *flags) -{ - return split_queue_lock_irqsave(folio_nid(folio), folio_memcg(folio), flags); -} - -static inline void split_queue_unlock(struct deferred_split *queue) -{ - spin_unlock(&queue->split_queue_lock); -} - -static inline void split_queue_unlock_irqrestore(struct deferred_split *queue, - unsigned long flags) -{ - spin_unlock_irqrestore(&queue->split_queue_lock, flags); -} - static inline bool is_transparent_hugepage(const struct folio *folio) { if (!folio_test_large(folio)) @@ -1189,7 +1279,7 @@ static inline bool is_transparent_hugepage(const struct folio *folio) static unsigned long __thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, loff_t off, unsigned long flags, unsigned long size, - vm_flags_t vm_flags) + vma_flags_t vma_flags) { loff_t off_end = off + len; loff_t off_align = round_up(off, size); @@ -1205,8 +1295,9 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, if (len_pad < len || (off + len_pad) < off) return 0; - ret = mm_get_unmapped_area_vmflags(filp, addr, len_pad, - off >> PAGE_SHIFT, flags, vm_flags); + ret = mm_get_unmapped_area_vmaflags(filp, addr, len_pad, + off >> PAGE_SHIFT, flags, + vma_flags); /* * The failure might be due to length padding. The caller will retry @@ -1231,25 +1322,27 @@ static unsigned long __thp_get_unmapped_area(struct file *filp, return ret; } -unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, +unsigned long thp_get_unmapped_area_vmaflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, - vm_flags_t vm_flags) + vma_flags_t vma_flags) { unsigned long ret; loff_t off = (loff_t)pgoff << PAGE_SHIFT; - ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, vm_flags); + ret = __thp_get_unmapped_area(filp, addr, len, off, flags, PMD_SIZE, + vma_flags); if (ret) return ret; - return mm_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, - vm_flags); + return mm_get_unmapped_area_vmaflags(filp, addr, len, pgoff, flags, + vma_flags); } unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags) { - return thp_get_unmapped_area_vmflags(filp, addr, len, pgoff, flags, 0); + return thp_get_unmapped_area_vmaflags(filp, addr, len, pgoff, flags, + EMPTY_VMA_FLAGS); } EXPORT_SYMBOL_GPL(thp_get_unmapped_area); @@ -1277,6 +1370,14 @@ static struct folio *vma_alloc_anon_folio_pmd(struct vm_area_struct *vma, count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE); return NULL; } + + if (folio_memcg_alloc_deferred(folio)) { + folio_put(folio); + count_vm_event(THP_FAULT_FALLBACK); + count_mthp_stat(order, MTHP_STAT_ANON_FAULT_FALLBACK); + return NULL; + } + folio_throttle_swaprate(folio, gfp); /* @@ -1622,7 +1723,7 @@ vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn, BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == (VM_PFNMAP|VM_MIXEDMAP)); - BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); + BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma)); pfnmap_setup_cachemode_pfn(pfn, &pgprot); @@ -1730,7 +1831,7 @@ vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, unsigned long pfn, BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP))); BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) == (VM_PFNMAP|VM_MIXEDMAP)); - BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags)); + BUG_ON((vma->vm_flags & VM_PFNMAP) && vma_is_cow_mapping(vma)); pfnmap_setup_cachemode_pfn(pfn, &pgprot); @@ -1804,11 +1905,11 @@ static void copy_huge_non_present_pmd( if (softleaf_is_migration_write(entry) || softleaf_is_migration_read_exclusive(entry)) { entry = make_readable_migration_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd_wp(*src_pmd)) - pmd = pmd_swp_mkuffd_wp(pmd); + if (pmd_swp_uffd(*src_pmd)) + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } else if (softleaf_is_device_private(entry)) { /* @@ -1817,12 +1918,12 @@ static void copy_huge_non_present_pmd( */ if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - pmd = swp_entry_to_pmd(entry); + pmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*src_pmd)) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd_wp(*src_pmd)) - pmd = pmd_swp_mkuffd_wp(pmd); + if (pmd_swp_uffd(*src_pmd)) + pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } @@ -1841,8 +1942,8 @@ static void copy_huge_non_present_pmd( add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR); mm_inc_nr_ptes(dst_mm); pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); - if (!userfaultfd_wp(dst_vma)) - pmd = pmd_swp_clear_uffd_wp(pmd); + if (!userfaultfd_protected(dst_vma)) + pmd = pmd_swp_clear_uffd(pmd); set_pmd_at(dst_mm, addr, dst_pmd, pmd); } @@ -1872,7 +1973,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, * applied special bit, or we made the PRIVATE mapping be * able to wrongly write to the backend MMIO. */ - VM_WARN_ON_ONCE(is_cow_mapping(src_vma->vm_flags) && pmd_write(pmd)); + VM_WARN_ON_ONCE(vma_is_cow_mapping(src_vma) && pmd_write(pmd)); goto set_pmd; } @@ -1936,9 +2037,15 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, out_zero_page: mm_inc_nr_ptes(dst_mm); pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable); + + /* See __copy_present_ptes(): restore accessible protection. */ + if (!userfaultfd_protected(dst_vma)) { + if (userfaultfd_rwp(src_vma) && pmd_uffd(pmd)) + pmd = pmd_modify(pmd, dst_vma->vm_page_prot); + pmd = pmd_clear_uffd(pmd); + } + pmdp_set_wrprotect(src_mm, addr, src_pmd); - if (!userfaultfd_wp(dst_vma)) - pmd = pmd_clear_uffd_wp(pmd); pmd = pmd_wrprotect(pmd); set_pmd: pmd = pmd_mkold(pmd); @@ -1987,7 +2094,7 @@ int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, * TODO: once we support anonymous pages, use * folio_try_dup_anon_rmap_*() and split if duplicating fails. */ - if (is_cow_mapping(vma->vm_flags) && pud_write(pud)) { + if (vma_is_cow_mapping(vma) && pud_write(pud)) { pudp_set_wrprotect(src_mm, addr, src_pud); pud = pud_wrprotect(pud); } @@ -2181,6 +2288,34 @@ static inline bool can_change_pmd_writable(struct vm_area_struct *vma, return pmd_dirty(pmd); } +vm_fault_t do_huge_pmd_uffd_rwp(struct vm_fault *vmf) +{ + struct vm_area_struct *vma = vmf->vma; + pmd_t pmd; + + if (!userfaultfd_rwp_async(vma)) + return handle_userfault(vmf, VM_UFFD_RWP); + + vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); + if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { + spin_unlock(vmf->ptl); + return 0; + } + pmd = pmd_modify(vmf->orig_pmd, vma->vm_page_prot); + /* pmd_modify() preserves _PAGE_UFFD; drop it on resolution */ + pmd = pmd_clear_uffd(pmd); + pmd = pmd_mkyoung(pmd); + if (!pmd_write(pmd) && + vma_wants_manual_pte_write_upgrade(vma) && + can_change_pmd_writable(vma, vmf->address, pmd)) + pmd = pmd_mkwrite(pmd, vma); + set_pmd_at(vma->vm_mm, vmf->address & HPAGE_PMD_MASK, + vmf->pmd, pmd); + update_mmu_cache_pmd(vma, vmf->address, vmf->pmd); + spin_unlock(vmf->ptl); + return 0; +} + /* NUMA hinting page fault entry point for trans huge pmds */ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf) { @@ -2282,8 +2417,8 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, goto out; if (unlikely(!pmd_present(orig_pmd))) { - VM_BUG_ON(thp_migration_supported() && - !pmd_is_migration_entry(orig_pmd)); + VM_WARN_ON_ONCE(!pmd_is_migration_entry(orig_pmd) && + !pmd_is_device_private_entry(orig_pmd)); goto out; } @@ -2341,17 +2476,89 @@ static inline void zap_deposited_table(struct mm_struct *mm, pmd_t *pmd) mm_dec_nr_ptes(mm); } -int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, +static void zap_huge_pmd_folio(struct mm_struct *mm, struct vm_area_struct *vma, + pmd_t pmdval, struct folio *folio, bool is_present) +{ + const bool is_device_private = folio_is_device_private(folio); + + /* Present and device private folios are rmappable. */ + if (is_present || is_device_private) + folio_remove_rmap_pmd(folio, &folio->page, vma); + + if (folio_test_anon(folio)) { + add_mm_counter(mm, MM_ANONPAGES, -HPAGE_PMD_NR); + } else { + add_mm_counter(mm, mm_counter_file(folio), + -HPAGE_PMD_NR); + + if (is_present && pmd_dirty(pmdval)) + folio_mark_dirty(folio); + if (is_present && pmd_young(pmdval) && + likely(vma_has_recency(vma))) + folio_mark_accessed(folio); + } + + /* Device private folios are pinned. */ + if (is_device_private) + folio_put(folio); +} + +static struct folio *normal_or_softleaf_folio_pmd(struct vm_area_struct *vma, + unsigned long addr, pmd_t pmdval, bool is_present) +{ + if (is_present) + return vm_normal_folio_pmd(vma, addr, pmdval); + + if (!thp_migration_supported()) + WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); + return pmd_to_softleaf_folio(pmdval); +} + +static bool has_deposited_pgtable(struct vm_area_struct *vma, pmd_t pmdval, + struct folio *folio) +{ + /* Some architectures require unconditional depositing. */ + if (arch_needs_pgtable_deposit()) + return true; + + /* + * Huge zero always deposited except for DAX which handles itself, see + * set_huge_zero_folio(). + */ + if (is_huge_zero_pmd(pmdval)) + return !vma_is_dax(vma); + + /* + * Otherwise, only anonymous folios are deposited, see + * __do_huge_pmd_anonymous_page(). + */ + return folio && folio_test_anon(folio); +} + +/** + * zap_huge_pmd - Zap a huge THP which is of PMD size. + * @tlb: The MMU gather TLB state associated with the operation. + * @vma: The VMA containing the range to zap. + * @pmd: A pointer to the leaf PMD entry. + * @addr: The virtual address for the range to zap. + * + * Returns: %true on success, %false otherwise. + */ +bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr) { - pmd_t orig_pmd; + struct mm_struct *mm = tlb->mm; + struct folio *folio = NULL; + bool is_present = false; + bool has_deposit; spinlock_t *ptl; + pmd_t orig_pmd; tlb_change_page_size(tlb, HPAGE_PMD_SIZE); ptl = __pmd_trans_huge_lock(pmd, vma); if (!ptl) - return 0; + return false; /* * For architectures like ppc64 we look at deposited pgtable * when calling pmdp_huge_get_and_clear. So do the @@ -2362,64 +2569,19 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, tlb->fullmm); arch_check_zapped_pmd(vma, orig_pmd); tlb_remove_pmd_tlb_entry(tlb, pmd, addr); - if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { - if (arch_needs_pgtable_deposit()) - zap_deposited_table(tlb->mm, pmd); - spin_unlock(ptl); - } else if (is_huge_zero_pmd(orig_pmd)) { - if (!vma_is_dax(vma) || arch_needs_pgtable_deposit()) - zap_deposited_table(tlb->mm, pmd); - spin_unlock(ptl); - } else { - struct folio *folio = NULL; - int flush_needed = 1; - if (pmd_present(orig_pmd)) { - struct page *page = pmd_page(orig_pmd); + is_present = pmd_present(orig_pmd); + folio = normal_or_softleaf_folio_pmd(vma, addr, orig_pmd, is_present); + has_deposit = has_deposited_pgtable(vma, orig_pmd, folio); + if (folio) + zap_huge_pmd_folio(mm, vma, orig_pmd, folio, is_present); + if (has_deposit) + zap_deposited_table(mm, pmd); - folio = page_folio(page); - folio_remove_rmap_pmd(folio, page, vma); - WARN_ON_ONCE(folio_mapcount(folio) < 0); - VM_BUG_ON_PAGE(!PageHead(page), page); - } else if (pmd_is_valid_softleaf(orig_pmd)) { - const softleaf_t entry = softleaf_from_pmd(orig_pmd); - - folio = softleaf_to_folio(entry); - flush_needed = 0; - - if (!thp_migration_supported()) - WARN_ONCE(1, "Non present huge pmd without pmd migration enabled!"); - } - - if (folio_test_anon(folio)) { - zap_deposited_table(tlb->mm, pmd); - add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR); - } else { - if (arch_needs_pgtable_deposit()) - zap_deposited_table(tlb->mm, pmd); - add_mm_counter(tlb->mm, mm_counter_file(folio), - -HPAGE_PMD_NR); - - /* - * Use flush_needed to indicate whether the PMD entry - * is present, instead of checking pmd_present() again. - */ - if (flush_needed && pmd_young(orig_pmd) && - likely(vma_has_recency(vma))) - folio_mark_accessed(folio); - } - - if (folio_is_device_private(folio)) { - folio_remove_rmap_pmd(folio, &folio->page, vma); - WARN_ON_ONCE(folio_mapcount(folio) < 0); - folio_put(folio); - } - - spin_unlock(ptl); - if (flush_needed) - tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE); - } - return 1; + spin_unlock(ptl); + if (is_present && folio) + tlb_remove_page_size(tlb, &folio->page, HPAGE_PMD_SIZE); + return true; } #ifndef pmd_move_must_withdraw @@ -2454,9 +2616,9 @@ static pmd_t clear_uffd_wp_pmd(pmd_t pmd) if (pmd_none(pmd)) return pmd; if (pmd_present(pmd)) - pmd = pmd_clear_uffd_wp(pmd); + pmd = pmd_clear_uffd(pmd); else - pmd = pmd_swp_clear_uffd_wp(pmd); + pmd = pmd_swp_clear_uffd(pmd); return pmd; } @@ -2499,8 +2661,19 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, pgtable_trans_huge_deposit(mm, new_pmd, pgtable); } pmd = move_soft_dirty_pmd(pmd); - if (vma_has_uffd_without_event_remap(vma)) + if (vma_has_uffd_without_event_remap(vma)) { + /* + * See __copy_present_ptes(): normalise the RWP marker + * so the destination starts accessible instead of + * taking a numa-hinting fault on first access. Only the + * marker (protnone + uffd) needs it; leave other present + * PMDs in the VMA untouched. + */ + if (pmd_present(pmd) && userfaultfd_rwp(vma) && + pmd_uffd(pmd)) + pmd = pmd_modify(pmd, vma->vm_page_prot); pmd = clear_uffd_wp_pmd(pmd); + } set_pmd_at(mm, new_addr, new_pmd, pmd); if (force_flush) flush_pmd_tlb_range(vma, old_addr, old_addr + PMD_SIZE); @@ -2513,15 +2686,16 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, } static void change_non_present_huge_pmd(struct mm_struct *mm, - unsigned long addr, pmd_t *pmd, bool uffd_wp, - bool uffd_wp_resolve) + unsigned long addr, pmd_t *pmd, bool uffd_prot, + bool uffd_prot_resolve) { softleaf_t entry = softleaf_from_pmd(*pmd); - const struct folio *folio = softleaf_to_folio(entry); pmd_t newpmd; VM_WARN_ON(!pmd_is_valid_softleaf(*pmd)); if (softleaf_is_migration_write(entry)) { + const struct folio *folio = softleaf_to_folio(entry); + /* * A protection check is difficult so * just be safe and disable write @@ -2530,20 +2704,22 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, entry = make_readable_exclusive_migration_entry(swp_offset(entry)); else entry = make_readable_migration_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); + newpmd = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pmd)) newpmd = pmd_swp_mksoft_dirty(newpmd); } else if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); - newpmd = swp_entry_to_pmd(entry); + newpmd = softleaf_to_pmd(entry); + if (pmd_swp_uffd(*pmd)) + newpmd = pmd_swp_mkuffd(newpmd); } else { newpmd = *pmd; } - if (uffd_wp) - newpmd = pmd_swp_mkuffd_wp(newpmd); - else if (uffd_wp_resolve) - newpmd = pmd_swp_clear_uffd_wp(newpmd); + if (uffd_prot) + newpmd = pmd_swp_mkuffd(newpmd); + else if (uffd_prot_resolve) + newpmd = pmd_swp_clear_uffd(newpmd); if (!pmd_same(*pmd, newpmd)) set_pmd_at(mm, addr, pmd, newpmd); } @@ -2563,8 +2739,9 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, spinlock_t *ptl; pmd_t oldpmd, entry; bool prot_numa = cp_flags & MM_CP_PROT_NUMA; - bool uffd_wp = cp_flags & MM_CP_UFFD_WP; - bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE; + bool uffd_prot = cp_flags & (MM_CP_UFFD_WP | MM_CP_UFFD_RWP); + bool uffd_prot_resolve = cp_flags & + (MM_CP_UFFD_WP_RESOLVE | MM_CP_UFFD_RWP_RESOLVE); int ret = 1; tlb_change_page_size(tlb, HPAGE_PMD_SIZE); @@ -2577,11 +2754,17 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, return 0; if (thp_migration_supported() && pmd_is_valid_softleaf(*pmd)) { - change_non_present_huge_pmd(mm, addr, pmd, uffd_wp, - uffd_wp_resolve); + change_non_present_huge_pmd(mm, addr, pmd, uffd_prot, + uffd_prot_resolve); goto unlock; } + /* Already in the desired state */ + if (prot_numa && pmd_protnone(*pmd)) + goto unlock; + if ((cp_flags & MM_CP_UFFD_RWP) && pmd_protnone(*pmd) && pmd_uffd(*pmd)) + goto unlock; + if (prot_numa) { /* @@ -2592,9 +2775,6 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, if (is_huge_zero_pmd(*pmd)) goto unlock; - if (pmd_protnone(*pmd)) - goto unlock; - if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, vma_is_single_threaded_private(vma))) goto unlock; @@ -2623,15 +2803,19 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, oldpmd = pmdp_invalidate_ad(vma, addr, pmd); entry = pmd_modify(oldpmd, newprot); - if (uffd_wp) - entry = pmd_mkuffd_wp(entry); - else if (uffd_wp_resolve) + if (uffd_prot) + entry = pmd_mkuffd(entry); + else if (uffd_prot_resolve) /* * Leave the write bit to be handled by PF interrupt * handler, then things like COW could be properly * handled. */ - entry = pmd_clear_uffd_wp(entry); + entry = pmd_clear_uffd(entry); + + /* See change_pte_range(): preserve RWP protection across mprotect() */ + if (userfaultfd_rwp(vma) && pmd_uffd(entry)) + entry = pmd_modify(entry, PAGE_NONE); /* See change_pte_range(). */ if ((cp_flags & MM_CP_TRY_CHANGE_WRITABLE) && !pmd_write(entry) && @@ -2671,10 +2855,10 @@ int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, return 1; /* - * Huge entries on userfault-wp only works with anonymous, while we - * don't have anonymous PUDs yet. + * Huge entries on userfault-wp or userfault-rwp only work with + * anonymous, while we don't have anonymous PUDs yet. */ - if (WARN_ON_ONCE(cp_flags & MM_CP_UFFD_WP_ALL)) + if (WARN_ON_ONCE(cp_flags & (MM_CP_UFFD_WP_ALL | MM_CP_UFFD_RWP_ALL))) return 1; ptl = __pud_trans_huge_lock(pudp, vma); @@ -2732,7 +2916,7 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm if (!pmd_trans_huge(src_pmdval)) { spin_unlock(src_ptl); if (pmd_is_migration_entry(src_pmdval)) { - pmd_migration_entry_wait(mm, &src_pmdval); + pmd_migration_entry_wait(mm, src_pmd); return -EAGAIN; } return -ENOENT; @@ -2790,15 +2974,23 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm } folio_move_anon_rmap(src_folio, dst_vma); - src_folio->index = linear_page_index(dst_vma, dst_addr); + src_folio->index = linear_anon_page_index(dst_vma, dst_addr); _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); /* Follow mremap() behavior and treat the entry dirty after the move */ _dst_pmd = pmd_mkwrite(pmd_mkdirty(_dst_pmd), dst_vma); } else { src_pmdval = pmdp_huge_clear_flush(src_vma, src_addr, src_pmd); - _dst_pmd = folio_mk_pmd(src_folio, dst_vma->vm_page_prot); + _dst_pmd = move_soft_dirty_pmd(src_pmdval); + _dst_pmd = clear_uffd_wp_pmd(_dst_pmd); } + + /* Re-arm RWP on the moved PMD if dst_vma is RWP-registered. */ + if (userfaultfd_rwp(dst_vma)) { + _dst_pmd = pmd_modify(_dst_pmd, PAGE_NONE); + _dst_pmd = pmd_mkuffd(_dst_pmd); + } + set_pmd_at(mm, dst_addr, dst_pmd, _dst_pmd); src_pgtable = pgtable_trans_huge_withdraw(mm, src_pmd); @@ -2863,7 +3055,7 @@ int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, orig_pud = pudp_huge_get_and_clear_full(vma, addr, pud, tlb->fullmm); arch_check_zapped_pud(vma, orig_pud); tlb_remove_pud_tlb_entry(tlb, pud, addr); - if (!vma_is_dax(vma) && vma_is_special_huge(vma)) { + if (vma_is_special_huge(vma)) { spin_unlock(ptl); /* No zero page support yet */ } else { @@ -2912,9 +3104,9 @@ static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, if (!folio_test_referenced(folio) && pud_young(old_pud)) folio_set_referenced(folio); folio_remove_rmap_pud(folio, page, vma); - folio_put(folio); add_mm_counter(vma->vm_mm, mm_counter_file(folio), -HPAGE_PUD_NR); + folio_put(folio); } void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, @@ -2971,10 +3163,15 @@ static void __split_huge_zero_page_pmd(struct vm_area_struct *vma, for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) { pte_t entry; - entry = pfn_pte(my_zero_pfn(addr), vma->vm_page_prot); + entry = pfn_pte(zero_pfn(addr), vma->vm_page_prot); entry = pte_mkspecial(entry); - if (pmd_uffd_wp(old_pmd)) - entry = pte_mkuffd_wp(entry); + if (pmd_uffd(old_pmd)) + entry = pte_mkuffd(entry); + + /* Restore PAGE_NONE so an RWP marker keeps trapping */ + if (userfaultfd_rwp(vma) && pmd_uffd(old_pmd)) + entry = pte_modify(entry, PAGE_NONE); + VM_BUG_ON(!pte_none(ptep_get(pte))); set_pte_at(mm, addr, pte, entry); pte++; @@ -3014,7 +3211,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, */ if (arch_needs_pgtable_deposit()) zap_deposited_table(mm, pmd); - if (!vma_is_dax(vma) && vma_is_special_huge(vma)) + if (vma_is_special_huge(vma)) return; if (unlikely(pmd_is_migration_entry(old_pmd))) { const softleaf_t old_entry = softleaf_from_pmd(old_pmd); @@ -3030,7 +3227,9 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (!folio_test_referenced(folio) && pmd_young(old_pmd)) folio_set_referenced(folio); folio_remove_rmap_pmd(folio, page, vma); + add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); folio_put(folio); + return; } add_mm_counter(mm, mm_counter_file(folio), -HPAGE_PMD_NR); return; @@ -3058,7 +3257,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); soft_dirty = pmd_swp_soft_dirty(old_pmd); - uffd_wp = pmd_swp_uffd_wp(old_pmd); + uffd_wp = pmd_swp_uffd(old_pmd); write = softleaf_is_migration_write(entry); if (PageAnon(page)) @@ -3074,7 +3273,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, folio = page_folio(page); soft_dirty = pmd_swp_soft_dirty(old_pmd); - uffd_wp = pmd_swp_uffd_wp(old_pmd); + uffd_wp = pmd_swp_uffd(old_pmd); write = softleaf_is_device_private_write(entry); anon_exclusive = PageAnonExclusive(page); @@ -3131,7 +3330,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, write = pmd_write(old_pmd); young = pmd_young(old_pmd); soft_dirty = pmd_soft_dirty(old_pmd); - uffd_wp = pmd_uffd_wp(old_pmd); + uffd_wp = pmd_uffd(old_pmd); VM_WARN_ON_FOLIO(!folio_ref_count(folio), folio); VM_WARN_ON_FOLIO(!folio_test_anon(folio), folio); @@ -3202,7 +3401,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); if (uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + entry = pte_swp_mkuffd(entry); VM_WARN_ON(!pte_none(ptep_get(pte + i))); set_pte_at(mm, addr, pte + i, entry); } @@ -3229,7 +3428,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_swp_mksoft_dirty(entry); if (uffd_wp) - entry = pte_swp_mkuffd_wp(entry); + entry = pte_swp_mkuffd(entry); VM_WARN_ON(!pte_none(ptep_get(pte + i))); set_pte_at(mm, addr, pte + i, entry); } @@ -3247,7 +3446,11 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, if (soft_dirty) entry = pte_mksoft_dirty(entry); if (uffd_wp) - entry = pte_mkuffd_wp(entry); + entry = pte_mkuffd(entry); + + /* Restore PAGE_NONE so an RWP marker keeps trapping */ + if (userfaultfd_rwp(vma) && uffd_wp) + entry = pte_modify(entry, PAGE_NONE); for (i = 0; i < HPAGE_PMD_NR; i++) VM_WARN_ON(!pte_none(ptep_get(pte + i))); @@ -3539,15 +3742,19 @@ static void __split_folio_to_order(struct folio *folio, int old_order, (1L << PG_arch_3) | #endif (1L << PG_dirty) | + (1L << PG_dropbehind) | LRU_GEN_MASK | LRU_REFS_MASK)); - if (handle_hwpoison && - page_range_has_hwpoisoned(new_head, new_nr_pages)) - folio_set_has_hwpoisoned(new_folio); - new_folio->mapping = folio->mapping; new_folio->index = folio->index + i; + /* + * page->private should not be set in tail pages. Warn once + * if private is unexpectedly set. Do it before swap.val assignment + * since private overlaps with swap.val. + */ + VM_WARN_ON_ONCE_PAGE(new_folio->private, new_head); + if (folio_test_swapcache(folio)) new_folio->swap.val = folio->swap.val + i; @@ -3566,6 +3773,14 @@ static void __split_folio_to_order(struct folio *folio, int old_order, folio_set_large_rmappable(new_folio); } + /* + * PG_has_hwpoisoned is on the 2nd page, so set it after + * the compound head is prepped. + */ + if (handle_hwpoison && + page_range_has_hwpoisoned(new_head, new_nr_pages)) + folio_set_has_hwpoisoned(new_folio); + if (folio_test_young(folio)) folio_set_young(new_folio); if (folio_test_idle(folio)) @@ -3631,6 +3846,7 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, const bool is_anon = folio_test_anon(folio); int old_order = folio_order(folio); int start_order = split_type == SPLIT_TYPE_UNIFORM ? new_order : old_order - 1; + struct folio *old_folio = folio; int split_order; /* @@ -3651,12 +3867,16 @@ static int __split_unmapped_folio(struct folio *folio, int new_order, * uniform split has xas_split_alloc() called before * irq is disabled to allocate enough memory, whereas * non-uniform split can handle ENOMEM. + * Use the to-be-split folio, so that a parallel + * folio_try_get() waits on it until xarray is updated + * with after-split folios and the original one is + * unfrozen. */ - if (split_type == SPLIT_TYPE_UNIFORM) - xas_split(xas, folio, old_order); - else { + if (split_type == SPLIT_TYPE_UNIFORM) { + xas_split(xas, old_folio, old_order); + } else { xas_set_order(xas, folio->index, split_order); - xas_try_split(xas, folio, old_order); + xas_try_split(xas, old_folio, old_order); if (xas_error(xas)) return xas_error(xas); } @@ -3713,33 +3933,9 @@ int folio_check_splittable(struct folio *folio, unsigned int new_order, if (!folio->mapping && !folio_test_anon(folio)) return -EBUSY; - if (folio_test_anon(folio)) { - /* order-1 is not supported for anonymous THP. */ - if (new_order == 1) - return -EINVAL; - } else if (split_type == SPLIT_TYPE_NON_UNIFORM || new_order) { - if (IS_ENABLED(CONFIG_READ_ONLY_THP_FOR_FS) && - !mapping_large_folio_support(folio->mapping)) { - /* - * We can always split a folio down to a single page - * (new_order == 0) uniformly. - * - * For any other scenario - * a) uniform split targeting a large folio - * (new_order > 0) - * b) any non-uniform split - * we must confirm that the file system supports large - * folios. - * - * Note that we might still have THPs in such - * mappings, which is created from khugepaged when - * CONFIG_READ_ONLY_THP_FOR_FS is enabled. But in that - * case, the mapping does not actually support large - * folios properly. - */ - return -EINVAL; - } - } + /* order-1 is not supported for anonymous THP. */ + if (folio_test_anon(folio) && new_order == 1) + return -EINVAL; /* * swapcache folio could only be split to order 0 @@ -3779,34 +3975,43 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n struct folio *end_folio = folio_next(folio); struct folio *new_folio, *next; int old_order = folio_order(folio); + struct list_lru_one *lru; + bool dequeue_deferred; int ret = 0; - struct deferred_split *ds_queue; VM_WARN_ON_ONCE(!mapping && end); - /* Prevent deferred_split_scan() touching ->_refcount */ - ds_queue = folio_split_queue_lock(folio); + /* + * If this folio can be on the deferred split queue, lock out + * the shrinker before freezing the ref. If the shrinker sees + * a 0-ref folio, it assumes it beat folio_put() to the list + * lock and must clean up the LRU state - the same dequeue we + * will do below as part of the split. + */ + dequeue_deferred = folio_test_anon(folio) && old_order > 1; + if (dequeue_deferred) { + struct mem_cgroup *memcg; + + rcu_read_lock(); + memcg = folio_memcg(folio); + lru = list_lru_lock(&deferred_split_lru, + folio_nid(folio), &memcg); + } if (folio_ref_freeze(folio, folio_cache_ref_count(folio) + 1)) { struct swap_cluster_info *ci = NULL; struct lruvec *lruvec; - if (old_order > 1) { - if (!list_empty(&folio->_deferred_list)) { - ds_queue->split_queue_len--; - /* - * Reinitialize page_deferred_list after removing the - * page from the split_queue, otherwise a subsequent - * split will see list corruption when checking the - * page_deferred_list. - */ - list_del_init(&folio->_deferred_list); - } + if (dequeue_deferred) { + __list_lru_del(&deferred_split_lru, lru, + &folio->_deferred_list, folio_nid(folio)); if (folio_test_partially_mapped(folio)) { folio_clear_partially_mapped(folio); mod_mthp_stat(old_order, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); } + list_lru_unlock(lru); + rcu_read_unlock(); } - split_queue_unlock(ds_queue); + if (mapping) { int nr = folio_nr_pages(folio); @@ -3818,7 +4023,6 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n } else { lruvec_stat_mod_folio(folio, NR_FILE_THPS, -nr); - filemap_nr_thps_dec(mapping); } } } @@ -3902,12 +4106,15 @@ static int __folio_freeze_and_split_unmapped(struct folio *folio, unsigned int n folio_ref_unfreeze(folio, folio_cache_ref_count(folio) + 1); if (do_lru) - unlock_page_lruvec(lruvec); + lruvec_unlock(lruvec); if (ci) swap_cluster_unlock(ci); } else { - split_queue_unlock(ds_queue); + if (dequeue_deferred) { + list_lru_unlock(lru); + rcu_read_unlock(); + } return -EAGAIN; } @@ -3942,34 +4149,42 @@ static int __folio_split(struct folio *folio, unsigned int new_order, XA_STATE(xas, &folio->mapping->i_pages, folio->index); struct folio *end_folio = folio_next(folio); bool is_anon = folio_test_anon(folio); + struct mem_cgroup *memcg, *old_memcg; struct address_space *mapping = NULL; struct anon_vma *anon_vma = NULL; int old_order = folio_order(folio); struct folio *new_folio, *next; int nr_shmem_dropped = 0; enum ttu_flags ttu_flags = 0; - int ret; pgoff_t end = 0; + int ret; VM_WARN_ON_ONCE_FOLIO(!folio_test_locked(folio), folio); VM_WARN_ON_ONCE_FOLIO(!folio_test_large(folio), folio); if (folio != page_folio(split_at) || folio != page_folio(lock_at)) { ret = -EINVAL; - goto out; + goto out_no_memcg; } if (new_order >= old_order) { ret = -EINVAL; - goto out; + goto out_no_memcg; } ret = folio_check_splittable(folio, new_order, split_type); if (ret) { VM_WARN_ONCE(ret == -EINVAL, "Tried to split an unsplittable folio"); - goto out; + goto out_no_memcg; } + /* + * switch to folio's memcg as xarray node allocation can happen and + * needs to charge to it. + */ + memcg = get_mem_cgroup_from_folio(folio); + old_memcg = set_active_memcg(memcg); + if (is_anon) { /* * The caller does not necessarily hold an mmap_lock that would @@ -3991,7 +4206,7 @@ static int __folio_split(struct folio *folio, unsigned int new_order, gfp_t gfp; mapping = folio->mapping; - min_order = mapping_min_folio_order(folio->mapping); + min_order = mapping_min_folio_order(mapping); if (new_order < min_order) { ret = -EINVAL; goto out; @@ -4005,6 +4220,8 @@ static int __folio_split(struct folio *folio, unsigned int new_order, goto out; } + mapping_set_update(&xas, mapping); + if (split_type == SPLIT_TYPE_UNIFORM) { xas_set_order(&xas, folio->index, new_order); xas_split_alloc(&xas, folio, old_order, gfp); @@ -4072,6 +4289,18 @@ fail: remap_page(folio, 1 << old_order, ttu_flags); /* + * Drop the mapping while the inode is still pinned. @folio stays + * locked and present in the page cache until the loop below, so + * eviction cannot free the inode yet; @lock_at is not enough, it may + * be a tail beyond EOF that the split already dropped from the page + * cache. Nothing past this point may touch the inode or the mapping. + */ + if (mapping) { + i_mmap_unlock_read(mapping); + mapping = NULL; + } + + /* * Unlock all after-split folios except the one containing * @lock_at page. If @folio is not split, it will be kept locked. */ @@ -4082,11 +4311,10 @@ fail: folio_unlock(new_folio); /* - * Subpages may be freed if there wasn't any mapping - * like if add_to_swap() is running on a lru page that - * had its mapping zapped. And freeing these pages - * requires taking the lru_lock so we do the put_page - * of the tail pages after the split is complete. + * Subpages whose mapping has been zapped may be freed + * earlier, but freeing them requires taking the + * lru_lock, so we defer put_page() on tail pages until + * after the split completes. */ free_folio_and_swap_cache(new_folio); } @@ -4099,8 +4327,12 @@ out_unlock: if (mapping) i_mmap_unlock_read(mapping); out: + /* restore to caller's old_memcg */ + set_active_memcg(old_memcg); + mem_cgroup_put(memcg); +out_no_memcg: xas_destroy(&xas); - if (old_order == HPAGE_PMD_ORDER) + if (is_pmd_order(old_order)) count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED); count_mthp_stat(old_order, !ret ? MTHP_STAT_SPLIT : MTHP_STAT_SPLIT_FAILED); return ret; @@ -4274,33 +4506,37 @@ int split_folio_to_list(struct folio *folio, struct list_head *list) * queueing THP splits, and that list is (racily observed to be) non-empty. * * It is unsafe to call folio_unqueue_deferred_split() until folio refcount is - * zero: because even when split_queue_lock is held, a non-empty _deferred_list - * might be in use on deferred_split_scan()'s unlocked on-stack list. + * zero: because even when the list_lru lock is held, a non-empty + * _deferred_list might be in use on deferred_split_scan()'s unlocked + * on-stack list. * - * If memory cgroups are enabled, split_queue_lock is in the mem_cgroup: it is - * therefore important to unqueue deferred split before changing folio memcg. + * The list_lru sublist is determined by folio's memcg: it is therefore + * important to unqueue deferred split before changing folio memcg. */ bool __folio_unqueue_deferred_split(struct folio *folio) { - struct deferred_split *ds_queue; + struct mem_cgroup *memcg; + struct list_lru_one *lru; + int nid = folio_nid(folio); unsigned long flags; bool unqueued = false; WARN_ON_ONCE(folio_ref_count(folio)); WARN_ON_ONCE(!mem_cgroup_disabled() && !folio_memcg_charged(folio)); - ds_queue = folio_split_queue_lock_irqsave(folio, &flags); - if (!list_empty(&folio->_deferred_list)) { - ds_queue->split_queue_len--; + rcu_read_lock(); + memcg = folio_memcg(folio); + lru = list_lru_lock_irqsave(&deferred_split_lru, nid, &memcg, &flags); + if (__list_lru_del(&deferred_split_lru, lru, &folio->_deferred_list, nid)) { if (folio_test_partially_mapped(folio)) { folio_clear_partially_mapped(folio); mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); } - list_del_init(&folio->_deferred_list); unqueued = true; } - split_queue_unlock_irqrestore(ds_queue, flags); + list_lru_unlock_irqrestore(lru, &flags); + rcu_read_unlock(); return unqueued; /* useful for debug warnings */ } @@ -4308,7 +4544,9 @@ bool __folio_unqueue_deferred_split(struct folio *folio) /* partially_mapped=false won't clear PG_partially_mapped folio flag */ void deferred_split_folio(struct folio *folio, bool partially_mapped) { - struct deferred_split *ds_queue; + struct list_lru_one *lru; + int nid; + struct mem_cgroup *memcg; unsigned long flags; /* @@ -4323,7 +4561,7 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped) /* * Exclude swapcache: originally to avoid a corrupt deferred split - * queue. Nowadays that is fully prevented by memcg1_swapout(); + * queue. Nowadays that is fully prevented by __memcg1_swapout(); * but if page reclaim is already handling the same folio, it is * unnecessary to handle it again in the shrinker, so excluding * swapcache here may still be a useful optimization. @@ -4331,7 +4569,11 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped) if (folio_test_swapcache(folio)) return; - ds_queue = folio_split_queue_lock_irqsave(folio, &flags); + nid = folio_nid(folio); + + rcu_read_lock(); + memcg = folio_memcg(folio); + lru = list_lru_lock_irqsave(&deferred_split_lru, nid, &memcg, &flags); if (partially_mapped) { if (!folio_test_partially_mapped(folio)) { folio_set_partially_mapped(folio); @@ -4339,36 +4581,23 @@ void deferred_split_folio(struct folio *folio, bool partially_mapped) count_vm_event(THP_DEFERRED_SPLIT_PAGE); count_mthp_stat(folio_order(folio), MTHP_STAT_SPLIT_DEFERRED); mod_mthp_stat(folio_order(folio), MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, 1); - } } else { /* partially mapped folios cannot become non-partially mapped */ VM_WARN_ON_FOLIO(folio_test_partially_mapped(folio), folio); } - if (list_empty(&folio->_deferred_list)) { - struct mem_cgroup *memcg; - - memcg = folio_split_queue_memcg(folio, ds_queue); - list_add_tail(&folio->_deferred_list, &ds_queue->split_queue); - ds_queue->split_queue_len++; - if (memcg) - set_shrinker_bit(memcg, folio_nid(folio), - shrinker_id(deferred_split_shrinker)); - } - split_queue_unlock_irqrestore(ds_queue, flags); + __list_lru_add(&deferred_split_lru, lru, &folio->_deferred_list, nid, memcg); + list_lru_unlock_irqrestore(lru, &flags); + rcu_read_unlock(); } static unsigned long deferred_split_count(struct shrinker *shrink, struct shrink_control *sc) { - struct pglist_data *pgdata = NODE_DATA(sc->nid); - struct deferred_split *ds_queue = &pgdata->deferred_split_queue; + unsigned long count; -#ifdef CONFIG_MEMCG - if (sc->memcg) - ds_queue = &sc->memcg->deferred_split_queue; -#endif - return READ_ONCE(ds_queue->split_queue_len); + count = list_lru_shrink_count(&deferred_split_lru, sc); + return count ?: SHRINK_EMPTY; } static bool thp_underused(struct folio *folio) @@ -4398,45 +4627,49 @@ static bool thp_underused(struct folio *folio) return false; } +static enum lru_status deferred_split_isolate(struct list_head *item, + struct list_lru_one *lru, + void *cb_arg) +{ + struct folio *folio = container_of(item, struct folio, _deferred_list); + struct list_head *freeable = cb_arg; + + if (folio_try_get(folio)) { + list_lru_isolate_move(lru, item, freeable); + return LRU_REMOVED; + } + + /* + * We lost race with folio_put(). Read folio state before the + * isolate: folio_unqueue_deferred_split() checks list_empty() + * locklessly, so once removed the folio can be freed any time. + */ + if (folio_test_partially_mapped(folio)) { + folio_clear_partially_mapped(folio); + mod_mthp_stat(folio_order(folio), + MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); + } + list_lru_isolate(lru, item); + return LRU_REMOVED; +} + static unsigned long deferred_split_scan(struct shrinker *shrink, struct shrink_control *sc) { - struct deferred_split *ds_queue; - unsigned long flags; + LIST_HEAD(dispose); struct folio *folio, *next; - int split = 0, i; - struct folio_batch fbatch; - - folio_batch_init(&fbatch); - -retry: - ds_queue = split_queue_lock_irqsave(sc->nid, sc->memcg, &flags); - /* Take pin on all head pages to avoid freeing them under us */ - list_for_each_entry_safe(folio, next, &ds_queue->split_queue, - _deferred_list) { - if (folio_try_get(folio)) { - folio_batch_add(&fbatch, folio); - } else if (folio_test_partially_mapped(folio)) { - /* We lost race with folio_put() */ - folio_clear_partially_mapped(folio); - mod_mthp_stat(folio_order(folio), - MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, -1); - } - list_del_init(&folio->_deferred_list); - ds_queue->split_queue_len--; - if (!--sc->nr_to_scan) - break; - if (!folio_batch_space(&fbatch)) - break; - } - split_queue_unlock_irqrestore(ds_queue, flags); + int split = 0; + unsigned long isolated; - for (i = 0; i < folio_batch_count(&fbatch); i++) { + isolated = list_lru_shrink_walk_irq(&deferred_split_lru, sc, + deferred_split_isolate, &dispose); + + list_for_each_entry_safe(folio, next, &dispose, _deferred_list) { bool did_split = false; bool underused = false; - struct deferred_split *fqueue; - folio = fbatch.folios[i]; + list_del_init(&folio->_deferred_list); + if (!folio_test_partially_mapped(folio)) { /* * See try_to_map_unused_to_zeropage(): we cannot @@ -4450,7 +4683,7 @@ retry: goto next; } if (!folio_trylock(folio)) - goto next; + goto requeue; if (!split_folio(folio)) { did_split = true; if (underused) @@ -4459,64 +4692,29 @@ retry: } folio_unlock(folio); next: - if (did_split || !folio_test_partially_mapped(folio)) - continue; /* - * Only add back to the queue if folio is partially mapped. - * If thp_underused returns false, or if split_folio fails - * in the case it was underused, then consider it used and - * don't add it back to split_queue. + * If thp_underused() returns false, or if split_folio() + * succeeds, or if split_folio() fails in the case it was + * underused, then consider it used and don't add it back to + * split_queue. */ - fqueue = folio_split_queue_lock_irqsave(folio, &flags); - if (list_empty(&folio->_deferred_list)) { - list_add_tail(&folio->_deferred_list, &fqueue->split_queue); - fqueue->split_queue_len++; + if (!did_split && folio_test_partially_mapped(folio)) { +requeue: + rcu_read_lock(); + list_lru_add_irq(&deferred_split_lru, + &folio->_deferred_list, + folio_nid(folio), + folio_memcg(folio)); + rcu_read_unlock(); } - split_queue_unlock_irqrestore(fqueue, flags); - } - folios_put(&fbatch); - - if (sc->nr_to_scan && !list_empty(&ds_queue->split_queue)) { - cond_resched(); - goto retry; + folio_put(folio); } - /* - * Stop shrinker if we didn't split any page, but the queue is empty. - * This can happen if pages were freed under us. - */ - if (!split && list_empty(&ds_queue->split_queue)) + if (!split && !isolated) return SHRINK_STOP; return split; } -#ifdef CONFIG_MEMCG -void reparent_deferred_split_queue(struct mem_cgroup *memcg) -{ - struct mem_cgroup *parent = parent_mem_cgroup(memcg); - struct deferred_split *ds_queue = &memcg->deferred_split_queue; - struct deferred_split *parent_ds_queue = &parent->deferred_split_queue; - int nid; - - spin_lock_irq(&ds_queue->split_queue_lock); - spin_lock_nested(&parent_ds_queue->split_queue_lock, SINGLE_DEPTH_NESTING); - - if (!ds_queue->split_queue_len) - goto unlock; - - list_splice_tail_init(&ds_queue->split_queue, &parent_ds_queue->split_queue); - parent_ds_queue->split_queue_len += ds_queue->split_queue_len; - ds_queue->split_queue_len = 0; - - for_each_node(nid) - set_shrinker_bit(parent, nid, shrinker_id(deferred_split_shrinker)); - -unlock: - spin_unlock(&parent_ds_queue->split_queue_lock); - spin_unlock_irq(&ds_queue->split_queue_lock); -} -#endif - #ifdef CONFIG_DEBUG_FS static void split_huge_pages_all(void) { @@ -4570,8 +4768,16 @@ next: static inline bool vma_not_suitable_for_thp_split(struct vm_area_struct *vma) { - return vma_is_special_huge(vma) || (vma->vm_flags & VM_IO) || - is_vm_hugetlb_page(vma); + if (vma_is_dax(vma)) + return true; + if (vma_is_special_huge(vma)) + return true; + if (vma_test(vma, VMA_IO_BIT)) + return true; + if (is_vm_hugetlb_page(vma)) + return true; + + return false; } static int split_huge_pages_pid(int pid, unsigned long vaddr_start, @@ -4852,7 +5058,7 @@ static int __init split_huge_pages_debugfs(void) late_initcall(split_huge_pages_debugfs); #endif -#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#ifdef CONFIG_ARCH_HAS_PMD_SOFTLEAVES int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct page *page) { @@ -4860,7 +5066,7 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, struct vm_area_struct *vma = pvmw->vma; struct mm_struct *mm = vma->vm_mm; unsigned long address = pvmw->address; - bool anon_exclusive; + bool anon_exclusive, present, writable, softdirty, uffd_wp; pmd_t pmdval; swp_entry_t entry; pmd_t pmdswp; @@ -4868,12 +5074,26 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, if (!(pvmw->pmd && !pvmw->pte)) return 0; - flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); - if (unlikely(!pmd_present(*pvmw->pmd))) - pmdval = pmdp_huge_get_and_clear(vma->vm_mm, address, pvmw->pmd); - else + present = pmd_present(*pvmw->pmd); + if (likely(present)) { + flush_cache_range(vma, address, address + HPAGE_PMD_SIZE); + pmdval = pmdp_invalidate(vma, address, pvmw->pmd); + writable = pmd_write(pmdval); + softdirty = pmd_soft_dirty(pmdval); + uffd_wp = pmd_uffd(pmdval); + } else { + softleaf_t old_entry; + + pmdval = pmdp_huge_get_and_clear(vma->vm_mm, address, pvmw->pmd); + old_entry = softleaf_from_pmd(pmdval); + + writable = softleaf_is_device_private_write(old_entry); + softdirty = pmd_swp_soft_dirty(pmdval); + uffd_wp = pmd_swp_uffd(pmdval); + } + /* See folio_try_share_anon_rmap_pmd(): invalidate PMD first. */ anon_exclusive = folio_test_anon(folio) && PageAnonExclusive(page); if (anon_exclusive && folio_try_share_anon_rmap_pmd(folio, page)) { @@ -4881,24 +5101,31 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, return -EBUSY; } - if (pmd_dirty(pmdval)) - folio_mark_dirty(folio); - if (pmd_write(pmdval)) + /* Determine type of migration entry. */ + if (writable) entry = make_writable_migration_entry(page_to_pfn(page)); else if (anon_exclusive) entry = make_readable_exclusive_migration_entry(page_to_pfn(page)); else entry = make_readable_migration_entry(page_to_pfn(page)); - if (pmd_young(pmdval)) + + /* Set A/D bits as necessary. */ + if (present && pmd_young(pmdval)) entry = make_migration_entry_young(entry); - if (pmd_dirty(pmdval)) + if (present && pmd_dirty(pmdval)) { + folio_mark_dirty(folio); entry = make_migration_entry_dirty(entry); - pmdswp = swp_entry_to_pmd(entry); - if (pmd_soft_dirty(pmdval)) + } + + /* Set PMD. */ + pmdswp = softleaf_to_pmd(entry); + if (softdirty) pmdswp = pmd_swp_mksoft_dirty(pmdswp); - if (pmd_uffd_wp(pmdval)) - pmdswp = pmd_swp_mkuffd_wp(pmdswp); + if (uffd_wp) + pmdswp = pmd_swp_mkuffd(pmdswp); set_pmd_at(mm, address, pvmw->pmd, pmdswp); + + /* Migration entry installed: cleanup rmap, folio. */ folio_remove_rmap_pmd(folio, page, vma); folio_put(folio); trace_set_migration_pmd(address, pmd_val(pmdswp)); @@ -4906,9 +5133,8 @@ int set_pmd_migration_entry(struct page_vma_mapped_walk *pvmw, return 0; } -void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) +void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct folio *folio) { - struct folio *folio = page_folio(new); struct vm_area_struct *vma = pvmw->vma; struct mm_struct *mm = vma->vm_mm; unsigned long address = pvmw->address; @@ -4927,8 +5153,13 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) pmde = pmd_mksoft_dirty(pmde); if (softleaf_is_migration_write(entry)) pmde = pmd_mkwrite(pmde, vma); - if (pmd_swp_uffd_wp(*pvmw->pmd)) - pmde = pmd_mkuffd_wp(pmde); + if (pmd_swp_uffd(*pvmw->pmd)) + pmde = pmd_mkuffd(pmde); + + /* See do_swap_page(): restore PAGE_NONE for RWP */ + if (pmd_swp_uffd(*pvmw->pmd) && userfaultfd_rwp(vma)) + pmde = pmd_modify(pmde, PAGE_NONE); + if (!softleaf_is_migration_young(entry)) pmde = pmd_mkold(pmde); /* NOTE: this may contain setting soft-dirty on some archs */ @@ -4939,17 +5170,15 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) swp_entry_t entry; if (pmd_write(pmde)) - entry = make_writable_device_private_entry( - page_to_pfn(new)); + entry = make_writable_device_private_entry(folio_pfn(folio)); else - entry = make_readable_device_private_entry( - page_to_pfn(new)); - pmde = swp_entry_to_pmd(entry); + entry = make_readable_device_private_entry(folio_pfn(folio)); + pmde = softleaf_to_pmd(entry); if (pmd_swp_soft_dirty(*pvmw->pmd)) pmde = pmd_swp_mksoft_dirty(pmde); - if (pmd_swp_uffd_wp(*pvmw->pmd)) - pmde = pmd_swp_mkuffd_wp(pmde); + if (pmd_swp_uffd(*pvmw->pmd)) + pmde = pmd_swp_mkuffd(pmde); } if (folio_test_anon(folio)) { @@ -4958,11 +5187,12 @@ void remove_migration_pmd(struct page_vma_mapped_walk *pvmw, struct page *new) if (!softleaf_is_migration_read(entry)) rmap_flags |= RMAP_EXCLUSIVE; - folio_add_anon_rmap_pmd(folio, new, vma, haddr, rmap_flags); + folio_add_anon_rmap_pmd(folio, &folio->page, vma, haddr, rmap_flags); } else { - folio_add_file_rmap_pmd(folio, new, vma); + folio_add_file_rmap_pmd(folio, &folio->page, vma); } - VM_BUG_ON(pmd_write(pmde) && folio_test_anon(folio) && !PageAnonExclusive(new)); + VM_WARN_ON_ONCE(pmd_write(pmde) && folio_test_anon(folio) && + !PageAnonExclusive(&folio->page)); set_pmd_at(mm, haddr, pvmw->pmd, pmde); /* No need to invalidate - it was non-present before */ |