diff options
Diffstat (limited to 'mm')
| -rw-r--r-- | mm/filemap.c | 2 | ||||
| -rw-r--r-- | mm/folio.c | 2 | ||||
| -rw-r--r-- | mm/huge_memory.c | 86 | ||||
| -rw-r--r-- | mm/memblock.c | 18 | ||||
| -rw-r--r-- | mm/memcontrol.c | 2 | ||||
| -rw-r--r-- | mm/mlock.c | 2 | ||||
| -rw-r--r-- | mm/mremap.c | 9 | ||||
| -rw-r--r-- | mm/shrinker.c | 2 | ||||
| -rw-r--r-- | mm/swapfile.c | 2 | ||||
| -rw-r--r-- | mm/vma.c | 6 |
10 files changed, 91 insertions, 40 deletions
diff --git a/mm/filemap.c b/mm/filemap.c index 6afec636881f..00fd89cf6f55 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -1616,7 +1616,7 @@ static void filemap_end_dropbehind(struct folio *folio) return; if (!folio_test_clear_dropbehind(folio)) return; - if (mapping) + if (mapping && !folio_mapped(folio)) folio_unmap_invalidate(mapping, folio, 0); } diff --git a/mm/folio.c b/mm/folio.c index c02dcea9c03c..50a6dbe55998 100644 --- a/mm/folio.c +++ b/mm/folio.c @@ -33,6 +33,7 @@ #include <linux/page_idle.h> #include <linux/local_lock.h> #include <linux/buffer_head.h> +#include <linux/kvm_types.h> #include "internal.h" #include "page_alloc.h" @@ -926,6 +927,7 @@ void lru_cache_drain_for_folio(const struct folio *folio, *drained = LRU_CACHE_DRAINED_ALL; } } +EXPORT_SYMBOL_FOR_KVM(lru_cache_drain_for_folio); atomic_t lru_disable_count = ATOMIC_INIT(0); diff --git a/mm/huge_memory.c b/mm/huge_memory.c index afbb5974bd22..1e5d68acf62a 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -92,7 +92,7 @@ unsigned long huge_anon_orders_madvise __read_mostly; unsigned long huge_anon_orders_inherit __read_mostly; static bool anon_orders_configured __initdata; -static inline bool file_thp_enabled(struct vm_area_struct *vma) +static inline bool file_thp_enabled(const struct vm_area_struct *vma) { struct inode *inode; @@ -118,6 +118,67 @@ static bool vma_is_special_huge(const struct vm_area_struct *vma) return vma_test_any(vma, VMA_PFNMAP_BIT, VMA_MIXEDMAP_BIT); } +static bool vma_file_bypass_thp_tuneables(const struct vm_area_struct *vma, + enum tva_type type) +{ + const bool has_huge_fault = vma->vm_ops->huge_fault; + + /* MADV_COLLAPSE ignores tuneables. */ + if (type == TVA_FORCED_COLLAPSE) + return true; + /* Huge PFN mappings are uncompactable so the policy doesn't apply. */ + if (vma_test(vma, VMA_PFNMAP_BIT) && has_huge_fault) + return true; + return false; +} + +static bool vma_file_allow_thp_tuneables(vm_flags_t vm_flags) +{ + /* THP=always? */ + if (hugepage_global_always()) + return true; + /* THP=madvise and marked MADV_HUGEPAGE? */ + if (hugepage_global_enabled() && (vm_flags & VM_HUGEPAGE)) + return true; + return false; +} + +static bool vma_file_check_thp_tuneables(const struct vm_area_struct *vma, + vm_flags_t vm_flags, enum tva_type type) +{ + return vma_file_bypass_thp_tuneables(vma, type) || + vma_file_allow_thp_tuneables(vm_flags); +} + +static bool vma_can_map_huge_file(const struct vm_area_struct *vma, + vm_flags_t vm_flags, enum tva_type type) +{ + const bool has_huge_fault = vma->vm_ops->huge_fault; + + /* + * Enforce THP collapse requirements as necessary. Anonymous vmas + * were already handled in thp_vma_allowable_orders(). + */ + if (!vma_file_check_thp_tuneables(vma, vm_flags, type)) + return false; + + switch (type) { + case TVA_PAGEFAULT: + /* + * Trust that ->huge_fault() handlers know what they are doing + * in fault path. + */ + return has_huge_fault; + case TVA_SMAPS: + if (has_huge_fault) + return true; + fallthrough; + default: + /* Only regular file is valid in collapse path. */ + return file_thp_enabled(vma); + } +} + unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type, @@ -190,27 +251,8 @@ unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vma, vma_start_pgoff(vma), 0, forced_collapse); - if (!vma_is_anonymous(vma)) { - /* - * Enforce THP collapse requirements as necessary. Anonymous vmas - * were already handled in thp_vma_allowable_orders(). - */ - if (!forced_collapse && - (!hugepage_global_enabled() || (!(vm_flags & VM_HUGEPAGE) && - !hugepage_global_always()))) - return 0; - - /* - * Trust that ->huge_fault() handlers know what they are doing - * in fault path. - */ - if (((in_pf || smaps)) && vma->vm_ops->huge_fault) - return orders; - /* Only regular file is valid in collapse path */ - if (((!in_pf || smaps)) && file_thp_enabled(vma)) - return orders; - return 0; - } + if (!vma_is_anonymous(vma)) + return vma_can_map_huge_file(vma, vm_flags, type) ? orders : 0; if (vma_is_temporary_stack(vma)) return 0; diff --git a/mm/memblock.c b/mm/memblock.c index 9ce86349a29f..021db49eb7fc 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -2908,14 +2908,18 @@ static int memblock_debug_show(struct seq_file *m, void *private) else seq_printf(m, "%4c ", 'x'); if (reg->flags) { - for (j = 0; j < count; j++) { - if (reg->flags & (1U << j)) { - seq_printf(m, "%s\n", flagname[j]); - break; - } + unsigned int flags = reg->flags; + bool first = true; + + for (j = 0; flags; j++, flags >>= 1) { + if (!(flags & 1)) + continue; + if (!first) + seq_putc(m, '|'); + seq_puts(m, j < count ? flagname[j] : "UNKNOWN"); + first = false; } - if (j == count) - seq_printf(m, "%s\n", "UNKNOWN"); + seq_putc(m, '\n'); } else { seq_printf(m, "%s\n", "NONE"); } diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 1271d390b617..856a7d07586c 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3158,7 +3158,7 @@ static int obj_cgroup_charge_pages(struct obj_cgroup *objcg, gfp_t gfp, memcg = get_mem_cgroup_from_objcg(objcg); - ret = try_charge_memcg(memcg, gfp, nr_pages); + ret = try_charge(memcg, gfp, nr_pages); if (ret) goto out; diff --git a/mm/mlock.c b/mm/mlock.c index efa6716e4dfb..39215a3eab1f 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -141,7 +141,7 @@ static struct lruvec *__munlock_folio(struct folio *folio, struct lruvec *lruvec munlock: if (folio_test_clear_mlocked(folio)) { - __zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages); + zone_stat_mod_folio(folio, NR_MLOCK, -nr_pages); if (isolated || !folio_test_unevictable(folio)) __count_vm_events(UNEVICTABLE_PGMUNLOCKED, nr_pages); else diff --git a/mm/mremap.c b/mm/mremap.c index 2b4b523a86b8..7c368440fafe 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -1355,12 +1355,11 @@ static void dontunmap_complete(struct vma_remap_struct *vrm, if (vma_is_anonymous(vma) && !vma->vm_file) vma_set_pgoff(vma, pgoff_unfaulted); } - - /* Because we won't unmap we don't need to touch locked_vm. */ } static unsigned long move_vma(struct vma_remap_struct *vrm) { + const bool is_dontunmap = vrm->flags & MREMAP_DONTUNMAP; struct mm_struct *mm = current->mm; struct vm_area_struct *new_vma; unsigned long hiwater_vm; @@ -1401,10 +1400,10 @@ static unsigned long move_vma(struct vma_remap_struct *vrm) */ hiwater_vm = mm->hiwater_vm; - vrm_stat_account(vrm, vrm->new_len); - if (unlikely(!err && (vrm->flags & MREMAP_DONTUNMAP))) + if (unlikely(is_dontunmap && !err)) dontunmap_complete(vrm, new_vma); - else + vrm_stat_account(vrm, vrm->new_len); + if (!is_dontunmap || err) unmap_source_vma(vrm); mm->hiwater_vm = hiwater_vm; diff --git a/mm/shrinker.c b/mm/shrinker.c index a70aab124a0e..7ec2a9704f6f 100644 --- a/mm/shrinker.c +++ b/mm/shrinker.c @@ -227,6 +227,8 @@ static int shrinker_memcg_alloc(struct shrinker *shrinker) { int id; + shrinker->id = -1; + if (mem_cgroup_disabled()) return -ENOSYS; if (mem_cgroup_kmem_disabled() && !(shrinker->flags & SHRINKER_NONSLAB)) diff --git a/mm/swapfile.c b/mm/swapfile.c index 53bf01d5f7f1..601979b97f95 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -156,7 +156,7 @@ static struct swap_info_struct *swap_entry_to_info(swp_entry_t entry) * This bit will be set if the device is not on the plist and not * usable, will be cleared if the device is on the plist. */ -#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_t) - 2)) +#define SWAP_USAGE_OFFLIST_BIT (1UL << (BITS_PER_TYPE(atomic_long_t) - 2)) #define SWAP_USAGE_COUNTER_MASK (~SWAP_USAGE_OFFLIST_BIT) static long swap_usage_in_pages(struct swap_info_struct *si) { @@ -2859,10 +2859,12 @@ static unsigned long __mmap_region(struct file *file, unsigned long addr, map.check_ksm_early = can_set_ksm_flags_early(&map); error = __mmap_setup(&map, &desc, uf); - if (!error && have_mmap_prepare) - error = call_mmap_prepare(&map, &desc); if (error) goto abort_munmap; + if (have_mmap_prepare) + error = call_mmap_prepare(&map, &desc); + if (error) + goto unacct_error; if (map.check_ksm_early) update_ksm_flags(&map); |