diff options
| -rw-r--r-- | mm/kfence/core.c | 12 | ||||
| -rw-r--r-- | mm/kfence/kfence.h | 3 | ||||
| -rw-r--r-- | mm/memcontrol.c | 8 | ||||
| -rw-r--r-- | mm/slab.h | 6 | ||||
| -rw-r--r-- | mm/slub.c | 31 |
5 files changed, 26 insertions, 34 deletions
diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 7da3be9fbf5d..90925c646c4c 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -636,14 +636,6 @@ static unsigned long kfence_init_pool(void) page = pfn_to_page(start_pfn + i); __SetPageSlab(page); -#ifdef CONFIG_MEMCG - struct slab *slab = page_slab(page); - slab->obj_exts = (unsigned long)&kfence_metadata_init[i / 2 - 1].obj_exts | - MEMCG_DATA_OBJEXTS; -#ifdef CONFIG_64BIT - slab->obj_exts_needs_objcg = 1; -#endif -#endif } /* @@ -707,10 +699,6 @@ reset_slab: continue; page = pfn_to_page(start_pfn + i); -#ifdef CONFIG_MEMCG - struct slab *slab = page_slab(page); - slab->obj_exts = 0; -#endif __ClearPageSlab(page); } diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h index 1f618f9b0d12..e6b4bf349ff7 100644 --- a/mm/kfence/kfence.h +++ b/mm/kfence/kfence.h @@ -102,9 +102,6 @@ struct kfence_metadata { struct kfence_track free_track __guarded_by(&lock); /* For updating alloc_covered on frees. */ u32 alloc_stack_hash __guarded_by(&lock); -#ifdef CONFIG_MEMCG - struct slabobj_ext obj_exts; -#endif }; #define KFENCE_METADATA_SIZE PAGE_ALIGN(sizeof(struct kfence_metadata) * \ diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 9971726406d6..d1312441a02b 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -3583,9 +3583,11 @@ bool __memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru, slab = virt_to_slab(p[i]); - if (!slab_obj_exts(slab) && - alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) { - continue; + if (!slab_obj_exts(slab)) { + if (is_kfence_address(p[i])) + continue; + if (alloc_slab_obj_exts(slab, s, flags, slab_alloc_flags)) + continue; } /* diff --git a/mm/slab.h b/mm/slab.h index f76cf2e2ed90..e64614749692 100644 --- a/mm/slab.h +++ b/mm/slab.h @@ -730,7 +730,11 @@ slab_obj_ext(struct kmem_cache *s, struct slab *slab, unsigned long obj_exts, VM_WARN_ON_ONCE(obj_exts != slab_obj_exts(slab)); - index = obj_to_index(s, slab, obj); + /* + * KFENCE objects have NULL obj_exts and thus can't reach this + * and we don't need obj_to_index() + */ + index = __obj_to_index(s, slab_address(slab), obj); if (!obj_exts_in_object(slab)) stride = slab_obj_ext_size(slab); diff --git a/mm/slub.c b/mm/slub.c index 9142f0f4c72f..9a0ee3d77230 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2067,9 +2067,6 @@ static inline void mark_obj_codetag_empty(const void *obj) if (!slab_obj_ext_has_codetag()) return; - if (is_kfence_address(obj)) - return; - obj_slab = virt_to_slab(obj); slab_exts = slab_obj_exts(obj_slab); if (slab_exts) { @@ -2332,11 +2329,15 @@ static inline unsigned long prepare_slab_obj_exts_hook(struct kmem_cache *s, struct slab *slab, gfp_t flags, unsigned int alloc_flags, void *p) { - if (!slab_obj_exts(slab) && - alloc_slab_obj_exts(slab, s, flags, alloc_flags)) { - pr_warn_once("%s, %s: Failed to create slab extension vector!\n", - __func__, s->name); - return 0; + if (!slab_obj_exts(slab)) { + if (is_kfence_address(p)) + return 0; + + if (alloc_slab_obj_exts(slab, s, flags, alloc_flags)) { + pr_warn_once("%s, %s: Failed to create slab extension vector!\n", + __func__, s->name); + return 0; + } } return slab_obj_exts(slab); @@ -2361,9 +2362,6 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags, if (alloc_flags & SLAB_ALLOC_NO_RECURSE) return; - if (is_kfence_address(object)) - return; - slab = virt_to_slab(object); obj_exts = prepare_slab_obj_exts_hook(s, slab, flags, alloc_flags, object); /* @@ -2383,7 +2381,13 @@ __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags, put_slab_obj_exts(obj_exts); } else { - alloc_tag_set_inaccurate(current->alloc_tag); + /* + * KFENCE allocations are rare and the amount of outstanding + * ones is limited to a small number so it's not worth setting + * tags as inaccurate because of them. + */ + if (!is_kfence_address(object)) + alloc_tag_set_inaccurate(current->alloc_tag); } } @@ -2414,9 +2418,6 @@ __alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p for (int i = 0; i < objects; i++) { struct slabobj_ext *ext; - if (is_kfence_address(p[i])) - continue; - ext = slab_obj_ext(s, slab, obj_exts, p[i]); alloc_tag_sub(slab_obj_ext_codetag_ref(slab, ext), s->size); } |