From b16b03ebb39d2b72376a5918ff1d6b367615be78 Mon Sep 17 00:00:00 2001 From: Julia Lawall Date: Sat, 1 Aug 2026 21:09:52 +0200 Subject: ring-buffer: drop unneeded semicolon When a function-like macro expands to an expression, that expression doesn't need a semicolon after it. All uses have been verified to have their own semicolons. This was found using the following Coccinelle semantic patch: @r@ identifier i : script:ocaml() { String.lowercase_ascii i = i }; expression e; @@ *#define i(...) e; Link: https://patch.msgid.link/20260801191002.1383835-6-Julia.Lawall@inria.fr Signed-off-by: Julia Lawall Acked-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 2667992f0aa2..8f34ff6fd066 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -405,11 +405,11 @@ static void free_buffer_page(struct buffer_page *bpage) */ #define alloc_cpu_buffer(cpu) (struct ring_buffer_per_cpu *) \ kzalloc_node(ALIGN(sizeof(struct ring_buffer_per_cpu), \ - cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); + cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)) #define alloc_cpu_page(cpu) (struct buffer_page *) \ kzalloc_node(ALIGN(sizeof(struct buffer_page), \ - cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)); + cache_line_size()), GFP_KERNEL, cpu_to_node(cpu)) static struct buffer_data_page *alloc_cpu_data(int cpu, int order) { -- cgit v1.2.3 From 234b1a72e9706fe20c08c96f4374ec8e83b934cb Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 13 Aug 2026 14:11:43 +0100 Subject: ring-buffer: Free cpu_buffer::free_page with subbuf_order When sub-buffers use an order greater than 0, cpu_buffer->free_page is allocated with subbuf_order. Use the correct order for cpu_buffer->free_page. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260813131152.3589632-2-vdonnefort@google.com Fixes: f9b94daa542a ("ring-buffer: Set new size of the ring buffer sub page") Reported-by: Sashiko Closes: https://sashiko.dev/#/patchset/20260806211306.3704194-1-vdonnefort%40google.com # patch 3 Signed-off-by: Vincent Donnefort Reviewed-by: Masami Hiramatsu (Google) Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 8f34ff6fd066..73a3fb76ba4d 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -2631,7 +2631,7 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) free_buffer_page(bpage); } - free_page((unsigned long)cpu_buffer->free_page); + free_pages((unsigned long)cpu_buffer->free_page, cpu_buffer->buffer->subbuf_order); kfree(cpu_buffer); } -- cgit v1.2.3 From 24974bd0da1b47fd56c975533ead50abf754e74d Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 13 Aug 2026 14:11:44 +0100 Subject: ring-buffer: Hold cpu_buffer::lock when resizing a subbuf Because, ring_buffer_subbuf_order_set() can clear cpu_buffer->free_page, hold cpu_buffer->lock to prevent races with ring_buffer_alloc_read_page() and ring_buffer_free_read_page(). Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260813131152.3589632-3-vdonnefort@google.com Fixes: 8e7b58c27b3c ("ring-buffer: Just update the subbuffers when changing their allocation order") Reported-by: Sashiko Closes: https://sashiko.dev/#/patchset/20260810125633.3344684-1-vdonnefort%40google.com # patch 3 Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 73a3fb76ba4d..0479c404db03 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -7431,8 +7431,10 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) cpu_buffer->nr_pages = cpu_buffer->nr_pages_to_update; cpu_buffer->nr_pages_to_update = 0; + arch_spin_lock(&cpu_buffer->lock); old_free_data_page = cpu_buffer->free_page; cpu_buffer->free_page = NULL; + arch_spin_unlock(&cpu_buffer->lock); rb_head_page_activate(cpu_buffer); -- cgit v1.2.3 From 7a1fb95de5404134f8758c1295ce88986bdf117c Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 13 Aug 2026 14:11:45 +0100 Subject: ring-buffer: Make cpu_buffer::free_page a buffer_data_read_page Discarding a cached reader page after a concurrent ring buffer resize uses the new global subbuf_order for the free_pages() call. This mismatched order may crashes the kernel or leaks memory because the cached page was allocated under the old size. Save the actual free_page order alongside the page address to ensure we always refer to the correct value and do not rely on the potentially stalled cpu_buffer->subbuf_order value. The simplest is to make free_page a buffer_data_read_page which already covers exactly what we need: a page address and a page order. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260813131152.3589632-4-vdonnefort@google.com Fixes: 8e7b58c27b3c ("ring-buffer: Just update the subbuffers when changing their allocation order") Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 0479c404db03..0d35fd90ea1d 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -507,7 +507,7 @@ struct ring_buffer_per_cpu { raw_spinlock_t reader_lock; /* serialize readers */ arch_spinlock_t lock; struct lock_class_key lock_key; - struct buffer_data_page *free_page; + struct buffer_data_read_page free_page; unsigned long nr_pages; unsigned int current_context; struct list_head *pages; @@ -2631,7 +2631,7 @@ static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer) free_buffer_page(bpage); } - free_pages((unsigned long)cpu_buffer->free_page, cpu_buffer->buffer->subbuf_order); + free_pages((unsigned long)cpu_buffer->free_page.data, cpu_buffer->free_page.order); kfree(cpu_buffer); } @@ -6962,9 +6962,9 @@ ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) local_irq_save(flags); arch_spin_lock(&cpu_buffer->lock); - if (cpu_buffer->free_page) { - bpage->data = cpu_buffer->free_page; - cpu_buffer->free_page = NULL; + if (cpu_buffer->free_page.data) { + *bpage = cpu_buffer->free_page; + cpu_buffer->free_page.data = NULL; } arch_spin_unlock(&cpu_buffer->lock); @@ -7016,8 +7016,8 @@ void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, local_irq_save(flags); arch_spin_lock(&cpu_buffer->lock); - if (!cpu_buffer->free_page) { - cpu_buffer->free_page = dpage; + if (!cpu_buffer->free_page.data) { + cpu_buffer->free_page = *data_page; dpage = NULL; } @@ -7390,7 +7390,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) } for_each_buffer_cpu(buffer, cpu) { - struct buffer_data_page *old_free_data_page; + struct buffer_data_read_page old_free_data_page; struct list_head old_pages; unsigned long flags; @@ -7433,7 +7433,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) arch_spin_lock(&cpu_buffer->lock); old_free_data_page = cpu_buffer->free_page; - cpu_buffer->free_page = NULL; + cpu_buffer->free_page.data = NULL; arch_spin_unlock(&cpu_buffer->lock); rb_head_page_activate(cpu_buffer); @@ -7445,7 +7445,7 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) list_del_init(&bpage->list); free_buffer_page(bpage); } - free_pages((unsigned long)old_free_data_page, old_order); + free_pages((unsigned long)old_free_data_page.data, old_free_data_page.order); rb_check_pages(cpu_buffer); } -- cgit v1.2.3 From 8a5f63637890f03177146efddaba5ec7a1b4d61f Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 13 Aug 2026 14:11:46 +0100 Subject: ring-buffer: Fix subbuf resize race with ring buffer readers trace_buffer subbuf_size is read lockless in ring_buffer_read_page() and ring_buffer_read_start(), while it can simultaneously be resized with ring_buffer_subbuf_order_set(). Instead of trace_buffer::subbuf_size, use bpage::order in ring_buffer_read_start() and ring_buffer_read_page(). In ring_buffer_read_start(), even with resize_disabled, there is still a possibility of a race with a buffer modification. Hold the trace_buffer mutex to synchronise with any pending ring buffer order modification. trace_buffer::subbuf_size is now actually useless, remove it. Also, create accessors rb_subbuf_capacity() and rb_page_capacity() which return the actual size available for storing events, while rb_subbuf_size() returns the actual subbuf page-size. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260813131152.3589632-5-vdonnefort@google.com Fixes: f9b94daa542a ("ring-buffer: Set new size of the ring buffer sub page") Reported-by: Sashiko Closes: https://sashiko.dev/#/patchset/20260805153225.2096152-1-vdonnefort%40google.com # patch 1 Acked-by: Masami Hiramatsu (Google) Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 167 ++++++++++++++++++++++++++++----------------- 1 file changed, 106 insertions(+), 61 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 0d35fd90ea1d..e436793ed5c1 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -391,6 +391,17 @@ static __always_inline unsigned int rb_page_size(struct buffer_page *bpage) return rb_data_page_size(bpage->page); } +/** + * rb_page_capacity - Get the capacity of a buffer page + * @bpage: The buffer page + * + * Return: The maximum size available for events in the given buffer page. + */ +static __always_inline unsigned int rb_page_capacity(struct buffer_page *bpage) +{ + return (PAGE_SIZE << bpage->order) - BUF_PAGE_HDR_SIZE; +} + static void free_buffer_page(struct buffer_page *bpage) { /* Range pages are not to be freed */ @@ -586,11 +597,42 @@ struct trace_buffer { struct ring_buffer_meta *meta; - unsigned int subbuf_size; unsigned int subbuf_order; unsigned int max_data_size; }; +static __always_inline unsigned int rb_subbuf_size(struct trace_buffer *buffer) +{ + return PAGE_SIZE << buffer->subbuf_order; +} + +/** + * rb_subbuf_capacity - Get the capacity of a subbuffer + * @buffer: A trace buffer + * + * Unsafe to use without holding trace_buffer::mutex or with resizing enabled. + * Consider rb_page_capacity() instead. + * + * Return: The maximum size available for events in a trace buffer subbuffer. + */ +static __always_inline unsigned int rb_subbuf_capacity(struct trace_buffer *buffer) +{ + return rb_subbuf_size(buffer) - BUF_PAGE_HDR_SIZE; +} + +/** + * rb_subbuf_start - Get the start address of a subbuffer + * @buffer: A trace buffer + * @addr: An address of an event on a subbuffer + * + * Return: The start of the subbuffer for where @addr sits + */ +static __always_inline +unsigned long rb_subbuf_start(struct trace_buffer *buffer, unsigned long addr) +{ + return addr & ~((unsigned long)(rb_subbuf_size(buffer) - 1)); +} + struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; @@ -630,7 +672,7 @@ int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq trace_seq_printf(s, "\tfield: char data;\t" "offset:%u;\tsize:%u;\tsigned:%u;\n", (unsigned int)offsetof(typeof(field), data), - (unsigned int)(buffer ? buffer->subbuf_size : + (unsigned int)(buffer ? rb_subbuf_capacity(buffer) : PAGE_SIZE - BUF_PAGE_HDR_SIZE), (unsigned int)is_signed_type(char)); @@ -1620,7 +1662,7 @@ rb_range_align_subbuf(unsigned long addr, int subbuf_size, int nr_subbufs) */ static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu) { - int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + int subbuf_size = rb_subbuf_size(buffer); struct ring_buffer_cpu_meta *meta; struct ring_buffer_meta *bmeta; unsigned long ptr; @@ -2432,8 +2474,8 @@ static int __rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer, bpage->id = i + 1; cpu_buffer->subbuf_ids[i + 1] = bpage; } else { - int order = cpu_buffer->buffer->subbuf_order; - bpage->page = alloc_cpu_data(cpu_buffer->cpu, order); + bpage->page = alloc_cpu_data(cpu_buffer->cpu, + cpu_buffer->buffer->subbuf_order); if (!bpage->page) goto free_pages; } @@ -2556,8 +2598,7 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) bpage->range = 1; cpu_buffer->subbuf_ids[0] = bpage; } else { - int order = cpu_buffer->buffer->subbuf_order; - bpage->page = alloc_cpu_data(cpu, order); + bpage->page = alloc_cpu_data(cpu, bpage->order); if (!bpage->page) goto fail_free_reader; } @@ -2731,10 +2772,9 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, buffer->subbuf_order = order; subbuf_size = (PAGE_SIZE << order); - buffer->subbuf_size = subbuf_size - BUF_PAGE_HDR_SIZE; /* Max payload is buffer page size - header (8bytes) */ - buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2); + buffer->max_data_size = rb_subbuf_capacity(buffer) - (sizeof(u32) * 2); buffer->flags = flags; buffer->clock = trace_clock_local; @@ -2818,9 +2858,8 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, if (nr_pages < 2) goto fail_free_buffers; } else { - /* need at least two pages */ - nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); + nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); if (nr_pages < 2) nr_pages = 2; } @@ -3203,7 +3242,7 @@ static void update_pages_handler(struct work_struct *work) * @size: the new size. * @cpu_id: the cpu buffer to resize * - * Minimum size is 2 * buffer->subbuf_size. + * Minimum size is 2 * rb_subbuf_capacity(buffer). * * Returns 0 on success and < 0 on failure. */ @@ -3225,12 +3264,6 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, !cpumask_test_cpu(cpu_id, buffer->cpumask)) return 0; - nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); - - /* we need a minimum of two pages */ - if (nr_pages < 2) - nr_pages = 2; - /* * Keep CPUs from coming online while resizing to synchronize * with new per CPU buffers being created. @@ -3241,6 +3274,12 @@ int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, mutex_lock(&buffer->mutex); atomic_inc(&buffer->resizing); + nr_pages = DIV_ROUND_UP(size, rb_subbuf_capacity(buffer)); + + /* we need a minimum of two pages */ + if (nr_pages < 2) + nr_pages = 2; + if (cpu_id == RING_BUFFER_ALL_CPUS) { /* * Don't succeed if resizing is disabled, as a reader might be @@ -3513,7 +3552,7 @@ rb_event_index(struct ring_buffer_per_cpu *cpu_buffer, struct ring_buffer_event { unsigned long addr = (unsigned long)event; - addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1; + addr &= (unsigned long)rb_subbuf_size(cpu_buffer->buffer) - 1; return addr - BUF_PAGE_HDR_SIZE; } @@ -3755,8 +3794,8 @@ static inline void rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer, unsigned long tail, struct rb_event_info *info) { - unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); struct buffer_page *tail_page = info->tail_page; + unsigned long bsize = rb_page_capacity(tail_page); struct ring_buffer_event *event; unsigned long length = info->length; @@ -4101,8 +4140,7 @@ rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer, new_index = rb_event_index(cpu_buffer, event); old_index = new_index + rb_event_ts_length(event); - addr = (unsigned long)event; - addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); + addr = rb_subbuf_start(cpu_buffer->buffer, (unsigned long)event); bpage = READ_ONCE(cpu_buffer->tail_page); @@ -4767,7 +4805,7 @@ __rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer, tail = write - info->length; /* See if we shot pass the end of this buffer page */ - if (unlikely(write > cpu_buffer->buffer->subbuf_size)) { + if (unlikely(write > rb_page_capacity(tail_page))) { check_buffer(cpu_buffer, info, CHECK_FULL_PAGE); return rb_move_tail(cpu_buffer, tail, info); } @@ -5012,7 +5050,7 @@ rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer, struct buffer_page *bpage = cpu_buffer->commit_page; struct buffer_page *start; - addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); + addr = rb_subbuf_start(cpu_buffer->buffer, addr); /* Do the likely case first */ if (likely(bpage->page == (void *)addr)) { @@ -5799,7 +5837,6 @@ static struct buffer_page * __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) { int max_loops = cpu_buffer->ring_meta ? cpu_buffer->nr_pages : 3; - unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); struct buffer_page *reader = NULL; unsigned long overwrite; unsigned long flags; @@ -5947,7 +5984,7 @@ __rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer) #define USECS_WAIT 1000000 for (nr_loops = 0; nr_loops < USECS_WAIT; nr_loops++) { /* If the write is past the end of page, a writer is still updating it */ - if (likely(!reader || rb_page_write(reader) <= bsize)) + if (likely(!reader || rb_page_write(reader) <= rb_page_capacity(reader))) break; udelay(1); @@ -6380,36 +6417,44 @@ EXPORT_SYMBOL_GPL(ring_buffer_consume); struct ring_buffer_iter * ring_buffer_read_start(struct trace_buffer *buffer, int cpu, gfp_t flags) { + struct ring_buffer_iter *iter __free(kfree) = kzalloc_obj(*iter, flags); struct ring_buffer_per_cpu *cpu_buffer; - struct ring_buffer_iter *iter; - - if (!cpumask_test_cpu(cpu, buffer->cpumask)) - return NULL; - iter = kzalloc_obj(*iter, flags); if (!iter) return NULL; - /* Holds the entire event: data and meta data */ - iter->event_size = buffer->subbuf_size; - iter->event = kmalloc(iter->event_size, flags); - if (!iter->event) { - kfree(iter); + if (!cpumask_test_cpu(cpu, buffer->cpumask)) return NULL; - } cpu_buffer = buffer->buffers[cpu]; - iter->cpu_buffer = cpu_buffer; + /* + * Only KDB is using GFP_ATOMIC, for the others, lock the buffer to + * prevent concurrent resizing. + */ + if (gfpflags_allow_blocking(flags)) + mutex_lock(&buffer->mutex); atomic_inc(&cpu_buffer->resize_disabled); + if (gfpflags_allow_blocking(flags)) + mutex_unlock(&buffer->mutex); + + /* Holds the entire event: data and meta data. */ + iter->event_size = rb_page_capacity(READ_ONCE(cpu_buffer->reader_page)); + iter->event = kmalloc(iter->event_size, flags); + if (!iter->event) { + atomic_dec(&cpu_buffer->resize_disabled); + return NULL; + } + iter->cpu_buffer = cpu_buffer; + guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); arch_spin_lock(&cpu_buffer->lock); rb_iter_reset(iter); arch_spin_unlock(&cpu_buffer->lock); - return iter; + return_ptr(iter); } EXPORT_SYMBOL_GPL(ring_buffer_read_start); @@ -6463,7 +6508,7 @@ unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu) if (!cpumask_test_cpu(cpu, buffer->cpumask)) return 0; - return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages; + return rb_subbuf_capacity(buffer) * buffer->buffers[cpu]->nr_pages; } EXPORT_SYMBOL_GPL(ring_buffer_size); @@ -7094,15 +7139,15 @@ int ring_buffer_read_page(struct trace_buffer *buffer, if (!data_page || !data_page->data) return -1; - if (data_page->order != buffer->subbuf_order) - return -1; - dpage = data_page->data; if (!dpage) return -1; guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock); + if (data_page->order != cpu_buffer->reader_page->order) + return -1; + reader = rb_get_reader_page(cpu_buffer); if (!reader) return -1; @@ -7228,7 +7273,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, * missed events, then record it there. */ if (missed_events > 0 && - buffer->subbuf_size - size >= sizeof(missed_events)) { + rb_page_capacity(reader) - size >= sizeof(missed_events)) { memcpy(&dpage->data[size], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); @@ -7248,8 +7293,8 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* * This page may be off to user land. Zero it out here. */ - if (size < buffer->subbuf_size) - memset(&dpage->data[size], 0, buffer->subbuf_size - size); + if (size < rb_page_capacity(reader)) + memset(&dpage->data[size], 0, rb_page_capacity(reader) - size); return read; } @@ -7275,7 +7320,7 @@ EXPORT_SYMBOL_GPL(ring_buffer_read_page_data); */ int ring_buffer_subbuf_size_get(struct trace_buffer *buffer) { - return buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + return rb_subbuf_size(buffer); } EXPORT_SYMBOL_GPL(ring_buffer_subbuf_size_get); @@ -7320,7 +7365,8 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) { struct ring_buffer_per_cpu *cpu_buffer; struct buffer_page *bpage, *tmp; - int old_order, old_size; + unsigned int old_capacity; + int old_order; int nr_pages; int psize; int err; @@ -7329,9 +7375,6 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (!buffer || order < 0) return -EINVAL; - if (buffer->subbuf_order == order) - return 0; - psize = (1 << order) * PAGE_SIZE; if (psize <= BUF_PAGE_HDR_SIZE) return -EINVAL; @@ -7340,18 +7383,21 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) if (psize > RB_WRITE_MASK + 1) return -EINVAL; - old_order = buffer->subbuf_order; - old_size = buffer->subbuf_size; - /* prevent another thread from changing buffer sizes */ guard(mutex)(&buffer->mutex); + + old_order = buffer->subbuf_order; + if (old_order == order) + return 0; + + old_capacity = rb_subbuf_capacity(buffer); + atomic_inc(&buffer->record_disabled); /* Make sure all commits have finished */ synchronize_rcu(); buffer->subbuf_order = order; - buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE; /* Make sure all new buffers are allocated, before deleting the old ones */ for_each_buffer_cpu(buffer, cpu) { @@ -7367,8 +7413,8 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) } /* Update the number of pages to match the new size */ - nr_pages = old_size * buffer->buffers[cpu]->nr_pages; - nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size); + nr_pages = old_capacity * buffer->buffers[cpu]->nr_pages; + nr_pages = DIV_ROUND_UP(nr_pages, rb_subbuf_capacity(buffer)); /* we need a minimum of two pages */ if (nr_pages < 2) @@ -7456,7 +7502,6 @@ int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) error: buffer->subbuf_order = old_order; - buffer->subbuf_size = old_size; atomic_dec(&buffer->record_disabled); @@ -7534,7 +7579,7 @@ static void rb_setup_ids_meta_page(struct ring_buffer_per_cpu *cpu_buffer, meta->meta_struct_len = sizeof(*meta); meta->nr_subbufs = nr_subbufs; - meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE; + meta->subbuf_size = rb_subbuf_size(cpu_buffer->buffer); meta->meta_page_size = meta->subbuf_size; rb_update_meta_page(cpu_buffer); @@ -7896,7 +7941,7 @@ consume: * missed events, then record it there. */ commit = rb_page_size(reader); - if (buffer->subbuf_size - commit >= sizeof(missed_events)) { + if (rb_subbuf_capacity(buffer) - commit >= sizeof(missed_events)) { memcpy(&dpage->data[commit], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); @@ -7928,7 +7973,7 @@ consume: out: /* Some archs do not have data cache coherency between kernel and user-space */ flush_kernel_vmap_range(cpu_buffer->reader_page->page, - buffer->subbuf_size + BUF_PAGE_HDR_SIZE); + rb_subbuf_size(buffer)); rb_update_meta_page(cpu_buffer); -- cgit v1.2.3 From e743527c5bfdceda1095bc0a9e596e2aebb6a9c3 Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 13 Aug 2026 14:11:47 +0100 Subject: ring-buffer: Fix subbuf resize race with ring_buffer_alloc_read_page() ring_buffer_alloc_read_page() is racy with ring_buffer_subbuf_order_set, it can allocate a reader page with an outdated order. This isn't a big issue, the user can still re-allocate a new reader page and try again. However, what is more problematic is if the value of subbuf_order changes in the middle of ring_buffer_alloc_read_page(). In that case, bpage->order might not match the actual allocated memory. Use bpage->order for the allocation to prevent this race. Cc: stable@vger.kernel.org Link: https://patch.msgid.link/20260813131152.3589632-6-vdonnefort@google.com Fixes: bce761d75745 ("ring-buffer: Read and write to ring buffers with custom sub buffer size") Reported-by: Sashiko Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index e436793ed5c1..796c10319d6a 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -7018,7 +7018,7 @@ ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) if (bpage->data) { rb_init_data_page(bpage->data); } else { - bpage->data = alloc_cpu_data(cpu, cpu_buffer->buffer->subbuf_order); + bpage->data = alloc_cpu_data(cpu, bpage->order); if (!bpage->data) { kfree(bpage); return ERR_PTR(-ENOMEM); -- cgit v1.2.3 From 472ac478b14d940b555153e50f3a34ac1c5e71ff Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 13 Aug 2026 14:11:49 +0100 Subject: ring-buffer: Dynamically calculate max_data_size The ring buffer order can be dynamically modified and temporarily disables writing to do so. It is therefore safe to use the updated value to calculate the maximum event size which can be written onto the ring buffer. However, notice it is hardly making any difference for trace_marker because of the TRACE_MARKER_MAX_SIZE limit. For an 8KiB subbuf size, trace_marker can take 4096 characters while it can 'only' take 4054 bytes for smaller subbufs. Link: https://patch.msgid.link/20260813131152.3589632-8-vdonnefort@google.com Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 796c10319d6a..2204a121ef7e 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -598,7 +598,6 @@ struct trace_buffer { struct ring_buffer_meta *meta; unsigned int subbuf_order; - unsigned int max_data_size; }; static __always_inline unsigned int rb_subbuf_size(struct trace_buffer *buffer) @@ -620,6 +619,23 @@ static __always_inline unsigned int rb_subbuf_capacity(struct trace_buffer *buff return rb_subbuf_size(buffer) - BUF_PAGE_HDR_SIZE; } +/** + * rb_subbuf_max_data_size - Get the maximum payload size of a single event + * @buffer: A trace buffer + * + * Return: The maximum data payload size that can be stored in a single event. + */ +static __always_inline unsigned int rb_subbuf_max_data_size(struct trace_buffer *buffer) +{ + struct ring_buffer_event *event; + + /* + * surely rb_subbuf_capacity() is bigger than + * RINGBUF_TYPE_DATA_TYPE_LEN_MAX (see ring_buffer_event_length). + */ + return rb_subbuf_capacity(buffer) - RB_EVNT_HDR_SIZE - sizeof(event->array[0]); +} + /** * rb_subbuf_start - Get the start address of a subbuffer * @buffer: A trace buffer @@ -2773,9 +2789,6 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, buffer->subbuf_order = order; subbuf_size = (PAGE_SIZE << order); - /* Max payload is buffer page size - header (8bytes) */ - buffer->max_data_size = rb_subbuf_capacity(buffer) - (sizeof(u32) * 2); - buffer->flags = flags; buffer->clock = trace_clock_local; buffer->reader_lock_key = key; @@ -4941,7 +4954,7 @@ rb_reserve_next_event(struct trace_buffer *buffer, if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) { add_ts_default = RB_ADD_STAMP_ABSOLUTE; info.length += RB_LEN_TIME_EXTEND; - if (info.length > cpu_buffer->buffer->max_data_size) + if (info.length > rb_subbuf_max_data_size(cpu_buffer->buffer)) goto out_fail; } else { add_ts_default = RB_ADD_STAMP_NONE; @@ -5016,7 +5029,7 @@ ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length) if (unlikely(atomic_read(&cpu_buffer->record_disabled))) goto out; - if (unlikely(length > buffer->max_data_size)) + if (unlikely(length > rb_subbuf_max_data_size(buffer))) goto out; if (unlikely(trace_recursive_lock(cpu_buffer))) @@ -5163,7 +5176,7 @@ int ring_buffer_write(struct trace_buffer *buffer, if (atomic_read(&cpu_buffer->record_disabled)) return -EBUSY; - if (length > buffer->max_data_size) + if (length > rb_subbuf_max_data_size(buffer)) return -EBUSY; if (unlikely(trace_recursive_lock(cpu_buffer))) @@ -6522,8 +6535,9 @@ unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer) { /* If abs timestamp is requested, events have a timestamp too */ if (ring_buffer_time_stamp_abs(buffer)) - return buffer->max_data_size - RB_LEN_TIME_EXTEND; - return buffer->max_data_size; + return rb_subbuf_max_data_size(buffer) - RB_LEN_TIME_EXTEND; + + return rb_subbuf_max_data_size(buffer); } EXPORT_SYMBOL_GPL(ring_buffer_max_event_size); @@ -7941,7 +7955,7 @@ consume: * missed events, then record it there. */ commit = rb_page_size(reader); - if (rb_subbuf_capacity(buffer) - commit >= sizeof(missed_events)) { + if (rb_page_capacity(reader) - commit >= sizeof(missed_events)) { memcpy(&dpage->data[commit], &missed_events, sizeof(missed_events)); local_add(RB_MISSED_STORED, &dpage->commit); -- cgit v1.2.3 From d9b5e22bf24d5c82522475306cd332b287ac43bc Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 13 Aug 2026 14:11:50 +0100 Subject: ring-buffer: Remove trace_buffer::cpus The 'cpus' field in struct trace_buffer became useless in commit 8e7b58c27b3c ("ring-buffer: Just update the subbuffers when changing their allocation order"). Remove it Link: https://patch.msgid.link/20260813131152.3589632-9-vdonnefort@google.com Fixes: 8e7b58c27b3c ("ring-buffer: Just update the subbuffers when changing their allocation order") Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 2204a121ef7e..f75aafc755bd 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -572,7 +572,6 @@ struct ring_buffer_per_cpu { struct trace_buffer { unsigned flags; - int cpus; atomic_t record_disabled; atomic_t resizing; cpumask_var_t cpumask; @@ -2796,7 +2795,6 @@ static struct trace_buffer *alloc_buffer(unsigned long size, unsigned flags, init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters); init_waitqueue_head(&buffer->irq_work.waiters); - buffer->cpus = nr_cpu_ids; bsize = sizeof(void *) * nr_cpu_ids; buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), -- cgit v1.2.3 From 8b502bf6eb3da15f4b954ad3632335ff10ed746a Mon Sep 17 00:00:00 2001 From: Vincent Donnefort Date: Thu, 13 Aug 2026 14:11:51 +0100 Subject: ring-buffer: Remove ring_buffer_per_cpu::mapped ring_buffer_per_cpu::mapped tracks if a ring-buffer is either mapped by user-space or if it is a persistent buffer. We already have user_mapped for the former and ring_meta for the latter. Get rid of mapped and instead create rb_is_static(). A static ring-buffer cannot be resized, swapped or have its pages extracted. Link: https://patch.msgid.link/20260813131152.3589632-10-vdonnefort@google.com Signed-off-by: Vincent Donnefort Signed-off-by: Steven Rostedt --- kernel/trace/ring_buffer.c | 45 +++++++++++++++++---------------------------- 1 file changed, 17 insertions(+), 28 deletions(-) (limited to 'kernel/trace') diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index f75aafc755bd..5fc009edc1ec 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -514,7 +514,7 @@ struct ring_buffer_per_cpu { int cpu; atomic_t record_disabled; atomic_t resize_disabled; - struct trace_buffer *buffer; + struct trace_buffer *buffer; raw_spinlock_t reader_lock; /* serialize readers */ arch_spinlock_t lock; struct lock_class_key lock_key; @@ -552,7 +552,6 @@ struct ring_buffer_per_cpu { /* pages removed since last reset */ unsigned long pages_removed; - unsigned int mapped; unsigned int user_mapped; /* user space mapping */ struct mutex mapping_lock; struct buffer_page **subbuf_ids; /* ID to subbuf VA */ @@ -648,6 +647,11 @@ unsigned long rb_subbuf_start(struct trace_buffer *buffer, unsigned long addr) return addr & ~((unsigned long)(rb_subbuf_size(buffer) - 1)); } +static bool rb_is_static(struct ring_buffer_per_cpu *cpu_buffer) +{ + return cpu_buffer->user_mapped || cpu_buffer->remote || cpu_buffer->ring_meta; +} + struct ring_buffer_iter { struct ring_buffer_per_cpu *cpu_buffer; unsigned long head; @@ -2578,7 +2582,6 @@ rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) * Range mapped buffers have the same restrictions as memory * mapped ones do. */ - cpu_buffer->mapped = 1; cpu_buffer->ring_meta = rb_range_meta(buffer, nr_pages, cpu); bpage->page = rb_range_buffer(cpu_buffer, 0); if (!bpage->page) @@ -6667,12 +6670,11 @@ rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer) rb_head_page_activate(cpu_buffer); cpu_buffer->pages_removed = 0; - if (cpu_buffer->mapped) { - rb_update_meta_page(cpu_buffer); - if (cpu_buffer->ring_meta) { - struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; - meta->commit_buffer = meta->head_buffer; - } + rb_update_meta_page(cpu_buffer); + if (cpu_buffer->ring_meta) { + struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta; + + meta->commit_buffer = meta->head_buffer; } } @@ -6921,8 +6923,8 @@ int ring_buffer_swap_cpu(struct trace_buffer *buffer_a, cpu_buffer_a = buffer_a->buffers[cpu]; cpu_buffer_b = buffer_b->buffers[cpu]; - /* It's up to the callers to not try to swap mapped buffers */ - if (WARN_ON_ONCE(cpu_buffer_a->mapped || cpu_buffer_b->mapped)) + /* It's up to the callers to not try to swap static buffers */ + if (WARN_ON_ONCE(rb_is_static(cpu_buffer_a) || rb_is_static(cpu_buffer_b))) return -EBUSY; /* At least make sure the two buffers are somewhat the same */ @@ -7134,7 +7136,6 @@ int ring_buffer_read_page(struct trace_buffer *buffer, unsigned int size; unsigned int read; u64 save_timestamp; - bool force_memcpy; if (!cpumask_test_cpu(cpu, buffer->cpumask)) return -1; @@ -7173,8 +7174,6 @@ int ring_buffer_read_page(struct trace_buffer *buffer, /* Check if any events were dropped */ missed_events = cpu_buffer->lost_events; - force_memcpy = cpu_buffer->mapped || cpu_buffer->remote; - /* * If this page has been partially read or * if len is not big enough to read the rest of the page or @@ -7184,7 +7183,7 @@ int ring_buffer_read_page(struct trace_buffer *buffer, */ if (read || (len < (size - read)) || cpu_buffer->reader_page == cpu_buffer->commit_page || - force_memcpy) { + rb_is_static(cpu_buffer)) { struct buffer_data_page *rpage = cpu_buffer->reader_page->page; unsigned int rpos = read; unsigned int pos = 0; @@ -7633,11 +7632,7 @@ static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer, lockdep_assert_held(&cpu_buffer->mapping_lock); - /* mapped is always greater or equal to user_mapped */ - if (WARN_ON(cpu_buffer->mapped < cpu_buffer->user_mapped)) - return -EINVAL; - - if (inc && cpu_buffer->mapped == UINT_MAX) + if (inc && cpu_buffer->user_mapped == UINT_MAX) return -EBUSY; if (WARN_ON(!inc && cpu_buffer->user_mapped == 0)) @@ -7646,13 +7641,10 @@ static int __rb_inc_dec_mapped(struct ring_buffer_per_cpu *cpu_buffer, mutex_lock(&cpu_buffer->buffer->mutex); raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); - if (inc) { + if (inc) cpu_buffer->user_mapped++; - cpu_buffer->mapped++; - } else { + else cpu_buffer->user_mapped--; - cpu_buffer->mapped--; - } raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); mutex_unlock(&cpu_buffer->buffer->mutex); @@ -7824,7 +7816,6 @@ int ring_buffer_map(struct trace_buffer *buffer, int cpu, if (!err) { raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); /* This is the first time it is mapped by user */ - cpu_buffer->mapped++; cpu_buffer->user_mapped = 1; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); } else { @@ -7881,8 +7872,6 @@ int ring_buffer_unmap(struct trace_buffer *buffer, int cpu) raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags); /* This is the last user space mapping */ - if (!WARN_ON_ONCE(cpu_buffer->mapped < cpu_buffer->user_mapped)) - cpu_buffer->mapped--; cpu_buffer->user_mapped = 0; raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags); -- cgit v1.2.3