From fd13f6c85144bb2026c534a35be1d7cb7628a64a Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Sun, 19 Oct 2008 21:00:09 +0200 Subject: oprofile: comment cleanup This fixes the coding style of some comments. Signed-off-by: Robert Richter --- drivers/oprofile/cpu_buffer.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'drivers/oprofile/cpu_buffer.c') diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index 01d38e78cde1..3958107723fb 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c @@ -127,9 +127,10 @@ void end_cpu_work(void) /* Resets the cpu buffer to a sane state. */ void cpu_buffer_reset(struct oprofile_cpu_buffer *cpu_buf) { - /* reset these to invalid values; the next sample - * collected will populate the buffer with proper - * values to initialize the buffer + /* + * reset these to invalid values; the next sample collected + * will populate the buffer with proper values to initialize + * the buffer */ cpu_buf->last_is_kernel = -1; cpu_buf->last_task = NULL; @@ -151,8 +152,10 @@ static void increment_head(struct oprofile_cpu_buffer *b) { unsigned long new_head = b->head_pos + 1; - /* Ensure anything written to the slot before we - * increment is visible */ + /* + * Ensure anything written to the slot before we increment is + * visible + */ wmb(); if (new_head < b->buffer_size) @@ -253,8 +256,10 @@ void oprofile_add_ext_sample(unsigned long pc, struct pt_regs * const regs, if (!oprofile_begin_trace(cpu_buf)) return; - /* if log_sample() fail we can't backtrace since we lost the source - * of this event */ + /* + * if log_sample() fail we can't backtrace since we lost the + * source of this event + */ if (log_sample(cpu_buf, pc, is_kernel, event)) oprofile_ops.backtrace(regs, backtrace_depth); oprofile_end_trace(cpu_buf); @@ -338,8 +343,10 @@ void oprofile_add_trace(unsigned long pc) return; } - /* broken frame can give an eip with the same value as an escape code, - * abort the trace if we get it */ + /* + * broken frame can give an eip with the same value as an + * escape code, abort the trace if we get it + */ if (pc == ESCAPE_CODE) { cpu_buf->tracing = 0; cpu_buf->backtrace_aborted++; -- cgit v1.2.3 From cdc1834d1aa2e5b574a25e66f82625b44cdd0d8f Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Fri, 26 Sep 2008 22:18:44 -0400 Subject: oprofile: whitspace changes only Signed-off-by: Robert Richter --- drivers/oprofile/cpu_buffer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/oprofile/cpu_buffer.c') diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index 3958107723fb..2c4d54187b90 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c @@ -277,8 +277,8 @@ void oprofile_add_sample(struct pt_regs * const regs, unsigned long event) #define MAX_IBS_SAMPLE_SIZE 14 -void oprofile_add_ibs_sample(struct pt_regs *const regs, - unsigned int *const ibs_sample, int ibs_code) +void oprofile_add_ibs_sample(struct pt_regs * const regs, + unsigned int * const ibs_sample, int ibs_code) { int is_kernel = !user_mode(regs); struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer); -- cgit v1.2.3 From 7d468abee0f1a7e918b5e2f23120436a54ba9f33 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Thu, 27 Nov 2008 10:57:09 +0100 Subject: oprofile: adding cpu buffer r/w access functions This is in preparation for changes in the cpu buffer implementation. Signed-off-by: Robert Richter --- drivers/oprofile/cpu_buffer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/oprofile/cpu_buffer.c') diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index 2c4d54187b90..7e5e650e409f 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c @@ -168,7 +168,7 @@ static inline void add_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, unsigned long event) { - struct op_sample *entry = &cpu_buf->buffer[cpu_buf->head_pos]; + struct op_sample *entry = cpu_buffer_write_entry(cpu_buf); entry->eip = pc; entry->event = event; increment_head(cpu_buf); -- cgit v1.2.3 From 229234ae4a5ed9376b2e0524da04b0e5edadbf76 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Thu, 27 Nov 2008 18:36:08 +0100 Subject: oprofile: adding cpu_buffer_write_commit() This is in preparation for changes in the cpu buffer implementation. Signed-off-by: Robert Richter --- drivers/oprofile/cpu_buffer.c | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) (limited to 'drivers/oprofile/cpu_buffer.c') diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index 7e5e650e409f..d6f5de686363 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c @@ -148,22 +148,6 @@ static unsigned long nr_available_slots(struct oprofile_cpu_buffer const *b) return tail + (b->buffer_size - head) - 1; } -static void increment_head(struct oprofile_cpu_buffer *b) -{ - unsigned long new_head = b->head_pos + 1; - - /* - * Ensure anything written to the slot before we increment is - * visible - */ - wmb(); - - if (new_head < b->buffer_size) - b->head_pos = new_head; - else - b->head_pos = 0; -} - static inline void add_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, unsigned long event) @@ -171,7 +155,7 @@ add_sample(struct oprofile_cpu_buffer *cpu_buf, struct op_sample *entry = cpu_buffer_write_entry(cpu_buf); entry->eip = pc; entry->event = event; - increment_head(cpu_buf); + cpu_buffer_write_commit(cpu_buf); } static inline void -- cgit v1.2.3 From fbc9bf9f0ed4f0fbc47dcb5b1c26c28c93b60e33 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Thu, 4 Dec 2008 16:27:00 +0100 Subject: oprofile: moving cpu_buffer_reset() to cpu_buffer.h This is in preparation for changes in the cpu buffer implementation. Signed-off-by: Robert Richter --- drivers/oprofile/cpu_buffer.c | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'drivers/oprofile/cpu_buffer.c') diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index d6f5de686363..5cf7efe38e67 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c @@ -124,18 +124,6 @@ void end_cpu_work(void) flush_scheduled_work(); } -/* Resets the cpu buffer to a sane state. */ -void cpu_buffer_reset(struct oprofile_cpu_buffer *cpu_buf) -{ - /* - * reset these to invalid values; the next sample collected - * will populate the buffer with proper values to initialize - * the buffer - */ - cpu_buf->last_is_kernel = -1; - cpu_buf->last_task = NULL; -} - /* compute number of available slots in cpu_buffer queue */ static unsigned long nr_available_slots(struct oprofile_cpu_buffer const *b) { -- cgit v1.2.3 From 6dad828b76c7224a22ddc9ce7aa495d994f03b31 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Tue, 9 Dec 2008 01:21:32 +0100 Subject: oprofile: port to the new ring_buffer This patch replaces the current oprofile cpu buffer implementation with the ring buffer provided by the tracing framework. The motivation here is to leave the pain of implementing ring buffers to others. Oh, no, there are more advantages. Main reason is the support of different sample sizes that could be stored in the buffer. Use cases for this are IBS and Cell spu profiling. Using the new ring buffer ensures valid and complete samples and allows copying the cpu buffer stateless without knowing its content. Second it will use generic kernel API and also reduce code size. And hopefully, there are less bugs. Since the new tracing ring buffer implementation uses spin locks to protect the buffer during read/write access, it is difficult to use the buffer in an NMI handler. In this case, writing to the buffer by the NMI handler (x86) could occur also during critical sections when reading the buffer. To avoid this, there are 2 buffers for independent read and write access. Read access is in process context only, write access only in the NMI handler. If the read buffer runs empty, both buffers are swapped atomically. There is potentially a small window during swapping where the buffers are disabled and samples could be lost. Using 2 buffers is a little bit overhead, but the solution is clear and does not require changes in the ring buffer implementation. It can be changed to a single buffer solution when the ring buffer access is implemented as non-locking atomic code. The new buffer requires more size to store the same amount of samples because each sample includes an u32 header. Also, there is more code to execute for buffer access. Nonetheless, the buffer implementation is proven in the ftrace environment and worth to use also in oprofile. Patches that changes the internal IBS buffer usage will follow. Cc: Steven Rostedt Signed-off-by: Robert Richter --- drivers/oprofile/cpu_buffer.c | 63 ++++++++++++++++++++++++++++++++----------- 1 file changed, 48 insertions(+), 15 deletions(-) (limited to 'drivers/oprofile/cpu_buffer.c') diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index 5cf7efe38e67..eb280ec96e24 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c @@ -28,6 +28,25 @@ #include "buffer_sync.h" #include "oprof.h" +#define OP_BUFFER_FLAGS 0 + +/* + * Read and write access is using spin locking. Thus, writing to the + * buffer by NMI handler (x86) could occur also during critical + * sections when reading the buffer. To avoid this, there are 2 + * buffers for independent read and write access. Read access is in + * process context only, write access only in the NMI handler. If the + * read buffer runs empty, both buffers are swapped atomically. There + * is potentially a small window during swapping where the buffers are + * disabled and samples could be lost. + * + * Using 2 buffers is a little bit overhead, but the solution is clear + * and does not require changes in the ring buffer implementation. It + * can be changed to a single buffer solution when the ring buffer + * access is implemented as non-locking atomic code. + */ +struct ring_buffer *op_ring_buffer_read; +struct ring_buffer *op_ring_buffer_write; DEFINE_PER_CPU(struct oprofile_cpu_buffer, cpu_buffer); static void wq_sync_buffer(struct work_struct *work); @@ -37,12 +56,12 @@ static int work_enabled; void free_cpu_buffers(void) { - int i; - - for_each_possible_cpu(i) { - vfree(per_cpu(cpu_buffer, i).buffer); - per_cpu(cpu_buffer, i).buffer = NULL; - } + if (op_ring_buffer_read) + ring_buffer_free(op_ring_buffer_read); + op_ring_buffer_read = NULL; + if (op_ring_buffer_write) + ring_buffer_free(op_ring_buffer_write); + op_ring_buffer_write = NULL; } unsigned long oprofile_get_cpu_buffer_size(void) @@ -64,14 +83,16 @@ int alloc_cpu_buffers(void) unsigned long buffer_size = fs_cpu_buffer_size; + op_ring_buffer_read = ring_buffer_alloc(buffer_size, OP_BUFFER_FLAGS); + if (!op_ring_buffer_read) + goto fail; + op_ring_buffer_write = ring_buffer_alloc(buffer_size, OP_BUFFER_FLAGS); + if (!op_ring_buffer_write) + goto fail; + for_each_possible_cpu(i) { struct oprofile_cpu_buffer *b = &per_cpu(cpu_buffer, i); - b->buffer = vmalloc_node(sizeof(struct op_sample) * buffer_size, - cpu_to_node(i)); - if (!b->buffer) - goto fail; - b->last_task = NULL; b->last_is_kernel = -1; b->tracing = 0; @@ -140,10 +161,22 @@ static inline void add_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, unsigned long event) { - struct op_sample *entry = cpu_buffer_write_entry(cpu_buf); - entry->eip = pc; - entry->event = event; - cpu_buffer_write_commit(cpu_buf); + struct op_entry entry; + + if (cpu_buffer_write_entry(&entry)) + goto Error; + + entry.sample->eip = pc; + entry.sample->event = event; + + if (cpu_buffer_write_commit(&entry)) + goto Error; + + return; + +Error: + cpu_buf->sample_lost_overflow++; + return; } static inline void -- cgit v1.2.3 From 1d7503b5dccf2b95babca050e4960e10d2633f2b Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Mon, 8 Dec 2008 11:59:52 +0100 Subject: oprofile: remove nr_available_slots() This function is no longer available after the port to the new ring buffer. Its removal can lead to incomplete sampling sequences since IBS samples and backtraces are transfered in multiple samples. Due to a full buffer, samples could be lost any time. The userspace daemon has to live with such incomplete sampling sequences as long as the data within one sample is consistent. This will be fixed by changing the internal buffer data there all data of one IBS sample or a backtrace is packed in a single ring buffer entry. This is possible since the new ring buffer supports variable data size. Signed-off-by: Robert Richter --- drivers/oprofile/cpu_buffer.c | 34 ---------------------------------- 1 file changed, 34 deletions(-) (limited to 'drivers/oprofile/cpu_buffer.c') diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index eb280ec96e24..7f7fc958297a 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c @@ -145,18 +145,6 @@ void end_cpu_work(void) flush_scheduled_work(); } -/* compute number of available slots in cpu_buffer queue */ -static unsigned long nr_available_slots(struct oprofile_cpu_buffer const *b) -{ - unsigned long head = b->head_pos; - unsigned long tail = b->tail_pos; - - if (tail > head) - return (tail - head) - 1; - - return tail + (b->buffer_size - head) - 1; -} - static inline void add_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, unsigned long event) @@ -206,11 +194,6 @@ static int log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, return 0; } - if (nr_available_slots(cpu_buf) < 3) { - cpu_buf->sample_lost_overflow++; - return 0; - } - is_kernel = !!is_kernel; task = current; @@ -233,11 +216,6 @@ static int log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, static int oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf) { - if (nr_available_slots(cpu_buf) < 4) { - cpu_buf->sample_lost_overflow++; - return 0; - } - add_code(cpu_buf, CPU_TRACE_BEGIN); cpu_buf->tracing = 1; return 1; @@ -291,12 +269,6 @@ void oprofile_add_ibs_sample(struct pt_regs * const regs, cpu_buf->sample_received++; - if (nr_available_slots(cpu_buf) < MAX_IBS_SAMPLE_SIZE) { - /* we can't backtrace since we lost the source of this event */ - cpu_buf->sample_lost_overflow++; - return; - } - /* notice a switch from user->kernel or vice versa */ if (cpu_buf->last_is_kernel != is_kernel) { cpu_buf->last_is_kernel = is_kernel; @@ -342,12 +314,6 @@ void oprofile_add_trace(unsigned long pc) if (!cpu_buf->tracing) return; - if (nr_available_slots(cpu_buf) < 1) { - cpu_buf->tracing = 0; - cpu_buf->sample_lost_overflow++; - return; - } - /* * broken frame can give an eip with the same value as an * escape code, abort the trace if we get it -- cgit v1.2.3 From 211117ff09b7d81d91b7857651587128ed8b13d9 Mon Sep 17 00:00:00 2001 From: Robert Richter Date: Tue, 9 Dec 2008 02:13:25 +0100 Subject: oprofile: fix lost sample counter The number of lost samples could be greater than the number of received samples. This patches fixes this. The implementation introduces return values for add_sample() and add_code(). Signed-off-by: Robert Richter --- drivers/oprofile/cpu_buffer.c | 83 +++++++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 30 deletions(-) (limited to 'drivers/oprofile/cpu_buffer.c') diff --git a/drivers/oprofile/cpu_buffer.c b/drivers/oprofile/cpu_buffer.c index 7f7fc958297a..61090969158f 100644 --- a/drivers/oprofile/cpu_buffer.c +++ b/drivers/oprofile/cpu_buffer.c @@ -145,32 +145,31 @@ void end_cpu_work(void) flush_scheduled_work(); } -static inline void +static inline int add_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, unsigned long event) { struct op_entry entry; + int ret; - if (cpu_buffer_write_entry(&entry)) - goto Error; + ret = cpu_buffer_write_entry(&entry); + if (ret) + return ret; entry.sample->eip = pc; entry.sample->event = event; - if (cpu_buffer_write_commit(&entry)) - goto Error; + ret = cpu_buffer_write_commit(&entry); + if (ret) + return ret; - return; - -Error: - cpu_buf->sample_lost_overflow++; - return; + return 0; } -static inline void +static inline int add_code(struct oprofile_cpu_buffer *buffer, unsigned long value) { - add_sample(buffer, ESCAPE_CODE, value); + return add_sample(buffer, ESCAPE_CODE, value); } /* This must be safe from any context. It's safe writing here @@ -201,17 +200,25 @@ static int log_sample(struct oprofile_cpu_buffer *cpu_buf, unsigned long pc, /* notice a switch from user->kernel or vice versa */ if (cpu_buf->last_is_kernel != is_kernel) { cpu_buf->last_is_kernel = is_kernel; - add_code(cpu_buf, is_kernel); + if (add_code(cpu_buf, is_kernel)) + goto fail; } /* notice a task switch */ if (cpu_buf->last_task != task) { cpu_buf->last_task = task; - add_code(cpu_buf, (unsigned long)task); + if (add_code(cpu_buf, (unsigned long)task)) + goto fail; } - add_sample(cpu_buf, pc, event); + if (add_sample(cpu_buf, pc, event)) + goto fail; + return 1; + +fail: + cpu_buf->sample_lost_overflow++; + return 0; } static int oprofile_begin_trace(struct oprofile_cpu_buffer *cpu_buf) @@ -266,37 +273,49 @@ void oprofile_add_ibs_sample(struct pt_regs * const regs, int is_kernel = !user_mode(regs); struct oprofile_cpu_buffer *cpu_buf = &__get_cpu_var(cpu_buffer); struct task_struct *task; + int fail = 0; cpu_buf->sample_received++; /* notice a switch from user->kernel or vice versa */ if (cpu_buf->last_is_kernel != is_kernel) { + if (add_code(cpu_buf, is_kernel)) + goto fail; cpu_buf->last_is_kernel = is_kernel; - add_code(cpu_buf, is_kernel); } /* notice a task switch */ if (!is_kernel) { task = current; if (cpu_buf->last_task != task) { + if (add_code(cpu_buf, (unsigned long)task)) + goto fail; cpu_buf->last_task = task; - add_code(cpu_buf, (unsigned long)task); } } - add_code(cpu_buf, ibs_code); - add_sample(cpu_buf, ibs_sample[0], ibs_sample[1]); - add_sample(cpu_buf, ibs_sample[2], ibs_sample[3]); - add_sample(cpu_buf, ibs_sample[4], ibs_sample[5]); + fail = fail || add_code(cpu_buf, ibs_code); + fail = fail || add_sample(cpu_buf, ibs_sample[0], ibs_sample[1]); + fail = fail || add_sample(cpu_buf, ibs_sample[2], ibs_sample[3]); + fail = fail || add_sample(cpu_buf, ibs_sample[4], ibs_sample[5]); if (ibs_code == IBS_OP_BEGIN) { - add_sample(cpu_buf, ibs_sample[6], ibs_sample[7]); - add_sample(cpu_buf, ibs_sample[8], ibs_sample[9]); - add_sample(cpu_buf, ibs_sample[10], ibs_sample[11]); + fail = fail || add_sample(cpu_buf, ibs_sample[6], ibs_sample[7]); + fail = fail || add_sample(cpu_buf, ibs_sample[8], ibs_sample[9]); + fail = fail || add_sample(cpu_buf, ibs_sample[10], ibs_sample[11]); } + if (fail) + goto fail; + if (backtrace_depth) oprofile_ops.backtrace(regs, backtrace_depth); + + return; + +fail: + cpu_buf->sample_lost_overflow++; + return; } #endif @@ -318,13 +337,17 @@ void oprofile_add_trace(unsigned long pc) * broken frame can give an eip with the same value as an * escape code, abort the trace if we get it */ - if (pc == ESCAPE_CODE) { - cpu_buf->tracing = 0; - cpu_buf->backtrace_aborted++; - return; - } + if (pc == ESCAPE_CODE) + goto fail; + + if (add_sample(cpu_buf, pc, 0)) + goto fail; - add_sample(cpu_buf, pc, 0); + return; +fail: + cpu_buf->tracing = 0; + cpu_buf->backtrace_aborted++; + return; } /* -- cgit v1.2.3