From 109704492ef637956265ec2eb72ae7b3b39eb6f4 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 20 Oct 2016 00:34:00 -0700 Subject: pstore: Make spinlock per zone instead of global Currently pstore has a global spinlock for all zones. Since the zones are independent and modify different areas of memory, there's no need to have a global lock, so we should use a per-zone lock as introduced here. Also, when ramoops's ftrace use-case has a FTRACE_PER_CPU flag introduced later, which splits the ftrace memory area into a single zone per CPU, it will eliminate the need for locking. In preparation for this, make the locking optional. Signed-off-by: Joel Fernandes [kees: updated commit message] Signed-off-by: Kees Cook --- include/linux/pstore_ram.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/linux/pstore_ram.h b/include/linux/pstore_ram.h index c668c861c96c..244d2423dbaf 100644 --- a/include/linux/pstore_ram.h +++ b/include/linux/pstore_ram.h @@ -40,6 +40,7 @@ struct persistent_ram_zone { void *vaddr; struct persistent_ram_buffer *buffer; size_t buffer_size; + raw_spinlock_t buffer_lock; /* ECC correction */ char *par_buffer; -- cgit v1.2.3 From 663deb47880f2283809669563c5a52ac7c6aef1a Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 20 Oct 2016 00:34:01 -0700 Subject: pstore: Allow prz to control need for locking In preparation of not locking at all for certain buffers depending on if there's contention, make locking optional depending on the initialization of the prz. Signed-off-by: Joel Fernandes [kees: moved locking flag into prz instead of via caller arguments] Signed-off-by: Kees Cook --- include/linux/pstore_ram.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/pstore_ram.h b/include/linux/pstore_ram.h index 244d2423dbaf..4058bf991868 100644 --- a/include/linux/pstore_ram.h +++ b/include/linux/pstore_ram.h @@ -24,6 +24,13 @@ #include #include +/* + * Choose whether access to the RAM zone requires locking or not. If a zone + * can be written to from different CPUs like with ftrace for example, then + * PRZ_FLAG_NO_LOCK is used. For all other cases, locking is required. + */ +#define PRZ_FLAG_NO_LOCK BIT(0) + struct persistent_ram_buffer; struct rs_control; @@ -40,6 +47,7 @@ struct persistent_ram_zone { void *vaddr; struct persistent_ram_buffer *buffer; size_t buffer_size; + u32 flags; raw_spinlock_t buffer_lock; /* ECC correction */ @@ -56,7 +64,7 @@ struct persistent_ram_zone { struct persistent_ram_zone *persistent_ram_new(phys_addr_t start, size_t size, u32 sig, struct persistent_ram_ecc_info *ecc_info, - unsigned int memtype); + unsigned int memtype, u32 flags); void persistent_ram_free(struct persistent_ram_zone *prz); void persistent_ram_zap(struct persistent_ram_zone *prz); -- cgit v1.2.3 From a1cf53ac6d156721afa86453d5e8423461881231 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 20 Oct 2016 00:34:04 -0700 Subject: ramoops: Split ftrace buffer space into per-CPU zones If the RAMOOPS_FLAG_FTRACE_PER_CPU flag is passed to ramoops pdata, split the ftrace space into multiple zones depending on the number of CPUs. This speeds up the performance of function tracing by about 280% in my tests as we avoid the locking. The trade off being lesser space available per CPU. Let the ramoops user decide which option they want based on pdata flag. Signed-off-by: Joel Fernandes [kees: added max_ftrace_cnt to track size, added DT logic and docs] Signed-off-by: Kees Cook --- include/linux/pstore_ram.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include') diff --git a/include/linux/pstore_ram.h b/include/linux/pstore_ram.h index 4058bf991868..9395f06e8372 100644 --- a/include/linux/pstore_ram.h +++ b/include/linux/pstore_ram.h @@ -86,6 +86,8 @@ ssize_t persistent_ram_ecc_string(struct persistent_ram_zone *prz, * @mem_address physical memory address to contain ramoops */ +#define RAMOOPS_FLAG_FTRACE_PER_CPU BIT(0) + struct ramoops_platform_data { unsigned long mem_size; phys_addr_t mem_address; @@ -95,6 +97,7 @@ struct ramoops_platform_data { unsigned long ftrace_size; unsigned long pmsg_size; int dump_oops; + u32 flags; struct persistent_ram_ecc_info ecc_info; }; -- cgit v1.2.3 From fbccdeb8d77d6830556bc4079eeed80298cc97dc Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Thu, 20 Oct 2016 00:34:05 -0700 Subject: pstore: Add ftrace timestamp counter In preparation for merging the per CPU buffers into one buffer when we retrieve the pstore ftrace data, we store the timestamp as a counter in the ftrace pstore record. We store the CPU number as well if !PSTORE_CPU_IN_IP, in this case we shift the counter and may lose ordering there but we preserve the same record size. The timestamp counter is also racy, and not doing any locking or synchronization here results in the benefit of lower overhead. Since we don't care much here for exact ordering of function traces across CPUs, we don't synchronize and may lose some counter updates but I'm ok with that. Using trace_clock() results in much lower performance so avoid using it since we don't want accuracy in timestamp and need a rough ordering to perform merge. Signed-off-by: Joel Fernandes [kees: updated commit message, added comments] Signed-off-by: Kees Cook --- include/linux/pstore.h | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) (limited to 'include') diff --git a/include/linux/pstore.h b/include/linux/pstore.h index 92013cc9cc8c..0da29cae009b 100644 --- a/include/linux/pstore.h +++ b/include/linux/pstore.h @@ -89,4 +89,80 @@ extern int pstore_register(struct pstore_info *); extern void pstore_unregister(struct pstore_info *); extern bool pstore_cannot_block_path(enum kmsg_dump_reason reason); +struct pstore_ftrace_record { + unsigned long ip; + unsigned long parent_ip; + u64 ts; +}; + +/* + * ftrace related stuff: Both backends and frontends need these so expose + * them here. + */ + +#if NR_CPUS <= 2 && defined(CONFIG_ARM_THUMB) +#define PSTORE_CPU_IN_IP 0x1 +#elif NR_CPUS <= 4 && defined(CONFIG_ARM) +#define PSTORE_CPU_IN_IP 0x3 +#endif + +#define TS_CPU_SHIFT 8 +#define TS_CPU_MASK (BIT(TS_CPU_SHIFT) - 1) + +/* + * If CPU number can be stored in IP, store it there, otherwise store it in + * the time stamp. This means more timestamp resolution is available when + * the CPU can be stored in the IP. + */ +#ifdef PSTORE_CPU_IN_IP +static inline void +pstore_ftrace_encode_cpu(struct pstore_ftrace_record *rec, unsigned int cpu) +{ + rec->ip |= cpu; +} + +static inline unsigned int +pstore_ftrace_decode_cpu(struct pstore_ftrace_record *rec) +{ + return rec->ip & PSTORE_CPU_IN_IP; +} + +static inline u64 +pstore_ftrace_read_timestamp(struct pstore_ftrace_record *rec) +{ + return rec->ts; +} + +static inline void +pstore_ftrace_write_timestamp(struct pstore_ftrace_record *rec, u64 val) +{ + rec->ts = val; +} +#else +static inline void +pstore_ftrace_encode_cpu(struct pstore_ftrace_record *rec, unsigned int cpu) +{ + rec->ts &= ~(TS_CPU_MASK); + rec->ts |= cpu; +} + +static inline unsigned int +pstore_ftrace_decode_cpu(struct pstore_ftrace_record *rec) +{ + return rec->ts & TS_CPU_MASK; +} + +static inline u64 +pstore_ftrace_read_timestamp(struct pstore_ftrace_record *rec) +{ + return rec->ts >> TS_CPU_SHIFT; +} + +static inline void +pstore_ftrace_write_timestamp(struct pstore_ftrace_record *rec, u64 val) +{ + rec->ts = (rec->ts & TS_CPU_MASK) | (val << TS_CPU_SHIFT); +} +#endif + #endif /*_LINUX_PSTORE_H*/ -- cgit v1.2.3 From d032ae8921ea792c1e6b2abb44022b2403f651f6 Mon Sep 17 00:00:00 2001 From: Joel Fernandes Date: Tue, 15 Nov 2016 12:31:20 -0800 Subject: ftrace: Provide API to use global filtering for ftrace ops Currently the global_ops filtering hash is not available to outside users registering for function tracing. Provide an API for those users to be able to choose global filtering. This is in preparation for pstore's ftrace feature to be able to use the global filters. Suggested-by: Steven Rostedt Cc: Anton Vorontsov Cc: Colin Cross Cc: Kees Cook Cc: Tony Luck Signed-off-by: Joel Fernandes Acked-by: Steven Rostedt Signed-off-by: Kees Cook --- include/linux/ftrace.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index b3d34d3e0e7e..d4a884db16a3 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -398,6 +398,7 @@ int ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf, void ftrace_set_global_filter(unsigned char *buf, int len, int reset); void ftrace_set_global_notrace(unsigned char *buf, int len, int reset); void ftrace_free_filter(struct ftrace_ops *ops); +void ftrace_ops_set_global_filter(struct ftrace_ops *ops); int register_ftrace_command(struct ftrace_func_command *cmd); int unregister_ftrace_command(struct ftrace_func_command *cmd); @@ -645,6 +646,7 @@ static inline unsigned long ftrace_location(unsigned long ip) #define ftrace_set_filter(ops, buf, len, reset) ({ -ENODEV; }) #define ftrace_set_notrace(ops, buf, len, reset) ({ -ENODEV; }) #define ftrace_free_filter(ops) do { } while (0) +#define ftrace_ops_set_global_filter(ops) do { } while (0) static inline ssize_t ftrace_filter_write(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) { return -ENODEV; } -- cgit v1.2.3