diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2026-08-18 12:31:07 -0700 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2026-08-18 12:31:07 -0700 |
| commit | 8915457146a11d20a6c0786396376afda65eec40 (patch) | |
| tree | 75a8c5625e800896bb4d977dede53590e60cce95 | |
| parent | a0acd94e3819fcd8346ae3c16df987e0fabb3128 (diff) | |
| parent | 917d558b151cad5b05991e5eaee22efab33525ca (diff) | |
Merge tag 'perf-core-2026-08-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull performance events updates from Ingo Molnar:
"uprobes updates:
- Fix a category of bugs with optimized uprobes that can clobber the
redzone area with call instruction storing return address on stack
where user code may keep temporary data without adjusting RSP.
Fix this by moving the optimized uprobes on top of 10-bytes NOP
instruction, so we can squeeze another instruction to escape the
redzone area before doing the call (Jiri Olsa, Andrii Nakryiko)
- Switch uretprobes_srcu to SRCU-fast-updown, to improve performance
(Puranjay Mohan)
Intel CPU PMU driver updates:
- Optimize ACR handling in match_prev_assignment() (Dapeng Mi)
- Fix various PMU driver bugs and data leaks (Dapeng Mi)
- Fix Intel PT stop/start with no update (Adrian Hunter)
Intel uncore PMU driver updates:
- Fix various uncore PMU setup robustness bugs (Zide Chen)
AMD uncore PMU driver updates:
- Add group validation (Sandipan Das)
.. and misc fixes and updates by Dapeng Mi, Randy Dunlap and Zide Chen"
* tag 'perf-core-2026-08-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (42 commits)
perf/x86: Optimize ACR handling in match_prev_assignment()
perf/x86/intel: Fix intel_cap handling on hybrid PMUs
perf/x86: Remove stale fixed counter helper and fix hybrid PMU access
perf/x86/intel: Unwind cpuc state if PEBS buffer setup fails
perf/x86: Guard intel_pmu_cpu_dead() against invalid hybrid PMU casts
perf/x86: Free hybrid state on PMU init failure
perf/x86: Unregister PMI handler on PMU init failure
perf/x86/intel/pt: Fix stop/start with no update
perf/x86/intel/pt: Use bitwise access for PERF_HES_STOPPED
perf/x86/intel/pt: Factor out pt_config_enable()
uprobes: Switch uretprobes_srcu to SRCU-fast-updown
srcu: Add lock guard for srcu_fast_updown flavor
perf/x86/intel/pt: Drop kernel-doc for deleted struct members
perf/x86/amd/uncore: Add group validation
selftests/bpf: Add tests for forked/cloned optimized uprobes
selftests/bpf: Add tests for uprobe nop10 red zone clobbering
selftests/bpf: Add reattach tests for uprobe syscall
selftests/bpf: Change uprobe/usdt trigger bench code to use nop10
selftests/bpf: Change uprobe syscall tests to use nop10
selftests/bpf: Emit nop,nop10 instructions combo for x86_64 arch
...
31 files changed, 1232 insertions, 584 deletions
diff --git a/arch/x86/events/amd/uncore.c b/arch/x86/events/amd/uncore.c index dbc00b6dd69e..222dfab9225f 100644 --- a/arch/x86/events/amd/uncore.c +++ b/arch/x86/events/amd/uncore.c @@ -265,6 +265,29 @@ static void amd_uncore_del(struct perf_event *event, int flags) hwc->idx = -1; } +static bool amd_uncore_group_valid(struct perf_event *event) +{ + struct amd_uncore_pmu *pmu = event_to_amd_uncore_pmu(event); + struct perf_event *leader = event->group_leader; + struct perf_event *sibling; + int counters = 0; + + if (leader->pmu == event->pmu) + counters++; + + for_each_sibling_event(sibling, leader) { + if (sibling->pmu == event->pmu && + sibling->state > PERF_EVENT_STATE_OFF) + counters++; + } + + /* + * When pmu->event_init() is called, the event is yet to be linked to + * its leader's sibling list, so it is counted separately + */ + return (counters + 1) <= pmu->num_counters; +} + static int amd_uncore_event_init(struct perf_event *event) { struct amd_uncore_pmu *pmu; @@ -283,6 +306,14 @@ static int amd_uncore_event_init(struct perf_event *event) return -ENODEV; /* + * Ensure that all events in a group can be scheduled together so that + * a failure can be reported at perf_event_open() time rather than + * silently at pmu->add() time when no free counter is found + */ + if (event->group_leader != event && !amd_uncore_group_valid(event)) + return -EINVAL; + + /* * NB and Last level cache counters (MSRs) are shared across all cores * that share the same NB / Last level cache. On family 16h and below, * Interrupts can be directed to a single target core, however, event diff --git a/arch/x86/events/core.c b/arch/x86/events/core.c index d1af33d96d0a..8b3ea0adb965 100644 --- a/arch/x86/events/core.c +++ b/arch/x86/events/core.c @@ -265,7 +265,7 @@ static void release_pmc_hardware(void) {} #endif -bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask, +bool check_hw_exists(unsigned long *cntr_mask, unsigned long *fixed_cntr_mask) { u64 val, val_fail = -1, val_new= ~0; @@ -297,8 +297,6 @@ bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask, if (ret) goto msr_fail; for_each_set_bit(i, fixed_cntr_mask, X86_PMC_IDX_MAX) { - if (fixed_counter_disabled(i, pmu)) - continue; if (val & (0x03ULL << i*4)) { bios_fail = 1; val_fail = val; @@ -790,6 +788,11 @@ int is_x86_event(struct perf_event *event) return false; } +inline struct pmu *x86_get_static_pmu(void) +{ + return &pmu; +} + struct pmu *x86_get_pmu(unsigned int cpu) { struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); @@ -1294,6 +1297,17 @@ int x86_perf_rdpmc_index(struct perf_event *event) return event->hw.event_base_rdpmc; } +static inline bool acr_match_prev_indices(struct perf_event *event, + struct cpu_hw_events *cpuc) +{ + struct hw_perf_event *hwc = &event->hw; + + if (!is_acr_event_group(event)) + return true; + /* ACR counter indices don't change. */ + return hwc->config1 == cpuc->acr_cfg_b[hwc->idx]; +} + static inline int match_prev_assignment(struct perf_event *event, struct cpu_hw_events *cpuc, int i) @@ -1303,7 +1317,7 @@ static inline int match_prev_assignment(struct perf_event *event, return hwc->idx == cpuc->assign[i] && hwc->last_cpu == smp_processor_id() && hwc->last_tag == cpuc->tags[i] && - !is_acr_event_group(event); + acr_match_prev_indices(event, cpuc); } static void x86_pmu_start(struct perf_event *event, int flags); @@ -1613,8 +1627,6 @@ void perf_event_print_debug(void) cpu, idx, prev_left); } for_each_set_bit(idx, fixed_cntr_mask, X86_PMC_IDX_MAX) { - if (fixed_counter_disabled(idx, cpuc->pmu)) - continue; rdmsrq(x86_pmu_fixed_ctr_addr(idx), pmc_count); pr_info("CPU#%d: fixed-PMC%d count: %016llx\n", @@ -2130,6 +2142,17 @@ void x86_pmu_show_pmu_cap(struct pmu *pmu) pr_info("... global_ctrl mask: %016llx\n", hybrid(pmu, intel_ctrl)); } +static void x86_pmu_free_hybrid(void) +{ + if (!x86_pmu.hybrid_pmu) + return; + + static_branch_disable(&perf_is_hybrid); + kfree(x86_pmu.hybrid_pmu); + x86_pmu.hybrid_pmu = NULL; + x86_pmu.num_hybrid_pmus = 0; +} + static int __init init_hw_perf_events(void) { struct x86_pmu_quirk *quirk; @@ -2164,7 +2187,7 @@ static int __init init_hw_perf_events(void) pmu_check_apic(); /* sanity check that the hardware exists or is emulated */ - if (!check_hw_exists(&pmu, x86_pmu.cntr_mask, x86_pmu.fixed_cntr_mask)) + if (!check_hw_exists(x86_pmu.cntr_mask, x86_pmu.fixed_cntr_mask)) goto out_bad_pmu; pr_cont("%s PMU driver.\n", x86_pmu.name); @@ -2219,7 +2242,7 @@ static int __init init_hw_perf_events(void) err = cpuhp_setup_state(CPUHP_PERF_X86_PREPARE, "perf/x86:prepare", x86_pmu_prepare_cpu, x86_pmu_dead_cpu); if (err) - return err; + goto pmi_unregister; err = cpuhp_setup_state(CPUHP_AP_PERF_X86_STARTING, "perf/x86:starting", x86_pmu_starting_cpu, @@ -2258,9 +2281,6 @@ static int __init init_hw_perf_events(void) for (j = 0; j < i; j++) perf_pmu_unregister(&x86_pmu.hybrid_pmu[j].pmu); pr_warn("Failed to register hybrid PMUs\n"); - kfree(x86_pmu.hybrid_pmu); - x86_pmu.hybrid_pmu = NULL; - x86_pmu.num_hybrid_pmus = 0; goto out2; } } @@ -2273,7 +2293,10 @@ out1: cpuhp_remove_state(CPUHP_AP_PERF_X86_STARTING); out: cpuhp_remove_state(CPUHP_PERF_X86_PREPARE); +pmi_unregister: + unregister_nmi_handler(NMI_LOCAL, "PMI"); out_bad_pmu: + x86_pmu_free_hybrid(); memset(&x86_pmu, 0, sizeof(x86_pmu)); return err; } @@ -2539,7 +2562,8 @@ static int x86_pmu_event_init(struct perf_event *event) } if (READ_ONCE(x86_pmu.attr_rdpmc) && - !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS)) + !(event->hw.flags & PERF_X86_EVENT_LARGE_PEBS) && + !(event->hw.config & ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE)) event->hw.flags |= PERF_EVENT_FLAG_USER_READ_CNT; return err; diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c index 2b35483e2b70..f6ee1819864e 100644 --- a/arch/x86/events/intel/core.c +++ b/arch/x86/events/intel/core.c @@ -3533,7 +3533,7 @@ static void intel_pmu_update_rdpmc_user_disable(struct perf_event *event) */ if (x86_pmu.attr_rdpmc == X86_USER_RDPMC_ALWAYS_ENABLE || (x86_pmu.attr_rdpmc == X86_USER_RDPMC_CONDITIONAL_ENABLE && - event->ctx->task)) + (event->attach_state & PERF_ATTACH_TASK))) event->hw.config &= ~ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE; else event->hw.config |= ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE; @@ -3547,8 +3547,6 @@ static void intel_pmu_enable_event(struct perf_event *event) struct hw_perf_event *hwc = &event->hw; int idx = hwc->idx; - intel_pmu_update_rdpmc_user_disable(event); - if (unlikely(event->attr.precise_ip)) static_call(x86_pmu_pebs_enable)(event); @@ -3715,8 +3713,6 @@ static void intel_pmu_reset(void) wrmsrq_safe(x86_pmu_event_addr(idx), 0ull); } for_each_set_bit(idx, fixed_cntr_mask, INTEL_PMC_MAX_FIXED) { - if (fixed_counter_disabled(idx, cpuc->pmu)) - continue; wrmsrq_safe(x86_pmu_fixed_ctr_addr(idx), 0ull); } @@ -5147,6 +5143,8 @@ static int intel_pmu_hw_config(struct perf_event *event) leader->hw.flags |= PERF_X86_EVENT_ACR; } + intel_pmu_update_rdpmc_user_disable(event); + if ((event->attr.type == PERF_TYPE_HARDWARE) || (event->attr.type == PERF_TYPE_HW_CACHE)) return 0; @@ -5924,13 +5922,20 @@ err: static int intel_pmu_cpu_prepare(int cpu) { + struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); int ret; - ret = intel_cpuc_prepare(&per_cpu(cpu_hw_events, cpu), cpu); + ret = intel_cpuc_prepare(cpuc, cpu); if (ret) return ret; - return alloc_arch_pebs_buf_on_cpu(cpu); + ret = alloc_arch_pebs_buf_on_cpu(cpu); + if (ret) { + intel_cpuc_finish(cpuc); + return ret; + } + + return 0; } static void flip_smm_bit(void *data) @@ -6142,8 +6147,14 @@ static void intel_pmu_check_extra_regs(struct extra_reg *extra_regs); static inline bool intel_pmu_broken_perf_cap(void) { - /* The Perf Metric (Bit 15) is always cleared */ - if (boot_cpu_data.x86_vfm == INTEL_METEORLAKE || + /* + * The Perf Metric (Bit 15) is always cleared on P-core of + * RPL and MTL. Details can be found in RPL018 erratum. + */ + if (boot_cpu_data.x86_vfm == INTEL_RAPTORLAKE || + boot_cpu_data.x86_vfm == INTEL_RAPTORLAKE_P || + boot_cpu_data.x86_vfm == INTEL_RAPTORLAKE_S || + boot_cpu_data.x86_vfm == INTEL_METEORLAKE || boot_cpu_data.x86_vfm == INTEL_METEORLAKE_L) return true; @@ -6178,7 +6189,7 @@ static inline void __intel_update_large_pebs_flags(struct pmu *pmu) #define counter_mask(_gp, _fixed) ((_gp) | ((u64)(_fixed) << INTEL_PMC_IDX_FIXED)) -static void update_pmu_cap(struct pmu *pmu) +static void update_pmu_cap_from_perfmonext(struct pmu *pmu) { unsigned int eax, ebx, ecx, edx; union cpuid35_eax eax_0; @@ -6236,10 +6247,24 @@ static void update_pmu_cap(struct pmu *pmu) WARN_ON(x86_pmu.arch_pebs == 1); x86_pmu.arch_pebs = 0; } +} - if (!intel_pmu_broken_perf_cap()) { - /* Perf Metric (Bit 15) and PEBS via PT (Bit 16) are hybrid enumeration */ - rdmsrq(MSR_IA32_PERF_CAPABILITIES, hybrid(pmu, intel_cap).capabilities); +static void intel_update_pmu_caps(struct pmu *pmu) +{ + if (this_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT)) + update_pmu_cap_from_perfmonext(pmu); + + if (is_hybrid() && this_cpu_has(X86_FEATURE_PDCM)) { + rdmsrq(MSR_IA32_PERF_CAPABILITIES, + hybrid(pmu, intel_cap).capabilities); + + /* + * Restore perf_metrics on platforms with broken + * perf_capablities. + */ + if (intel_pmu_broken_perf_cap() && + hybrid_pmu(pmu)->pmu_type == hybrid_big) + hybrid(pmu, intel_cap).perf_metrics = 1; } } @@ -6324,13 +6349,13 @@ static bool init_hybrid_pmu(int cpu) if (!cpumask_empty(&pmu->supported_cpus)) goto end; - if (this_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT)) - update_pmu_cap(&pmu->pmu); - + intel_update_pmu_caps(&pmu->pmu); intel_pmu_check_hybrid_pmus(pmu); - if (!check_hw_exists(&pmu->pmu, pmu->cntr_mask, pmu->fixed_cntr_mask)) + if (!check_hw_exists(pmu->cntr_mask, pmu->fixed_cntr_mask)) { + cpuc->pmu = NULL; return false; + } pr_info("%s PMU driver: ", pmu->name); @@ -6475,11 +6500,12 @@ void intel_cpuc_finish(struct cpu_hw_events *cpuc) static void intel_pmu_cpu_dead(int cpu) { struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu); + struct pmu *pmu = x86_get_static_pmu(); release_arch_pebs_buf_on_cpu(cpu); intel_cpuc_finish(cpuc); - if (is_hybrid() && cpuc->pmu) + if (is_hybrid() && cpuc->pmu && cpuc->pmu != pmu) cpumask_clear_cpu(cpu, &hybrid_pmu(cpuc->pmu)->supported_cpus); } @@ -7679,8 +7705,10 @@ static __always_inline int intel_pmu_init_hybrid(enum hybrid_pmu_type pmus) x86_pmu.num_hybrid_pmus = hweight_long(pmus_mask); x86_pmu.hybrid_pmu = kzalloc_objs(struct x86_hybrid_pmu, x86_pmu.num_hybrid_pmus); - if (!x86_pmu.hybrid_pmu) + if (!x86_pmu.hybrid_pmu) { + x86_pmu.num_hybrid_pmus = 0; return -ENOMEM; + } static_branch_enable(&perf_is_hybrid); x86_pmu.filter = intel_pmu_filter; @@ -7863,14 +7891,14 @@ __init int intel_pmu_init(void) struct attribute **td_attr = &empty_attrs; struct attribute **mem_attr = &empty_attrs; struct attribute **tsx_attr = &empty_attrs; + struct x86_hybrid_pmu *pmu; + unsigned int fixed_mask; union cpuid10_edx edx; union cpuid10_eax eax; union cpuid10_ebx ebx; - unsigned int fixed_mask; + int version, i, ret; bool pmem = false; - int version, i; char *name; - struct x86_hybrid_pmu *pmu; /* Architectural Perfmon was introduced starting with Core "Yonah" */ if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON)) { @@ -7940,19 +7968,10 @@ __init int intel_pmu_init(void) x86_pmu.lbr_read = intel_pmu_lbr_read_32; } - if (boot_cpu_has(X86_FEATURE_ARCH_LBR)) - intel_pmu_arch_lbr_init(); - intel_pebs_init(); x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */ - if (version >= 5) { - x86_pmu.intel_cap.anythread_deprecated = edx.split.anythread_deprecated; - if (x86_pmu.intel_cap.anythread_deprecated) - pr_cont(" AnyThread deprecated, "); - } - /* The perf side of core PMU is ready to support the mediated vPMU. */ x86_get_pmu(smp_processor_id())->capabilities |= PERF_PMU_CAP_MEDIATED_VPMU; @@ -8546,7 +8565,9 @@ __init int intel_pmu_init(void) * * Initialize the common PerfMon capabilities here. */ - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; x86_pmu.pebs_latency_data = grt_latency_data; x86_pmu.get_event_constraints = adl_get_event_constraints; @@ -8604,7 +8625,9 @@ __init int intel_pmu_init(void) case INTEL_METEORLAKE: case INTEL_METEORLAKE_L: case INTEL_ARROWLAKE_U: - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; x86_pmu.pebs_latency_data = cmt_latency_data; x86_pmu.get_event_constraints = mtl_get_event_constraints; @@ -8635,7 +8658,9 @@ __init int intel_pmu_init(void) pr_cont("Pantherlake Hybrid events, "); name = "pantherlake_hybrid"; - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; @@ -8650,7 +8675,9 @@ __init int intel_pmu_init(void) pr_cont("Arrowlake Hybrid events, "); name = "arrowlake_hybrid"; - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; @@ -8667,7 +8694,9 @@ __init int intel_pmu_init(void) pr_cont("Lunarlake Hybrid events, "); name = "lunarlake_hybrid"; - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; /* Initialize big core specific PerfMon capabilities.*/ pmu = &x86_pmu.hybrid_pmu[X86_HYBRID_PMU_CORE_IDX]; @@ -8692,7 +8721,9 @@ __init int intel_pmu_init(void) break; case INTEL_ARROWLAKE_H: - intel_pmu_init_hybrid(hybrid_big_small_tiny); + ret = intel_pmu_init_hybrid(hybrid_big_small_tiny); + if (ret) + return ret; x86_pmu.pebs_latency_data = arl_h_latency_data; x86_pmu.get_event_constraints = arl_h_get_event_constraints; @@ -8727,7 +8758,9 @@ __init int intel_pmu_init(void) case INTEL_NOVALAKE_L: pr_cont("Novalake Hybrid events, "); name = "novalake_hybrid"; - intel_pmu_init_hybrid(hybrid_big_small); + ret = intel_pmu_init_hybrid(hybrid_big_small); + if (ret) + return ret; x86_pmu.pebs_latency_data = nvl_latency_data; x86_pmu.get_event_constraints = mtl_get_event_constraints; @@ -8813,8 +8846,8 @@ __init int intel_pmu_init(void) * from the leaf 0xa. The core specific update will be done later * when a new type is online. */ - if (!is_hybrid() && boot_cpu_has(X86_FEATURE_ARCH_PERFMON_EXT)) - update_pmu_cap(NULL); + if (!is_hybrid()) + intel_update_pmu_caps(NULL); if (x86_pmu.arch_pebs) { static_call_update(intel_pmu_disable_event_ext, @@ -8829,8 +8862,13 @@ __init int intel_pmu_init(void) &x86_pmu.intel_ctrl); /* AnyThread may be deprecated on arch perfmon v5 or later */ - if (x86_pmu.intel_cap.anythread_deprecated) + if (version >= 5 && edx.split.anythread_deprecated) { x86_pmu.format_attrs = intel_arch_formats_attr; + pr_cont("AnyThread deprecated, "); + } + + if (boot_cpu_has(X86_FEATURE_ARCH_LBR)) + intel_pmu_arch_lbr_init(); intel_pmu_check_event_constraints_all(NULL); diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c index 91a093d8cf2e..e86e4ba91e1b 100644 --- a/arch/x86/events/intel/ds.c +++ b/arch/x86/events/intel/ds.c @@ -1444,10 +1444,6 @@ struct event_constraint intel_skl_pebs_event_constraints[] = { }; struct event_constraint intel_icl_pebs_event_constraints[] = { - INTEL_FLAGS_UEVENT_CONSTRAINT(0x01c0, 0x100000000ULL), /* old INST_RETIRED.PREC_DIST */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x0100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), /* SLOTS */ - INTEL_PLD_CONSTRAINT(0x1cd, 0xff), /* MEM_TRANS_RETIRED.LOAD_LATENCY */ INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_ST(0x12d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_STORES */ @@ -1470,9 +1466,6 @@ struct event_constraint intel_icl_pebs_event_constraints[] = { }; struct event_constraint intel_glc_pebs_event_constraints[] = { - INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), - INTEL_FLAGS_EVENT_CONSTRAINT(0xc0, 0xfe), INTEL_PLD_CONSTRAINT(0x1cd, 0xfe), INTEL_PSD_CONSTRAINT(0x2cd, 0x1), @@ -1497,9 +1490,6 @@ struct event_constraint intel_glc_pebs_event_constraints[] = { }; struct event_constraint intel_lnc_pebs_event_constraints[] = { - INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), - INTEL_FLAGS_UEVENT_CONSTRAINT(0x012a, 0x1), /* OCR.* events */ INTEL_FLAGS_UEVENT_CONSTRAINT(0x012b, 0x1), /* OCR.* events */ @@ -1531,9 +1521,6 @@ struct event_constraint intel_lnc_pebs_event_constraints[] = { }; struct event_constraint intel_pnc_pebs_event_constraints[] = { - INTEL_FLAGS_UEVENT_CONSTRAINT(0x100, 0x100000000ULL), /* INST_RETIRED.PREC_DIST */ - INTEL_FLAGS_UEVENT_CONSTRAINT(0x0400, 0x800000000ULL), - INTEL_HYBRID_LDLAT_CONSTRAINT(0x1cd, 0xfc), INTEL_HYBRID_STLAT_CONSTRAINT(0x2cd, 0x3), INTEL_FLAGS_UEVENT_CONSTRAINT_DATALA_LD(0x11d0, 0xf), /* MEM_INST_RETIRED.STLB_MISS_LOADS */ diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c index cae2e02fe6cc..f8fadb0b16a4 100644 --- a/arch/x86/events/intel/lbr.c +++ b/arch/x86/events/intel/lbr.c @@ -1213,7 +1213,7 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) { u64 from, to; int br_sel = cpuc->br_sel; - int i, j, type, to_plm; + int i, j, type, from_plm, to_plm; bool compress = false; /* if sampling all branches, then nothing to filter */ @@ -1232,7 +1232,7 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) * Doesn't support OTHER_BRANCH decoding for now. * OTHER_BRANCH branch type still rely on software decoding. */ - if (static_cpu_has(X86_FEATURE_ARCH_LBR) && + if (static_branch_likely(&x86_lbr_type) && type <= ARCH_LBR_BR_TYPE_KNOWN_MAX) { to_plm = kernel_ip(to) ? X86_BR_KERNEL : X86_BR_USER; type = arch_lbr_br_type_map[type] | to_plm; @@ -1245,8 +1245,14 @@ intel_pmu_lbr_filter(struct cpu_hw_events *cpuc) type |= X86_BR_NO_TX; } - /* if type does not correspond, then discard */ - if (type == X86_BR_NONE || (br_sel & type) != type) { + from_plm = kernel_ip(from) ? X86_BR_KERNEL : X86_BR_USER; + /* + * If type does not correspond, then discard. + * Specifically reject entries whose from address is in + * kernel space when only X86_BR_USER is requested. + */ + if (type == X86_BR_NONE || (br_sel & type) != type || + (!(br_sel & X86_BR_KERNEL) && (from_plm & X86_BR_KERNEL))) { cpuc->lbr_entries[i].from = 0; compress = true; } diff --git a/arch/x86/events/intel/pt.c b/arch/x86/events/intel/pt.c index b5726b50e77d..5754cd405562 100644 --- a/arch/x86/events/intel/pt.c +++ b/arch/x86/events/intel/pt.c @@ -502,6 +502,29 @@ static u64 pt_config_filters(struct perf_event *event) return rtit_ctl; } +static void pt_config_enable(struct perf_event *event) +{ + struct pt *pt = this_cpu_ptr(&pt_ctx); + + /* + * Allow resume before starting so as not to overwrite a value set by a + * PMI. + */ + barrier(); + WRITE_ONCE(pt->resume_allowed, 1); + /* Configuration is complete, it is now OK to handle an NMI */ + barrier(); + WRITE_ONCE(pt->handle_nmi, 1); + barrier(); + pt_config_start(event); + barrier(); + /* + * Allow pause after starting so its pt_config_stop() doesn't race with + * pt_config_start(). + */ + WRITE_ONCE(pt->pause_allowed, 1); +} + static void pt_config(struct perf_event *event) { struct pt *pt = this_cpu_ptr(&pt_ctx); @@ -541,23 +564,7 @@ static void pt_config(struct perf_event *event) event->hw.aux_config = reg; - /* - * Allow resume before starting so as not to overwrite a value set by a - * PMI. - */ - barrier(); - WRITE_ONCE(pt->resume_allowed, 1); - /* Configuration is complete, it is now OK to handle an NMI */ - barrier(); - WRITE_ONCE(pt->handle_nmi, 1); - barrier(); - pt_config_start(event); - barrier(); - /* - * Allow pause after starting so its pt_config_stop() doesn't race with - * pt_config_start(). - */ - WRITE_ONCE(pt->pause_allowed, 1); + pt_config_enable(event); } static void pt_config_stop(struct perf_event *event) @@ -1533,12 +1540,14 @@ void intel_pt_interrupt(void) perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0)); - if (!event->hw.state) { + event->hw.state |= PERF_HES_UPTODATE; + + if (!(event->hw.state & PERF_HES_STOPPED)) { int ret; buf = perf_aux_output_begin(&pt->handle, event); if (!buf) { - event->hw.state = PERF_HES_STOPPED; + event->hw.state |= PERF_HES_STOPPED; WRITE_ONCE(pt->resume_allowed, 0); return; } @@ -1554,6 +1563,8 @@ void intel_pt_interrupt(void) pt_config_buffer(buf); pt_config_start(event); + + event->hw.state &= ~PERF_HES_UPTODATE; } } @@ -1622,6 +1633,18 @@ static void pt_event_start(struct perf_event *event, int mode) return; } + /* + * Re-start subsequent to a call to pt_event_stop() without the + * PERF_EF_UPDATE flag. Absence of PERF_HES_UPTODATE indicates that + * perf_aux_output_begin() has already been called. This path can + * come about only in snapshot/overwrite mode - see pt_event_stop(). + */ + if (!(hwc->state & PERF_HES_UPTODATE)) { + hwc->state &= ~PERF_HES_STOPPED; + pt_config_enable(event); + return; + } + buf = perf_aux_output_begin(&pt->handle, event); if (!buf) goto fail_stop; @@ -1632,7 +1655,7 @@ static void pt_event_start(struct perf_event *event, int mode) goto fail_end_stop; } - hwc->state = 0; + hwc->state &= ~(PERF_HES_STOPPED | PERF_HES_UPTODATE); pt_config_buffer(buf); pt_config(event); @@ -1642,12 +1665,13 @@ static void pt_event_start(struct perf_event *event, int mode) fail_end_stop: perf_aux_output_end(&pt->handle, 0); fail_stop: - hwc->state = PERF_HES_STOPPED; + hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; } static void pt_event_stop(struct perf_event *event, int mode) { struct pt *pt = this_cpu_ptr(&pt_ctx); + struct pt_buffer *buf; if (mode & PERF_EF_PAUSE) { if (READ_ONCE(pt->pause_allowed)) @@ -1673,17 +1697,24 @@ static void pt_event_stop(struct perf_event *event, int mode) pt_config_stop(event); - if (event->hw.state == PERF_HES_STOPPED) - return; - - event->hw.state = PERF_HES_STOPPED; + event->hw.state |= PERF_HES_STOPPED; - if (mode & PERF_EF_UPDATE) { - struct pt_buffer *buf = perf_get_aux(&pt->handle); + if (event->hw.state & PERF_HES_UPTODATE) + return; - if (!buf) - return; + buf = perf_get_aux(&pt->handle); + if (!buf) + return; + /* + * When not in snapshot/overwrite mode, there is a possibility that the + * buffer has run out of space. The accounting for that is handled by + * the update, so always update in that case. Snapshot/overwrite mode is + * treated differently to allow for pt_event_snapshot_aux() which can + * still get called if the AUX-sampling event is not stopped until after + * PT is stopped. + */ + if ((mode & PERF_EF_UPDATE) || !buf->snapshot) { if (WARN_ON_ONCE(pt->handle.event != event)) return; @@ -1698,6 +1729,7 @@ static void pt_event_stop(struct perf_event *event, int mode) local_xchg(&buf->data_size, buf->nr_pages << PAGE_SHIFT); perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0)); + event->hw.state |= PERF_HES_UPTODATE; } } @@ -1768,13 +1800,15 @@ static int pt_event_add(struct perf_event *event, int mode) if (pt->handle.event) goto fail; + event->hw.state |= PERF_HES_UPTODATE; + if (mode & PERF_EF_START) { pt_event_start(event, 0); ret = -EINVAL; - if (hwc->state == PERF_HES_STOPPED) + if (hwc->state & PERF_HES_STOPPED) goto fail; } else { - hwc->state = PERF_HES_STOPPED; + hwc->state |= PERF_HES_STOPPED; } ret = 0; diff --git a/arch/x86/events/intel/pt.h b/arch/x86/events/intel/pt.h index 2ac36250b656..700e1a915d25 100644 --- a/arch/x86/events/intel/pt.h +++ b/arch/x86/events/intel/pt.h @@ -58,7 +58,6 @@ struct pt_pmu { * @cur_idx: current output region's index within @cur table * @output_off: offset within the current output region * @data_size: running total of the amount of data in this buffer - * @lost: if data was lost/truncated * @head: logical write offset inside the buffer * @snapshot: if this is for a snapshot/overwrite counter * @single: use Single Range Output instead of ToPA @@ -68,7 +67,6 @@ struct pt_pmu { * @stop_te: STOP topa entry pointer * @intr_te: INT topa entry pointer * @data_pages: array of pages from perf - * @topa_index: table of topa entries indexed by page offset */ struct pt_buffer { struct list_head tables; diff --git a/arch/x86/events/intel/uncore.c b/arch/x86/events/intel/uncore.c index 7857959c6e82..9f8a80c9dcb6 100644 --- a/arch/x86/events/intel/uncore.c +++ b/arch/x86/events/intel/uncore.c @@ -757,7 +757,7 @@ static int uncore_pmu_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ @@ -953,16 +953,18 @@ static int uncore_pmu_register(struct intel_uncore_pmu *pmu) ret = perf_pmu_register(&pmu->pmu, pmu->name, -1); if (!ret) - pmu->registered = true; + uncore_pmu_set_registered(pmu); return ret; } static void uncore_pmu_unregister(struct intel_uncore_pmu *pmu) { - if (!pmu->registered) + if (!uncore_pmu_registered(pmu)) return; perf_pmu_unregister(&pmu->pmu); - pmu->registered = false; + + /* Keep PMU_BROKEN_BIT sticky. */ + uncore_pmu_clear_registered(pmu); } static void uncore_free_boxes(struct intel_uncore_pmu *pmu) @@ -1148,6 +1150,44 @@ uncore_pci_find_dev_pmu(struct pci_dev *pdev, const struct pci_device_id *ids) return pmu; } +static int uncore_box_setup(struct intel_uncore_pmu *pmu, + struct intel_uncore_box *box) +{ + int ret; + + if (uncore_pmu_broken(pmu)) + return -ENODEV; + + ret = uncore_box_init(box); + if (ret) + goto err; + + /* First active box registers the pmu. */ + if (atomic_inc_return(&pmu->activeboxes) > 1) + return 0; + + ret = uncore_pmu_register(pmu); + if (ret) { + atomic_dec(&pmu->activeboxes); + goto err; + } + + return 0; +err: + /* + * If any box fails, mark the per-package PMU as broken regardless of + * whether it was registered or not. + * + * Don't decrement refcnt to avoid other in-die CPUs from trying to set + * up the PMU box again. + * + * Don't kfree box; MSR and MMIO boxes are freed at module exit only. + */ + uncore_pmu_set_broken(pmu); + uncore_box_exit(box); + return ret; +} + /* * Register the PMU for a PCI device * @pdev: The PCI device. @@ -1167,26 +1207,22 @@ static int uncore_pci_pmu_register(struct pci_dev *pdev, return -EINVAL; box = uncore_alloc_box(type, NUMA_NO_NODE); - if (!box) + if (!box) { + uncore_pmu_set_broken(pmu); return -ENOMEM; + } atomic_inc(&box->refcnt); box->dieid = die; box->pci_dev = pdev; box->pmu = pmu; - uncore_box_init(box); - - pmu->boxes[die] = box; - if (atomic_inc_return(&pmu->activeboxes) > 1) - return 0; - /* First active box registers the pmu */ - ret = uncore_pmu_register(pmu); - if (ret) { - pmu->boxes[die] = NULL; - uncore_box_exit(box); + ret = uncore_box_setup(pmu, box); + if (!ret) + pmu->boxes[die] = box; + else kfree(box); - } + return ret; } @@ -1248,11 +1284,16 @@ static void uncore_pci_pmu_unregister(struct intel_uncore_pmu *pmu, int die) { struct intel_uncore_box *box = pmu->boxes[die]; + if (!box) + return; + pmu->boxes[die] = NULL; if (atomic_dec_return(&pmu->activeboxes) == 0) uncore_pmu_unregister(pmu); - uncore_box_exit(box); - kfree(box); + if (atomic_dec_return(&box->refcnt) == 0) { + uncore_box_exit(box); + kfree(box); + } } static void uncore_pci_remove(struct pci_dev *pdev) @@ -1272,7 +1313,6 @@ static void uncore_pci_remove(struct pci_dev *pdev) break; } } - WARN_ON_ONCE(i >= UNCORE_EXTRA_PCI_DEV_MAX); return; } @@ -1486,7 +1526,8 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, if (old_cpu < 0) { WARN_ON_ONCE(box->cpu != -1); - if (uncore_die_has_box(type, die, pmu->pmu_idx)) { + if (uncore_die_has_box(type, die, pmu->pmu_idx) && + !uncore_pmu_broken(pmu)) { box->cpu = new_cpu; cpumask_set_cpu(new_cpu, &pmu->cpu_mask); } @@ -1494,12 +1535,14 @@ static void uncore_change_type_ctx(struct intel_uncore_type *type, int old_cpu, } WARN_ON_ONCE(box->cpu != -1 && box->cpu != old_cpu); - box->cpu = -1; cpumask_clear_cpu(old_cpu, &pmu->cpu_mask); - if (new_cpu < 0) + if (new_cpu < 0) { + box->cpu = -1; continue; + } - if (!uncore_die_has_box(type, die, pmu->pmu_idx)) + /* An inactive box doesn't need migration. */ + if (box->cpu == -1) continue; uncore_pmu_cancel_hrtimer(box); perf_pmu_migrate_context(&pmu->pmu, old_cpu, new_cpu); @@ -1515,7 +1558,7 @@ static void uncore_change_context(struct intel_uncore_type **uncores, uncore_change_type_ctx(*uncores, old_cpu, new_cpu); } -static void uncore_box_unref(struct intel_uncore_type **types, int id) +static void uncore_box_unref(struct intel_uncore_type **types, int die) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; @@ -1526,9 +1569,14 @@ static void uncore_box_unref(struct intel_uncore_type **types, int id) type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; - if (box && box->cpu >= 0 && atomic_dec_return(&box->refcnt) == 0) + box = pmu->boxes[die]; + if (box && box->cpu >= 0 && + atomic_dec_return(&box->refcnt) == 0) { + if (uncore_box_active(box) && + atomic_dec_return(&pmu->activeboxes) == 0) + uncore_pmu_unregister(pmu); uncore_box_exit(box); + } } } } @@ -1537,9 +1585,15 @@ static int uncore_event_cpu_offline(unsigned int cpu) { int die, target; + /* Clear the references */ + die = topology_logical_die_id(cpu); + uncore_box_unref(uncore_msr_uncores, die); + uncore_box_unref(uncore_mmio_uncores, die); + /* Check if exiting cpu is used for collecting uncore events */ if (!cpumask_test_and_clear_cpu(cpu, &uncore_cpu_mask)) - goto unref; + return 0; + /* Find a new cpu to collect uncore events */ target = cpumask_any_but(topology_die_cpumask(cpu), cpu); @@ -1552,16 +1606,10 @@ static int uncore_event_cpu_offline(unsigned int cpu) uncore_change_context(uncore_msr_uncores, cpu, target); uncore_change_context(uncore_mmio_uncores, cpu, target); uncore_change_context(uncore_pci_uncores, cpu, target); - -unref: - /* Clear the references */ - die = topology_logical_die_id(cpu); - uncore_box_unref(uncore_msr_uncores, die); - uncore_box_unref(uncore_mmio_uncores, die); return 0; } -static int allocate_boxes(struct intel_uncore_type **types, +static void allocate_boxes(struct intel_uncore_type **types, unsigned int die, unsigned int cpu) { struct intel_uncore_box *box, *tmp; @@ -1575,11 +1623,13 @@ static int allocate_boxes(struct intel_uncore_type **types, type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - if (pmu->boxes[die]) + if (pmu->boxes[die] || uncore_pmu_broken(pmu)) continue; box = uncore_alloc_box(type, cpu_to_node(cpu)); - if (!box) + if (!box) { + uncore_pmu_set_broken(pmu); goto cleanup; + } box->pmu = pmu; box->dieid = die; list_add(&box->active_list, &allocated); @@ -1590,35 +1640,30 @@ static int allocate_boxes(struct intel_uncore_type **types, list_del_init(&box->active_list); box->pmu->boxes[die] = box; } - return 0; + return; cleanup: list_for_each_entry_safe(box, tmp, &allocated, active_list) { list_del_init(&box->active_list); kfree(box); } - return -ENOMEM; } static int uncore_box_ref(struct intel_uncore_type **types, - int id, unsigned int cpu) + int die, unsigned int cpu) { struct intel_uncore_type *type; struct intel_uncore_pmu *pmu; struct intel_uncore_box *box; - int i, ret; - - ret = allocate_boxes(types, id, cpu); - if (ret) - return ret; + int i; for (; *types; types++) { type = *types; pmu = type->pmus; for (i = 0; i < type->num_boxes; i++, pmu++) { - box = pmu->boxes[id]; + box = pmu->boxes[die]; if (box && box->cpu >= 0 && atomic_inc_return(&box->refcnt) == 1) - uncore_box_init(box); + uncore_box_setup(pmu, box); } } return 0; @@ -1626,93 +1671,35 @@ static int uncore_box_ref(struct intel_uncore_type **types, static int uncore_event_cpu_online(unsigned int cpu) { - int die, target, msr_ret, mmio_ret; + int die, target; die = topology_logical_die_id(cpu); - msr_ret = uncore_box_ref(uncore_msr_uncores, die, cpu); - mmio_ret = uncore_box_ref(uncore_mmio_uncores, die, cpu); - if (msr_ret && mmio_ret) - return -ENOMEM; + allocate_boxes(uncore_msr_uncores, die, cpu); + allocate_boxes(uncore_mmio_uncores, die, cpu); /* * Check if there is an online cpu in the package * which collects uncore events already. */ target = cpumask_any_and(&uncore_cpu_mask, topology_die_cpumask(cpu)); - if (target < nr_cpu_ids) - return 0; - - cpumask_set_cpu(cpu, &uncore_cpu_mask); - - if (!msr_ret) + if (target >= nr_cpu_ids) { + cpumask_set_cpu(cpu, &uncore_cpu_mask); uncore_change_context(uncore_msr_uncores, -1, cpu); - if (!mmio_ret) uncore_change_context(uncore_mmio_uncores, -1, cpu); - uncore_change_context(uncore_pci_uncores, -1, cpu); - return 0; -} - -static int __init type_pmu_register(struct intel_uncore_type *type) -{ - int i, ret; - - for (i = 0; i < type->num_boxes; i++) { - ret = uncore_pmu_register(&type->pmus[i]); - if (ret) - return ret; + uncore_change_context(uncore_pci_uncores, -1, cpu); } - return 0; -} - -static int __init uncore_msr_pmus_register(void) -{ - struct intel_uncore_type **types = uncore_msr_uncores; - int ret; - for (; *types; types++) { - ret = type_pmu_register(*types); - if (ret) - return ret; - } - return 0; -} - -static int __init uncore_cpu_init(void) -{ - int ret; - - ret = uncore_types_init(uncore_msr_uncores); - if (ret) - goto err; - - ret = uncore_msr_pmus_register(); - if (ret) - goto err; + uncore_box_ref(uncore_msr_uncores, die, cpu); + uncore_box_ref(uncore_mmio_uncores, die, cpu); return 0; -err: - uncore_types_exit(uncore_msr_uncores); - uncore_msr_uncores = empty_uncore; - return ret; } -static int __init uncore_mmio_init(void) +static int __init uncore_pmu_types_init(struct intel_uncore_type **types) { - struct intel_uncore_type **types = uncore_mmio_uncores; - int ret; - - ret = uncore_types_init(types); + int ret = uncore_types_init(types); if (ret) - goto err; + uncore_types_exit(types); - for (; *types; types++) { - ret = type_pmu_register(*types); - if (ret) - goto err; - } - return 0; -err: - uncore_types_exit(uncore_mmio_uncores); - uncore_mmio_uncores = empty_uncore; return ret; } @@ -2013,12 +2000,16 @@ static int __init intel_uncore_init(void) if (uncore_init->cpu_init) { uncore_init->cpu_init(); - cret = uncore_cpu_init(); + cret = uncore_pmu_types_init(uncore_msr_uncores); + if (cret) + uncore_msr_uncores = empty_uncore; } if (uncore_init->mmio_init) { uncore_init->mmio_init(); - mret = uncore_mmio_init(); + mret = uncore_pmu_types_init(uncore_mmio_uncores); + if (mret) + uncore_mmio_uncores = empty_uncore; } if (cret && pret && mret) { diff --git a/arch/x86/events/intel/uncore.h b/arch/x86/events/intel/uncore.h index c2e5ccb1d72c..c8dfa2d21bfd 100644 --- a/arch/x86/events/intel/uncore.h +++ b/arch/x86/events/intel/uncore.h @@ -129,7 +129,7 @@ struct intel_uncore_type { #define events_group attr_groups[2] struct intel_uncore_ops { - void (*init_box)(struct intel_uncore_box *); + int (*init_box)(struct intel_uncore_box *); void (*exit_box)(struct intel_uncore_box *); void (*disable_box)(struct intel_uncore_box *); void (*enable_box)(struct intel_uncore_box *); @@ -146,13 +146,24 @@ struct intel_uncore_pmu { struct pmu pmu; char name[UNCORE_PMU_NAME_LEN]; int pmu_idx; - bool registered; + unsigned long flags; atomic_t activeboxes; cpumask_t cpu_mask; struct intel_uncore_type *type; struct intel_uncore_box **boxes; }; +#define PMU_REGISTERED_BIT 0 +#define PMU_BROKEN_BIT 1 + +#define uncore_pmu_registered(pmu) test_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_broken(pmu) test_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_available(pmu) (uncore_pmu_registered(pmu) && \ + !uncore_pmu_broken(pmu)) +#define uncore_pmu_set_registered(pmu) set_bit(PMU_REGISTERED_BIT, &(pmu)->flags) +#define uncore_pmu_set_broken(pmu) set_bit(PMU_BROKEN_BIT, &(pmu)->flags) +#define uncore_pmu_clear_registered(pmu) clear_bit(PMU_REGISTERED_BIT, &(pmu)->flags) + struct intel_uncore_extra_reg { raw_spinlock_t lock; u64 config, config1, config2; @@ -185,7 +196,7 @@ struct intel_uncore_box { #define CFL_UNC_CBO_7_PERFEVTSEL0 0xf70 #define CFL_UNC_CBO_7_PER_CTR0 0xf76 -#define UNCORE_BOX_FLAG_INITIATED 0 +#define UNCORE_BOX_FLAG_INITIALIZED 0 /* event config registers are 8-byte apart */ #define UNCORE_BOX_FLAG_CTL_OFFS8 1 /* CFL 8th CBOX has different MSR space */ @@ -557,17 +568,29 @@ static inline u64 uncore_read_counter(struct intel_uncore_box *box, return box->pmu->type->ops->read_counter(box, event); } -static inline void uncore_box_init(struct intel_uncore_box *box) +static inline bool uncore_box_active(struct intel_uncore_box *box) { - if (!test_and_set_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { - if (box->pmu->type->ops->init_box) - box->pmu->type->ops->init_box(box); + return (!box->pmu->type->ops->init_box || + test_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)); +} + +static inline int uncore_box_init(struct intel_uncore_box *box) +{ + int ret = 0; + + if (!test_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags) && + box->pmu->type->ops->init_box) { + ret = box->pmu->type->ops->init_box(box); + if (!ret) + __set_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags); } + + return ret; } static inline void uncore_box_exit(struct intel_uncore_box *box) { - if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIATED, &box->flags)) { + if (test_and_clear_bit(UNCORE_BOX_FLAG_INITIALIZED, &box->flags)) { if (box->pmu->type->ops->exit_box) box->pmu->type->ops->exit_box(box); } diff --git a/arch/x86/events/intel/uncore_discovery.c b/arch/x86/events/intel/uncore_discovery.c index e50776222256..0a22edf4d509 100644 --- a/arch/x86/events/intel/uncore_discovery.c +++ b/arch/x86/events/intel/uncore_discovery.c @@ -489,14 +489,15 @@ static u64 intel_generic_uncore_box_ctl(struct intel_uncore_box *box) return unit->addr; } -void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_msr_init_box(struct intel_uncore_box *box) { u64 box_ctl = intel_generic_uncore_box_ctl(box); if (!box_ctl) - return; + return -ENODEV; wrmsrq(box_ctl, GENERIC_PMON_BOX_CTL_INT); + return 0; } void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box) @@ -578,15 +579,16 @@ static inline int intel_pci_uncore_box_ctl(struct intel_uncore_box *box) return UNCORE_DISCOVERY_PCI_BOX_CTRL(intel_generic_uncore_box_ctl(box)); } -void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_pci_init_box(struct intel_uncore_box *box) { int box_ctl = intel_pci_uncore_box_ctl(box); if (!box_ctl) - return; + return -ENODEV; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(box->pci_dev, box_ctl, GENERIC_PMON_BOX_CTL_INT); + return pci_write_config_dword(box->pci_dev, box_ctl, + GENERIC_PMON_BOX_CTL_INT); } void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box) @@ -648,7 +650,7 @@ static struct intel_uncore_ops generic_uncore_pci_ops = { #define UNCORE_GENERIC_MMIO_SIZE 0x4000 -void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) +int intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) { static struct intel_uncore_discovery_unit *unit; struct intel_uncore_type *type = box->pmu->type; @@ -658,13 +660,13 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) if (!unit) { pr_warn("Uncore type %d id %d: Cannot find box control address.\n", type->type_id, box->pmu->pmu_idx); - return; + return -ENODEV; } if (!unit->addr) { pr_warn("Uncore type %d box %d: Invalid box control address.\n", type->type_id, unit->id); - return; + return -ENODEV; } addr = unit->addr; @@ -672,10 +674,11 @@ void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box) if (!box->io_addr) { pr_warn("Uncore type %d box %d: ioremap error for 0x%llx.\n", type->type_id, unit->id, (unsigned long long)addr); - return; + return -ENOMEM; } writel(GENERIC_PMON_BOX_CTL_INT, box->io_addr); + return 0; } void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box) diff --git a/arch/x86/events/intel/uncore_discovery.h b/arch/x86/events/intel/uncore_discovery.h index e1330342b92e..142e1b56cfc2 100644 --- a/arch/x86/events/intel/uncore_discovery.h +++ b/arch/x86/events/intel/uncore_discovery.h @@ -148,11 +148,11 @@ void intel_uncore_generic_uncore_cpu_init(void); int intel_uncore_generic_uncore_pci_init(void); void intel_uncore_generic_uncore_mmio_init(void); -void intel_generic_uncore_msr_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_msr_init_box(struct intel_uncore_box *box); void intel_generic_uncore_msr_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_msr_enable_box(struct intel_uncore_box *box); -void intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_mmio_init_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_enable_box(struct intel_uncore_box *box); void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, @@ -160,7 +160,7 @@ void intel_generic_uncore_mmio_disable_event(struct intel_uncore_box *box, void intel_generic_uncore_mmio_enable_event(struct intel_uncore_box *box, struct perf_event *event); -void intel_generic_uncore_pci_init_box(struct intel_uncore_box *box); +int intel_generic_uncore_pci_init_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_disable_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_enable_box(struct intel_uncore_box *box); void intel_generic_uncore_pci_disable_event(struct intel_uncore_box *box, diff --git a/arch/x86/events/intel/uncore_nhmex.c b/arch/x86/events/intel/uncore_nhmex.c index 8962e7cb21e3..7a6855281102 100644 --- a/arch/x86/events/intel/uncore_nhmex.c +++ b/arch/x86/events/intel/uncore_nhmex.c @@ -199,9 +199,10 @@ DEFINE_UNCORE_FORMAT_ATTR(counter, counter, "config:6-7"); DEFINE_UNCORE_FORMAT_ATTR(match, match, "config1:0-63"); DEFINE_UNCORE_FORMAT_ATTR(mask, mask, "config2:0-63"); -static void nhmex_uncore_msr_init_box(struct intel_uncore_box *box) +static int nhmex_uncore_msr_init_box(struct intel_uncore_box *box) { wrmsrq(NHMEX_U_MSR_PMON_GLOBAL_CTL, NHMEX_U_PMON_GLOBAL_EN_ALL); + return 0; } static void nhmex_uncore_msr_exit_box(struct intel_uncore_box *box) diff --git a/arch/x86/events/intel/uncore_snb.c b/arch/x86/events/intel/uncore_snb.c index edddd4f9ab5f..055131c508ff 100644 --- a/arch/x86/events/intel/uncore_snb.c +++ b/arch/x86/events/intel/uncore_snb.c @@ -295,12 +295,14 @@ static void snb_uncore_msr_disable_event(struct intel_uncore_box *box, struct pe wrmsrq(event->hw.config_base, 0); } -static void snb_uncore_msr_init_box(struct intel_uncore_box *box) +static int snb_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) { wrmsrq(SNB_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN | SNB_UNC_GLOBAL_CTL_CORE_ALL); } + + return 0; } static void snb_uncore_msr_enable_box(struct intel_uncore_box *box) @@ -394,7 +396,7 @@ void snb_uncore_cpu_init(void) snb_uncore_cbox.num_boxes = topology_num_cores_per_package(); } -static void skl_uncore_msr_init_box(struct intel_uncore_box *box) +static int skl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) { wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, @@ -404,6 +406,8 @@ static void skl_uncore_msr_init_box(struct intel_uncore_box *box) /* The 8th CBOX has different MSR space */ if (box->pmu->pmu_idx == 7) __set_bit(UNCORE_BOX_FLAG_CFL8_CBOX_MSR_OFFS, &box->flags); + + return 0; } static void skl_uncore_msr_enable_box(struct intel_uncore_box *box) @@ -547,10 +551,12 @@ static struct intel_uncore_type *tgl_msr_uncores[] = { NULL, }; -static void rkl_uncore_msr_init_box(struct intel_uncore_box *box) +static int rkl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) wrmsrq(SKL_UNC_PERF_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + + return 0; } void tgl_uncore_cpu_init(void) @@ -707,9 +713,10 @@ static struct intel_uncore_type mtl_uncore_hac_cbox = { .format_group = &adl_uncore_format_group, }; -static void mtl_uncore_msr_init_box(struct intel_uncore_box *box) +static int mtl_uncore_msr_init_box(struct intel_uncore_box *box) { wrmsrq(uncore_msr_box_ctl(box), SNB_UNC_GLOBAL_CTL_EN); + return 0; } static struct intel_uncore_ops mtl_uncore_msr_ops = { @@ -773,10 +780,12 @@ static struct intel_uncore_type *lnl_msr_uncores[] = { #define LNL_UNC_MSR_GLOBAL_CTL 0x240e -static void lnl_uncore_msr_init_box(struct intel_uncore_box *box) +static int lnl_uncore_msr_init_box(struct intel_uncore_box *box) { if (box->pmu->pmu_idx == 0) wrmsrq(LNL_UNC_MSR_GLOBAL_CTL, SNB_UNC_GLOBAL_CTL_EN); + + return 0; } static struct intel_uncore_ops lnl_uncore_msr_ops = { @@ -874,7 +883,7 @@ static const struct attribute_group snb_uncore_imc_format_group = { .attrs = snb_uncore_imc_formats_attr, }; -static void snb_uncore_imc_init_box(struct intel_uncore_box *box) +static int snb_uncore_imc_init_box(struct intel_uncore_box *box) { struct intel_uncore_type *type = box->pmu->type; struct pci_dev *pdev = box->pci_dev; @@ -893,10 +902,13 @@ static void snb_uncore_imc_init_box(struct intel_uncore_box *box) addr &= ~(PAGE_SIZE - 1); box->io_addr = ioremap(addr, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); + return -ENOMEM; + } box->hrtimer_duration = UNCORE_SNB_IMC_HRTIMER_INTERVAL; + return 0; } static void snb_uncore_imc_enable_box(struct intel_uncore_box *box) @@ -928,7 +940,7 @@ static int snb_uncore_imc_event_init(struct perf_event *event) pmu = uncore_event_to_pmu(event); /* no device found for this pmu */ - if (!pmu->registered) + if (!uncore_pmu_available(pmu)) return -ENOENT; /* Sampling not supported yet */ @@ -1532,7 +1544,7 @@ static struct pci_dev *tgl_uncore_get_mc_dev(void) #define TGL_UNCORE_MMIO_IMC_MEM_OFFSET 0x10000 #define TGL_UNCORE_PCI_IMC_MAP_SIZE 0xe000 -static void +static int uncore_get_box_mmio_addr(struct intel_uncore_box *box, unsigned int base_offset, int bar_offset, int step) @@ -1541,19 +1553,20 @@ uncore_get_box_mmio_addr(struct intel_uncore_box *box, struct intel_uncore_pmu *pmu = box->pmu; struct intel_uncore_type *type = pmu->type; resource_size_t addr; + int ret = 0; u32 bar; if (!pdev) { pr_warn("perf uncore: Cannot find matched IMC device.\n"); - return; + return -ENODEV; } pci_read_config_dword(pdev, bar_offset, &bar); if (!(bar & BIT(0))) { pr_warn("perf uncore: BAR 0x%x is disabled. Failed to map %s counters.\n", bar_offset, type->name); - pci_dev_put(pdev); - return; + ret = -ENODEV; + goto out; } bar &= ~BIT(0); addr = (resource_size_t)(bar + step * pmu->pmu_idx); @@ -1565,23 +1578,26 @@ uncore_get_box_mmio_addr(struct intel_uncore_box *box, addr += base_offset; box->io_addr = ioremap(addr, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { + ret = -ENOMEM; pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); - + } +out: pci_dev_put(pdev); + return ret; } -static void __uncore_imc_init_box(struct intel_uncore_box *box, +static int __uncore_imc_init_box(struct intel_uncore_box *box, unsigned int base_offset) { - uncore_get_box_mmio_addr(box, base_offset, + return uncore_get_box_mmio_addr(box, base_offset, SNB_UNCORE_PCI_IMC_BAR_OFFSET, TGL_UNCORE_MMIO_IMC_MEM_OFFSET); } -static void tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int tgl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, 0); + return __uncore_imc_init_box(box, 0); } static struct intel_uncore_ops tgl_uncore_imc_freerunning_ops = { @@ -1648,13 +1664,15 @@ void tgl_uncore_mmio_init(void) #define ADL_UNCORE_IMC_CTL_INT (ADL_UNCORE_IMC_CTL_RST_CTRL | \ ADL_UNCORE_IMC_CTL_RST_CTRS) -static void adl_uncore_imc_init_box(struct intel_uncore_box *box) +static int adl_uncore_imc_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE); + int ret = __uncore_imc_init_box(box, ADL_UNCORE_IMC_BASE); /* The global control in MC1 can control both MCs. */ - if (box->io_addr && (box->pmu->pmu_idx == 1)) + if (!ret && (box->pmu->pmu_idx == 1)) writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + ADL_UNCORE_IMC_GLOBAL_CTL); + + return ret; } static void adl_uncore_mmio_disable_box(struct intel_uncore_box *box) @@ -1731,9 +1749,9 @@ static struct freerunning_counters adl_uncore_imc_freerunning[] = { [ADL_MMIO_UNCORE_IMC_DATA_WRITE] = { 0xA0, 0x0, 0x0, 1, 64 }, }; -static void adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int adl_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { - __uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE); + return __uncore_imc_init_box(box, ADL_UNCORE_IMC_FREERUNNING_BASE); } static struct intel_uncore_ops adl_uncore_imc_freerunning_ops = { @@ -1803,9 +1821,9 @@ static const struct attribute_group lnl_uncore_format_group = { .attrs = lnl_uncore_formats_attr, }; -static void lnl_uncore_hbo_init_box(struct intel_uncore_box *box) +static int lnl_uncore_hbo_init_box(struct intel_uncore_box *box) { - uncore_get_box_mmio_addr(box, LNL_UNCORE_HBO_BASE, + return uncore_get_box_mmio_addr(box, LNL_UNCORE_HBO_BASE, LNL_UNCORE_PCI_SAFBAR_OFFSET, LNL_UNCORE_HBO_OFFSET); } @@ -1829,14 +1847,16 @@ static struct intel_uncore_type lnl_uncore_hbo = { .format_group = &lnl_uncore_format_group, }; -static void lnl_uncore_sncu_init_box(struct intel_uncore_box *box) +static int lnl_uncore_sncu_init_box(struct intel_uncore_box *box) { - uncore_get_box_mmio_addr(box, LNL_UNCORE_SNCU_BASE, + int ret = uncore_get_box_mmio_addr(box, LNL_UNCORE_SNCU_BASE, LNL_UNCORE_PCI_SAFBAR_OFFSET, 0); - if (box->io_addr) + if (!ret) writel(ADL_UNCORE_IMC_CTL_INT, box->io_addr + LNL_UNCORE_GLOBAL_CTL); + + return ret; } static struct intel_uncore_ops lnl_uncore_sncu_ops = { @@ -1887,13 +1907,15 @@ static struct intel_uncore_type ptl_uncore_imc = { .mmio_map_size = 0xf00, }; -static void ptl_uncore_sncu_init_box(struct intel_uncore_box *box) +static int ptl_uncore_sncu_init_box(struct intel_uncore_box *box) { - intel_generic_uncore_mmio_init_box(box); + int ret = intel_generic_uncore_mmio_init_box(box); /* Clear the global freeze bit */ if (box->io_addr) writel(0, box->io_addr + PTL_UNCORE_GLOBAL_CTL_OFFSET); + + return ret; } static struct intel_uncore_ops ptl_uncore_sncu_ops = { diff --git a/arch/x86/events/intel/uncore_snbep.c b/arch/x86/events/intel/uncore_snbep.c index 334dc384b5b9..a97cd029db36 100644 --- a/arch/x86/events/intel/uncore_snbep.c +++ b/arch/x86/events/intel/uncore_snbep.c @@ -627,12 +627,12 @@ static u64 snbep_uncore_pci_read_counter(struct intel_uncore_box *box, struct pe return count; } -static void snbep_uncore_pci_init_box(struct intel_uncore_box *box) +static int snbep_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); - pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, box_ctl, SNBEP_PMON_BOX_CTL_INT); } static void snbep_uncore_msr_disable_box(struct intel_uncore_box *box) @@ -680,12 +680,14 @@ static void snbep_uncore_msr_disable_event(struct intel_uncore_box *box, wrmsrq(hwc->config_base, hwc->config); } -static void snbep_uncore_msr_init_box(struct intel_uncore_box *box) +static int snbep_uncore_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); if (msr) wrmsrq(msr, SNBEP_PMON_BOX_CTL_INT); + + return 0; } static struct attribute *snbep_uncore_formats_attr[] = { @@ -1507,18 +1509,21 @@ int snbep_uncore_pci_init(void) /* end of Sandy Bridge-EP uncore support */ /* IvyTown uncore support */ -static void ivbep_uncore_msr_init_box(struct intel_uncore_box *box) +static int ivbep_uncore_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); if (msr) wrmsrq(msr, IVBEP_PMON_BOX_CTL_INT); + + return 0; } -static void ivbep_uncore_pci_init_box(struct intel_uncore_box *box) +static int ivbep_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; - pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } #define IVBEP_UNCORE_MSR_OPS_COMMON_INIT() \ @@ -2784,7 +2789,7 @@ static struct intel_uncore_type hswep_uncore_cbox = { /* * Write SBOX Initialization register bit by bit to avoid spurious #GPs */ -static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) +static int hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) { unsigned msr = uncore_msr_box_ctl(box); @@ -2798,6 +2803,8 @@ static void hswep_uncore_sbox_msr_init_box(struct intel_uncore_box *box) wrmsrq(msr, flags); } } + + return 0; } static struct intel_uncore_ops hswep_uncore_sbox_msr_ops = { @@ -4162,12 +4169,13 @@ static const struct attribute_group skx_upi_uncore_format_group = { .attrs = skx_upi_uncore_formats_attr, }; -static void skx_upi_uncore_pci_init_box(struct intel_uncore_box *box) +static int skx_upi_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SKX_UPI_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops skx_upi_uncore_pci_ops = { @@ -4323,12 +4331,13 @@ static struct intel_uncore_type skx_uncore_upi = { .cleanup_mapping = skx_upi_cleanup_mapping, }; -static void skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box) +static int skx_m2m_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, SKX_M2M_PCI_PMON_BOX_CTL, + IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops skx_m2m_uncore_pci_ops = { @@ -4831,13 +4840,13 @@ void snr_uncore_cpu_init(void) uncore_msr_uncores = snr_msr_uncores; } -static void snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box) +static int snr_m2m_uncore_pci_init_box(struct intel_uncore_box *box) { struct pci_dev *pdev = box->pci_dev; int box_ctl = uncore_pci_box_ctl(box); __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT); + return pci_write_config_dword(pdev, box_ctl, IVBEP_PMON_BOX_CTL_INT); } static struct intel_uncore_ops snr_m2m_uncore_pci_ops = { @@ -5010,17 +5019,22 @@ static int snr_uncore_mmio_map(struct intel_uncore_box *box, return 0; } -static void __snr_uncore_mmio_init_box(struct intel_uncore_box *box, +static int __snr_uncore_mmio_init_box(struct intel_uncore_box *box, unsigned int box_ctl, int mem_offset, unsigned int device) { - if (!snr_uncore_mmio_map(box, box_ctl, mem_offset, device)) + int ret; + + ret = snr_uncore_mmio_map(box, box_ctl, mem_offset, device); + if (!ret) writel(IVBEP_PMON_BOX_CTL_INT, box->io_addr); + + return ret; } -static void snr_uncore_mmio_init_box(struct intel_uncore_box *box) +static int snr_uncore_mmio_init_box(struct intel_uncore_box *box) { - __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), + return __snr_uncore_mmio_init_box(box, uncore_mmio_box_ctl(box), SNR_IMC_MMIO_MEM0_OFFSET, SNR_MC_DEVICE_ID); } @@ -5637,14 +5651,14 @@ int icx_uncore_pci_init(void) return 0; } -static void icx_uncore_imc_init_box(struct intel_uncore_box *box) +static int icx_uncore_imc_init_box(struct intel_uncore_box *box) { unsigned int box_ctl = box->pmu->type->box_ctl + box->pmu->type->mmio_offset * (box->pmu->pmu_idx % ICX_NUMBER_IMC_CHN); int mem_offset = (box->pmu->pmu_idx / ICX_NUMBER_IMC_CHN) * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - __snr_uncore_mmio_init_box(box, box_ctl, mem_offset, + return __snr_uncore_mmio_init_box(box, box_ctl, mem_offset, SNR_MC_DEVICE_ID); } @@ -5701,12 +5715,12 @@ static struct uncore_event_desc icx_uncore_imc_freerunning_events[] = { { /* end: all zeroes */ }, }; -static void icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int icx_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), + return snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), mem_offset, SNR_MC_DEVICE_ID); } @@ -6003,10 +6017,10 @@ static struct intel_uncore_type spr_uncore_mdf = { .name = "mdf", }; -static void spr_uncore_mmio_offs8_init_box(struct intel_uncore_box *box) +static int spr_uncore_mmio_offs8_init_box(struct intel_uncore_box *box) { __set_bit(UNCORE_BOX_FLAG_CTL_OFFS8, &box->flags); - intel_generic_uncore_mmio_init_box(box); + return intel_generic_uncore_mmio_init_box(box); } static struct intel_uncore_ops spr_uncore_mmio_offs8_ops = { @@ -6187,12 +6201,11 @@ static struct uncore_event_desc spr_uncore_imc_freerunning_events[] = { #define SPR_MC_DEVICE_ID 0x3251 -static void spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) +static int spr_uncore_imc_freerunning_init_box(struct intel_uncore_box *box) { int mem_offset = box->pmu->pmu_idx * ICX_IMC_MEM_STRIDE + SNR_IMC_MMIO_MEM0_OFFSET; - - snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), - mem_offset, SPR_MC_DEVICE_ID); + return snr_uncore_mmio_map(box, uncore_mmio_box_ctl(box), + mem_offset, SPR_MC_DEVICE_ID); } static struct intel_uncore_ops spr_uncore_imc_freerunning_ops = { @@ -6881,20 +6894,24 @@ static unsigned int dmr_iio_freerunning_box_offsets[] = { 0x0, 0x8000, 0x18000, 0x20000 }; -static void dmr_uncore_freerunning_init_box(struct intel_uncore_box *box) +static int dmr_uncore_freerunning_init_box(struct intel_uncore_box *box) { struct intel_uncore_type *type = box->pmu->type; u64 mmio_base; if (box->pmu->pmu_idx >= type->num_boxes) - return; + return -ENODEV; mmio_base = DMR_IMH1_HIOP_MMIO_BASE; mmio_base += dmr_iio_freerunning_box_offsets[box->pmu->pmu_idx]; box->io_addr = ioremap(mmio_base, type->mmio_map_size); - if (!box->io_addr) + if (!box->io_addr) { pr_warn("perf uncore: Failed to ioremap for %s.\n", type->name); + return -ENOMEM; + } + + return 0; } static struct intel_uncore_ops dmr_uncore_freerunning_ops = { diff --git a/arch/x86/events/perf_event.h b/arch/x86/events/perf_event.h index eae24bb35dc1..fa381110f7a7 100644 --- a/arch/x86/events/perf_event.h +++ b/arch/x86/events/perf_event.h @@ -141,7 +141,7 @@ static inline bool is_acr_self_reload_event(struct perf_event *event) { struct hw_perf_event *hwc = &event->hw; - if (hwc->idx < 0) + if (hwc->idx < 0 || !is_acr_event_group(event)) return false; return test_bit(hwc->idx, (unsigned long *)&hwc->config1); @@ -668,7 +668,7 @@ union perf_capabilities { u64 perf_metrics:1; u64 pebs_output_pt_available:1; u64 pebs_timing_info:1; - u64 anythread_deprecated:1; + u64 __reserved:1; u64 rdpmc_metrics_clear:1; }; u64 capabilities; @@ -1161,6 +1161,7 @@ static struct perf_pmu_format_hybrid_attr format_attr_hybrid_##_name = {\ .pmu_type = _pmu, \ } +struct pmu *x86_get_static_pmu(void); struct pmu *x86_get_pmu(unsigned int cpu); extern struct x86_pmu x86_pmu __read_mostly; @@ -1242,7 +1243,7 @@ static inline int x86_pmu_rdpmc_index(int index) return x86_pmu.rdpmc_index ? x86_pmu.rdpmc_index(index) : index; } -bool check_hw_exists(struct pmu *pmu, unsigned long *cntr_mask, +bool check_hw_exists(unsigned long *cntr_mask, unsigned long *fixed_cntr_mask); int x86_add_exclusive(unsigned int what); @@ -1344,7 +1345,7 @@ static inline u64 x86_pmu_get_event_config(struct perf_event *event) static inline bool x86_pmu_has_rdpmc_user_disable(struct pmu *pmu) { return !!(hybrid(pmu, config_mask) & - ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE); + ARCH_PERFMON_EVENTSEL_RDPMC_USER_DISABLE); } extern struct event_constraint emptyconstraint; @@ -1455,13 +1456,6 @@ ssize_t events_hybrid_sysfs_show(struct device *dev, struct device_attribute *attr, char *page); -static inline bool fixed_counter_disabled(int i, struct pmu *pmu) -{ - u64 intel_ctrl = hybrid(pmu, intel_ctrl); - - return !(intel_ctrl >> (i + INTEL_PMC_IDX_FIXED)); -} - #ifdef CONFIG_CPU_SUP_AMD int amd_pmu_init(void); diff --git a/arch/x86/kernel/uprobes.c b/arch/x86/kernel/uprobes.c index 3af979fb41d3..65a2de82ecd2 100644 --- a/arch/x86/kernel/uprobes.c +++ b/arch/x86/kernel/uprobes.c @@ -276,15 +276,9 @@ static bool is_prefix_bad(struct insn *insn) return false; } -static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool x86_64) +static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn) { - enum insn_mode m = x86_64 ? INSN_MODE_64 : INSN_MODE_32; u32 volatile *good_insns; - int ret; - - ret = insn_decode(insn, auprobe->insn, sizeof(auprobe->insn), m); - if (ret < 0) - return -ENOEXEC; if (is_prefix_bad(insn)) return -ENOTSUPP; @@ -293,7 +287,7 @@ static int uprobe_init_insn(struct arch_uprobe *auprobe, struct insn *insn, bool if (insn_masking_exception(insn)) return -ENOTSUPP; - if (x86_64) + if (insn->x86_64) good_insns = good_insns_64; else good_insns = good_insns_32; @@ -631,14 +625,29 @@ static struct vm_special_mapping tramp_mapping = { .pages = tramp_mapping_pages, }; -struct uprobe_trampoline { - struct hlist_node node; - unsigned long vaddr; -}; + +#define LEA_INSN_SIZE 5 +#define OPT_INSN_SIZE (LEA_INSN_SIZE + CALL_INSN_SIZE) +#define REDZONE_SIZE 0x80 + +static const u8 lea_rsp[] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; + +static bool is_opt_insns(const uprobe_opcode_t *insn) +{ + return !memcmp(insn, lea_rsp, LEA_INSN_SIZE) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} + +static bool is_swbp_opt_insns(uprobe_opcode_t *insn) +{ + return is_swbp_insn(&insn[0]) && + !memcmp(&insn[1], &lea_rsp[1], LEA_INSN_SIZE - 1) && + insn[LEA_INSN_SIZE] == CALL_INSN_OPCODE; +} static bool is_reachable_by_call(unsigned long vtramp, unsigned long vaddr) { - long delta = (long)(vaddr + 5 - vtramp); + long delta = (long)(vaddr + OPT_INSN_SIZE - vtramp); return delta >= INT_MIN && delta <= INT_MAX; } @@ -651,7 +660,7 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) }; unsigned long low_limit, high_limit; unsigned long low_tramp, high_tramp; - unsigned long call_end = vaddr + 5; + unsigned long call_end = vaddr + OPT_INSN_SIZE; if (check_add_overflow(call_end, INT_MIN, &low_limit)) low_limit = PAGE_SIZE; @@ -682,83 +691,32 @@ static unsigned long find_nearest_trampoline(unsigned long vaddr) return high_tramp; } -static struct uprobe_trampoline *create_uprobe_trampoline(unsigned long vaddr) +static struct vm_area_struct *get_uprobe_trampoline(struct mm_struct *mm, unsigned long vaddr, + bool *new_mapping) { - struct pt_regs *regs = task_pt_regs(current); - struct mm_struct *mm = current->mm; - struct uprobe_trampoline *tramp; + VMA_ITERATOR(vmi, mm, 0); struct vm_area_struct *vma; - if (!user_64bit_mode(regs)) - return NULL; - - vaddr = find_nearest_trampoline(vaddr); - if (IS_ERR_VALUE(vaddr)) - return NULL; - - tramp = kzalloc_obj(*tramp); - if (unlikely(!tramp)) - return NULL; - - tramp->vaddr = vaddr; - vma = _install_special_mapping(mm, tramp->vaddr, PAGE_SIZE, - VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_DONTCOPY|VM_IO, - &tramp_mapping); - if (IS_ERR(vma)) { - kfree(tramp); - return NULL; - } - return tramp; -} - -static struct uprobe_trampoline *get_uprobe_trampoline(unsigned long vaddr, bool *new) -{ - struct uprobes_state *state = ¤t->mm->uprobes_state; - struct uprobe_trampoline *tramp = NULL; + *new_mapping = false; if (vaddr > TASK_SIZE || vaddr < PAGE_SIZE) - return NULL; + return ERR_PTR(-EINVAL); - hlist_for_each_entry(tramp, &state->head_tramps, node) { - if (is_reachable_by_call(tramp->vaddr, vaddr)) { - *new = false; - return tramp; - } + for_each_vma(vmi, vma) { + if (!vma_is_special_mapping(vma, &tramp_mapping)) + continue; + if (is_reachable_by_call(vma->vm_start, vaddr)) + return vma; } - tramp = create_uprobe_trampoline(vaddr); - if (!tramp) - return NULL; - - *new = true; - hlist_add_head(&tramp->node, &state->head_tramps); - return tramp; -} - -static void destroy_uprobe_trampoline(struct uprobe_trampoline *tramp) -{ - /* - * We do not unmap and release uprobe trampoline page itself, - * because there's no easy way to make sure none of the threads - * is still inside the trampoline. - */ - hlist_del(&tramp->node); - kfree(tramp); -} - -void arch_uprobe_init_state(struct mm_struct *mm) -{ - INIT_HLIST_HEAD(&mm->uprobes_state.head_tramps); -} - -void arch_uprobe_clear_state(struct mm_struct *mm) -{ - struct uprobes_state *state = &mm->uprobes_state; - struct uprobe_trampoline *tramp; - struct hlist_node *n; + vaddr = find_nearest_trampoline(vaddr); + if (IS_ERR_VALUE(vaddr)) + return ERR_PTR(vaddr); - hlist_for_each_entry_safe(tramp, n, &state->head_tramps, node) - destroy_uprobe_trampoline(tramp); + *new_mapping = true; + return _install_special_mapping(mm, vaddr, PAGE_SIZE, + VM_READ|VM_EXEC|VM_MAYEXEC|VM_MAYREAD|VM_IO, + &tramp_mapping); } static bool __in_uprobe_trampoline(struct mm_struct *mm, unsigned long ip) @@ -810,7 +768,7 @@ SYSCALL_DEFINE0(uprobe) /* Allow execution only from uprobe trampolines. */ if (!in_uprobe_trampoline(regs->ip)) - return -ENXIO; + return -EPROTO; err = copy_from_user(&args, (void __user *)regs->sp, sizeof(args)); if (err) @@ -826,8 +784,8 @@ SYSCALL_DEFINE0(uprobe) regs->ax = args.ax; regs->r11 = args.r11; regs->cx = args.cx; - regs->ip = args.retaddr - 5; - regs->sp += sizeof(args); + regs->ip = args.retaddr - OPT_INSN_SIZE; + regs->sp += sizeof(args) + REDZONE_SIZE; regs->orig_ax = -1; sp = regs->sp; @@ -844,12 +802,12 @@ SYSCALL_DEFINE0(uprobe) */ if (regs->sp != sp) { /* skip the trampoline call */ - if (args.retaddr - 5 == regs->ip) - regs->ip += 5; + if (args.retaddr - OPT_INSN_SIZE == regs->ip) + regs->ip += OPT_INSN_SIZE; return regs->ax; } - regs->sp -= sizeof(args); + regs->sp -= sizeof(args) + REDZONE_SIZE; /* for the case uprobe_consumer has changed ax/r11/cx */ args.ax = regs->ax; @@ -857,7 +815,7 @@ SYSCALL_DEFINE0(uprobe) args.cx = regs->cx; /* keep return address unless we are instructed otherwise */ - if (args.retaddr - 5 != regs->ip) + if (args.retaddr - OPT_INSN_SIZE != regs->ip) args.retaddr = regs->ip; if (shstk_push(args.retaddr) == -EFAULT) @@ -891,7 +849,7 @@ asm ( "pop %rax\n" "pop %r11\n" "pop %rcx\n" - "ret\n" + "ret $" __stringify(REDZONE_SIZE) "\n" "int3\n" ".balign " __stringify(PAGE_SIZE) "\n" ".popsection\n" @@ -909,7 +867,8 @@ late_initcall(arch_uprobes_init); enum { EXPECT_SWBP, - EXPECT_CALL, + EXPECT_OPTIMIZED, + EXPECT_SWBP_OPTIMIZED, }; struct write_opcode_ctx { @@ -917,30 +876,29 @@ struct write_opcode_ctx { int expect; }; -static int is_call_insn(uprobe_opcode_t *insn) -{ - return *insn == CALL_INSN_OPCODE; -} - /* - * Verification callback used by int3_update uprobe_write calls to make sure - * the underlying instruction is as expected - either int3 or call. + * Verification callback used by uprobe_write calls to make sure the underlying + * instruction is in the expected stage of the INT3 update sequence. */ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t *new_opcode, int nbytes, void *data) { struct write_opcode_ctx *ctx = data; - uprobe_opcode_t old_opcode[5]; + uprobe_opcode_t old_opcode[OPT_INSN_SIZE]; - uprobe_copy_from_page(page, ctx->base, (uprobe_opcode_t *) &old_opcode, 5); + uprobe_copy_from_page(page, ctx->base, old_opcode, OPT_INSN_SIZE); switch (ctx->expect) { case EXPECT_SWBP: if (is_swbp_insn(&old_opcode[0])) return 1; break; - case EXPECT_CALL: - if (is_call_insn(&old_opcode[0])) + case EXPECT_OPTIMIZED: + if (is_opt_insns(&old_opcode[0])) + return 1; + break; + case EXPECT_SWBP_OPTIMIZED: + if (is_swbp_opt_insns(&old_opcode[0])) return 1; break; } @@ -949,48 +907,137 @@ static int verify_insn(struct page *page, unsigned long vaddr, uprobe_opcode_t * } /* - * Modify multi-byte instructions by using INT3 breakpoints on SMP. + * Modify the optimized instruction by using INT3 breakpoints on SMP. * We completely avoid using stop_machine() here, and achieve the * synchronization using INT3 breakpoints and SMP cross-calls. * (borrowed comment from smp_text_poke_batch_finish) * - * The way it is done: - * - Add an INT3 trap to the address that will be patched - * - SMP sync all CPUs - * - Update all but the first byte of the patched range - * - SMP sync all CPUs - * - Replace the first byte (INT3) by the first byte of the replacing opcode - * - SMP sync all CPUs + * For optimization (int3_update_optimize): + * 1) Start with the uprobe INT3 trap already installed + * 2) Update everything but the first byte + * 3) Replace the first INT3 by the first byte of the LEA instruction + * + * For unoptimization (int3_update_unoptimize): + * 1) Start with the optimized uprobe lea/call instructions + * 2) Add an INT3 trap to the address that will be patched + * 3) Restore the NOP bytes before the call opcode + * 4) Replace the first INT3 by the first byte of the NOP instruction + * + * Note that unoptimization deliberately keeps the call opcode and displacement + * in bytes 5..9. Those bytes become operands of the restored 10-byte NOP. + * + * Since there is only a single target uprobe-trampoline for the given nop10 + * instruction address, the CALL instruction will not be changed across + * unoptimization/optimization cycles. + * Therefore, any task that is preempted at the CALL instruction is guaranteed + * to observe that CALL and not anything else. */ -static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, - unsigned long vaddr, char *insn, bool optimize) +static int int3_update_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) { - uprobe_opcode_t int3 = UPROBE_SWBP_INSN; struct write_opcode_ctx ctx = { .base = vaddr, }; int err; /* - * Write int3 trap. + * 1) Initial state after set_swbp() installed the uprobe: + * cc 2e 0f 1f 84 00 00 00 00 00 * - * The swbp_optimize path comes with breakpoint already installed, - * so we can skip this step for optimize == true. + * After a previous unoptimization bytes 5..9 may still contain the + * old call instruction, which remains valid for threads already there. */ - if (!optimize) { - ctx.expect = EXPECT_CALL; - err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, - true /* is_register */, false /* do_update_ref_ctr */, - &ctx); - if (err) - return err; - } + smp_text_poke_sync_each_cpu(); + + /* + * 2) Rewrite the LEA tail and call displacement: + * cc [8d 64 24 80 e8 d0 d1 d2 d3] + */ + ctx.expect = EXPECT_SWBP; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + OPT_INSN_SIZE - 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; smp_text_poke_sync_each_cpu(); - /* Write all but the first byte of the patched range. */ + /* + * 3) Publish the first LEA byte: + * [48] 8d 64 24 80 e8 d0 d1 d2 d3 + * + * From offset 0 this is: + * lea -0x80(%rsp), %rsp + * call <uprobe-trampoline> + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + goto error; + + smp_text_poke_sync_each_cpu(); + return 0; + +error: + /* + * In all intermediate states byte 0 is INT3, so EXPECT_SWBP covers every + * case. Restore NOP bytes 1..4, but keep the valid CALL at bytes 5..9 + * for a thread that had already executed the LEA before a previous + * unoptimization. + */ ctx.expect = EXPECT_SWBP; - err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, 4, verify_insn, + uprobe_write(auprobe, vma, vaddr + 1, auprobe->insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true, false, &ctx); + smp_text_poke_sync_each_cpu(); + return err; +} + +static int int3_update_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, + unsigned long vaddr, uprobe_opcode_t *insn) +{ + uprobe_opcode_t int3 = UPROBE_SWBP_INSN; + struct write_opcode_ctx ctx = { + .base = vaddr, + .expect = EXPECT_OPTIMIZED, + }; + int err; + + /* + * Note the first two uprobe_write calls use is_register=true, because they + * are intermediate patching states while the probe is still active, so + * we force the exclusive anonymous page for the update. + * Also we use do_update_ref_ctr=false because refctr was already updated by + * the initial int3 install. + * + * The last uprobe_write to nop10 instruction is called with is_register=false + * and do_update_ref_ctr=true to trigger the refctr update and to instruct + * uprobe_write to zap the anonymous page if it now matches the file page. + * + * 1) Initial optimized state: + * 48 8d 64 24 80 e8 d0 d1 d2 d3 + * + * 2) Trap new entries before restoring the NOP bytes: + * [cc] 8d 64 24 80 e8 d0 d1 d2 d3 + */ + err = uprobe_write(auprobe, vma, vaddr, &int3, 1, verify_insn, + true /* is_register */, false /* do_update_ref_ctr */, + &ctx); + if (err) + return err; + + smp_text_poke_sync_each_cpu(); + + /* + * 3) Restore bytes 1..4 of the original NOP while keeping byte 0 trapped + * and byte 5 as CALL: + * cc [2e 0f 1f 84] e8 d0 d1 d2 d3 + */ + ctx.expect = EXPECT_SWBP_OPTIMIZED; + err = uprobe_write(auprobe, vma, vaddr + 1, insn + 1, + LEA_INSN_SIZE - 1, verify_insn, true /* is_register */, false /* do_update_ref_ctr */, &ctx); if (err) @@ -999,13 +1046,16 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, smp_text_poke_sync_each_cpu(); /* - * Write first byte. + * 4) Publish the first byte of the original NOP: + * [66] 2e 0f 1f 84 e8 d0 d1 d2 d3 * - * The swbp_unoptimize needs to finish uprobe removal together - * with ref_ctr update, using uprobe_write with proper flags. + * From offset 0 this is the restored 10-byte NOP; the CALL opcode and + * displacement are now only NOP operands. Offset 5 still decodes as + * CALL for a thread that was already there. */ + ctx.expect = EXPECT_SWBP; err = uprobe_write(auprobe, vma, vaddr, insn, 1, verify_insn, - optimize /* is_register */, !optimize /* do_update_ref_ctr */, + false /* is_register */, true /* do_update_ref_ctr */, &ctx); if (err) return err; @@ -1017,17 +1067,25 @@ static int int3_update(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int swbp_optimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr, unsigned long tramp) { - u8 call[5]; + u8 insn[OPT_INSN_SIZE], *call = &insn[LEA_INSN_SIZE]; - __text_gen_insn(call, CALL_INSN_OPCODE, (const void *) vaddr, + /* + * We have nop10 instruction (with first byte overwritten to int3), + * changing it to: + * lea -0x80(%rsp), %rsp + * call tramp + */ + memcpy(insn, lea_rsp, LEA_INSN_SIZE); + __text_gen_insn(call, CALL_INSN_OPCODE, + (const void *) (vaddr + LEA_INSN_SIZE), (const void *) tramp, CALL_INSN_SIZE); - return int3_update(auprobe, vma, vaddr, call, true /* optimize */); + return int3_update_optimize(auprobe, vma, vaddr, insn); } static int swbp_unoptimize(struct arch_uprobe *auprobe, struct vm_area_struct *vma, unsigned long vaddr) { - return int3_update(auprobe, vma, vaddr, auprobe->insn, false /* optimize */); + return int3_update_unoptimize(auprobe, vma, vaddr, auprobe->insn); } static int copy_from_vaddr(struct mm_struct *mm, unsigned long vaddr, void *dst, int len) @@ -1049,19 +1107,19 @@ static bool __is_optimized(struct mm_struct *mm, uprobe_opcode_t *insn, unsigned struct __packed __arch_relative_insn { u8 op; s32 raddr; - } *call = (struct __arch_relative_insn *) insn; + } *call = (struct __arch_relative_insn *)(insn + LEA_INSN_SIZE); - if (!is_call_insn(insn)) + if (!is_opt_insns(insn)) return false; - return __in_uprobe_trampoline(mm, vaddr + 5 + call->raddr); + return __in_uprobe_trampoline(mm, vaddr + OPT_INSN_SIZE + call->raddr); } static int is_optimized(struct mm_struct *mm, unsigned long vaddr) { - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; int err; - err = copy_from_vaddr(mm, vaddr, &insn, 5); + err = copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE); if (err) return err; return __is_optimized(mm, (uprobe_opcode_t *)&insn, vaddr); @@ -1111,27 +1169,29 @@ int set_orig_insn(struct arch_uprobe *auprobe, struct vm_area_struct *vma, static int __arch_uprobe_optimize(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long vaddr) { - struct uprobe_trampoline *tramp; - struct vm_area_struct *vma; - bool new = false; - int err = 0; + struct pt_regs *regs = task_pt_regs(current); + struct vm_area_struct *vma, *tramp; + bool new_mapping; + int ret; + if (!user_64bit_mode(regs)) + return -EINVAL; vma = find_vma(mm, vaddr); if (!vma) return -EINVAL; - tramp = get_uprobe_trampoline(vaddr, &new); - if (!tramp) - return -EINVAL; - err = swbp_optimize(auprobe, vma, vaddr, tramp->vaddr); - if (WARN_ON_ONCE(err) && new) - destroy_uprobe_trampoline(tramp); - return err; + tramp = get_uprobe_trampoline(mm, vaddr, &new_mapping); + if (IS_ERR(tramp)) + return PTR_ERR(tramp); + ret = swbp_optimize(auprobe, vma, vaddr, tramp->vm_start); + if (WARN_ON_ONCE(ret) && new_mapping) + WARN_ON_ONCE(do_munmap(mm, tramp->vm_start, PAGE_SIZE, NULL)); + return ret; } void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) { struct mm_struct *mm = current->mm; - uprobe_opcode_t insn[5]; + uprobe_opcode_t insn[OPT_INSN_SIZE]; if (!should_optimize(auprobe)) return; @@ -1142,7 +1202,7 @@ void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr) * Check if some other thread already optimized the uprobe for us, * if it's the case just go away silently. */ - if (copy_from_vaddr(mm, vaddr, &insn, 5)) + if (copy_from_vaddr(mm, vaddr, &insn, OPT_INSN_SIZE)) goto unlock; if (!is_swbp_insn((uprobe_opcode_t*) &insn)) goto unlock; @@ -1158,16 +1218,32 @@ unlock: mmap_write_unlock(mm); } +static bool is_optimizable_nop10(struct insn *insn) +{ + static const u8 nop10_prefix[] = { + 0x66, 0x2e, 0x0f, 0x1f, 0x84 + }; + + /* + * Restrict this to the 10-byte NOP form whose last 5 bytes are + * SIB/displacement operands. Unoptimization keeps the call opcode and + * displacement in those bytes, so other NOP encodings are not safe. + */ + return insn->length == OPT_INSN_SIZE && + insn_is_nop(insn) && + !memcmp(insn->kaddr, nop10_prefix, ARRAY_SIZE(nop10_prefix)); +} + static bool can_optimize(struct insn *insn, unsigned long vaddr) { - if (!insn->x86_64 || insn->length != 5) + if (!insn->x86_64) return false; - if (!insn_is_nop(insn)) + if (!is_optimizable_nop10(insn)) return false; /* We can't do cross page atomic writes yet. */ - return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= 5; + return PAGE_SIZE - (vaddr & ~PAGE_MASK) >= OPT_INSN_SIZE; } #else /* 32-bit: */ /* @@ -1549,16 +1625,26 @@ static int push_setup_xol_ops(struct arch_uprobe *auprobe, struct insn *insn) */ int arch_uprobe_analyze_insn(struct arch_uprobe *auprobe, struct mm_struct *mm, unsigned long addr) { + enum insn_mode m = is_64bit_mm(mm) ? INSN_MODE_64 : INSN_MODE_32; u8 fix_ip_or_call = UPROBE_FIX_IP; struct insn insn; int ret; - ret = uprobe_init_insn(auprobe, &insn, is_64bit_mm(mm)); - if (ret) - return ret; + ret = insn_decode(&insn, auprobe->insn, sizeof(auprobe->insn), m); + if (ret < 0) + return -ENOEXEC; - if (can_optimize(&insn, addr)) + /* + * No need to check instruction in uprobe_init_insn in case we + * are on top of optimizable nop10. + */ + if (can_optimize(&insn, addr)) { set_bit(ARCH_UPROBE_FLAG_CAN_OPTIMIZE, &auprobe->flags); + } else { + ret = uprobe_init_insn(auprobe, &insn); + if (ret) + return ret; + } ret = branch_setup_xol_ops(auprobe, &insn); if (ret != -ENOSYS) diff --git a/include/linux/srcu.h b/include/linux/srcu.h index a54ce9e808b9..72c86d3b23f2 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -638,4 +638,11 @@ DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct, DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) #define class_srcu_fast_notrace_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, _T) +DEFINE_LOCK_GUARD_1(srcu_fast_updown, struct srcu_struct, + _T->scp = srcu_read_lock_fast_updown(_T->lock), + srcu_read_unlock_fast_updown(_T->lock, _T->scp), + struct srcu_ctr __percpu *scp) +DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_updown, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) +#define class_srcu_fast_updown_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_updown, _T) + #endif diff --git a/include/linux/uprobes.h b/include/linux/uprobes.h index f548fea2adec..d34dbc0fbbfe 100644 --- a/include/linux/uprobes.h +++ b/include/linux/uprobes.h @@ -25,6 +25,7 @@ struct mm_struct; struct inode; struct notifier_block; struct page; +struct srcu_ctr; /* * Allowed return values from uprobe consumer's handler callback @@ -106,7 +107,7 @@ enum hprobe_state { * underlying uprobe is not guaranteed anymore. __UPROBE_DEAD is just an * internal marker and is handled transparently by hprobe_fetch() helper. * - * When uprobe is SRCU-protected, we also record srcu_idx value, necessary for + * When uprobe is SRCU-protected, we also record srcu_scp value, necessary for * SRCU unlocking. * * See hprobe_expire() and hprobe_fetch() for details of race-free uprobe @@ -115,7 +116,7 @@ enum hprobe_state { */ struct hprobe { enum hprobe_state state; - int srcu_idx; + struct srcu_ctr __percpu *srcu_scp; struct uprobe *uprobe; }; @@ -186,9 +187,6 @@ struct xol_area; struct uprobes_state { struct xol_area *xol_area; -#ifdef CONFIG_X86_64 - struct hlist_head head_tramps; -#endif }; typedef int (*uprobe_write_verify_t)(struct page *page, unsigned long vaddr, @@ -238,8 +236,6 @@ extern void uprobe_handle_trampoline(struct pt_regs *regs); extern void *arch_uretprobe_trampoline(unsigned long *psize); extern unsigned long uprobe_get_trampoline_vaddr(void); extern void uprobe_copy_from_page(struct page *page, unsigned long vaddr, void *dst, int len); -extern void arch_uprobe_clear_state(struct mm_struct *mm); -extern void arch_uprobe_init_state(struct mm_struct *mm); extern void handle_syscall_uprobe(struct pt_regs *regs, unsigned long bp_vaddr); extern void arch_uprobe_optimize(struct arch_uprobe *auprobe, unsigned long vaddr); extern unsigned long arch_uprobe_get_xol_area(void); diff --git a/kernel/events/core.c b/kernel/events/core.c index 1a73ba0747df..2eee83cdb43d 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -7800,10 +7800,20 @@ unsigned long perf_misc_flags(struct perf_event *event, unsigned long perf_instruction_pointer(struct perf_event *event, struct pt_regs *regs) { - if (should_sample_guest(event)) - return perf_guest_get_ip(); + /* + * Hardware skid can lead to a scenario where a PMI is + * delivered after the CPU has already entered kernel mode. + * In that case, user-space sampling must not expose kernel + * register state. + */ + if (should_sample_guest(event)) { + return event->attr.exclude_kernel && + !(perf_guest_state() & PERF_GUEST_USER) ? + 0 : perf_guest_get_ip(); + } - return perf_arch_instruction_pointer(regs); + return event->attr.exclude_kernel && !user_mode(regs) ? + 0 : perf_arch_instruction_pointer(regs); } static void @@ -7837,10 +7847,22 @@ static void perf_sample_regs_user(struct perf_regs *regs_user, } static void perf_sample_regs_intr(struct perf_regs *regs_intr, - struct pt_regs *regs) + struct pt_regs *regs, + bool exclude_kernel) { - regs_intr->regs = regs; - regs_intr->abi = perf_reg_abi(current); + /* + * Hardware skid can lead to a scenario where a PMI is + * delivered after the CPU has already entered kernel mode. + * In that case, user-space sampling must not expose kernel + * register state. + */ + if (exclude_kernel && !user_mode(regs)) { + regs_intr->abi = PERF_SAMPLE_REGS_ABI_NONE; + regs_intr->regs = NULL; + } else { + regs_intr->regs = regs; + regs_intr->abi = perf_reg_abi(current); + } } @@ -8731,7 +8753,8 @@ void perf_prepare_sample(struct perf_sample_data *data, /* regs dump ABI info */ int size = sizeof(u64); - perf_sample_regs_intr(&data->regs_intr, regs); + perf_sample_regs_intr(&data->regs_intr, regs, + event->attr.exclude_kernel); if (data->regs_intr.regs) { u64 mask = event->attr.sample_regs_intr; @@ -13918,7 +13941,9 @@ SYSCALL_DEFINE5(perf_event_open, if (err) return err; - if (!attr.exclude_kernel) { + if (!attr.exclude_kernel || + ((attr.sample_type & PERF_SAMPLE_CALLCHAIN) && + !attr.exclude_callchain_kernel)) { err = perf_allow_kernel(); if (err) return err; diff --git a/kernel/events/uprobes.c b/kernel/events/uprobes.c index 6300b216012c..a18529ab2b87 100644 --- a/kernel/events/uprobes.c +++ b/kernel/events/uprobes.c @@ -54,7 +54,7 @@ static struct mutex uprobes_mmap_mutex[UPROBES_HASH_SZ]; DEFINE_STATIC_PERCPU_RWSEM(dup_mmap_sem); /* Covers return_instance's uprobe lifetime. */ -DEFINE_STATIC_SRCU(uretprobes_srcu); +DEFINE_STATIC_SRCU_FAST_UPDOWN(uretprobes_srcu); /* Have a copy of original instruction */ #define UPROBE_COPY_INSN 0 @@ -707,12 +707,13 @@ static void put_uprobe(struct uprobe *uprobe) } /* Initialize hprobe as SRCU-protected "leased" uprobe */ -static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe, int srcu_idx) +static void hprobe_init_leased(struct hprobe *hprobe, struct uprobe *uprobe, + struct srcu_ctr __percpu *srcu_scp) { WARN_ON(!uprobe); hprobe->state = HPROBE_LEASED; hprobe->uprobe = uprobe; - hprobe->srcu_idx = srcu_idx; + hprobe->srcu_scp = srcu_scp; } /* Initialize hprobe as refcounted ("stable") uprobe (uprobe can be NULL). */ @@ -720,7 +721,7 @@ static void hprobe_init_stable(struct hprobe *hprobe, struct uprobe *uprobe) { hprobe->state = uprobe ? HPROBE_STABLE : HPROBE_GONE; hprobe->uprobe = uprobe; - hprobe->srcu_idx = -1; + hprobe->srcu_scp = NULL; } /* @@ -757,7 +758,7 @@ static void hprobe_finalize(struct hprobe *hprobe, enum hprobe_state hstate) { switch (hstate) { case HPROBE_LEASED: - __srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx); + srcu_up_read_fast(&uretprobes_srcu, hprobe->srcu_scp); break; case HPROBE_STABLE: put_uprobe(hprobe->uprobe); @@ -829,7 +830,7 @@ static struct uprobe *hprobe_expire(struct hprobe *hprobe, bool get) */ if (try_cmpxchg(&hprobe->state, &hstate, uprobe ? HPROBE_STABLE : HPROBE_GONE)) { /* We won the race, we are the ones to unlock SRCU */ - __srcu_read_unlock(&uretprobes_srcu, hprobe->srcu_idx); + srcu_up_read_fast(&uretprobes_srcu, hprobe->srcu_scp); return get && uprobe ? get_uprobe(uprobe) : uprobe; } @@ -1806,14 +1807,6 @@ static struct xol_area *get_xol_area(void) return area; } -void __weak arch_uprobe_clear_state(struct mm_struct *mm) -{ -} - -void __weak arch_uprobe_init_state(struct mm_struct *mm) -{ -} - /* * uprobe_clear_state - Free the area allocated for slots. */ @@ -1825,8 +1818,6 @@ void uprobe_clear_state(struct mm_struct *mm) delayed_uprobe_remove(NULL, mm); mutex_unlock(&delayed_uprobe_lock); - arch_uprobe_clear_state(mm); - if (!area) return; @@ -2045,7 +2036,7 @@ static void ri_timer(struct timer_list *timer) struct return_instance *ri; /* SRCU protects uprobe from reuse for the cmpxchg() inside hprobe_expire(). */ - guard(srcu)(&uretprobes_srcu); + guard(srcu_fast_updown)(&uretprobes_srcu); /* RCU protects return_instance from freeing. */ guard(rcu)(); @@ -2142,7 +2133,7 @@ static int dup_utask(struct task_struct *t, struct uprobe_task *o_utask) t->utask = n_utask; /* protect uprobes from freeing, we'll need try_get_uprobe() them */ - guard(srcu)(&uretprobes_srcu); + guard(srcu_fast_updown)(&uretprobes_srcu); p = &n_utask->return_instances; for (o = o_utask->return_instances; o; o = o->next) { @@ -2254,8 +2245,8 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs, { struct uprobe_task *utask = current->utask; unsigned long orig_ret_vaddr, trampoline_vaddr; + struct srcu_ctr __percpu *srcu_scp; bool chained; - int srcu_idx; if (!get_xol_area()) goto free; @@ -2293,8 +2284,12 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs, orig_ret_vaddr = utask->return_instances->orig_ret_vaddr; } - /* __srcu_read_lock() because SRCU lock survives switch to user space */ - srcu_idx = __srcu_read_lock(&uretprobes_srcu); + /* + * Use srcu_down_read_fast() because the SRCU lock survives a switch to + * user space and can be unlocked from a different context by ri_timer() + * or dup_utask(). + */ + srcu_scp = srcu_down_read_fast(&uretprobes_srcu); ri->func = instruction_pointer(regs); ri->stack = user_stack_pointer(regs); @@ -2303,7 +2298,7 @@ static void prepare_uretprobe(struct uprobe *uprobe, struct pt_regs *regs, utask->depth++; - hprobe_init_leased(&ri->hprobe, uprobe, srcu_idx); + hprobe_init_leased(&ri->hprobe, uprobe, srcu_scp); ri->next = utask->return_instances; rcu_assign_pointer(utask->return_instances, ri); diff --git a/kernel/fork.c b/kernel/fork.c index 94e021eabf1d..175c73bbe2bf 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -1069,7 +1069,6 @@ static void mm_init_uprobes_state(struct mm_struct *mm) { #ifdef CONFIG_UPROBES mm->uprobes_state.xol_area = NULL; - arch_uprobe_init_state(mm); #endif } diff --git a/tools/lib/bpf/features.c b/tools/lib/bpf/features.c index b7e388f99d0b..e5641fa60163 100644 --- a/tools/lib/bpf/features.c +++ b/tools/lib/bpf/features.c @@ -577,10 +577,10 @@ static int probe_ldimm64_full_range_off(int token_fd) static int probe_uprobe_syscall(int token_fd) { /* - * If kernel supports uprobe() syscall, it will return -ENXIO when called + * If kernel supports uprobe() syscall, it will return -EPROTO when called * from the outside of a kernel-generated uprobe trampoline. */ - return syscall(__NR_uprobe) < 0 && errno == ENXIO; + return syscall(__NR_uprobe) < 0 && errno == EPROTO; } #else static int probe_uprobe_syscall(int token_fd) diff --git a/tools/lib/bpf/usdt.c b/tools/lib/bpf/usdt.c index 57fb82bb81b5..d2ecd3daab96 100644 --- a/tools/lib/bpf/usdt.c +++ b/tools/lib/bpf/usdt.c @@ -305,7 +305,7 @@ struct usdt_manager *usdt_manager_new(struct bpf_object *obj) /* * Detect kernel support for uprobe() syscall, it's presence means we can - * take advantage of faster nop5 uprobe handling. + * take advantage of faster nop10 uprobe handling. * Added in: 56101b69c919 ("uprobes/x86: Add uprobe syscall to speed up uprobe") */ man->has_uprobe_syscall = kernel_supports(obj, FEAT_UPROBE_SYSCALL); @@ -604,14 +604,14 @@ static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, #if defined(__x86_64__) static bool has_nop_combo(int fd, long off) { - unsigned char nop_combo[6] = { - 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 /* nop,nop5 */ + unsigned char nop_combo[11] = { + 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, }; - unsigned char buf[6]; + unsigned char buf[11]; - if (pread(fd, buf, 6, off) != 6) + if (pread(fd, buf, 11, off) != 11) return false; - return memcmp(buf, nop_combo, 6) == 0; + return memcmp(buf, nop_combo, 11) == 0; } #else static bool has_nop_combo(int fd, long off) @@ -822,8 +822,8 @@ static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, memset(target, 0, sizeof(*target)); /* - * We have uprobe syscall and usdt with nop,nop5 instructions combo, - * so we can place the uprobe directly on nop5 (+1) and get this probe + * We have uprobe syscall and usdt with nop,nop10 instructions combo, + * so we can place the uprobe directly on nop10 (+1) and get this probe * optimized. */ if (man->has_uprobe_syscall && has_nop_combo(elf_fd->fd, usdt_rel_ip)) { diff --git a/tools/testing/selftests/bpf/bench.c b/tools/testing/selftests/bpf/bench.c index 3d9d2cd7764b..c4a3a6b3eb83 100644 --- a/tools/testing/selftests/bpf/bench.c +++ b/tools/testing/selftests/bpf/bench.c @@ -539,12 +539,12 @@ extern const struct bench bench_trig_uretprobe_multi_push; extern const struct bench bench_trig_uprobe_multi_ret; extern const struct bench bench_trig_uretprobe_multi_ret; #ifdef __x86_64__ -extern const struct bench bench_trig_uprobe_nop5; -extern const struct bench bench_trig_uretprobe_nop5; -extern const struct bench bench_trig_uprobe_multi_nop5; -extern const struct bench bench_trig_uretprobe_multi_nop5; +extern const struct bench bench_trig_uprobe_nop10; +extern const struct bench bench_trig_uretprobe_nop10; +extern const struct bench bench_trig_uprobe_multi_nop10; +extern const struct bench bench_trig_uretprobe_multi_nop10; extern const struct bench bench_trig_usdt_nop; -extern const struct bench bench_trig_usdt_nop5; +extern const struct bench bench_trig_usdt_nop10; #endif extern const struct bench bench_rb_libbpf; @@ -622,12 +622,12 @@ static const struct bench *benchs[] = { &bench_trig_uprobe_multi_ret, &bench_trig_uretprobe_multi_ret, #ifdef __x86_64__ - &bench_trig_uprobe_nop5, - &bench_trig_uretprobe_nop5, - &bench_trig_uprobe_multi_nop5, - &bench_trig_uretprobe_multi_nop5, + &bench_trig_uprobe_nop10, + &bench_trig_uretprobe_nop10, + &bench_trig_uprobe_multi_nop10, + &bench_trig_uretprobe_multi_nop10, &bench_trig_usdt_nop, - &bench_trig_usdt_nop5, + &bench_trig_usdt_nop10, #endif /* ringbuf/perfbuf benchmarks */ &bench_rb_libbpf, diff --git a/tools/testing/selftests/bpf/benchs/bench_trigger.c b/tools/testing/selftests/bpf/benchs/bench_trigger.c index 2f22ec61667b..61513efc167a 100644 --- a/tools/testing/selftests/bpf/benchs/bench_trigger.c +++ b/tools/testing/selftests/bpf/benchs/bench_trigger.c @@ -396,15 +396,15 @@ static void *uprobe_producer_ret(void *input) } #ifdef __x86_64__ -__nocf_check __weak void uprobe_target_nop5(void) +__nocf_check __weak void uprobe_target_nop10(void) { - asm volatile (".byte 0x0f, 0x1f, 0x44, 0x00, 0x00"); + asm volatile (".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00"); } -static void *uprobe_producer_nop5(void *input) +static void *uprobe_producer_nop10(void *input) { while (true) - uprobe_target_nop5(); + uprobe_target_nop10(); return NULL; } @@ -418,7 +418,7 @@ static void *uprobe_producer_usdt_nop(void *input) return NULL; } -static void *uprobe_producer_usdt_nop5(void *input) +static void *uprobe_producer_usdt_nop10(void *input) { while (true) usdt_2(); @@ -542,24 +542,24 @@ static void uretprobe_multi_ret_setup(void) } #ifdef __x86_64__ -static void uprobe_nop5_setup(void) +static void uprobe_nop10_setup(void) { - usetup(false, false /* !use_multi */, &uprobe_target_nop5); + usetup(false, false /* !use_multi */, &uprobe_target_nop10); } -static void uretprobe_nop5_setup(void) +static void uretprobe_nop10_setup(void) { - usetup(true, false /* !use_multi */, &uprobe_target_nop5); + usetup(true, false /* !use_multi */, &uprobe_target_nop10); } -static void uprobe_multi_nop5_setup(void) +static void uprobe_multi_nop10_setup(void) { - usetup(false, true /* use_multi */, &uprobe_target_nop5); + usetup(false, true /* use_multi */, &uprobe_target_nop10); } -static void uretprobe_multi_nop5_setup(void) +static void uretprobe_multi_nop10_setup(void) { - usetup(true, true /* use_multi */, &uprobe_target_nop5); + usetup(true, true /* use_multi */, &uprobe_target_nop10); } static void usdt_setup(const char *name) @@ -598,7 +598,7 @@ static void usdt_nop_setup(void) usdt_setup("usdt_1"); } -static void usdt_nop5_setup(void) +static void usdt_nop10_setup(void) { usdt_setup("usdt_2"); } @@ -665,10 +665,10 @@ BENCH_TRIG_USERMODE(uretprobe_multi_nop, nop, "uretprobe-multi-nop"); BENCH_TRIG_USERMODE(uretprobe_multi_push, push, "uretprobe-multi-push"); BENCH_TRIG_USERMODE(uretprobe_multi_ret, ret, "uretprobe-multi-ret"); #ifdef __x86_64__ -BENCH_TRIG_USERMODE(uprobe_nop5, nop5, "uprobe-nop5"); -BENCH_TRIG_USERMODE(uretprobe_nop5, nop5, "uretprobe-nop5"); -BENCH_TRIG_USERMODE(uprobe_multi_nop5, nop5, "uprobe-multi-nop5"); -BENCH_TRIG_USERMODE(uretprobe_multi_nop5, nop5, "uretprobe-multi-nop5"); +BENCH_TRIG_USERMODE(uprobe_nop10, nop10, "uprobe-nop10"); +BENCH_TRIG_USERMODE(uretprobe_nop10, nop10, "uretprobe-nop10"); +BENCH_TRIG_USERMODE(uprobe_multi_nop10, nop10, "uprobe-multi-nop10"); +BENCH_TRIG_USERMODE(uretprobe_multi_nop10, nop10, "uretprobe-multi-nop10"); BENCH_TRIG_USERMODE(usdt_nop, usdt_nop, "usdt-nop"); -BENCH_TRIG_USERMODE(usdt_nop5, usdt_nop5, "usdt-nop5"); +BENCH_TRIG_USERMODE(usdt_nop10, usdt_nop10, "usdt-nop10"); #endif diff --git a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh index 9ec59423b949..e490b337e960 100755 --- a/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh +++ b/tools/testing/selftests/bpf/benchs/run_bench_uprobes.sh @@ -2,7 +2,7 @@ set -eufo pipefail -for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop5} usdt-nop usdt-nop5 +for i in usermode-count syscall-count {uprobe,uretprobe}-{nop,push,ret,nop10} usdt-nop usdt-nop10 do summary=$(sudo ./bench -w2 -d5 -a trig-$i | tail -n1 | cut -d'(' -f1 | cut -d' ' -f3-) printf "%-15s: %s\n" $i "$summary" diff --git a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c index 955a37751b52..e193206fc5d2 100644 --- a/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c +++ b/tools/testing/selftests/bpf/prog_tests/uprobe_syscall.c @@ -4,6 +4,8 @@ #ifdef __x86_64__ +#define _GNU_SOURCE +#include <sched.h> #include <unistd.h> #include <asm/ptrace.h> #include <linux/compiler.h> @@ -13,11 +15,12 @@ #include <sys/syscall.h> #include <sys/prctl.h> #include <asm/prctl.h> +#include <stdnoreturn.h> #include "uprobe_syscall.skel.h" #include "uprobe_syscall_executed.skel.h" #include "bpf/libbpf_internal.h" -#define USDT_NOP .byte 0x0f, 0x1f, 0x44, 0x00, 0x00 +#define USDT_NOP .byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 #include "usdt.h" #pragma GCC diagnostic ignored "-Wattributes" @@ -26,7 +29,7 @@ __attribute__((aligned(16))) __nocf_check __weak __naked unsigned long uprobe_regs_trigger(void) { asm volatile ( - ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00\n" /* nop5 */ + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ "movq $0xdeadbeef, %rax\n" "ret\n" ); @@ -345,9 +348,9 @@ cleanup: __attribute__((aligned(16))) __nocf_check __weak __naked void uprobe_test(void) { - asm volatile (" \n" - ".byte 0x0f, 0x1f, 0x44, 0x00, 0x00 \n" - "ret \n" + asm volatile ( + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ + "ret\n" ); } @@ -357,6 +360,50 @@ __nocf_check __weak void usdt_test(void) USDT(optimized_uprobe, usdt); } +/* + * Assembly-level red zone clobbering test. Stores known values in the + * red zone (below RSP), executes a nop10 (uprobe site), and checks that + * the values survived. Returns 0 if intact, 1 if clobbered. + * + * The nop5 optimization used CALL (which pushes a return address to + * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization + * should escape that by moving stackpointer below the redzone before + * doing the CALL. + * + * Align the code at 64 bytes, to make sure nop10 is not on page boundary. + */ +__attribute__((aligned(64))) +__nocf_check __weak __naked unsigned long uprobe_red_zone_test(void) +{ + asm volatile ( + "movabs $0x1111111111111111, %%rax\n" + "movq %%rax, -8(%%rsp)\n" + "movabs $0x2222222222222222, %%rax\n" + "movq %%rax, -16(%%rsp)\n" + "movabs $0x3333333333333333, %%rax\n" + "movq %%rax, -24(%%rsp)\n" + + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10: uprobe site */ + + "movabs $0x1111111111111111, %%rax\n" + "cmpq %%rax, -8(%%rsp)\n" + "jne 1f\n" + "movabs $0x2222222222222222, %%rax\n" + "cmpq %%rax, -16(%%rsp)\n" + "jne 1f\n" + "movabs $0x3333333333333333, %%rax\n" + "cmpq %%rax, -24(%%rsp)\n" + "jne 1f\n" + + "xorl %%eax, %%eax\n" + "retq\n" + "1:\n" + "movl $1, %%eax\n" + "retq\n" + ::: "rax", "memory" + ); +} + static int find_uprobes_trampoline(void *tramp_addr) { void *start, *end; @@ -388,14 +435,15 @@ static int find_uprobes_trampoline(void *tramp_addr) return ret; } -static unsigned char nop5[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 }; +static unsigned char nop10[10] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; +static unsigned char lea_rsp[5] = { 0x48, 0x8d, 0x64, 0x24, 0x80 }; -static void *find_nop5(void *fn) +static void *find_nop10(void *fn) { int i; - for (i = 0; i < 10; i++) { - if (!memcmp(nop5, fn + i, 5)) + for (i = 0; i < 128; i++) { + if (!memcmp(nop10, fn + i, 10)) return fn + i; } return NULL; @@ -403,6 +451,16 @@ static void *find_nop5(void *fn) typedef void (__attribute__((nocf_check)) *trigger_t)(void); +static void check_attach_notrigger(struct uprobe_syscall_executed *skel, + void *addr, int executed) +{ + unsigned char *op = addr; + + /* Make sure bpf program was not executed. */ + ASSERT_EQ(skel->bss->executed, executed, "executed"); + ASSERT_EQ(*op, 0xcc, "int3"); +} + static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigger, void *addr, int executed) { @@ -420,7 +478,8 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge ASSERT_EQ(skel->bss->executed, executed, "executed"); /* .. and check the trampoline is as expected. */ - call = (struct __arch_relative_insn *) addr; + ASSERT_OK(memcmp(addr, lea_rsp, 5), "lea_rsp"); + call = (struct __arch_relative_insn *)(addr + 5); tramp = (void *) (call + 1) + call->raddr; ASSERT_EQ(call->op, 0xe8, "call"); ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); @@ -428,21 +487,26 @@ static void *check_attach(struct uprobe_syscall_executed *skel, trigger_t trigge return tramp; } -static void check_detach(void *addr, void *tramp) +static bool check_detach(void *addr, void *tramp) { + static const unsigned char nop10_prefix[] = { 0x66, 0x2e, 0x0f, 0x1f, 0x84 }; + bool ok = true; + /* [uprobes_trampoline] stays after detach */ - ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); - ASSERT_OK(memcmp(addr, nop5, 5), "nop5"); + ok &= ASSERT_OK(find_uprobes_trampoline(tramp), "uprobes_trampoline"); + ok &= ASSERT_OK(memcmp(addr, nop10_prefix, 5), "nop10_prefix"); + return ok; } -static void check(struct uprobe_syscall_executed *skel, struct bpf_link *link, - trigger_t trigger, void *addr, int executed) +static void *check(struct uprobe_syscall_executed *skel, struct bpf_link *link, + trigger_t trigger, void *addr, int executed) { void *tramp; tramp = check_attach(skel, trigger, addr, executed); bpf_link__destroy(link); check_detach(addr, tramp); + return tramp; } static void test_uprobe_legacy(void) @@ -453,6 +517,7 @@ static void test_uprobe_legacy(void) ); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -470,7 +535,30 @@ static void test_uprobe_legacy(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 2); + tramp = check(skel, link, uprobe_test, uprobe_test, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 2); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_opts")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 4); /* uretprobe */ skel->bss->executed = 0; @@ -492,6 +580,7 @@ static void test_uprobe_multi(void) LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -512,7 +601,30 @@ static void test_uprobe_multi(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 2); + tramp = check(skel, link, uprobe_test, uprobe_test, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 2); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_multi, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 4); /* uretprobe.multi */ skel->bss->executed = 0; @@ -536,6 +648,7 @@ static void test_uprobe_session(void) ); struct bpf_link *link; unsigned long offset; + void *tramp; offset = get_uprobe_offset(&uprobe_test); if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) @@ -555,7 +668,30 @@ static void test_uprobe_session(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) goto cleanup; - check(skel, link, uprobe_test, uprobe_test, 4); + tramp = check(skel, link, uprobe_test, uprobe_test, 4); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check_attach_notrigger(skel, uprobe_test, 4); + + bpf_link__destroy(link); + if (!check_detach(uprobe_test, tramp)) + goto cleanup; + + uprobe_test(); + ASSERT_EQ(skel->bss->executed, 4, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_uprobe_multi(skel->progs.test_uprobe_session, + 0, "/proc/self/exe", NULL, &opts); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_uprobe_multi")) + goto cleanup; + + check(skel, link, uprobe_test, uprobe_test, 8); cleanup: uprobe_syscall_executed__destroy(skel); @@ -565,11 +701,11 @@ static void test_uprobe_usdt(void) { struct uprobe_syscall_executed *skel; struct bpf_link *link; - void *addr; + void *addr, *tramp; errno = 0; - addr = find_nop5(usdt_test); - if (!ASSERT_OK_PTR(addr, "find_nop5")) + addr = find_nop10(usdt_test); + if (!ASSERT_OK_PTR(addr, "find_nop10")) return; skel = uprobe_syscall_executed__open_and_load(); @@ -584,7 +720,32 @@ static void test_uprobe_usdt(void) if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) goto cleanup; - check(skel, link, usdt_test, addr, 2); + tramp = check(skel, link, usdt_test, addr, 2); + + /* reattach and detach without triggering optimization */ + link = bpf_program__attach_usdt(skel->progs.test_usdt, + -1 /* all PIDs */, "/proc/self/exe", + "optimized_uprobe", "usdt", NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) + goto cleanup; + + check_attach_notrigger(skel, addr, 2); + + bpf_link__destroy(link); + if (!check_detach(addr, tramp)) + goto cleanup; + + usdt_test(); + ASSERT_EQ(skel->bss->executed, 2, "executed_no_probe"); + + /* reattach with triggering optimization */ + link = bpf_program__attach_usdt(skel->progs.test_usdt, + -1 /* all PIDs */, "/proc/self/exe", + "optimized_uprobe", "usdt", NULL); + if (!ASSERT_OK_PTR(link, "bpf_program__attach_usdt")) + goto cleanup; + + check(skel, link, usdt_test, addr, 4); cleanup: uprobe_syscall_executed__destroy(skel); @@ -757,12 +918,124 @@ cleanup: #define __NR_uprobe 336 #endif +static void test_uprobe_red_zone(void) +{ + struct uprobe_syscall_executed *skel; + struct bpf_link *link; + void *nop10_addr; + size_t offset; + int i; + + nop10_addr = find_nop10(uprobe_red_zone_test); + if (!ASSERT_NEQ(nop10_addr, NULL, "find_nop10")) + return; + + skel = uprobe_syscall_executed__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + offset = get_uprobe_offset(nop10_addr); + link = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + 0, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(link, "attach_uprobe")) + goto cleanup; + + for (i = 0; i < 10; i++) + ASSERT_EQ(uprobe_red_zone_test(), 0, "red_zone_intact"); + + bpf_link__destroy(link); + +cleanup: + uprobe_syscall_executed__destroy(skel); +} + static void test_uprobe_error(void) { long err = syscall(__NR_uprobe); ASSERT_EQ(err, -1, "error"); - ASSERT_EQ(errno, ENXIO, "errno"); + ASSERT_EQ(errno, EPROTO, "errno"); +} + +__attribute__((aligned(16))) +__nocf_check __weak __naked void uprobe_fork_test(void) +{ + asm volatile ( + ".byte 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00\n" /* nop10 */ + "ret\n" + ); +} + +static noreturn int child_func(void *arg) +{ + struct uprobe_syscall_executed *skel = arg; + + /* Make sure the child's probe is still there and optimized.. */ + if (memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp))) + _exit(1); + + skel->bss->pid = getpid(); + + /* .. and it executes properly. */ + uprobe_fork_test(); + + if (skel->bss->executed != 3) + _exit(2); + + _exit(0); +} + +static void test_uprobe_fork_optimized(bool clone_vm) +{ + struct uprobe_syscall_executed *skel = NULL; + unsigned long offset; + int pid, status, err; + char stack[65535]; + + offset = get_uprobe_offset(&uprobe_fork_test); + if (!ASSERT_GE(offset, 0, "get_uprobe_offset")) + return; + + skel = uprobe_syscall_executed__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + goto cleanup; + + skel->links.test_uprobe = bpf_program__attach_uprobe_opts(skel->progs.test_uprobe, + -1, "/proc/self/exe", offset, NULL); + if (!ASSERT_OK_PTR(skel->links.test_uprobe, "attach_uprobe")) + goto cleanup; + + skel->bss->pid = getpid(); + + /* Trigger optimization of uprobe in uprobe_fork_test. */ + uprobe_fork_test(); + uprobe_fork_test(); + + /* Make sure it got optimied. */ + if (!ASSERT_OK(memcmp(uprobe_fork_test, lea_rsp, sizeof(lea_rsp)), "optimized")) + goto cleanup; + + if (clone_vm) { + pid = clone(child_func, stack + sizeof(stack), CLONE_VM|SIGCHLD, skel); + if (!ASSERT_GT(pid, 0, "clone")) + goto cleanup; + } else { + pid = fork(); + if (!ASSERT_GE(pid, 0, "fork")) + goto cleanup; + if (pid == 0) + child_func(skel); + } + + /* Wait for the child and verify it exited properly with 0. */ + err = waitpid(pid, &status, 0); + if (ASSERT_EQ(err, pid, "waitpid")) { + ASSERT_EQ(WIFEXITED(status), 1, "child_exited"); + ASSERT_EQ(WEXITSTATUS(status), 0, "child_exit_code"); + } + +cleanup: + uprobe_syscall_executed__destroy(skel); } static void __test_uprobe_syscall(void) @@ -783,6 +1056,12 @@ static void __test_uprobe_syscall(void) test_uprobe_usdt(); if (test__start_subtest("uprobe_race")) test_uprobe_race(); + if (test__start_subtest("uprobe_red_zone")) + test_uprobe_red_zone(); + if (test__start_subtest("uprobe_optimized_fork")) + test_uprobe_fork_optimized(false); + if (test__start_subtest("uprobe_optimized_clone_vm")) + test_uprobe_fork_optimized(true); if (test__start_subtest("uprobe_error")) test_uprobe_error(); if (test__start_subtest("uprobe_regs_equal")) diff --git a/tools/testing/selftests/bpf/prog_tests/usdt.c b/tools/testing/selftests/bpf/prog_tests/usdt.c index 69759b27794d..8004c9568ffa 100644 --- a/tools/testing/selftests/bpf/prog_tests/usdt.c +++ b/tools/testing/selftests/bpf/prog_tests/usdt.c @@ -250,9 +250,10 @@ cleanup: #ifdef __x86_64__ extern void usdt_1(void); extern void usdt_2(void); +extern void usdt_red_zone_trigger(void); static unsigned char nop1[1] = { 0x90 }; -static unsigned char nop1_nop5_combo[6] = { 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x00 }; +static unsigned char nop1_nop10_combo[11] = { 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 }; static void *find_instr(void *fn, unsigned char *instr, size_t cnt) { @@ -271,17 +272,17 @@ static void subtest_optimized_attach(void) __u8 *addr_1, *addr_2; /* usdt_1 USDT probe has single nop instruction */ - addr_1 = find_instr(usdt_1, nop1_nop5_combo, 6); - if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop5_combo")) + addr_1 = find_instr(usdt_1, nop1_nop10_combo, 11); + if (!ASSERT_NULL(addr_1, "usdt_1_find_nop1_nop10_combo")) return; addr_1 = find_instr(usdt_1, nop1, 1); if (!ASSERT_OK_PTR(addr_1, "usdt_1_find_nop1")) return; - /* usdt_2 USDT probe has nop,nop5 instructions combo */ - addr_2 = find_instr(usdt_2, nop1_nop5_combo, 6); - if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop5_combo")) + /* usdt_2 USDT probe has nop,nop10 instructions combo */ + addr_2 = find_instr(usdt_2, nop1_nop10_combo, 11); + if (!ASSERT_OK_PTR(addr_2, "usdt_2_find_nop1_nop10_combo")) return; skel = test_usdt__open_and_load(); @@ -309,12 +310,12 @@ static void subtest_optimized_attach(void) bpf_link__destroy(skel->links.usdt_executed); - /* we expect the nop5 ip */ + /* we expect the nop10 ip */ skel->bss->expected_ip = (unsigned long) addr_2 + 1; /* * Attach program on top of usdt_2 which is probe defined on top - * of nop1,nop5 combo, so the probe gets optimized on top of nop5. + * of nop1,nop10 combo, so the probe gets optimized on top of nop10. */ skel->links.usdt_executed = bpf_program__attach_usdt(skel->progs.usdt_executed, 0 /*self*/, "/proc/self/exe", @@ -328,13 +329,64 @@ static void subtest_optimized_attach(void) /* nop stays on addr_2 address */ ASSERT_EQ(*addr_2, 0x90, "nop"); - /* call is on addr_2 + 1 address */ - ASSERT_EQ(*(addr_2 + 1), 0xe8, "call"); + /* + * lea -0x80(%rsp), %rsp + * call ... + */ + static unsigned char expected[] = { 0x48, 0x8d, 0x64, 0x24, 0x80, 0xe8 }; + + ASSERT_MEMEQ(addr_2 + 1, expected, sizeof(expected), "lea_and_call"); ASSERT_EQ(skel->bss->executed, 4, "executed"); cleanup: test_usdt__destroy(skel); } + +/* + * Test that USDT arguments survive nop10 optimization in a function where + * the compiler places operands in the red zone. + * + * Signal handlers are prone to having the compiler place USDT argument + * operands in the red zone (below rsp). + * + * The nop5 optimization used CALL (which pushes a return address to + * [rsp-8]), the value at -8(%rsp) was overwritten. The nop10 optimization + * should escape that by moving stackpointer below the redzone before + * doing the CALL. + */ +static void subtest_optimized_red_zone(void) +{ + struct test_usdt *skel; + int i; + + skel = test_usdt__open_and_load(); + if (!ASSERT_OK_PTR(skel, "open_and_load")) + return; + + skel->bss->expected_arg[0] = 0xDEADBEEF; + skel->bss->expected_arg[1] = 0xCAFEBABE; + skel->bss->expected_arg[2] = 0xFEEDFACE; + skel->bss->expected_pid = getpid(); + + skel->links.usdt_check_arg = bpf_program__attach_usdt( + skel->progs.usdt_check_arg, 0, "/proc/self/exe", + "optimized_attach", "usdt_red_zone", NULL); + if (!ASSERT_OK_PTR(skel->links.usdt_check_arg, "attach_usdt_red_zone")) + goto cleanup; + + for (i = 0; i < 10; i++) + usdt_red_zone_trigger(); + + ASSERT_EQ(skel->bss->arg_total, 10, "arg_total"); + ASSERT_EQ(skel->bss->arg_bad, 0, "arg_bad"); + ASSERT_EQ(skel->bss->arg_last[0], 0xDEADBEEF, "arg_last_1"); + ASSERT_EQ(skel->bss->arg_last[1], 0xCAFEBABE, "arg_last_2"); + ASSERT_EQ(skel->bss->arg_last[2], 0xFEEDFACE, "arg_last_3"); + +cleanup: + test_usdt__destroy(skel); +} + #endif unsigned short test_usdt_100_semaphore SEC(".probes"); @@ -608,6 +660,8 @@ void test_usdt(void) subtest_basic_usdt(true); if (test__start_subtest("optimized_attach")) subtest_optimized_attach(); + if (test__start_subtest("optimized_red_zone")) + subtest_optimized_red_zone(); #endif if (test__start_subtest("multispec")) subtest_multispec_usdt(); diff --git a/tools/testing/selftests/bpf/progs/test_usdt.c b/tools/testing/selftests/bpf/progs/test_usdt.c index f00cb52874e0..0ee78fb050a1 100644 --- a/tools/testing/selftests/bpf/progs/test_usdt.c +++ b/tools/testing/selftests/bpf/progs/test_usdt.c @@ -149,5 +149,30 @@ int usdt_executed(struct pt_regs *ctx) executed++; return 0; } + +int arg_total; +int arg_bad; +long arg_last[3]; +long expected_arg[3]; +int expected_pid; + +SEC("usdt") +int BPF_USDT(usdt_check_arg, long arg1, long arg2, long arg3) +{ + if (expected_pid != (bpf_get_current_pid_tgid() >> 32)) + return 0; + + __sync_fetch_and_add(&arg_total, 1); + arg_last[0] = arg1; + arg_last[1] = arg2; + arg_last[2] = arg3; + + if (arg1 != expected_arg[0] || + arg2 != expected_arg[1] || + arg3 != expected_arg[2]) + __sync_fetch_and_add(&arg_bad, 1); + + return 0; +} #endif char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/usdt.h b/tools/testing/selftests/bpf/usdt.h index c71e21df38b3..75687f50f4e2 100644 --- a/tools/testing/selftests/bpf/usdt.h +++ b/tools/testing/selftests/bpf/usdt.h @@ -313,7 +313,7 @@ struct usdt_sema { volatile unsigned short active; }; #if defined(__ia64__) || defined(__s390__) || defined(__s390x__) #define USDT_NOP nop 0 #elif defined(__x86_64__) -#define USDT_NOP .byte 0x90, 0x0f, 0x1f, 0x44, 0x00, 0x0 /* nop, nop5 */ +#define USDT_NOP .byte 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00 /* nop, nop10 */ #else #define USDT_NOP nop #endif diff --git a/tools/testing/selftests/bpf/usdt_2.c b/tools/testing/selftests/bpf/usdt_2.c index 789883aaca4c..5e38f8605b02 100644 --- a/tools/testing/selftests/bpf/usdt_2.c +++ b/tools/testing/selftests/bpf/usdt_2.c @@ -3,7 +3,7 @@ #if defined(__x86_64__) /* - * Include usdt.h with default nop,nop5 instructions combo. + * Include usdt.h with default nop,nop10 instructions combo. */ #include "usdt.h" @@ -13,4 +13,17 @@ void usdt_2(void) USDT(optimized_attach, usdt_2); } +static volatile unsigned long usdt_red_zone_arg1 = 0xDEADBEEF; +static volatile unsigned long usdt_red_zone_arg2 = 0xCAFEBABE; +static volatile unsigned long usdt_red_zone_arg3 = 0xFEEDFACE; + +void __attribute__((noinline)) usdt_red_zone_trigger(void) +{ + unsigned long a1 = usdt_red_zone_arg1; + unsigned long a2 = usdt_red_zone_arg2; + unsigned long a3 = usdt_red_zone_arg3; + + USDT(optimized_attach, usdt_red_zone, a1, a2, a3); +} + #endif |