diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/cpus/aarch64/cortex_a75.S | 103 | ||||
| -rw-r--r-- | lib/cpus/aarch64/cortex_a75_pubsub.c | 61 | ||||
| -rw-r--r-- | lib/cpus/aarch64/cpuamu.c | 70 | ||||
| -rw-r--r-- | lib/cpus/aarch64/cpuamu_helpers.S | 107 | ||||
| -rw-r--r-- | lib/extensions/amu/aarch32/amu.c | 23 | ||||
| -rw-r--r-- | lib/extensions/amu/aarch32/amu_helpers.S | 174 | ||||
| -rw-r--r-- | lib/extensions/amu/aarch64/amu.c | 16 | ||||
| -rw-r--r-- | lib/extensions/spe/spe.c | 89 | ||||
| -rw-r--r-- | lib/extensions/sve/sve.c | 175 | ||||
| -rw-r--r-- | lib/psci/aarch32/psci_helpers.S | 24 | ||||
| -rw-r--r-- | lib/psci/aarch64/psci_helpers.S | 24 | ||||
| -rw-r--r-- | lib/xlat_tables/xlat_tables_common.c | 2 | ||||
| -rw-r--r-- | lib/xlat_tables_v2/aarch32/xlat_tables_arch.c | 2 | ||||
| -rw-r--r-- | lib/xlat_tables_v2/aarch64/xlat_tables_arch.c | 2 |
14 files changed, 476 insertions, 396 deletions
diff --git a/lib/cpus/aarch64/cortex_a75.S b/lib/cpus/aarch64/cortex_a75.S index 946f9884..12ea304d 100644 --- a/lib/cpus/aarch64/cortex_a75.S +++ b/lib/cpus/aarch64/cortex_a75.S @@ -6,108 +6,9 @@ #include <arch.h> #include <asm_macros.S> -#include <bl_common.h> -#include <cpu_macros.S> -#include <plat_macros.S> #include <cortex_a75.h> - - .globl cortex_a75_amu_cnt_read - .globl cortex_a75_amu_cnt_write - .globl cortex_a75_amu_read_cpuamcntenset_el0 - .globl cortex_a75_amu_read_cpuamcntenclr_el0 - .globl cortex_a75_amu_write_cpuamcntenset_el0 - .globl cortex_a75_amu_write_cpuamcntenclr_el0 - -/* - * uint64_t cortex_a75_amu_cnt_read(int idx); - * - * Given `idx`, read the corresponding AMU counter - * and return it in `x0`. - */ -func cortex_a75_amu_cnt_read - adr x1, 1f - lsl x0, x0, #3 - add x1, x1, x0 - br x1 - -1: - mrs x0, CPUAMEVCNTR0_EL0 - ret - mrs x0, CPUAMEVCNTR1_EL0 - ret - mrs x0, CPUAMEVCNTR2_EL0 - ret - mrs x0, CPUAMEVCNTR3_EL0 - ret - mrs x0, CPUAMEVCNTR4_EL0 - ret -endfunc cortex_a75_amu_cnt_read - -/* - * void cortex_a75_amu_cnt_write(int idx, uint64_t val); - * - * Given `idx`, write `val` to the corresponding AMU counter. - */ -func cortex_a75_amu_cnt_write - adr x2, 1f - lsl x0, x0, #3 - add x2, x2, x0 - br x2 - -1: - msr CPUAMEVCNTR0_EL0, x0 - ret - msr CPUAMEVCNTR1_EL0, x0 - ret - msr CPUAMEVCNTR2_EL0, x0 - ret - msr CPUAMEVCNTR3_EL0, x0 - ret - msr CPUAMEVCNTR4_EL0, x0 - ret -endfunc cortex_a75_amu_cnt_write - -/* - * unsigned int cortex_a75_amu_read_cpuamcntenset_el0(void); - * - * Read the `CPUAMCNTENSET_EL0` CPU register and return - * it in `x0`. - */ -func cortex_a75_amu_read_cpuamcntenset_el0 - mrs x0, CPUAMCNTENSET_EL0 - ret -endfunc cortex_a75_amu_read_cpuamcntenset_el0 - -/* - * unsigned int cortex_a75_amu_read_cpuamcntenclr_el0(void); - * - * Read the `CPUAMCNTENCLR_EL0` CPU register and return - * it in `x0`. - */ -func cortex_a75_amu_read_cpuamcntenclr_el0 - mrs x0, CPUAMCNTENCLR_EL0 - ret -endfunc cortex_a75_amu_read_cpuamcntenclr_el0 - -/* - * void cortex_a75_amu_write_cpuamcntenset_el0(unsigned int mask); - * - * Write `mask` to the `CPUAMCNTENSET_EL0` CPU register. - */ -func cortex_a75_amu_write_cpuamcntenset_el0 - msr CPUAMCNTENSET_EL0, x0 - ret -endfunc cortex_a75_amu_write_cpuamcntenset_el0 - -/* - * void cortex_a75_amu_write_cpuamcntenclr_el0(unsigned int mask); - * - * Write `mask` to the `CPUAMCNTENCLR_EL0` CPU register. - */ -func cortex_a75_amu_write_cpuamcntenclr_el0 - mrs x0, CPUAMCNTENCLR_EL0 - ret -endfunc cortex_a75_amu_write_cpuamcntenclr_el0 +#include <cpuamu.h> +#include <cpu_macros.S> func cortex_a75_reset_func #if IMAGE_BL31 && WORKAROUND_CVE_2017_5715 diff --git a/lib/cpus/aarch64/cortex_a75_pubsub.c b/lib/cpus/aarch64/cortex_a75_pubsub.c index a1ffcb04..16f62f47 100644 --- a/lib/cpus/aarch64/cortex_a75_pubsub.c +++ b/lib/cpus/aarch64/cortex_a75_pubsub.c @@ -1,73 +1,24 @@ /* - * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ #include <cortex_a75.h> -#include <platform.h> +#include <cpuamu.h> #include <pubsub_events.h> -struct amu_ctx { - uint64_t cnts[CORTEX_A75_AMU_NR_COUNTERS]; - uint16_t mask; -}; - -static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT]; - static void *cortex_a75_context_save(const void *arg) { - struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; - unsigned int midr; - unsigned int midr_mask; - int i; - - midr = read_midr(); - midr_mask = (MIDR_IMPL_MASK << MIDR_IMPL_SHIFT) | - (MIDR_PN_MASK << MIDR_PN_SHIFT); - if ((midr & midr_mask) != (CORTEX_A75_MIDR & midr_mask)) - return 0; - - /* Save counter configuration */ - ctx->mask = cortex_a75_amu_read_cpuamcntenset_el0(); - - /* Ensure counters are disabled */ - cortex_a75_amu_write_cpuamcntenclr_el0(ctx->mask); - isb(); - - /* Save counters */ - for (i = 0; i < CORTEX_A75_AMU_NR_COUNTERS; i++) - ctx->cnts[i] = cortex_a75_amu_cnt_read(i); - + if (midr_match(CORTEX_A75_MIDR) != 0) + cpuamu_context_save(CORTEX_A75_AMU_NR_COUNTERS); return 0; } static void *cortex_a75_context_restore(const void *arg) { - struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; - unsigned int midr; - unsigned int midr_mask; - int i; - - midr = read_midr(); - midr_mask = (MIDR_IMPL_MASK << MIDR_IMPL_SHIFT) | - (MIDR_PN_MASK << MIDR_PN_SHIFT); - if ((midr & midr_mask) != (CORTEX_A75_MIDR & midr_mask)) - return 0; - - ctx = &amu_ctxs[plat_my_core_pos()]; - - /* Counters were disabled in `cortex_a75_context_save()` */ - assert(cortex_a75_amu_read_cpuamcntenset_el0() == 0); - - /* Restore counters */ - for (i = 0; i < CORTEX_A75_AMU_NR_COUNTERS; i++) - cortex_a75_amu_cnt_write(i, ctx->cnts[i]); - isb(); - - /* Restore counter configuration */ - cortex_a75_amu_write_cpuamcntenset_el0(ctx->mask); - + if (midr_match(CORTEX_A75_MIDR) != 0) + cpuamu_context_restore(CORTEX_A75_AMU_NR_COUNTERS); return 0; } diff --git a/lib/cpus/aarch64/cpuamu.c b/lib/cpus/aarch64/cpuamu.c new file mode 100644 index 00000000..b9bad860 --- /dev/null +++ b/lib/cpus/aarch64/cpuamu.c @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2018, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <cpuamu.h> +#include <platform.h> +#include <pubsub_events.h> + +#define CPUAMU_NR_COUNTERS 5U + +struct amu_ctx { + uint64_t cnts[CPUAMU_NR_COUNTERS]; + unsigned int mask; +}; + +static struct amu_ctx amu_ctxs[PLATFORM_CORE_COUNT]; + +int midr_match(unsigned int cpu_midr) +{ + unsigned int midr, midr_mask; + + midr = (unsigned int)read_midr(); + midr_mask = (MIDR_IMPL_MASK << MIDR_IMPL_SHIFT) | + (MIDR_PN_MASK << MIDR_PN_SHIFT); + return ((midr & midr_mask) == (cpu_midr & midr_mask)); +} + +void cpuamu_context_save(unsigned int nr_counters) +{ + struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; + unsigned int i; + + assert(nr_counters <= CPUAMU_NR_COUNTERS); + + /* Save counter configuration */ + ctx->mask = cpuamu_read_cpuamcntenset_el0(); + + /* Disable counters */ + cpuamu_write_cpuamcntenclr_el0(ctx->mask); + isb(); + + /* Save counters */ + for (i = 0; i < nr_counters; i++) + ctx->cnts[i] = cpuamu_cnt_read(i); +} + +void cpuamu_context_restore(unsigned int nr_counters) +{ + struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; + unsigned int i; + + assert(nr_counters <= CPUAMU_NR_COUNTERS); + + /* + * Disable counters. They were enabled early in the + * CPU reset function. + */ + cpuamu_write_cpuamcntenclr_el0(ctx->mask); + isb(); + + /* Restore counters */ + for (i = 0; i < nr_counters; i++) + cpuamu_cnt_write(i, ctx->cnts[i]); + isb(); + + /* Restore counter configuration */ + cpuamu_write_cpuamcntenset_el0(ctx->mask); +} diff --git a/lib/cpus/aarch64/cpuamu_helpers.S b/lib/cpus/aarch64/cpuamu_helpers.S new file mode 100644 index 00000000..8965d6d0 --- /dev/null +++ b/lib/cpus/aarch64/cpuamu_helpers.S @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2018, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <asm_macros.S> +#include <cpuamu.h> + + .globl cpuamu_cnt_read + .globl cpuamu_cnt_write + .globl cpuamu_read_cpuamcntenset_el0 + .globl cpuamu_read_cpuamcntenclr_el0 + .globl cpuamu_write_cpuamcntenset_el0 + .globl cpuamu_write_cpuamcntenclr_el0 + +/* + * uint64_t cpuamu_cnt_read(int idx); + * + * Given `idx`, read the corresponding AMU counter + * and return it in `x0`. + */ +func cpuamu_cnt_read + adr x1, 1f + lsl x0, x0, #3 + add x1, x1, x0 + br x1 + +1: + mrs x0, CPUAMEVCNTR0_EL0 + ret + mrs x0, CPUAMEVCNTR1_EL0 + ret + mrs x0, CPUAMEVCNTR2_EL0 + ret + mrs x0, CPUAMEVCNTR3_EL0 + ret + mrs x0, CPUAMEVCNTR4_EL0 + ret +endfunc cpuamu_cnt_read + +/* + * void cpuamu_cnt_write(int idx, uint64_t val); + * + * Given `idx`, write `val` to the corresponding AMU counter. + */ +func cpuamu_cnt_write + adr x2, 1f + lsl x0, x0, #3 + add x2, x2, x0 + br x2 + +1: + msr CPUAMEVCNTR0_EL0, x0 + ret + msr CPUAMEVCNTR1_EL0, x0 + ret + msr CPUAMEVCNTR2_EL0, x0 + ret + msr CPUAMEVCNTR3_EL0, x0 + ret + msr CPUAMEVCNTR4_EL0, x0 + ret +endfunc cpuamu_cnt_write + +/* + * unsigned int cpuamu_read_cpuamcntenset_el0(void); + * + * Read the `CPUAMCNTENSET_EL0` CPU register and return + * it in `x0`. + */ +func cpuamu_read_cpuamcntenset_el0 + mrs x0, CPUAMCNTENSET_EL0 + ret +endfunc cpuamu_read_cpuamcntenset_el0 + +/* + * unsigned int cpuamu_read_cpuamcntenclr_el0(void); + * + * Read the `CPUAMCNTENCLR_EL0` CPU register and return + * it in `x0`. + */ +func cpuamu_read_cpuamcntenclr_el0 + mrs x0, CPUAMCNTENCLR_EL0 + ret +endfunc cpuamu_read_cpuamcntenclr_el0 + +/* + * void cpuamu_write_cpuamcntenset_el0(unsigned int mask); + * + * Write `mask` to the `CPUAMCNTENSET_EL0` CPU register. + */ +func cpuamu_write_cpuamcntenset_el0 + msr CPUAMCNTENSET_EL0, x0 + ret +endfunc cpuamu_write_cpuamcntenset_el0 + +/* + * void cpuamu_write_cpuamcntenclr_el0(unsigned int mask); + * + * Write `mask` to the `CPUAMCNTENCLR_EL0` CPU register. + */ +func cpuamu_write_cpuamcntenclr_el0 + msr CPUAMCNTENCLR_EL0, x0 + ret +endfunc cpuamu_write_cpuamcntenclr_el0 diff --git a/lib/extensions/amu/aarch32/amu.c b/lib/extensions/amu/aarch32/amu.c index 68cc4b34..05c98f1c 100644 --- a/lib/extensions/amu/aarch32/amu.c +++ b/lib/extensions/amu/aarch32/amu.c @@ -30,7 +30,7 @@ int amu_supported(void) void amu_enable(int el2_unused) { - if (!amu_supported()) + if (amu_supported() == 0) return; if (el2_unused) { @@ -54,7 +54,7 @@ void amu_enable(int el2_unused) /* Read the group 0 counter identified by the given `idx`. */ uint64_t amu_group0_cnt_read(int idx) { - assert(amu_supported()); + assert(amu_supported() != 0); assert(idx >= 0 && idx < AMU_GROUP0_NR_COUNTERS); return amu_group0_cnt_read_internal(idx); @@ -63,7 +63,7 @@ uint64_t amu_group0_cnt_read(int idx) /* Write the group 0 counter identified by the given `idx` with `val`. */ void amu_group0_cnt_write(int idx, uint64_t val) { - assert(amu_supported()); + assert(amu_supported() != 0); assert(idx >= 0 && idx < AMU_GROUP0_NR_COUNTERS); amu_group0_cnt_write_internal(idx, val); @@ -73,7 +73,7 @@ void amu_group0_cnt_write(int idx, uint64_t val) /* Read the group 1 counter identified by the given `idx`. */ uint64_t amu_group1_cnt_read(int idx) { - assert(amu_supported()); + assert(amu_supported() != 0); assert(idx >= 0 && idx < AMU_GROUP1_NR_COUNTERS); return amu_group1_cnt_read_internal(idx); @@ -82,7 +82,7 @@ uint64_t amu_group1_cnt_read(int idx) /* Write the group 1 counter identified by the given `idx` with `val`. */ void amu_group1_cnt_write(int idx, uint64_t val) { - assert(amu_supported()); + assert(amu_supported() != 0); assert(idx >= 0 && idx < AMU_GROUP1_NR_COUNTERS); amu_group1_cnt_write_internal(idx, val); @@ -91,7 +91,7 @@ void amu_group1_cnt_write(int idx, uint64_t val) void amu_group1_set_evtype(int idx, unsigned int val) { - assert(amu_supported()); + assert(amu_supported() != 0); assert(idx >= 0 && idx < AMU_GROUP1_NR_COUNTERS); amu_group1_set_evtype_internal(idx, val); @@ -103,13 +103,14 @@ static void *amu_context_save(const void *arg) struct amu_ctx *ctx; int i; - if (!amu_supported()) + if (amu_supported() == 0) return (void *)-1; ctx = &amu_ctxs[plat_my_core_pos()]; /* Assert that group 0 counter configuration is what we expect */ - assert(read_amcntenset0() == AMU_GROUP0_COUNTERS_MASK); + assert(read_amcntenset0() == AMU_GROUP0_COUNTERS_MASK && + read_amcntenset1() == AMU_GROUP1_COUNTERS_MASK); /* * Disable group 0 counters to avoid other observers like SCP sampling @@ -131,17 +132,15 @@ static void *amu_context_save(const void *arg) static void *amu_context_restore(const void *arg) { struct amu_ctx *ctx; - uint64_t features; int i; - features = read_id_pfr0() >> ID_PFR0_AMU_SHIFT; - if ((features & ID_PFR0_AMU_MASK) != 1) + if (amu_supported() == 0) return (void *)-1; ctx = &amu_ctxs[plat_my_core_pos()]; /* Counters were disabled in `amu_context_save()` */ - assert(read_amcntenset0() == 0); + assert(read_amcntenset0() == 0 && read_amcntenset1() == 0); /* Restore group 0 counters */ for (i = 0; i < AMU_GROUP0_NR_COUNTERS; i++) diff --git a/lib/extensions/amu/aarch32/amu_helpers.S b/lib/extensions/amu/aarch32/amu_helpers.S index 84dca04c..effb8e50 100644 --- a/lib/extensions/amu/aarch32/amu_helpers.S +++ b/lib/extensions/amu/aarch32/amu_helpers.S @@ -18,7 +18,7 @@ * uint64_t amu_group0_cnt_read_internal(int idx); * * Given `idx`, read the corresponding AMU counter - * and return it in `r0`. + * and return it in `r0` and `r1`. */ func amu_group0_cnt_read_internal #if ENABLE_ASSERTIONS @@ -52,13 +52,15 @@ endfunc amu_group0_cnt_read_internal * void amu_group0_cnt_write_internal(int idx, uint64_t val); * * Given `idx`, write `val` to the corresponding AMU counter. + * `idx` is passed in `r0` and `val` is passed in `r2` and `r3`. + * `r1` is used as a scratch register. */ func amu_group0_cnt_write_internal #if ENABLE_ASSERTIONS /* `idx` should be between [0, 3] */ - mov r2, r0 - lsr r2, r2, #2 - cmp r2, #0 + mov r1, r0 + lsr r1, r1, #2 + cmp r1, #0 ASM_ASSERT(eq) #endif @@ -66,19 +68,19 @@ func amu_group0_cnt_write_internal * Given `idx` calculate address of stcopr16/bx lr instruction pair * in the table below. */ - adr r2, 1f + adr r1, 1f lsl r0, r0, #3 /* each stcopr16/bx lr sequence is 8 bytes */ - add r2, r2, r0 - bx r2 + add r1, r1, r0 + bx r1 1: - stcopr16 r0,r1, AMEVCNTR00 /* index 0 */ + stcopr16 r2, r3, AMEVCNTR00 /* index 0 */ bx lr - stcopr16 r0,r1, AMEVCNTR01 /* index 1 */ + stcopr16 r2, r3, AMEVCNTR01 /* index 1 */ bx lr - stcopr16 r0,r1, AMEVCNTR02 /* index 2 */ + stcopr16 r2, r3, AMEVCNTR02 /* index 2 */ bx lr - stcopr16 r0,r1, AMEVCNTR03 /* index 3 */ + stcopr16 r2, r3, AMEVCNTR03 /* index 3 */ bx lr endfunc amu_group0_cnt_write_internal @@ -86,14 +88,14 @@ endfunc amu_group0_cnt_write_internal * uint64_t amu_group1_cnt_read_internal(int idx); * * Given `idx`, read the corresponding AMU counter - * and return it in `r0`. + * and return it in `r0` and `r1`. */ func amu_group1_cnt_read_internal #if ENABLE_ASSERTIONS /* `idx` should be between [0, 15] */ - mov r2, r0 - lsr r2, r2, #4 - cmp r2, #0 + mov r1, r0 + lsr r1, r1, #4 + cmp r1, #0 ASM_ASSERT(eq) #endif @@ -107,51 +109,53 @@ func amu_group1_cnt_read_internal bx r1 1: - ldcopr16 r0,r1, AMEVCNTR10 /* index 0 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR11 /* index 1 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR12 /* index 2 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR13 /* index 3 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR14 /* index 4 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR15 /* index 5 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR16 /* index 6 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR17 /* index 7 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR18 /* index 8 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR19 /* index 9 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR1A /* index 10 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR1B /* index 11 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR1C /* index 12 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR1D /* index 13 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR1E /* index 14 */ - bx lr - ldcopr16 r0,r1, AMEVCNTR1F /* index 15 */ - bx lr + ldcopr16 r0, r1, AMEVCNTR10 /* index 0 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR11 /* index 1 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR12 /* index 2 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR13 /* index 3 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR14 /* index 4 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR15 /* index 5 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR16 /* index 6 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR17 /* index 7 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR18 /* index 8 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR19 /* index 9 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1A /* index 10 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1B /* index 11 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1C /* index 12 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1D /* index 13 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1E /* index 14 */ + bx lr + ldcopr16 r0, r1, AMEVCNTR1F /* index 15 */ + bx lr endfunc amu_group1_cnt_read_internal /* * void amu_group1_cnt_write_internal(int idx, uint64_t val); * * Given `idx`, write `val` to the corresponding AMU counter. + * `idx` is passed in `r0` and `val` is passed in `r2` and `r3`. + * `r1` is used as a scratch register. */ func amu_group1_cnt_write_internal #if ENABLE_ASSERTIONS /* `idx` should be between [0, 15] */ - mov r2, r0 - lsr r2, r2, #4 - cmp r2, #0 + mov r1, r0 + lsr r1, r1, #4 + cmp r1, #0 ASM_ASSERT(eq) #endif @@ -159,43 +163,43 @@ func amu_group1_cnt_write_internal * Given `idx` calculate address of ldcopr16/bx lr instruction pair * in the table below. */ - adr r2, 1f + adr r1, 1f lsl r0, r0, #3 /* each stcopr16/bx lr sequence is 8 bytes */ - add r2, r2, r0 - bx r2 + add r1, r1, r0 + bx r1 1: - stcopr16 r0,r1, AMEVCNTR10 /* index 0 */ + stcopr16 r2, r3, AMEVCNTR10 /* index 0 */ bx lr - stcopr16 r0,r1, AMEVCNTR11 /* index 1 */ + stcopr16 r2, r3, AMEVCNTR11 /* index 1 */ bx lr - stcopr16 r0,r1, AMEVCNTR12 /* index 2 */ + stcopr16 r2, r3, AMEVCNTR12 /* index 2 */ bx lr - stcopr16 r0,r1, AMEVCNTR13 /* index 3 */ + stcopr16 r2, r3, AMEVCNTR13 /* index 3 */ bx lr - stcopr16 r0,r1, AMEVCNTR14 /* index 4 */ + stcopr16 r2, r3, AMEVCNTR14 /* index 4 */ bx lr - stcopr16 r0,r1, AMEVCNTR15 /* index 5 */ + stcopr16 r2, r3, AMEVCNTR15 /* index 5 */ bx lr - stcopr16 r0,r1, AMEVCNTR16 /* index 6 */ + stcopr16 r2, r3, AMEVCNTR16 /* index 6 */ bx lr - stcopr16 r0,r1, AMEVCNTR17 /* index 7 */ + stcopr16 r2, r3, AMEVCNTR17 /* index 7 */ bx lr - stcopr16 r0,r1, AMEVCNTR18 /* index 8 */ + stcopr16 r2, r3, AMEVCNTR18 /* index 8 */ bx lr - stcopr16 r0,r1, AMEVCNTR19 /* index 9 */ + stcopr16 r2, r3, AMEVCNTR19 /* index 9 */ bx lr - stcopr16 r0,r1, AMEVCNTR1A /* index 10 */ + stcopr16 r2, r3, AMEVCNTR1A /* index 10 */ bx lr - stcopr16 r0,r1, AMEVCNTR1B /* index 11 */ + stcopr16 r2, r3, AMEVCNTR1B /* index 11 */ bx lr - stcopr16 r0,r1, AMEVCNTR1C /* index 12 */ + stcopr16 r2, r3, AMEVCNTR1C /* index 12 */ bx lr - stcopr16 r0,r1, AMEVCNTR1D /* index 13 */ + stcopr16 r2, r3, AMEVCNTR1D /* index 13 */ bx lr - stcopr16 r0,r1, AMEVCNTR1E /* index 14 */ + stcopr16 r2, r3, AMEVCNTR1E /* index 14 */ bx lr - stcopr16 r0,r1, AMEVCNTR1F /* index 15 */ + stcopr16 r2, r3, AMEVCNTR1F /* index 15 */ bx lr endfunc amu_group1_cnt_write_internal @@ -230,36 +234,36 @@ func amu_group1_set_evtype_internal bx r2 1: - stcopr r0, AMEVTYPER10 /* index 0 */ + stcopr r1, AMEVTYPER10 /* index 0 */ bx lr - stcopr r0, AMEVTYPER11 /* index 1 */ + stcopr r1, AMEVTYPER11 /* index 1 */ bx lr - stcopr r0, AMEVTYPER12 /* index 2 */ + stcopr r1, AMEVTYPER12 /* index 2 */ bx lr - stcopr r0, AMEVTYPER13 /* index 3 */ + stcopr r1, AMEVTYPER13 /* index 3 */ bx lr - stcopr r0, AMEVTYPER14 /* index 4 */ + stcopr r1, AMEVTYPER14 /* index 4 */ bx lr - stcopr r0, AMEVTYPER15 /* index 5 */ + stcopr r1, AMEVTYPER15 /* index 5 */ bx lr - stcopr r0, AMEVTYPER16 /* index 6 */ + stcopr r1, AMEVTYPER16 /* index 6 */ bx lr - stcopr r0, AMEVTYPER17 /* index 7 */ + stcopr r1, AMEVTYPER17 /* index 7 */ bx lr - stcopr r0, AMEVTYPER18 /* index 8 */ + stcopr r1, AMEVTYPER18 /* index 8 */ bx lr - stcopr r0, AMEVTYPER19 /* index 9 */ + stcopr r1, AMEVTYPER19 /* index 9 */ bx lr - stcopr r0, AMEVTYPER1A /* index 10 */ + stcopr r1, AMEVTYPER1A /* index 10 */ bx lr - stcopr r0, AMEVTYPER1B /* index 11 */ + stcopr r1, AMEVTYPER1B /* index 11 */ bx lr - stcopr r0, AMEVTYPER1C /* index 12 */ + stcopr r1, AMEVTYPER1C /* index 12 */ bx lr - stcopr r0, AMEVTYPER1D /* index 13 */ + stcopr r1, AMEVTYPER1D /* index 13 */ bx lr - stcopr r0, AMEVTYPER1E /* index 14 */ + stcopr r1, AMEVTYPER1E /* index 14 */ bx lr - stcopr r0, AMEVTYPER1F /* index 15 */ + stcopr r1, AMEVTYPER1F /* index 15 */ bx lr endfunc amu_group1_set_evtype_internal diff --git a/lib/extensions/amu/aarch64/amu.c b/lib/extensions/amu/aarch64/amu.c index 7d39f35c..5d556e5d 100644 --- a/lib/extensions/amu/aarch64/amu.c +++ b/lib/extensions/amu/aarch64/amu.c @@ -37,7 +37,7 @@ void amu_enable(int el2_unused) { uint64_t v; - if (!amu_supported()) + if (amu_supported() == 0) return; if (el2_unused) { @@ -67,7 +67,7 @@ void amu_enable(int el2_unused) /* Read the group 0 counter identified by the given `idx`. */ uint64_t amu_group0_cnt_read(int idx) { - assert(amu_supported()); + assert(amu_supported() != 0); assert(idx >= 0 && idx < AMU_GROUP0_NR_COUNTERS); return amu_group0_cnt_read_internal(idx); @@ -76,7 +76,7 @@ uint64_t amu_group0_cnt_read(int idx) /* Write the group 0 counter identified by the given `idx` with `val`. */ void amu_group0_cnt_write(int idx, uint64_t val) { - assert(amu_supported()); + assert(amu_supported() != 0); assert(idx >= 0 && idx < AMU_GROUP0_NR_COUNTERS); amu_group0_cnt_write_internal(idx, val); @@ -86,7 +86,7 @@ void amu_group0_cnt_write(int idx, uint64_t val) /* Read the group 1 counter identified by the given `idx`. */ uint64_t amu_group1_cnt_read(int idx) { - assert(amu_supported()); + assert(amu_supported() != 0); assert(idx >= 0 && idx < AMU_GROUP1_NR_COUNTERS); return amu_group1_cnt_read_internal(idx); @@ -95,7 +95,7 @@ uint64_t amu_group1_cnt_read(int idx) /* Write the group 1 counter identified by the given `idx` with `val`. */ void amu_group1_cnt_write(int idx, uint64_t val) { - assert(amu_supported()); + assert(amu_supported() != 0); assert(idx >= 0 && idx < AMU_GROUP1_NR_COUNTERS); amu_group1_cnt_write_internal(idx, val); @@ -108,7 +108,7 @@ void amu_group1_cnt_write(int idx, uint64_t val) */ void amu_group1_set_evtype(int idx, unsigned int val) { - assert(amu_supported()); + assert(amu_supported() != 0); assert (idx >= 0 && idx < AMU_GROUP1_NR_COUNTERS); amu_group1_set_evtype_internal(idx, val); @@ -120,7 +120,7 @@ static void *amu_context_save(const void *arg) struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; int i; - if (!amu_supported()) + if (amu_supported() == 0) return (void *)-1; /* Assert that group 0/1 counter configuration is what we expect */ @@ -154,7 +154,7 @@ static void *amu_context_restore(const void *arg) struct amu_ctx *ctx = &amu_ctxs[plat_my_core_pos()]; int i; - if (!amu_supported()) + if (amu_supported() == 0) return (void *)-1; /* Counters were disabled in `amu_context_save()` */ diff --git a/lib/extensions/spe/spe.c b/lib/extensions/spe/spe.c index a1ed7b70..dc358403 100644 --- a/lib/extensions/spe/spe.c +++ b/lib/extensions/spe/spe.c @@ -15,71 +15,72 @@ */ #define psb_csync() asm volatile("hint #17") -void spe_enable(int el2_unused) +int spe_supported(void) { uint64_t features; features = read_id_aa64dfr0_el1() >> ID_AA64DFR0_PMS_SHIFT; - if ((features & ID_AA64DFR0_PMS_MASK) == 1) { - uint64_t v; + return (features & ID_AA64DFR0_PMS_MASK) == 1; +} - if (el2_unused) { - /* - * MDCR_EL2.TPMS (ARM v8.2): Do not trap statistical - * profiling controls to EL2. - * - * MDCR_EL2.E2PB (ARM v8.2): SPE enabled in Non-secure - * state. Accesses to profiling buffer controls at - * Non-secure EL1 are not trapped to EL2. - */ - v = read_mdcr_el2(); - v &= ~MDCR_EL2_TPMS; - v |= MDCR_EL2_E2PB(MDCR_EL2_E2PB_EL1); - write_mdcr_el2(v); - } +void spe_enable(int el2_unused) +{ + uint64_t v; + if (spe_supported() == 0) + return; + + if (el2_unused) { /* - * MDCR_EL2.NSPB (ARM v8.2): SPE enabled in Non-secure state - * and disabled in secure state. Accesses to SPE registers at - * S-EL1 generate trap exceptions to EL3. + * MDCR_EL2.TPMS (ARM v8.2): Do not trap statistical + * profiling controls to EL2. + * + * MDCR_EL2.E2PB (ARM v8.2): SPE enabled in Non-secure + * state. Accesses to profiling buffer controls at + * Non-secure EL1 are not trapped to EL2. */ - v = read_mdcr_el3(); - v |= MDCR_NSPB(MDCR_NSPB_EL1); - write_mdcr_el3(v); + v = read_mdcr_el2(); + v &= ~MDCR_EL2_TPMS; + v |= MDCR_EL2_E2PB(MDCR_EL2_E2PB_EL1); + write_mdcr_el2(v); } + + /* + * MDCR_EL2.NSPB (ARM v8.2): SPE enabled in Non-secure state + * and disabled in secure state. Accesses to SPE registers at + * S-EL1 generate trap exceptions to EL3. + */ + v = read_mdcr_el3(); + v |= MDCR_NSPB(MDCR_NSPB_EL1); + write_mdcr_el3(v); } void spe_disable(void) { - uint64_t features; + uint64_t v; - features = read_id_aa64dfr0_el1() >> ID_AA64DFR0_PMS_SHIFT; - if ((features & ID_AA64DFR0_PMS_MASK) == 1) { - uint64_t v; + if (spe_supported() == 0) + return; - /* Drain buffered data */ - psb_csync(); - dsbnsh(); + /* Drain buffered data */ + psb_csync(); + dsbnsh(); - /* Disable profiling buffer */ - v = read_pmblimitr_el1(); - v &= ~(1ULL << 0); - write_pmblimitr_el1(v); - isb(); - } + /* Disable profiling buffer */ + v = read_pmblimitr_el1(); + v &= ~(1ULL << 0); + write_pmblimitr_el1(v); + isb(); } static void *spe_drain_buffers_hook(const void *arg) { - uint64_t features; - - features = read_id_aa64dfr0_el1() >> ID_AA64DFR0_PMS_SHIFT; - if ((features & ID_AA64DFR0_PMS_MASK) == 1) { - /* Drain buffered data */ - psb_csync(); - dsbnsh(); - } + if (spe_supported() == 0) + return (void *)-1; + /* Drain buffered data */ + psb_csync(); + dsbnsh(); return 0; } diff --git a/lib/extensions/sve/sve.c b/lib/extensions/sve/sve.c index 14e51bd8..64424878 100644 --- a/lib/extensions/sve/sve.c +++ b/lib/extensions/sve/sve.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2017-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -9,117 +9,120 @@ #include <pubsub.h> #include <sve.h> -static void *disable_sve_hook(const void *arg) +int sve_supported(void) { uint64_t features; features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT; - if ((features & ID_AA64PFR0_SVE_MASK) == 1) { - uint64_t cptr; - - /* - * Disable SVE, SIMD and FP access for the Secure world. - * As the SIMD/FP registers are part of the SVE Z-registers, any - * use of SIMD/FP functionality will corrupt the SVE registers. - * Therefore it is necessary to prevent use of SIMD/FP support - * in the Secure world as well as SVE functionality. - */ - cptr = read_cptr_el3(); - cptr = (cptr | TFP_BIT) & ~(CPTR_EZ_BIT); - write_cptr_el3(cptr); + return (features & ID_AA64PFR0_SVE_MASK) == 1; +} - /* - * No explicit ISB required here as ERET to switch to Secure - * world covers it - */ - } +static void *disable_sve_hook(const void *arg) +{ + uint64_t cptr; + + if (sve_supported() == 0) + return (void *)-1; + + /* + * Disable SVE, SIMD and FP access for the Secure world. + * As the SIMD/FP registers are part of the SVE Z-registers, any + * use of SIMD/FP functionality will corrupt the SVE registers. + * Therefore it is necessary to prevent use of SIMD/FP support + * in the Secure world as well as SVE functionality. + */ + cptr = read_cptr_el3(); + cptr = (cptr | TFP_BIT) & ~(CPTR_EZ_BIT); + write_cptr_el3(cptr); + + /* + * No explicit ISB required here as ERET to switch to Secure + * world covers it + */ return 0; } static void *enable_sve_hook(const void *arg) { - uint64_t features; - - features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT; - if ((features & ID_AA64PFR0_SVE_MASK) == 1) { - uint64_t cptr; - - /* - * Enable SVE, SIMD and FP access for the Non-secure world. - */ - cptr = read_cptr_el3(); - cptr = (cptr | CPTR_EZ_BIT) & ~(TFP_BIT); - write_cptr_el3(cptr); - - /* - * No explicit ISB required here as ERET to switch to Non-secure - * world covers it - */ - } + uint64_t cptr; + + if (sve_supported() == 0) + return (void *)-1; + + /* + * Enable SVE, SIMD and FP access for the Non-secure world. + */ + cptr = read_cptr_el3(); + cptr = (cptr | CPTR_EZ_BIT) & ~(TFP_BIT); + write_cptr_el3(cptr); + + /* + * No explicit ISB required here as ERET to switch to Non-secure + * world covers it + */ return 0; } void sve_enable(int el2_unused) { - uint64_t features; + uint64_t cptr; + + if (sve_supported() == 0) + return; - features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT; - if ((features & ID_AA64PFR0_SVE_MASK) == 1) { - uint64_t cptr; #if CTX_INCLUDE_FPREGS - /* - * CTX_INCLUDE_FPREGS is not supported on SVE enabled systems. - */ - assert(0); + /* + * CTX_INCLUDE_FPREGS is not supported on SVE enabled systems. + */ + assert(0); #endif + /* + * Update CPTR_EL3 to enable access to SVE functionality for the + * Non-secure world. + * NOTE - assumed that CPTR_EL3.TFP is set to allow access to + * the SIMD, floating-point and SVE support. + * + * CPTR_EL3.EZ: Set to 1 to enable access to SVE functionality + * in the Non-secure world. + */ + cptr = read_cptr_el3(); + cptr |= CPTR_EZ_BIT; + write_cptr_el3(cptr); + + /* + * Need explicit ISB here to guarantee that update to ZCR_ELx + * and CPTR_EL2.TZ do not result in trap to EL3. + */ + isb(); + + /* + * Ensure lower ELs have access to full vector length. + */ + write_zcr_el3(ZCR_EL3_LEN_MASK); + + if (el2_unused) { /* - * Update CPTR_EL3 to enable access to SVE functionality for the - * Non-secure world. - * NOTE - assumed that CPTR_EL3.TFP is set to allow access to - * the SIMD, floating-point and SVE support. + * Update CPTR_EL2 to enable access to SVE functionality + * for Non-secure world, EL2 and Non-secure EL1 and EL0. + * NOTE - assumed that CPTR_EL2.TFP is set to allow + * access to the SIMD, floating-point and SVE support. * - * CPTR_EL3.EZ: Set to 1 to enable access to SVE functionality - * in the Non-secure world. + * CPTR_EL2.TZ: Set to 0 to enable access to SVE support + * for EL2 and Non-secure EL1 and EL0. */ - cptr = read_cptr_el3(); - cptr |= CPTR_EZ_BIT; - write_cptr_el3(cptr); - - /* - * Need explicit ISB here to guarantee that update to ZCR_ELx - * and CPTR_EL2.TZ do not result in trap to EL3. - */ - isb(); + cptr = read_cptr_el2(); + cptr &= ~(CPTR_EL2_TZ_BIT); + write_cptr_el2(cptr); /* * Ensure lower ELs have access to full vector length. */ - write_zcr_el3(ZCR_EL3_LEN_MASK); - - if (el2_unused) { - /* - * Update CPTR_EL2 to enable access to SVE functionality - * for Non-secure world, EL2 and Non-secure EL1 and EL0. - * NOTE - assumed that CPTR_EL2.TFP is set to allow - * access to the SIMD, floating-point and SVE support. - * - * CPTR_EL2.TZ: Set to 0 to enable access to SVE support - * for EL2 and Non-secure EL1 and EL0. - */ - cptr = read_cptr_el2(); - cptr &= ~(CPTR_EL2_TZ_BIT); - write_cptr_el2(cptr); - - /* - * Ensure lower ELs have access to full vector length. - */ - write_zcr_el2(ZCR_EL2_LEN_MASK); - } - /* - * No explicit ISB required here as ERET to switch to - * Non-secure world covers it. - */ + write_zcr_el2(ZCR_EL2_LEN_MASK); } + /* + * No explicit ISB required here as ERET to switch to + * Non-secure world covers it. + */ } SUBSCRIBE_TO_EVENT(cm_exited_normal_world, disable_sve_hook); diff --git a/lib/psci/aarch32/psci_helpers.S b/lib/psci/aarch32/psci_helpers.S index 9373d4f1..a29a29c4 100644 --- a/lib/psci/aarch32/psci_helpers.S +++ b/lib/psci/aarch32/psci_helpers.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2016-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -91,6 +91,28 @@ func psci_do_pwrup_cache_maintenance stcopr r0, SCTLR isb +#if PLAT_XLAT_TABLES_DYNAMIC + /* --------------------------------------------- + * During warm boot the MMU is enabled with data + * cache disabled, then the interconnect is set + * up and finally the data cache is enabled. + * + * During this period, if another CPU modifies + * the translation tables, the MMU table walker + * may read the old entries. This is only a + * problem for dynamic regions, the warm boot + * code isn't affected because it is static. + * + * Invalidate all TLB entries loaded while the + * CPU wasn't coherent with the rest of the + * system. + * --------------------------------------------- + */ + stcopr r0, TLBIALL + dsb ish + isb +#endif + pop {r12, pc} endfunc psci_do_pwrup_cache_maintenance diff --git a/lib/psci/aarch64/psci_helpers.S b/lib/psci/aarch64/psci_helpers.S index afe21ebe..d37ca764 100644 --- a/lib/psci/aarch64/psci_helpers.S +++ b/lib/psci/aarch64/psci_helpers.S @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014-2016, ARM Limited and Contributors. All rights reserved. + * Copyright (c) 2014-2018, ARM Limited and Contributors. All rights reserved. * * SPDX-License-Identifier: BSD-3-Clause */ @@ -115,6 +115,28 @@ func psci_do_pwrup_cache_maintenance msr sctlr_el3, x0 isb +#if PLAT_XLAT_TABLES_DYNAMIC + /* --------------------------------------------- + * During warm boot the MMU is enabled with data + * cache disabled, then the interconnect is set + * up and finally the data cache is enabled. + * + * During this period, if another CPU modifies + * the translation tables, the MMU table walker + * may read the old entries. This is only a + * problem for dynamic regions, the warm boot + * code isn't affected because it is static. + * + * Invalidate all TLB entries loaded while the + * CPU wasn't coherent with the rest of the + * system. + * --------------------------------------------- + */ + tlbi alle3 + dsb ish + isb +#endif + ldp x29, x30, [sp], #16 ret endfunc psci_do_pwrup_cache_maintenance diff --git a/lib/xlat_tables/xlat_tables_common.c b/lib/xlat_tables/xlat_tables_common.c index c6fa10ee..13099362 100644 --- a/lib/xlat_tables/xlat_tables_common.c +++ b/lib/xlat_tables/xlat_tables_common.c @@ -31,7 +31,7 @@ #define debug_print(...) ((void)0) #endif -#define UNSET_DESC ~0ull +#define UNSET_DESC ~0ULL static uint64_t xlat_tables[MAX_XLAT_TABLES][XLAT_TABLE_ENTRIES] __aligned(XLAT_TABLE_SIZE) __section("xlat_table"); diff --git a/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c b/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c index fc7ca46a..7d67a4ad 100644 --- a/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c +++ b/lib/xlat_tables_v2/aarch32/xlat_tables_arch.c @@ -22,7 +22,7 @@ unsigned long long xlat_arch_get_max_supported_pa(void) { /* Physical address space size for long descriptor format. */ - return (1ull << 40) - 1ull; + return (1ULL << 40) - 1ULL; } #endif /* ENABLE_ASSERTIONS*/ diff --git a/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c b/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c index aa5b9e54..ffbc665f 100644 --- a/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c +++ b/lib/xlat_tables_v2/aarch64/xlat_tables_arch.c @@ -62,7 +62,7 @@ unsigned long long xlat_arch_get_max_supported_pa(void) /* All other values are reserved */ assert(pa_range < ARRAY_SIZE(pa_range_bits_arr)); - return (1ull << pa_range_bits_arr[pa_range]) - 1ull; + return (1ULL << pa_range_bits_arr[pa_range]) - 1ULL; } #endif /* ENABLE_ASSERTIONS*/ |