diff options
| -rw-r--r-- | Makefile | 2 | ||||
| -rw-r--r-- | bl31/bl31.mk | 4 | ||||
| -rw-r--r-- | docs/user-guide.rst | 11 | ||||
| -rw-r--r-- | include/common/aarch64/el3_common_macros.S | 6 | ||||
| -rw-r--r-- | include/lib/aarch64/arch.h | 17 | ||||
| -rw-r--r-- | include/lib/aarch64/arch_helpers.h | 3 | ||||
| -rw-r--r-- | include/lib/extensions/sve.h | 12 | ||||
| -rw-r--r-- | lib/el3_runtime/aarch64/context_mgmt.c | 5 | ||||
| -rw-r--r-- | lib/extensions/sve/sve.c | 126 | ||||
| -rw-r--r-- | make_helpers/defaults.mk | 9 |
10 files changed, 192 insertions, 3 deletions
@@ -463,6 +463,7 @@ $(eval $(call assert_boolean,ENABLE_PMF)) $(eval $(call assert_boolean,ENABLE_PSCI_STAT)) $(eval $(call assert_boolean,ENABLE_RUNTIME_INSTRUMENTATION)) $(eval $(call assert_boolean,ENABLE_SPE_FOR_LOWER_ELS)) +$(eval $(call assert_boolean,ENABLE_SVE_FOR_NS)) $(eval $(call assert_boolean,ERROR_DEPRECATED)) $(eval $(call assert_boolean,GENERATE_COT)) $(eval $(call assert_boolean,GICV2_G0_FOR_EL3)) @@ -503,6 +504,7 @@ $(eval $(call add_define,ENABLE_PMF)) $(eval $(call add_define,ENABLE_PSCI_STAT)) $(eval $(call add_define,ENABLE_RUNTIME_INSTRUMENTATION)) $(eval $(call add_define,ENABLE_SPE_FOR_LOWER_ELS)) +$(eval $(call add_define,ENABLE_SVE_FOR_NS)) $(eval $(call add_define,ERROR_DEPRECATED)) $(eval $(call add_define,GICV2_G0_FOR_EL3)) $(eval $(call add_define,HW_ASSISTED_COHERENCY)) diff --git a/bl31/bl31.mk b/bl31/bl31.mk index ebd0e71a..fdcc9313 100644 --- a/bl31/bl31.mk +++ b/bl31/bl31.mk @@ -54,6 +54,10 @@ ifeq (${ENABLE_AMU},1) BL31_SOURCES += lib/extensions/amu/aarch64/amu.c endif +ifeq (${ENABLE_SVE_FOR_NS},1) +BL31_SOURCES += lib/extensions/sve/sve.c +endif + BL31_LINKERFILE := bl31/bl31.ld.S # Flag used to indicate if Crash reporting via console should be included diff --git a/docs/user-guide.rst b/docs/user-guide.rst index 79d660d0..04acbe7f 100644 --- a/docs/user-guide.rst +++ b/docs/user-guide.rst @@ -354,6 +354,17 @@ Common build options The default is 1 but is automatically disabled when the target architecture is AArch32. +- ``ENABLE_SVE_FOR_NS``: Boolean option to enable Scalable Vector Extension + (SVE) for the Non-secure world only. SVE is an optional architectural feature + for AArch64. Note that when SVE is enabled for the Non-secure world, access + to SIMD and floating-point functionality from the Secure world is disabled. + This is to avoid corruption of the Non-secure world data in the Z-registers + which are aliased by the SIMD and FP registers. The build option is not + compatible with the ``CTX_INCLUDE_FPREGS`` build option, and will raise an + assert on platforms where SVE is implemented and ``ENABLE_SVE_FOR_NS`` set to + 1. The default is 1 but is automatically disabled when the target + architecture is AArch32. + - ``ENABLE_STACK_PROTECTOR``: String option to enable the stack protection checks in GCC. Allowed values are "all", "strong" and "0" (default). "strong" is the recommended stack protection level if this feature is diff --git a/include/common/aarch64/el3_common_macros.S b/include/common/aarch64/el3_common_macros.S index ed35df82..63a0fa77 100644 --- a/include/common/aarch64/el3_common_macros.S +++ b/include/common/aarch64/el3_common_macros.S @@ -127,9 +127,9 @@ * CPTR_EL3.TTA: Set to zero so that System register accesses to the * trace registers do not trap to EL3. * - * CPTR_EL3.TFP: Set to zero so that accesses to Advanced SIMD and - * floating-point functionality do not trap to EL3. - * --------------------------------------------------------------------- + * CPTR_EL3.TFP: Set to zero so that accesses to the V- or Z- registers + * by Advanced SIMD, floating-point or SVE instructions (if implemented) + * do not trap to EL3. */ mov_imm x0, (CPTR_EL3_RESET_VAL & ~(TCPAC_BIT | TTA_BIT | TFP_BIT)) msr cptr_el3, x0 diff --git a/include/lib/aarch64/arch.h b/include/lib/aarch64/arch.h index 65e9fc1b..96e2d5fe 100644 --- a/include/lib/aarch64/arch.h +++ b/include/lib/aarch64/arch.h @@ -114,6 +114,9 @@ #define ID_AA64PFR0_AMU_LENGTH U(4) #define ID_AA64PFR0_AMU_MASK U(0xf) #define ID_AA64PFR0_ELX_MASK U(0xf) +#define ID_AA64PFR0_SVE_SHIFT U(32) +#define ID_AA64PFR0_SVE_MASK U(0xf) +#define ID_AA64PFR0_SVE_LENGTH U(4) /* ID_AA64DFR0_EL1.PMS definitions (for ARMv8.2+) */ #define ID_AA64DFR0_PMS_SHIFT U(32) @@ -301,6 +304,7 @@ #define TAM_BIT (U(1) << 30) #define TTA_BIT (U(1) << 20) #define TFP_BIT (U(1) << 10) +#define CPTR_EZ_BIT (U(1) << 8) #define CPTR_EL3_RESET_VAL U(0x0) /* CPTR_EL2 definitions */ @@ -309,6 +313,7 @@ #define CPTR_EL2_TAM_BIT (U(1) << 30) #define CPTR_EL2_TTA_BIT (U(1) << 20) #define CPTR_EL2_TFP_BIT (U(1) << 10) +#define CPTR_EL2_TZ_BIT (U(1) << 8) #define CPTR_EL2_RESET_VAL CPTR_EL2_RES1 /* CPSR/SPSR definitions */ @@ -556,6 +561,18 @@ #define PMCR_EL0_D_BIT (U(1) << 3) /******************************************************************************* + * Definitions for system register interface to SVE + ******************************************************************************/ +#define ZCR_EL3 S3_6_C1_C2_0 +#define ZCR_EL2 S3_4_C1_C2_0 + +/* ZCR_EL3 definitions */ +#define ZCR_EL3_LEN_MASK U(0xf) + +/* ZCR_EL2 definitions */ +#define ZCR_EL2_LEN_MASK U(0xf) + +/******************************************************************************* * Definitions of MAIR encodings for device and normal memory ******************************************************************************/ /* diff --git a/include/lib/aarch64/arch_helpers.h b/include/lib/aarch64/arch_helpers.h index b6be1675..831dfb06 100644 --- a/include/lib/aarch64/arch_helpers.h +++ b/include/lib/aarch64/arch_helpers.h @@ -329,6 +329,9 @@ DEFINE_RENAME_SYSREG_RW_FUNCS(amcntenset1_el0, AMCNTENSET1_EL0) DEFINE_RENAME_SYSREG_RW_FUNCS(pmblimitr_el1, PMBLIMITR_EL1) +DEFINE_RENAME_SYSREG_WRITE_FUNC(zcr_el3, ZCR_EL3) +DEFINE_RENAME_SYSREG_WRITE_FUNC(zcr_el2, ZCR_EL2) + #define IS_IN_EL(x) \ (GET_EL(read_CurrentEl()) == MODE_EL##x) diff --git a/include/lib/extensions/sve.h b/include/lib/extensions/sve.h new file mode 100644 index 00000000..28923e3f --- /dev/null +++ b/include/lib/extensions/sve.h @@ -0,0 +1,12 @@ +/* + * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#ifndef __SVE_H__ +#define __SVE_H__ + +void sve_enable(int el2_unused); + +#endif /* __SVE_H__ */ diff --git a/lib/el3_runtime/aarch64/context_mgmt.c b/lib/el3_runtime/aarch64/context_mgmt.c index b892729e..c6c2249a 100644 --- a/lib/el3_runtime/aarch64/context_mgmt.c +++ b/lib/el3_runtime/aarch64/context_mgmt.c @@ -18,6 +18,7 @@ #include <smcc_helpers.h> #include <spe.h> #include <string.h> +#include <sve.h> #include <utils.h> @@ -225,6 +226,10 @@ static void enable_extensions_nonsecure(int el2_unused) #if ENABLE_AMU amu_enable(el2_unused); #endif + +#if ENABLE_SVE_FOR_NS + sve_enable(el2_unused); +#endif #endif } diff --git a/lib/extensions/sve/sve.c b/lib/extensions/sve/sve.c new file mode 100644 index 00000000..14e51bd8 --- /dev/null +++ b/lib/extensions/sve/sve.c @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2017, ARM Limited and Contributors. All rights reserved. + * + * SPDX-License-Identifier: BSD-3-Clause + */ + +#include <arch.h> +#include <arch_helpers.h> +#include <pubsub.h> +#include <sve.h> + +static void *disable_sve_hook(const void *arg) +{ + uint64_t features; + + features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT; + if ((features & ID_AA64PFR0_SVE_MASK) == 1) { + uint64_t cptr; + + /* + * Disable SVE, SIMD and FP access for the Secure world. + * As the SIMD/FP registers are part of the SVE Z-registers, any + * use of SIMD/FP functionality will corrupt the SVE registers. + * Therefore it is necessary to prevent use of SIMD/FP support + * in the Secure world as well as SVE functionality. + */ + cptr = read_cptr_el3(); + cptr = (cptr | TFP_BIT) & ~(CPTR_EZ_BIT); + write_cptr_el3(cptr); + + /* + * No explicit ISB required here as ERET to switch to Secure + * world covers it + */ + } + return 0; +} + +static void *enable_sve_hook(const void *arg) +{ + uint64_t features; + + features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT; + if ((features & ID_AA64PFR0_SVE_MASK) == 1) { + uint64_t cptr; + + /* + * Enable SVE, SIMD and FP access for the Non-secure world. + */ + cptr = read_cptr_el3(); + cptr = (cptr | CPTR_EZ_BIT) & ~(TFP_BIT); + write_cptr_el3(cptr); + + /* + * No explicit ISB required here as ERET to switch to Non-secure + * world covers it + */ + } + return 0; +} + +void sve_enable(int el2_unused) +{ + uint64_t features; + + features = read_id_aa64pfr0_el1() >> ID_AA64PFR0_SVE_SHIFT; + if ((features & ID_AA64PFR0_SVE_MASK) == 1) { + uint64_t cptr; +#if CTX_INCLUDE_FPREGS + /* + * CTX_INCLUDE_FPREGS is not supported on SVE enabled systems. + */ + assert(0); +#endif + /* + * Update CPTR_EL3 to enable access to SVE functionality for the + * Non-secure world. + * NOTE - assumed that CPTR_EL3.TFP is set to allow access to + * the SIMD, floating-point and SVE support. + * + * CPTR_EL3.EZ: Set to 1 to enable access to SVE functionality + * in the Non-secure world. + */ + cptr = read_cptr_el3(); + cptr |= CPTR_EZ_BIT; + write_cptr_el3(cptr); + + /* + * Need explicit ISB here to guarantee that update to ZCR_ELx + * and CPTR_EL2.TZ do not result in trap to EL3. + */ + isb(); + + /* + * Ensure lower ELs have access to full vector length. + */ + write_zcr_el3(ZCR_EL3_LEN_MASK); + + if (el2_unused) { + /* + * Update CPTR_EL2 to enable access to SVE functionality + * for Non-secure world, EL2 and Non-secure EL1 and EL0. + * NOTE - assumed that CPTR_EL2.TFP is set to allow + * access to the SIMD, floating-point and SVE support. + * + * CPTR_EL2.TZ: Set to 0 to enable access to SVE support + * for EL2 and Non-secure EL1 and EL0. + */ + cptr = read_cptr_el2(); + cptr &= ~(CPTR_EL2_TZ_BIT); + write_cptr_el2(cptr); + + /* + * Ensure lower ELs have access to full vector length. + */ + write_zcr_el2(ZCR_EL2_LEN_MASK); + } + /* + * No explicit ISB required here as ERET to switch to + * Non-secure world covers it. + */ + } +} + +SUBSCRIBE_TO_EVENT(cm_exited_normal_world, disable_sve_hook); +SUBSCRIBE_TO_EVENT(cm_entering_normal_world, enable_sve_hook); diff --git a/make_helpers/defaults.mk b/make_helpers/defaults.mk index 9f7abed1..1d20080e 100644 --- a/make_helpers/defaults.mk +++ b/make_helpers/defaults.mk @@ -158,3 +158,12 @@ ifeq (${ARCH},aarch32) endif ENABLE_AMU := 0 + +# By default, enable Scalable Vector Extension if implemented for Non-secure +# lower ELs +# Note SVE is only supported on AArch64 - therefore do not enable in AArch32 +ifneq (${ARCH},aarch32) + ENABLE_SVE_FOR_NS := 1 +else + override ENABLE_SVE_FOR_NS := 0 +endif |