diff options
Diffstat (limited to 'arch/mips/mach-octeon/include/mach/cvmx-fau.h')
| -rw-r--r-- | arch/mips/mach-octeon/include/mach/cvmx-fau.h | 581 |
1 files changed, 581 insertions, 0 deletions
diff --git a/arch/mips/mach-octeon/include/mach/cvmx-fau.h b/arch/mips/mach-octeon/include/mach/cvmx-fau.h new file mode 100644 index 00000000000..d795ff6e9b0 --- /dev/null +++ b/arch/mips/mach-octeon/include/mach/cvmx-fau.h @@ -0,0 +1,581 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2018-2022 Marvell International Ltd. + * + * Interface to the hardware Fetch and Add Unit. + */ + +#ifndef __CVMX_FAU_H__ +#define __CVMX_FAU_H__ + +extern u8 *cvmx_fau_regs_ptr; + +/** + * Initializes fau, on devices with FAU hw this is a noop. + */ +int cvmx_fau_init(void); + +/** + * Return the location of emulated FAU register + */ +static inline u8 *__cvmx_fau_sw_addr(int reg) +{ + if (cvmx_unlikely(!cvmx_fau_regs_ptr)) + cvmx_fau_init(); + return (cvmx_fau_regs_ptr + reg); +} + +/** + * Perform an atomic 64 bit add + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 8 for 64 bit access. + * @param value Signed value to add. + * Note: Only the low 22 bits are available. + * @return Value of the register before the update + */ +static inline int64_t cvmx_fau_fetch_and_add64(cvmx_fau_reg64_t reg, + int64_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) + return cvmx_hwfau_fetch_and_add64(reg, value); + + return __atomic_fetch_add(CASTPTR(int64_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 32 bit add + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 4 for 32 bit access. + * @param value Signed value to add. + * Note: Only the low 22 bits are available. + * @return Value of the register before the update + */ +static inline int32_t cvmx_fau_fetch_and_add32(cvmx_fau_reg32_t reg, + int32_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) + return cvmx_hwfau_fetch_and_add32(reg, value); + + reg ^= SWIZZLE_32; + return __atomic_fetch_add(CASTPTR(int32_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 16 bit add + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 2 for 16 bit access. + * @param value Signed value to add. + * @return Value of the register before the update + */ +static inline int16_t cvmx_fau_fetch_and_add16(cvmx_fau_reg16_t reg, + int16_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) + return cvmx_hwfau_fetch_and_add16(reg, value); + + reg ^= SWIZZLE_16; + return __atomic_fetch_add(CASTPTR(int16_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 8 bit add + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * @param value Signed value to add. + * @return Value of the register before the update + */ +static inline int8_t cvmx_fau_fetch_and_add8(cvmx_fau_reg8_t reg, int8_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) + return cvmx_hwfau_fetch_and_add8(reg, value); + + reg ^= SWIZZLE_8; + return __atomic_fetch_add(CASTPTR(int8_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 64 bit add after the current tag switch + * completes + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 8 for 64 bit access. + * @param value Signed value to add. + * Note: Only the low 22 bits are available. + * @return If a timeout occurs, the error bit will be set. Otherwise + * the value of the register before the update will be + * returned + */ +static inline cvmx_fau_tagwait64_t +cvmx_fau_tagwait_fetch_and_add64(cvmx_fau_reg64_t reg, int64_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) + return cvmx_hwfau_tagwait_fetch_and_add64(reg, value); + + /* not implemented yet.*/ + return (cvmx_fau_tagwait64_t){ 1, 0 }; +} + +/** + * Perform an atomic 32 bit add after the current tag switch + * completes + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 4 for 32 bit access. + * @param value Signed value to add. + * Note: Only the low 22 bits are available. + * @return If a timeout occurs, the error bit will be set. Otherwise + * the value of the register before the update will be + * returned + */ +static inline cvmx_fau_tagwait32_t +cvmx_fau_tagwait_fetch_and_add32(cvmx_fau_reg32_t reg, int32_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) + return cvmx_hwfau_tagwait_fetch_and_add32(reg, value); + + /* not implemented yet.*/ + return (cvmx_fau_tagwait32_t){ 1, 0 }; +} + +/** + * Perform an atomic 16 bit add after the current tag switch + * completes + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 2 for 16 bit access. + * @param value Signed value to add. + * @return If a timeout occurs, the error bit will be set. Otherwise + * the value of the register before the update will be + * returned + */ +static inline cvmx_fau_tagwait16_t +cvmx_fau_tagwait_fetch_and_add16(cvmx_fau_reg16_t reg, int16_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) + return cvmx_hwfau_tagwait_fetch_and_add16(reg, value); + + /* not implemented yet.*/ + return (cvmx_fau_tagwait16_t){ 1, 0 }; +} + +/** + * Perform an atomic 8 bit add after the current tag switch + * completes + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * @param value Signed value to add. + * @return If a timeout occurs, the error bit will be set. Otherwise + * the value of the register before the update will be + * returned + */ +static inline cvmx_fau_tagwait8_t +cvmx_fau_tagwait_fetch_and_add8(cvmx_fau_reg8_t reg, int8_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) + return cvmx_hwfau_tagwait_fetch_and_add8(reg, value); + + /* not implemented yet.*/ + return (cvmx_fau_tagwait8_t){ 1, 0 }; +} + +/** + * Perform an async atomic 64 bit add. The old value is + * placed in the scratch memory at byte address scraddr. + * + * @param scraddr Scratch memory byte address to put response in. + * Must be 8 byte aligned. + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 8 for 64 bit access. + * @param value Signed value to add. + * Note: Only the low 22 bits are available. + * @return Placed in the scratch pad register + */ +static inline void +cvmx_fau_async_fetch_and_add64(u64 scraddr, cvmx_fau_reg64_t reg, int64_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_async_fetch_and_add64(scraddr, reg, value); + return; + } + cvmx_scratch_write64( + scraddr, + __atomic_fetch_add(CASTPTR(int64_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST)); +} + +/** + * Perform an async atomic 32 bit add. The old value is + * placed in the scratch memory at byte address scraddr. + * + * @param scraddr Scratch memory byte address to put response in. + * Must be 8 byte aligned. + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 4 for 32 bit access. + * @param value Signed value to add. + * Note: Only the low 22 bits are available. + * @return Placed in the scratch pad register + */ +static inline void +cvmx_fau_async_fetch_and_add32(u64 scraddr, cvmx_fau_reg32_t reg, int32_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_async_fetch_and_add32(scraddr, reg, value); + return; + } + cvmx_scratch_write64( + scraddr, + __atomic_fetch_add(CASTPTR(int32_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST)); +} + +/** + * Perform an async atomic 16 bit add. The old value is + * placed in the scratch memory at byte address scraddr. + * + * @param scraddr Scratch memory byte address to put response in. + * Must be 8 byte aligned. + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 2 for 16 bit access. + * @param value Signed value to add. + * @return Placed in the scratch pad register + */ +static inline void +cvmx_fau_async_fetch_and_add16(u64 scraddr, cvmx_fau_reg16_t reg, int16_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_async_fetch_and_add16(scraddr, reg, value); + return; + } + cvmx_scratch_write64( + scraddr, + __atomic_fetch_add(CASTPTR(int16_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST)); +} + +/** + * Perform an async atomic 8 bit add. The old value is + * placed in the scratch memory at byte address scraddr. + * + * @param scraddr Scratch memory byte address to put response in. + * Must be 8 byte aligned. + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * @param value Signed value to add. + * @return Placed in the scratch pad register + */ +static inline void +cvmx_fau_async_fetch_and_add8(u64 scraddr, cvmx_fau_reg8_t reg, int8_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_async_fetch_and_add8(scraddr, reg, value); + return; + } + cvmx_scratch_write64( + scraddr, + __atomic_fetch_add(CASTPTR(int8_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST)); +} + +/** + * Perform an async atomic 64 bit add after the current tag + * switch completes. + * + * @param scraddr Scratch memory byte address to put response in. + * Must be 8 byte aligned. + * If a timeout occurs, the error bit (63) will be set. Otherwise + * the value of the register before the update will be + * returned + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 8 for 64 bit access. + * @param value Signed value to add. + * Note: Only the low 22 bits are available. + * @return Placed in the scratch pad register + */ +static inline void cvmx_fau_async_tagwait_fetch_and_add64(u64 scraddr, + cvmx_fau_reg64_t reg, + int64_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_async_tagwait_fetch_and_add64(scraddr, reg, value); + return; + } + + /* Broken. Where is the tag wait? */ + cvmx_scratch_write64( + scraddr, + __atomic_fetch_add(CASTPTR(int64_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST)); +} + +/** + * Perform an async atomic 32 bit add after the current tag + * switch completes. + * + * @param scraddr Scratch memory byte address to put response in. + * Must be 8 byte aligned. + * If a timeout occurs, the error bit (63) will be set. Otherwise + * the value of the register before the update will be + * returned + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 4 for 32 bit access. + * @param value Signed value to add. + * Note: Only the low 22 bits are available. + * @return Placed in the scratch pad register + */ +static inline void cvmx_fau_async_tagwait_fetch_and_add32(u64 scraddr, + cvmx_fau_reg32_t reg, + int32_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_async_tagwait_fetch_and_add32(scraddr, reg, value); + return; + } + /* Broken. Where is the tag wait? */ + cvmx_scratch_write64( + scraddr, + __atomic_fetch_add(CASTPTR(int32_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST)); +} + +/** + * Perform an async atomic 16 bit add after the current tag + * switch completes. + * + * @param scraddr Scratch memory byte address to put response in. + * Must be 8 byte aligned. + * If a timeout occurs, the error bit (63) will be set. Otherwise + * the value of the register before the update will be + * returned + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 2 for 16 bit access. + * @param value Signed value to add. + * @return Placed in the scratch pad register + */ +static inline void cvmx_fau_async_tagwait_fetch_and_add16(u64 scraddr, + cvmx_fau_reg16_t reg, + int16_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_async_tagwait_fetch_and_add16(scraddr, reg, value); + return; + } + /* Broken. Where is the tag wait? */ + cvmx_scratch_write64( + scraddr, + __atomic_fetch_add(CASTPTR(int16_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST)); +} + +/** + * Perform an async atomic 8 bit add after the current tag + * switch completes. + * + * @param scraddr Scratch memory byte address to put response in. + * Must be 8 byte aligned. + * If a timeout occurs, the error bit (63) will be set. Otherwise + * the value of the register before the update will be + * returned + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * @param value Signed value to add. + * @return Placed in the scratch pad register + */ +static inline void cvmx_fau_async_tagwait_fetch_and_add8(u64 scraddr, + cvmx_fau_reg8_t reg, + int8_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_async_tagwait_fetch_and_add8(scraddr, reg, value); + return; + } + /* Broken. Where is the tag wait? */ + cvmx_scratch_write64( + scraddr, + __atomic_fetch_add(CASTPTR(int8_t, __cvmx_fau_sw_addr(reg)), + value, __ATOMIC_SEQ_CST)); +} + +/** + * Perform an atomic 64 bit add + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 8 for 64 bit access. + * @param value Signed value to add. + */ +static inline void cvmx_fau_atomic_add64(cvmx_fau_reg64_t reg, int64_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_atomic_add64(reg, value); + return; + } + /* Ignored fetch values should be optimized away */ + __atomic_add_fetch(CASTPTR(int64_t, __cvmx_fau_sw_addr(reg)), value, + __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 32 bit add + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 4 for 32 bit access. + * @param value Signed value to add. + */ +static inline void cvmx_fau_atomic_add32(cvmx_fau_reg32_t reg, int32_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_atomic_add32(reg, value); + return; + } + reg ^= SWIZZLE_32; + /* Ignored fetch values should be optimized away */ + __atomic_add_fetch(CASTPTR(int32_t, __cvmx_fau_sw_addr(reg)), value, + __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 16 bit add + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 2 for 16 bit access. + * @param value Signed value to add. + */ +static inline void cvmx_fau_atomic_add16(cvmx_fau_reg16_t reg, int16_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_atomic_add16(reg, value); + return; + } + reg ^= SWIZZLE_16; + /* Ignored fetch values should be optimized away */ + __atomic_add_fetch(CASTPTR(int16_t, __cvmx_fau_sw_addr(reg)), value, + __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 8 bit add + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * @param value Signed value to add. + */ +static inline void cvmx_fau_atomic_add8(cvmx_fau_reg8_t reg, int8_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_atomic_add8(reg, value); + return; + } + reg ^= SWIZZLE_8; + /* Ignored fetch values should be optimized away */ + __atomic_add_fetch(CASTPTR(int8_t, __cvmx_fau_sw_addr(reg)), value, + __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 64 bit write + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 8 for 64 bit access. + * @param value Signed value to write. + */ +static inline void cvmx_fau_atomic_write64(cvmx_fau_reg64_t reg, int64_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_atomic_write64(reg, value); + return; + } + __atomic_store_n(CASTPTR(int64_t, __cvmx_fau_sw_addr(reg)), value, + __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 32 bit write + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 4 for 32 bit access. + * @param value Signed value to write. + */ +static inline void cvmx_fau_atomic_write32(cvmx_fau_reg32_t reg, int32_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_atomic_write32(reg, value); + return; + } + reg ^= SWIZZLE_32; + __atomic_store_n(CASTPTR(int32_t, __cvmx_fau_sw_addr(reg)), value, + __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 16 bit write + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * - Step by 2 for 16 bit access. + * @param value Signed value to write. + */ +static inline void cvmx_fau_atomic_write16(cvmx_fau_reg16_t reg, int16_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_atomic_write16(reg, value); + return; + } + reg ^= SWIZZLE_16; + __atomic_store_n(CASTPTR(int16_t, __cvmx_fau_sw_addr(reg)), value, + __ATOMIC_SEQ_CST); +} + +/** + * Perform an atomic 8 bit write + * + * @param reg FAU atomic register to access. 0 <= reg < 2048. + * @param value Signed value to write. + */ +static inline void cvmx_fau_atomic_write8(cvmx_fau_reg8_t reg, int8_t value) +{ + if (octeon_has_feature(OCTEON_FEATURE_FAU)) { + cvmx_hwfau_atomic_write8(reg, value); + return; + } + reg ^= SWIZZLE_8; + __atomic_store_n(CASTPTR(int8_t, __cvmx_fau_sw_addr(reg)), value, + __ATOMIC_SEQ_CST); +} + +/** Allocates 64bit FAU register. + * @param reserve base address to reserve + * @return value is the base address of allocated FAU register + */ +int cvmx_fau64_alloc(int reserve); + +/** Allocates 32bit FAU register. + * @param reserve base address to reserve + * @return value is the base address of allocated FAU register + */ +int cvmx_fau32_alloc(int reserve); + +/** Allocates 16bit FAU register. + * @param reserve base address to reserve + * @return value is the base address of allocated FAU register + */ +int cvmx_fau16_alloc(int reserve); + +/** Allocates 8bit FAU register. + * @param reserve base address to reserve + * @return value is the base address of allocated FAU register + */ +int cvmx_fau8_alloc(int reserve); + +/** Frees the specified FAU register. + * @param address base address of register to release. + * @return 0 on success; -1 on failure + */ +int cvmx_fau_free(int address); + +/** Display the fau registers array + */ +void cvmx_fau_show(void); + +#endif /* __CVMX_FAU_H__ */ |