From f30fe6314698d107edbb9db50bc3c3443a30ec80 Mon Sep 17 00:00:00 2001 From: Tianjia Zhang Date: Mon, 26 Sep 2022 17:14:40 +0800 Subject: crypto: scatterwalk - remove duplicate function declarations scatterwalk_map() is an inline function already defined in the header file, it is necessary to delete the re-declaration at the same location, which was left out in the header file by an earlier modification. Signed-off-by: Tianjia Zhang Signed-off-by: Herbert Xu --- include/crypto/scatterwalk.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include') diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index ccdb05f68a75..f2c42b4111b1 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -93,7 +93,6 @@ static inline void scatterwalk_done(struct scatter_walk *walk, int out, void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, size_t nbytes, int out); -void *scatterwalk_map(struct scatter_walk *walk); void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, unsigned int start, unsigned int nbytes, int out); -- cgit v1.2.3 From 7984ceb134bf31aa9a597f10ed52d831d5aede14 Mon Sep 17 00:00:00 2001 From: Frederick Lawler Date: Mon, 17 Oct 2022 14:25:00 -0500 Subject: crypto: af_alg - Support symmetric encryption via keyring keys We want to leverage keyring to store sensitive keys, and then use those keys for symmetric encryption via the crypto API. Among the key types we wish to support are: user, logon, encrypted, and trusted. User key types are already able to have their data copied to user space, but logon does not support this. Further, trusted and encrypted keys will return their encrypted data back to user space on read, which does not make them ideal for symmetric encryption. To support symmetric encryption for these key types, add a new ALG_SET_KEY_BY_KEY_SERIAL setsockopt() option to the crypto API. This allows users to pass a key_serial_t to the crypto API to perform symmetric encryption. The behavior is the same as ALG_SET_KEY, but the crypto key data is copied in kernel space from a keyring key, which allows for the support of logon, encrypted, and trusted key types. Keyring keys must have the KEY_(POS|USR|GRP|OTH)_SEARCH permission set to leverage this feature. This follows the asymmetric_key type where key lookup calls eventually lead to keyring_search_rcu() without the KEYRING_SEARCH_NO_CHECK_PERM flag set. Signed-off-by: Frederick Lawler Signed-off-by: Herbert Xu --- include/uapi/linux/if_alg.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/uapi/linux/if_alg.h b/include/uapi/linux/if_alg.h index 578b18aab821..0824fbc026a1 100644 --- a/include/uapi/linux/if_alg.h +++ b/include/uapi/linux/if_alg.h @@ -52,6 +52,7 @@ struct af_alg_iv { #define ALG_SET_AEAD_ASSOCLEN 4 #define ALG_SET_AEAD_AUTHSIZE 5 #define ALG_SET_DRBG_ENTROPY 6 +#define ALG_SET_KEY_BY_KEY_SERIAL 7 /* Operations */ #define ALG_OP_DECRYPT 0 -- cgit v1.2.3 From 520af5da664a8edc4f4c1cd8e6e8488ecccdb7e5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 3 Nov 2022 20:22:59 +0100 Subject: crypto: lib/aesgcm - Provide minimal library implementation Implement a minimal library version of AES-GCM based on the existing library implementations of AES and multiplication in GF(2^128). Using these primitives, GCM can be implemented in a straight-forward manner. GCM has a couple of sharp edges, i.e., the amount of input data processed with the same initialization vector (IV) should be capped to protect the counter from 32-bit rollover (or carry), and the size of the authentication tag should be fixed for a given key. [0] The former concern is addressed trivially, given that the function call API uses 32-bit signed types for the input lengths. It is still up to the caller to avoid IV reuse in general, but this is not something we can police at the implementation level. As for the latter concern, let's make the authentication tag size part of the key schedule, and only permit it to be configured as part of the key expansion routine. Note that table based AES implementations are susceptible to known plaintext timing attacks on the encryption key. The AES library already attempts to mitigate this to some extent, but given that the counter mode encryption used by GCM operates exclusively on known plaintext by construction (the IV and therefore the initial counter value are known to an attacker), let's take some extra care to mitigate this, by calling the AES library with interrupts disabled. [0] https://nvlpubs.nist.gov/nistpubs/legacy/sp/nistspecialpublication800-38d.pdf Link: https://lore.kernel.org/all/c6fb9b25-a4b6-2e4a-2dd1-63adda055a49@amd.com/ Signed-off-by: Ard Biesheuvel Tested-by: Nikunj A Dadhania Signed-off-by: Herbert Xu --- include/crypto/gcm.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'include') diff --git a/include/crypto/gcm.h b/include/crypto/gcm.h index 9d7eff04f224..fd9df607a836 100644 --- a/include/crypto/gcm.h +++ b/include/crypto/gcm.h @@ -3,6 +3,9 @@ #include +#include +#include + #define GCM_AES_IV_SIZE 12 #define GCM_RFC4106_IV_SIZE 8 #define GCM_RFC4543_IV_SIZE 8 @@ -60,4 +63,23 @@ static inline int crypto_ipsec_check_assoclen(unsigned int assoclen) return 0; } + +struct aesgcm_ctx { + be128 ghash_key; + struct crypto_aes_ctx aes_ctx; + unsigned int authsize; +}; + +int aesgcm_expandkey(struct aesgcm_ctx *ctx, const u8 *key, + unsigned int keysize, unsigned int authsize); + +void aesgcm_encrypt(const struct aesgcm_ctx *ctx, u8 *dst, const u8 *src, + int crypt_len, const u8 *assoc, int assoc_len, + const u8 iv[GCM_AES_IV_SIZE], u8 *authtag); + +bool __must_check aesgcm_decrypt(const struct aesgcm_ctx *ctx, u8 *dst, + const u8 *src, int crypt_len, const u8 *assoc, + int assoc_len, const u8 iv[GCM_AES_IV_SIZE], + const u8 *authtag); + #endif -- cgit v1.2.3 From 16bdbae394280f1d97933d919023eccbf0b564bd Mon Sep 17 00:00:00 2001 From: "Jason A. Donenfeld" Date: Mon, 7 Nov 2022 13:24:55 +0100 Subject: hwrng: core - treat default_quality as a maximum and default to 1024 Most hw_random devices return entropy which is assumed to be of full quality, but driver authors don't bother setting the quality knob. Some hw_random devices return less than full quality entropy, and then driver authors set the quality knob. Therefore, the entropy crediting should be opt-out rather than opt-in per-driver, to reflect the actual reality on the ground. For example, the two Raspberry Pi RNG drivers produce full entropy randomness, and both EDK2 and U-Boot's drivers for these treat them as such. The result is that EFI then uses these numbers and passes the to Linux, and Linux credits them as boot, thereby initializing the RNG. Yet, in Linux, the quality knob was never set to anything, and so on the chance that Linux is booted without EFI, nothing is ever credited. That's annoying. The same pattern appears to repeat itself throughout various drivers. In fact, very very few drivers have bothered setting quality=1024. Looking at the git history of existing drivers and corresponding mailing list discussion, this conclusion tracks. There's been a decent amount of discussion about drivers that set quality < 1024 -- somebody read and interepreted a datasheet, or made some back of the envelope calculation somehow. But there's been very little, if any, discussion about most drivers where the quality is just set to 1024 or unset (or set to 1000 when the authors misunderstood the API and assumed it was base-10 rather than base-2); in both cases the intent was fairly clear of, "this is a hardware random device; it's fine." So let's invert this logic. A hw_random struct's quality knob now controls the maximum quality a driver can produce, or 0 to specify 1024. Then, the module-wide switch called "default_quality" is changed to represent the maximum quality of any driver. By default it's 1024, and the quality of any particular driver is then given by: min(default_quality, rng->quality ?: 1024); This way, the user can still turn this off for weird reasons (and we can replace whatever driver-specific disabling hacks existed in the past), yet we get proper crediting for relevant RNGs. Cc: Dominik Brodowski Cc: Ard Biesheuvel Cc: Herbert Xu Signed-off-by: Jason A. Donenfeld Signed-off-by: Herbert Xu --- include/linux/hw_random.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index 77c2885c4c13..8a3115516a1b 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -34,7 +34,7 @@ * @priv: Private data, for use by the RNG driver. * @quality: Estimation of true entropy in RNG's bitstream * (in bits of entropy per 1024 bits of input; - * valid values: 1 to 1024, or 0 for unknown). + * valid values: 1 to 1024, or 0 for maximum). */ struct hwrng { const char *name; -- cgit v1.2.3 From cc7710d0d4ebc6998f04035cde4f32c5ddbe9d7f Mon Sep 17 00:00:00 2001 From: Xiongfeng Wang Date: Fri, 11 Nov 2022 18:00:36 +0800 Subject: crypto: hisilicon/qm - add missing pci_dev_put() in q_num_set() pci_get_device() will increase the reference count for the returned pci_dev. We need to use pci_dev_put() to decrease the reference count before q_num_set() returns. Fixes: c8b4b477079d ("crypto: hisilicon - add HiSilicon HPRE accelerator") Signed-off-by: Xiongfeng Wang Reviewed-by: Weili Qian Signed-off-by: Herbert Xu --- include/linux/hisi_acc_qm.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index e230c7c46110..c3618255b150 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -384,14 +384,14 @@ struct hisi_qp { static inline int q_num_set(const char *val, const struct kernel_param *kp, unsigned int device) { - struct pci_dev *pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, - device, NULL); + struct pci_dev *pdev; u32 n, q_num; int ret; if (!val) return -EINVAL; + pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, device, NULL); if (!pdev) { q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2); pr_info("No device found currently, suppose queue number is %u\n", @@ -401,6 +401,8 @@ static inline int q_num_set(const char *val, const struct kernel_param *kp, q_num = QM_QNUM_V1; else q_num = QM_QNUM_V2; + + pci_dev_put(pdev); } ret = kstrtou32(val, 10, &n); -- cgit v1.2.3 From e6cb02bd0a52457e486a752da5db7b67f2540c16 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 11 Nov 2022 18:05:41 +0800 Subject: crypto: skcipher - Allow sync algorithms with large request contexts Some sync algorithms may require a large amount of temporary space during its operations. There is no reason why they should be limited just because some legacy users want to place all temporary data on the stack. Such algorithms can now set a flag to indicate that they need extra request context, which will cause them to be invisible to users that go through the sync_skcipher interface. Signed-off-by: Herbert Xu --- include/crypto/internal/skcipher.h | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'include') diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index a2339f80a615..2a97540156bb 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -14,6 +14,14 @@ #include #include +/* + * Set this if your algorithm is sync but needs a reqsize larger + * than MAX_SYNC_SKCIPHER_REQSIZE. + * + * Reuse bit that is specific to hash algorithms. + */ +#define CRYPTO_ALG_SKCIPHER_REQSIZE_LARGE CRYPTO_ALG_OPTIONAL_KEY + struct aead_request; struct rtattr; -- cgit v1.2.3 From b40b62ed7b0ffe8eb2e6fe8bcfb47027c9a93e93 Mon Sep 17 00:00:00 2001 From: Kai Ye Date: Sat, 12 Nov 2022 02:12:51 +0000 Subject: crypto: hisilicon/qm - modify the process of regs dfx The last register logic and different register logic are combined. Use "u32" instead of 'int' in the regs function input parameter to simplify some checks. Signed-off-by: Kai Ye Signed-off-by: Herbert Xu --- include/linux/hisi_acc_qm.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index c3618255b150..be3aedaa96dc 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -471,11 +471,11 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen); int hisi_qm_sriov_configure(struct pci_dev *pdev, int num_vfs); void hisi_qm_dev_err_init(struct hisi_qm *qm); void hisi_qm_dev_err_uninit(struct hisi_qm *qm); -int hisi_qm_diff_regs_init(struct hisi_qm *qm, - struct dfx_diff_registers *dregs, int reg_len); -void hisi_qm_diff_regs_uninit(struct hisi_qm *qm, int reg_len); +int hisi_qm_regs_debugfs_init(struct hisi_qm *qm, + struct dfx_diff_registers *dregs, u32 reg_len); +void hisi_qm_regs_debugfs_uninit(struct hisi_qm *qm, u32 reg_len); void hisi_qm_acc_diff_regs_dump(struct hisi_qm *qm, struct seq_file *s, - struct dfx_diff_registers *dregs, int regs_len); + struct dfx_diff_registers *dregs, u32 regs_len); pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, pci_channel_state_t state); -- cgit v1.2.3 From c060e16ddb51a92b1f7fa84c628d287ea5799864 Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 18 Nov 2022 11:44:21 -0800 Subject: Revert "crypto: shash - avoid comparing pointers to exported functions under CFI" This reverts commit 22ca9f4aaf431a9413dcc115dd590123307f274f because CFI no longer breaks cross-module function address equality, so crypto_shash_alg_has_setkey() can now be an inline function like before. This commit should not be backported to kernels that don't have the new CFI implementation. Acked-by: Peter Zijlstra (Intel) Reviewed-by: Sami Tolvanen Signed-off-by: Eric Biggers Signed-off-by: Herbert Xu --- include/crypto/internal/hash.h | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 25806141db59..0a288dddcf5b 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -75,7 +75,13 @@ void crypto_unregister_ahashes(struct ahash_alg *algs, int count); int ahash_register_instance(struct crypto_template *tmpl, struct ahash_instance *inst); -bool crypto_shash_alg_has_setkey(struct shash_alg *alg); +int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, + unsigned int keylen); + +static inline bool crypto_shash_alg_has_setkey(struct shash_alg *alg) +{ + return alg->setkey != shash_no_setkey; +} static inline bool crypto_shash_alg_needs_key(struct shash_alg *alg) { -- cgit v1.2.3 From 56861cbde1b9f3b34d300e6ba87f2c3de1a9c309 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 22 Nov 2022 17:24:01 +0800 Subject: crypto: kpp - Add helper to set reqsize The value of reqsize should only be changed through a helper. To do so we need to first add a helper for this. Signed-off-by: Herbert Xu --- include/crypto/internal/kpp.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'include') diff --git a/include/crypto/internal/kpp.h b/include/crypto/internal/kpp.h index 9cb0662ebe87..31ff3c1986ef 100644 --- a/include/crypto/internal/kpp.h +++ b/include/crypto/internal/kpp.h @@ -50,6 +50,12 @@ static inline void *kpp_request_ctx(struct kpp_request *req) return req->__ctx; } +static inline void kpp_set_reqsize(struct crypto_kpp *kpp, + unsigned int reqsize) +{ + crypto_kpp_alg(kpp)->reqsize = reqsize; +} + static inline void *kpp_tfm_ctx(struct crypto_kpp *tfm) { return tfm->base.__crt_ctx; -- cgit v1.2.3 From 3e71e5b0efcc730216f4450b796df4fdd627ecd0 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 22 Nov 2022 18:03:35 +0800 Subject: crypto: akcipher - Move reqsize into tfm The value of reqsize cannot be determined in case of fallbacks. Therefore it must be stored in the tfm and not the alg object. Signed-off-by: Herbert Xu --- include/crypto/akcipher.h | 7 ++++--- include/crypto/internal/akcipher.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/crypto/akcipher.h b/include/crypto/akcipher.h index 5764b46bd1ec..734c213918bd 100644 --- a/include/crypto/akcipher.h +++ b/include/crypto/akcipher.h @@ -43,9 +43,12 @@ struct akcipher_request { * struct crypto_akcipher - user-instantiated objects which encapsulate * algorithms and core processing logic * + * @reqsize: Request context size required by algorithm implementation * @base: Common crypto API algorithm data structure */ struct crypto_akcipher { + unsigned int reqsize; + struct crypto_tfm base; }; @@ -86,7 +89,6 @@ struct crypto_akcipher { * counterpart to @init, used to remove various changes set in * @init. * - * @reqsize: Request context size required by algorithm implementation * @base: Common crypto API algorithm data structure */ struct akcipher_alg { @@ -102,7 +104,6 @@ struct akcipher_alg { int (*init)(struct crypto_akcipher *tfm); void (*exit)(struct crypto_akcipher *tfm); - unsigned int reqsize; struct crypto_alg base; }; @@ -155,7 +156,7 @@ static inline struct akcipher_alg *crypto_akcipher_alg( static inline unsigned int crypto_akcipher_reqsize(struct crypto_akcipher *tfm) { - return crypto_akcipher_alg(tfm)->reqsize; + return tfm->reqsize; } static inline void akcipher_request_set_tfm(struct akcipher_request *req, diff --git a/include/crypto/internal/akcipher.h b/include/crypto/internal/akcipher.h index 8d3220c9ab77..1474a2d890fc 100644 --- a/include/crypto/internal/akcipher.h +++ b/include/crypto/internal/akcipher.h @@ -36,7 +36,7 @@ static inline void *akcipher_request_ctx(struct akcipher_request *req) static inline void akcipher_set_reqsize(struct crypto_akcipher *akcipher, unsigned int reqsize) { - crypto_akcipher_alg(akcipher)->reqsize = reqsize; + akcipher->reqsize = reqsize; } static inline void *akcipher_tfm_ctx(struct crypto_akcipher *tfm) -- cgit v1.2.3 From 4d2b225a67e6df962bbeaad473bfd8f97cfbf478 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Tue, 22 Nov 2022 18:09:16 +0800 Subject: crypto: kpp - Move reqsize into tfm The value of reqsize cannot be determined in case of fallbacks. Therefore it must be stored in the tfm and not the alg object. Signed-off-by: Herbert Xu --- include/crypto/internal/kpp.h | 2 +- include/crypto/kpp.h | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'include') diff --git a/include/crypto/internal/kpp.h b/include/crypto/internal/kpp.h index 31ff3c1986ef..167662407e36 100644 --- a/include/crypto/internal/kpp.h +++ b/include/crypto/internal/kpp.h @@ -53,7 +53,7 @@ static inline void *kpp_request_ctx(struct kpp_request *req) static inline void kpp_set_reqsize(struct crypto_kpp *kpp, unsigned int reqsize) { - crypto_kpp_alg(kpp)->reqsize = reqsize; + kpp->reqsize = reqsize; } static inline void *kpp_tfm_ctx(struct crypto_kpp *tfm) diff --git a/include/crypto/kpp.h b/include/crypto/kpp.h index 24d01e9877c1..33ff32878802 100644 --- a/include/crypto/kpp.h +++ b/include/crypto/kpp.h @@ -37,9 +37,13 @@ struct kpp_request { * struct crypto_kpp - user-instantiated object which encapsulate * algorithms and core processing logic * + * @reqsize: Request context size required by algorithm + * implementation * @base: Common crypto API algorithm data structure */ struct crypto_kpp { + unsigned int reqsize; + struct crypto_tfm base; }; @@ -64,8 +68,6 @@ struct crypto_kpp { * put in place here. * @exit: Undo everything @init did. * - * @reqsize: Request context size required by algorithm - * implementation * @base: Common crypto API algorithm data structure */ struct kpp_alg { @@ -79,7 +81,6 @@ struct kpp_alg { int (*init)(struct crypto_kpp *tfm); void (*exit)(struct crypto_kpp *tfm); - unsigned int reqsize; struct crypto_alg base; }; @@ -128,7 +129,7 @@ static inline struct kpp_alg *crypto_kpp_alg(struct crypto_kpp *tfm) static inline unsigned int crypto_kpp_reqsize(struct crypto_kpp *tfm) { - return crypto_kpp_alg(tfm)->reqsize; + return tfm->reqsize; } static inline void kpp_request_set_tfm(struct kpp_request *req, -- cgit v1.2.3 From 14386d471322a204344ae81a28738b71e261d3a0 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Nov 2022 12:36:28 +0800 Subject: crypto: Prepare to move crypto_tfm_ctx The helper crypto_tfm_ctx is only used by the Crypto API algorithm code and should really be in algapi.h. However, for historical reasons many files relied on it to be in crypto.h. This patch changes those files to use algapi.h instead in prepartion for a move. Signed-off-by: Herbert Xu --- include/crypto/aria.h | 2 +- include/crypto/internal/acompress.h | 2 ++ include/crypto/internal/scompress.h | 3 ++- 3 files changed, 5 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/crypto/aria.h b/include/crypto/aria.h index 254da46cc385..73295146be11 100644 --- a/include/crypto/aria.h +++ b/include/crypto/aria.h @@ -18,11 +18,11 @@ #ifndef _CRYPTO_ARIA_H #define _CRYPTO_ARIA_H +#include #include #include #include #include -#include #include #define ARIA_MIN_KEY_SIZE 16 diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index cfc47e18820f..49339003bd2c 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -8,7 +8,9 @@ */ #ifndef _CRYPTO_ACOMP_INT_H #define _CRYPTO_ACOMP_INT_H + #include +#include /* * Transform internal helpers. diff --git a/include/crypto/internal/scompress.h b/include/crypto/internal/scompress.h index f834274c2493..252cc949d4ee 100644 --- a/include/crypto/internal/scompress.h +++ b/include/crypto/internal/scompress.h @@ -8,7 +8,8 @@ */ #ifndef _CRYPTO_SCOMP_INT_H #define _CRYPTO_SCOMP_INT_H -#include + +#include #define SCOMP_SCRATCH_SIZE 131072 -- cgit v1.2.3 From e634ac4a8aaab37bdc69177df9b40acf92eccc6d Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Nov 2022 12:36:31 +0800 Subject: crypto: api - Add crypto_tfm_ctx_dma This patch adds the helpers crypto_tfm_ctx_aligned and crypto_tfm_ctx_dma. The first aligns the tfm context to the value cra_alignmask. The second sets the alignment according to dma_cache_get_alignment(); This patch also moves crypto_tfm_ctx into algapi.h. Signed-off-by: Herbert Xu --- include/crypto/algapi.h | 41 +++++++++++++++++++++++++++++++++++++++-- include/linux/crypto.h | 5 ----- 2 files changed, 39 insertions(+), 7 deletions(-) (limited to 'include') diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index f50c5d1725da..4c99eb66e654 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -7,6 +7,7 @@ #ifndef _CRYPTO_ALGAPI_H #define _CRYPTO_ALGAPI_H +#include #include #include #include @@ -25,6 +26,14 @@ #define MAX_CIPHER_BLOCKSIZE 16 #define MAX_CIPHER_ALIGNMASK 15 +#ifdef ARCH_DMA_MINALIGN +#define CRYPTO_DMA_ALIGN ARCH_DMA_MINALIGN +#else +#define CRYPTO_DMA_ALIGN CRYPTO_MINALIGN +#endif + +#define CRYPTO_DMA_PADDING ((CRYPTO_DMA_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1)) + struct crypto_aead; struct crypto_instance; struct module; @@ -189,10 +198,38 @@ static inline void crypto_xor_cpy(u8 *dst, const u8 *src1, const u8 *src2, } } +static inline void *crypto_tfm_ctx(struct crypto_tfm *tfm) +{ + return tfm->__crt_ctx; +} + +static inline void *crypto_tfm_ctx_align(struct crypto_tfm *tfm, + unsigned int align) +{ + if (align <= crypto_tfm_ctx_alignment()) + align = 1; + + return PTR_ALIGN(crypto_tfm_ctx(tfm), align); +} + static inline void *crypto_tfm_ctx_aligned(struct crypto_tfm *tfm) { - return PTR_ALIGN(crypto_tfm_ctx(tfm), - crypto_tfm_alg_alignmask(tfm) + 1); + return crypto_tfm_ctx_align(tfm, crypto_tfm_alg_alignmask(tfm) + 1); +} + +static inline unsigned int crypto_dma_align(void) +{ + return CRYPTO_DMA_ALIGN; +} + +static inline unsigned int crypto_dma_padding(void) +{ + return (crypto_dma_align() - 1) & ~(crypto_tfm_ctx_alignment() - 1); +} + +static inline void *crypto_tfm_ctx_dma(struct crypto_tfm *tfm) +{ + return crypto_tfm_ctx_align(tfm, crypto_dma_align()); } static inline struct crypto_instance *crypto_tfm_alg_instance( diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 2324ab6f1846..5d1e961f810e 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -714,11 +714,6 @@ static inline void crypto_tfm_clear_flags(struct crypto_tfm *tfm, u32 flags) tfm->crt_flags &= ~flags; } -static inline void *crypto_tfm_ctx(struct crypto_tfm *tfm) -{ - return tfm->__crt_ctx; -} - static inline unsigned int crypto_tfm_ctx_alignment(void) { struct crypto_tfm *tfm; -- cgit v1.2.3 From f8e4d1d0ac832de8efc98f302acf9476bbfffb55 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Nov 2022 12:36:33 +0800 Subject: crypto: aead - Add ctx helpers with DMA alignment This patch adds helpers to access the aead context structure and request context structure with an added alignment for DMA access. Signed-off-by: Herbert Xu --- include/crypto/internal/aead.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'include') diff --git a/include/crypto/internal/aead.h b/include/crypto/internal/aead.h index d482017f3e20..cd8cb1e921b7 100644 --- a/include/crypto/internal/aead.h +++ b/include/crypto/internal/aead.h @@ -39,6 +39,11 @@ static inline void *crypto_aead_ctx(struct crypto_aead *tfm) return crypto_tfm_ctx(&tfm->base); } +static inline void *crypto_aead_ctx_dma(struct crypto_aead *tfm) +{ + return crypto_tfm_ctx_dma(&tfm->base); +} + static inline struct crypto_instance *aead_crypto_instance( struct aead_instance *inst) { @@ -65,6 +70,16 @@ static inline void *aead_request_ctx(struct aead_request *req) return req->__ctx; } +static inline void *aead_request_ctx_dma(struct aead_request *req) +{ + unsigned int align = crypto_dma_align(); + + if (align <= crypto_tfm_ctx_alignment()) + align = 1; + + return PTR_ALIGN(aead_request_ctx(req), align); +} + static inline void aead_request_complete(struct aead_request *req, int err) { req->base.complete(&req->base, err); @@ -108,6 +123,13 @@ static inline void crypto_aead_set_reqsize(struct crypto_aead *aead, aead->reqsize = reqsize; } +static inline void crypto_aead_set_reqsize_dma(struct crypto_aead *aead, + unsigned int reqsize) +{ + reqsize += crypto_dma_align() & ~(crypto_tfm_ctx_alignment() - 1); + aead->reqsize = reqsize; +} + static inline void aead_init_queue(struct aead_queue *queue, unsigned int max_qlen) { -- cgit v1.2.3 From b5f755fbd5d1102104c502ae213e9b42478c098f Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Nov 2022 12:36:35 +0800 Subject: crypto: hash - Add ctx helpers with DMA alignment This patch adds helpers to access the ahash context structure and request context structure with an added alignment for DMA access. Signed-off-by: Herbert Xu --- include/crypto/internal/hash.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'include') diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 0a288dddcf5b..1a2a41b79253 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -140,6 +140,11 @@ static inline void *crypto_ahash_ctx(struct crypto_ahash *tfm) return crypto_tfm_ctx(crypto_ahash_tfm(tfm)); } +static inline void *crypto_ahash_ctx_dma(struct crypto_ahash *tfm) +{ + return crypto_tfm_ctx_dma(crypto_ahash_tfm(tfm)); +} + static inline struct ahash_alg *__crypto_ahash_alg(struct crypto_alg *alg) { return container_of(__crypto_hash_alg_common(alg), struct ahash_alg, @@ -152,6 +157,13 @@ static inline void crypto_ahash_set_reqsize(struct crypto_ahash *tfm, tfm->reqsize = reqsize; } +static inline void crypto_ahash_set_reqsize_dma(struct crypto_ahash *ahash, + unsigned int reqsize) +{ + reqsize += crypto_dma_align() & ~(crypto_tfm_ctx_alignment() - 1); + ahash->reqsize = reqsize; +} + static inline struct crypto_instance *ahash_crypto_instance( struct ahash_instance *inst) { @@ -175,6 +187,16 @@ static inline void *ahash_instance_ctx(struct ahash_instance *inst) return crypto_instance_ctx(ahash_crypto_instance(inst)); } +static inline void *ahash_request_ctx_dma(struct ahash_request *req) +{ + unsigned int align = crypto_dma_align(); + + if (align <= crypto_tfm_ctx_alignment()) + align = 1; + + return PTR_ALIGN(ahash_request_ctx(req), align); +} + static inline void ahash_request_complete(struct ahash_request *req, int err) { req->base.complete(&req->base, err); -- cgit v1.2.3 From 12658ac5e612214023c26f0689e6bbe8bbea0871 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Nov 2022 12:36:37 +0800 Subject: crypto: skcipher - Add ctx helpers with DMA alignment This patch adds helpers to access the skcipher context structure and request context structure with an added alignment for DMA access. Signed-off-by: Herbert Xu --- include/crypto/internal/skcipher.h | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'include') diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index 2a97540156bb..06d0a5491cf3 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -130,6 +130,13 @@ static inline void crypto_skcipher_set_reqsize( skcipher->reqsize = reqsize; } +static inline void crypto_skcipher_set_reqsize_dma( + struct crypto_skcipher *skcipher, unsigned int reqsize) +{ + reqsize += crypto_dma_align() & ~(crypto_tfm_ctx_alignment() - 1); + skcipher->reqsize = reqsize; +} + int crypto_register_skcipher(struct skcipher_alg *alg); void crypto_unregister_skcipher(struct skcipher_alg *alg); int crypto_register_skciphers(struct skcipher_alg *algs, int count); @@ -159,11 +166,26 @@ static inline void *crypto_skcipher_ctx(struct crypto_skcipher *tfm) return crypto_tfm_ctx(&tfm->base); } +static inline void *crypto_skcipher_ctx_dma(struct crypto_skcipher *tfm) +{ + return crypto_tfm_ctx_dma(&tfm->base); +} + static inline void *skcipher_request_ctx(struct skcipher_request *req) { return req->__ctx; } +static inline void *skcipher_request_ctx_dma(struct skcipher_request *req) +{ + unsigned int align = crypto_dma_align(); + + if (align <= crypto_tfm_ctx_alignment()) + align = 1; + + return PTR_ALIGN(skcipher_request_ctx(req), align); +} + static inline u32 skcipher_request_flags(struct skcipher_request *req) { return req->base.flags; -- cgit v1.2.3 From 1c799571976da15e055f32a0e244697500e97f64 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Nov 2022 12:36:39 +0800 Subject: crypto: api - Increase MAX_ALGAPI_ALIGNMASK to 127 Previously we limited the maximum alignment mask to 63. This is mostly due to stack usage for shash. This patch introduces a separate limit for shash algorithms and increases the general limit to 127 which is the value that we need for DMA allocations on arm64. Signed-off-by: Herbert Xu --- include/crypto/algapi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 4c99eb66e654..8722fd67f40a 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -22,7 +22,7 @@ * algs and architectures. Ciphers have a lower maximum size. */ #define MAX_ALGAPI_BLOCKSIZE 160 -#define MAX_ALGAPI_ALIGNMASK 63 +#define MAX_ALGAPI_ALIGNMASK 127 #define MAX_CIPHER_BLOCKSIZE 16 #define MAX_CIPHER_ALIGNMASK 15 -- cgit v1.2.3 From 4ac3377645e98d319cb5404e72d40a4aa69d252c Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Nov 2022 12:36:41 +0800 Subject: crypto: akcipher - Add ctx helpers with DMA alignment This patch adds helpers to access the akcipher context structure and request context structure with an added alignment for DMA access. Signed-off-by: Herbert Xu --- include/crypto/internal/akcipher.h | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/crypto/internal/akcipher.h b/include/crypto/internal/akcipher.h index 1474a2d890fc..aaf1092b93b8 100644 --- a/include/crypto/internal/akcipher.h +++ b/include/crypto/internal/akcipher.h @@ -33,15 +33,37 @@ static inline void *akcipher_request_ctx(struct akcipher_request *req) return req->__ctx; } +static inline void *akcipher_request_ctx_dma(struct akcipher_request *req) +{ + unsigned int align = crypto_dma_align(); + + if (align <= crypto_tfm_ctx_alignment()) + align = 1; + + return PTR_ALIGN(akcipher_request_ctx(req), align); +} + static inline void akcipher_set_reqsize(struct crypto_akcipher *akcipher, unsigned int reqsize) { akcipher->reqsize = reqsize; } +static inline void akcipher_set_reqsize_dma(struct crypto_akcipher *akcipher, + unsigned int reqsize) +{ + reqsize += crypto_dma_align() & ~(crypto_tfm_ctx_alignment() - 1); + akcipher->reqsize = reqsize; +} + static inline void *akcipher_tfm_ctx(struct crypto_akcipher *tfm) { - return tfm->base.__crt_ctx; + return crypto_tfm_ctx(&tfm->base); +} + +static inline void *akcipher_tfm_ctx_dma(struct crypto_akcipher *tfm) +{ + return crypto_tfm_ctx_dma(&tfm->base); } static inline void akcipher_request_complete(struct akcipher_request *req, -- cgit v1.2.3 From a5a49249effb6f03086214b25719d415cc867b3d Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Fri, 25 Nov 2022 12:36:43 +0800 Subject: crypto: kpp - Add ctx helpers with DMA alignment This patch adds helpers to access the kpp context structure and request context structure with an added alignment for DMA access. Signed-off-by: Herbert Xu --- include/crypto/internal/kpp.h | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/crypto/internal/kpp.h b/include/crypto/internal/kpp.h index 167662407e36..3c9726e89f53 100644 --- a/include/crypto/internal/kpp.h +++ b/include/crypto/internal/kpp.h @@ -50,15 +50,37 @@ static inline void *kpp_request_ctx(struct kpp_request *req) return req->__ctx; } +static inline void *kpp_request_ctx_dma(struct kpp_request *req) +{ + unsigned int align = crypto_dma_align(); + + if (align <= crypto_tfm_ctx_alignment()) + align = 1; + + return PTR_ALIGN(kpp_request_ctx(req), align); +} + static inline void kpp_set_reqsize(struct crypto_kpp *kpp, unsigned int reqsize) { kpp->reqsize = reqsize; } +static inline void kpp_set_reqsize_dma(struct crypto_kpp *kpp, + unsigned int reqsize) +{ + reqsize += crypto_dma_align() & ~(crypto_tfm_ctx_alignment() - 1); + kpp->reqsize = reqsize; +} + static inline void *kpp_tfm_ctx(struct crypto_kpp *tfm) { - return tfm->base.__crt_ctx; + return crypto_tfm_ctx(&tfm->base); +} + +static inline void *kpp_tfm_ctx_dma(struct crypto_kpp *tfm) +{ + return crypto_tfm_ctx_dma(&tfm->base); } static inline void kpp_request_complete(struct kpp_request *req, int err) -- cgit v1.2.3 From eaade84a6302f139aede74fe5a568a70adb9baa2 Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Mon, 5 Dec 2022 12:31:44 +0800 Subject: crypto: api - Use linux/cache.h instead of asm/cache.h Directly including asm/cache.h leads to build failures on powerpc so replace it with linux/cache.h instead. Fixes: e634ac4a8aaa ("crypto: api - Add crypto_tfm_ctx_dma") Reported-by: Stephen Rothwell Signed-off-by: Herbert Xu --- include/crypto/algapi.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 8722fd67f40a..61b327206b55 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -7,8 +7,8 @@ #ifndef _CRYPTO_ALGAPI_H #define _CRYPTO_ALGAPI_H -#include #include +#include #include #include #include -- cgit v1.2.3 From 5fc8041e56782e4d44682f8c2e4d822817a4dae6 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Mon, 28 Nov 2022 12:21:22 +0000 Subject: crypto: acomp - define max size for destination The acomp API allows to send requests with a NULL destination buffer. In this case, the algorithm implementation needs to allocate the destination scatter list, perform the operation and return the buffer to the user. For decompression, data is likely to expand and be bigger than the allocated buffer. Define the maximum size (128KB) that acomp implementations will allocate for decompression operations as destination buffer when they receive a request with a NULL destination buffer. Suggested-by: Herbert Xu Signed-off-by: Giovanni Cabiddu Signed-off-by: Herbert Xu --- include/crypto/acompress.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/crypto/acompress.h b/include/crypto/acompress.h index cb3d6b1c655d..e4bc96528902 100644 --- a/include/crypto/acompress.h +++ b/include/crypto/acompress.h @@ -11,6 +11,7 @@ #include #define CRYPTO_ACOMP_ALLOC_OUTPUT 0x00000001 +#define CRYPTO_ACOMP_DST_MAX 131072 /** * struct acomp_req - asynchronous (de)compression request -- cgit v1.2.3