From 9552389c465ed1ded39edf4a5642a861b53c2955 Mon Sep 17 00:00:00 2001 From: Gilad Ben-Yossef Date: Tue, 2 Jul 2019 14:39:20 +0300 Subject: crypto: fips - add FIPS test failure notification chain Crypto test failures in FIPS mode cause an immediate panic, but on some system the cryptographic boundary extends beyond just the Linux controlled domain. Add a simple atomic notification chain to allow interested parties to register to receive notification prior to us kicking the bucket. Signed-off-by: Gilad Ben-Yossef Signed-off-by: Herbert Xu --- include/linux/fips.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'include') diff --git a/include/linux/fips.h b/include/linux/fips.h index afeeece92302..c6961e932fef 100644 --- a/include/linux/fips.h +++ b/include/linux/fips.h @@ -4,8 +4,15 @@ #ifdef CONFIG_CRYPTO_FIPS extern int fips_enabled; +extern struct atomic_notifier_head fips_fail_notif_chain; + +void fips_fail_notify(void); + #else #define fips_enabled 0 + +static inline void fips_fail_notify(void) {} + #endif #endif -- cgit v1.2.3 From e59c1c98745637796df824c0177f279b6e9cad94 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 2 Jul 2019 21:41:22 +0200 Subject: crypto: aes - create AES library based on the fixed time AES code Take the existing small footprint and mostly time invariant C code and turn it into a AES library that can be used for non-performance critical, casual use of AES, and as a fallback for, e.g., SIMD code that needs a secondary path that can be taken in contexts where the SIMD unit is off limits (e.g., in hard interrupts taken from kernel context) Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/aes.h | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'include') diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 0fdb542c70cd..d0067fca0cd0 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -37,4 +37,38 @@ int crypto_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len); int crypto_aes_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len); + +/** + * aes_expandkey - Expands the AES key as described in FIPS-197 + * @ctx: The location where the computed key will be stored. + * @in_key: The supplied key. + * @key_len: The length of the supplied key. + * + * Returns 0 on success. The function fails only if an invalid key size (or + * pointer) is supplied. + * The expanded key size is 240 bytes (max of 14 rounds with a unique 16 bytes + * key schedule plus a 16 bytes key which is used before the first round). + * The decryption key is prepared for the "Equivalent Inverse Cipher" as + * described in FIPS-197. The first slot (16 bytes) of each key (enc or dec) is + * for the initial combination, the second slot for the first round and so on. + */ +int aes_expandkey(struct crypto_aes_ctx *ctx, const u8 *in_key, + unsigned int key_len); + +/** + * aes_encrypt - Encrypt a single AES block + * @ctx: Context struct containing the key schedule + * @out: Buffer to store the ciphertext + * @in: Buffer containing the plaintext + */ +void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); + +/** + * aes_decrypt - Decrypt a single AES block + * @ctx: Context struct containing the key schedule + * @out: Buffer to store the plaintext + * @in: Buffer containing the ciphertext + */ +void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); + #endif -- cgit v1.2.3 From 5bb12d7825adf0e80b849a273834f3131a6cc4e1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 2 Jul 2019 21:41:33 +0200 Subject: crypto: aes-generic - drop key expansion routine in favor of library version Drop aes-generic's version of crypto_aes_expand_key(), and switch to the key expansion routine provided by the AES library. AES key expansion is not performance critical, and it is better to have a single version shared by all AES implementations. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/aes.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include') diff --git a/include/crypto/aes.h b/include/crypto/aes.h index d0067fca0cd0..0a64a977f9b3 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -35,8 +35,6 @@ extern const u32 crypto_il_tab[4][256] ____cacheline_aligned; int crypto_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len); -int crypto_aes_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key, - unsigned int key_len); /** * aes_expandkey - Expands the AES key as described in FIPS-197 -- cgit v1.2.3 From d9ec772d9550b6e513c51bc4e6fa1e3ffb50181c Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 2 Jul 2019 21:41:34 +0200 Subject: crypto: ctr - add helper for performing a CTR encryption walk Add a static inline helper modeled after crypto_cbc_encrypt_walk() that can be reused for SIMD algorithms that need to implement a non-SIMD fallback for performing CTR encryption. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/ctr.h | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) (limited to 'include') diff --git a/include/crypto/ctr.h b/include/crypto/ctr.h index 06984a26c8cf..a1c66d1001af 100644 --- a/include/crypto/ctr.h +++ b/include/crypto/ctr.h @@ -8,8 +8,58 @@ #ifndef _CRYPTO_CTR_H #define _CRYPTO_CTR_H +#include +#include +#include +#include + #define CTR_RFC3686_NONCE_SIZE 4 #define CTR_RFC3686_IV_SIZE 8 #define CTR_RFC3686_BLOCK_SIZE 16 +static inline int crypto_ctr_encrypt_walk(struct skcipher_request *req, + void (*fn)(struct crypto_skcipher *, + const u8 *, u8 *)) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + int blocksize = crypto_skcipher_chunksize(tfm); + u8 buf[MAX_CIPHER_BLOCKSIZE]; + struct skcipher_walk walk; + int err; + + /* avoid integer division due to variable blocksize parameter */ + if (WARN_ON_ONCE(!is_power_of_2(blocksize))) + return -EINVAL; + + err = skcipher_walk_virt(&walk, req, false); + + while (walk.nbytes > 0) { + u8 *dst = walk.dst.virt.addr; + u8 *src = walk.src.virt.addr; + int nbytes = walk.nbytes; + int tail = 0; + + if (nbytes < walk.total) { + tail = walk.nbytes & (blocksize - 1); + nbytes -= tail; + } + + do { + int bsize = min(nbytes, blocksize); + + fn(tfm, walk.iv, buf); + + crypto_xor_cpy(dst, src, buf, bsize); + crypto_inc(walk.iv, blocksize); + + dst += bsize; + src += bsize; + nbytes -= bsize; + } while (nbytes > 0); + + err = skcipher_walk_done(&walk, tail); + } + return err; +} + #endif /* _CRYPTO_CTR_H */ -- cgit v1.2.3 From 1e25ca02a0619c10bc0eae975846926902381ce5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 2 Jul 2019 21:41:45 +0200 Subject: crypto: aes-generic - unexport last-round AES tables The versions of the AES lookup tables that are only used during the last round are never used outside of the driver, so there is no need to export their symbols. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/aes.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include') diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 0a64a977f9b3..df8426fd8051 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -29,9 +29,7 @@ struct crypto_aes_ctx { }; extern const u32 crypto_ft_tab[4][256] ____cacheline_aligned; -extern const u32 crypto_fl_tab[4][256] ____cacheline_aligned; extern const u32 crypto_it_tab[4][256] ____cacheline_aligned; -extern const u32 crypto_il_tab[4][256] ____cacheline_aligned; int crypto_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len); -- cgit v1.2.3 From 9467a3150cf4afca638673e099af71e8c493a3a0 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 2 Jul 2019 21:41:46 +0200 Subject: crypto: lib/aes - export sbox and inverse sbox There are a few copies of the AES S-boxes floating around, so export the ones from the AES library so that we can reuse them in other modules. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/aes.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include') diff --git a/include/crypto/aes.h b/include/crypto/aes.h index df8426fd8051..8e0f4cf948e5 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -67,4 +67,7 @@ void aes_encrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); */ void aes_decrypt(const struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); +extern const u8 crypto_aes_sbox[]; +extern const u8 crypto_aes_inv_sbox[]; + #endif -- cgit v1.2.3 From 5cb97700beaa005ceb2a127b6f53536a4544c9d8 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 3 Jul 2019 10:55:06 +0200 Subject: crypto: morus - remove generic and x86 implementations MORUS was not selected as a winner in the CAESAR competition, which is not surprising since it is considered to be cryptographically broken [0]. (Note that this is not an implementation defect, but a flaw in the underlying algorithm). Since it is unlikely to be in use currently, let's remove it before we're stuck with it. [0] https://eprint.iacr.org/2019/172.pdf Reviewed-by: Ondrej Mosnacek Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/morus1280_glue.h | 97 ----------------------------------------- include/crypto/morus640_glue.h | 97 ----------------------------------------- include/crypto/morus_common.h | 18 -------- 3 files changed, 212 deletions(-) delete mode 100644 include/crypto/morus1280_glue.h delete mode 100644 include/crypto/morus640_glue.h delete mode 100644 include/crypto/morus_common.h (limited to 'include') diff --git a/include/crypto/morus1280_glue.h b/include/crypto/morus1280_glue.h deleted file mode 100644 index 5cefddb1991f..000000000000 --- a/include/crypto/morus1280_glue.h +++ /dev/null @@ -1,97 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * The MORUS-1280 Authenticated-Encryption Algorithm - * Common glue skeleton -- header file - * - * Copyright (c) 2016-2018 Ondrej Mosnacek - * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. - */ - -#ifndef _CRYPTO_MORUS1280_GLUE_H -#define _CRYPTO_MORUS1280_GLUE_H - -#include -#include -#include -#include -#include - -#define MORUS1280_WORD_SIZE 8 -#define MORUS1280_BLOCK_SIZE (MORUS_BLOCK_WORDS * MORUS1280_WORD_SIZE) - -struct morus1280_block { - u8 bytes[MORUS1280_BLOCK_SIZE]; -}; - -struct morus1280_glue_ops { - void (*init)(void *state, const void *key, const void *iv); - void (*ad)(void *state, const void *data, unsigned int length); - void (*enc)(void *state, const void *src, void *dst, unsigned int length); - void (*dec)(void *state, const void *src, void *dst, unsigned int length); - void (*enc_tail)(void *state, const void *src, void *dst, unsigned int length); - void (*dec_tail)(void *state, const void *src, void *dst, unsigned int length); - void (*final)(void *state, void *tag_xor, u64 assoclen, u64 cryptlen); -}; - -struct morus1280_ctx { - const struct morus1280_glue_ops *ops; - struct morus1280_block key; -}; - -void crypto_morus1280_glue_init_ops(struct crypto_aead *aead, - const struct morus1280_glue_ops *ops); -int crypto_morus1280_glue_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int keylen); -int crypto_morus1280_glue_setauthsize(struct crypto_aead *tfm, - unsigned int authsize); -int crypto_morus1280_glue_encrypt(struct aead_request *req); -int crypto_morus1280_glue_decrypt(struct aead_request *req); - -#define MORUS1280_DECLARE_ALG(id, driver_name, priority) \ - static const struct morus1280_glue_ops crypto_morus1280_##id##_ops = {\ - .init = crypto_morus1280_##id##_init, \ - .ad = crypto_morus1280_##id##_ad, \ - .enc = crypto_morus1280_##id##_enc, \ - .enc_tail = crypto_morus1280_##id##_enc_tail, \ - .dec = crypto_morus1280_##id##_dec, \ - .dec_tail = crypto_morus1280_##id##_dec_tail, \ - .final = crypto_morus1280_##id##_final, \ - }; \ - \ - static int crypto_morus1280_##id##_init_tfm(struct crypto_aead *tfm) \ - { \ - crypto_morus1280_glue_init_ops(tfm, &crypto_morus1280_##id##_ops); \ - return 0; \ - } \ - \ - static void crypto_morus1280_##id##_exit_tfm(struct crypto_aead *tfm) \ - { \ - } \ - \ - static struct aead_alg crypto_morus1280_##id##_alg = { \ - .setkey = crypto_morus1280_glue_setkey, \ - .setauthsize = crypto_morus1280_glue_setauthsize, \ - .encrypt = crypto_morus1280_glue_encrypt, \ - .decrypt = crypto_morus1280_glue_decrypt, \ - .init = crypto_morus1280_##id##_init_tfm, \ - .exit = crypto_morus1280_##id##_exit_tfm, \ - \ - .ivsize = MORUS_NONCE_SIZE, \ - .maxauthsize = MORUS_MAX_AUTH_SIZE, \ - .chunksize = MORUS1280_BLOCK_SIZE, \ - \ - .base = { \ - .cra_flags = CRYPTO_ALG_INTERNAL, \ - .cra_blocksize = 1, \ - .cra_ctxsize = sizeof(struct morus1280_ctx), \ - .cra_alignmask = 0, \ - .cra_priority = priority, \ - \ - .cra_name = "__morus1280", \ - .cra_driver_name = "__"driver_name, \ - \ - .cra_module = THIS_MODULE, \ - } \ - } - -#endif /* _CRYPTO_MORUS1280_GLUE_H */ diff --git a/include/crypto/morus640_glue.h b/include/crypto/morus640_glue.h deleted file mode 100644 index 0ee6266cb26c..000000000000 --- a/include/crypto/morus640_glue.h +++ /dev/null @@ -1,97 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * The MORUS-640 Authenticated-Encryption Algorithm - * Common glue skeleton -- header file - * - * Copyright (c) 2016-2018 Ondrej Mosnacek - * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. - */ - -#ifndef _CRYPTO_MORUS640_GLUE_H -#define _CRYPTO_MORUS640_GLUE_H - -#include -#include -#include -#include -#include - -#define MORUS640_WORD_SIZE 4 -#define MORUS640_BLOCK_SIZE (MORUS_BLOCK_WORDS * MORUS640_WORD_SIZE) - -struct morus640_block { - u8 bytes[MORUS640_BLOCK_SIZE]; -}; - -struct morus640_glue_ops { - void (*init)(void *state, const void *key, const void *iv); - void (*ad)(void *state, const void *data, unsigned int length); - void (*enc)(void *state, const void *src, void *dst, unsigned int length); - void (*dec)(void *state, const void *src, void *dst, unsigned int length); - void (*enc_tail)(void *state, const void *src, void *dst, unsigned int length); - void (*dec_tail)(void *state, const void *src, void *dst, unsigned int length); - void (*final)(void *state, void *tag_xor, u64 assoclen, u64 cryptlen); -}; - -struct morus640_ctx { - const struct morus640_glue_ops *ops; - struct morus640_block key; -}; - -void crypto_morus640_glue_init_ops(struct crypto_aead *aead, - const struct morus640_glue_ops *ops); -int crypto_morus640_glue_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int keylen); -int crypto_morus640_glue_setauthsize(struct crypto_aead *tfm, - unsigned int authsize); -int crypto_morus640_glue_encrypt(struct aead_request *req); -int crypto_morus640_glue_decrypt(struct aead_request *req); - -#define MORUS640_DECLARE_ALG(id, driver_name, priority) \ - static const struct morus640_glue_ops crypto_morus640_##id##_ops = {\ - .init = crypto_morus640_##id##_init, \ - .ad = crypto_morus640_##id##_ad, \ - .enc = crypto_morus640_##id##_enc, \ - .enc_tail = crypto_morus640_##id##_enc_tail, \ - .dec = crypto_morus640_##id##_dec, \ - .dec_tail = crypto_morus640_##id##_dec_tail, \ - .final = crypto_morus640_##id##_final, \ - }; \ - \ - static int crypto_morus640_##id##_init_tfm(struct crypto_aead *tfm) \ - { \ - crypto_morus640_glue_init_ops(tfm, &crypto_morus640_##id##_ops); \ - return 0; \ - } \ - \ - static void crypto_morus640_##id##_exit_tfm(struct crypto_aead *tfm) \ - { \ - } \ - \ - static struct aead_alg crypto_morus640_##id##_alg = {\ - .setkey = crypto_morus640_glue_setkey, \ - .setauthsize = crypto_morus640_glue_setauthsize, \ - .encrypt = crypto_morus640_glue_encrypt, \ - .decrypt = crypto_morus640_glue_decrypt, \ - .init = crypto_morus640_##id##_init_tfm, \ - .exit = crypto_morus640_##id##_exit_tfm, \ - \ - .ivsize = MORUS_NONCE_SIZE, \ - .maxauthsize = MORUS_MAX_AUTH_SIZE, \ - .chunksize = MORUS640_BLOCK_SIZE, \ - \ - .base = { \ - .cra_flags = CRYPTO_ALG_INTERNAL, \ - .cra_blocksize = 1, \ - .cra_ctxsize = sizeof(struct morus640_ctx), \ - .cra_alignmask = 0, \ - .cra_priority = priority, \ - \ - .cra_name = "__morus640", \ - .cra_driver_name = "__"driver_name, \ - \ - .cra_module = THIS_MODULE, \ - } \ - } - -#endif /* _CRYPTO_MORUS640_GLUE_H */ diff --git a/include/crypto/morus_common.h b/include/crypto/morus_common.h deleted file mode 100644 index 969510a9a56c..000000000000 --- a/include/crypto/morus_common.h +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-or-later */ -/* - * The MORUS Authenticated-Encryption Algorithm - * Common definitions - * - * Copyright (c) 2016-2018 Ondrej Mosnacek - * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. - */ - -#ifndef _CRYPTO_MORUS_COMMON_H -#define _CRYPTO_MORUS_COMMON_H - -#define MORUS_BLOCK_WORDS 4 -#define MORUS_STATE_BLOCKS 5 -#define MORUS_NONCE_SIZE 16 -#define MORUS_MAX_AUTH_SIZE 16 - -#endif /* _CRYPTO_MORUS_COMMON_H */ -- cgit v1.2.3 From 91b05a7e7d8033a90a64f5fc0e3808db423e420a Mon Sep 17 00:00:00 2001 From: Ondrej Mosnacek Date: Tue, 9 Jul 2019 13:11:24 +0200 Subject: crypto: user - make NETLINK_CRYPTO work inside netns Currently, NETLINK_CRYPTO works only in the init network namespace. It doesn't make much sense to cut it out of the other network namespaces, so do the minor plumbing work necessary to make it work in any network namespace. Code inspired by net/core/sock_diag.c. Tested using kcapi-dgst from libkcapi [1]: Before: # unshare -n kcapi-dgst -c sha256 Signed-off-by: Herbert Xu --- include/crypto/internal/cryptouser.h | 2 -- include/net/net_namespace.h | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/crypto/internal/cryptouser.h b/include/crypto/internal/cryptouser.h index 8c602b187c58..40623f4457df 100644 --- a/include/crypto/internal/cryptouser.h +++ b/include/crypto/internal/cryptouser.h @@ -1,8 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include -extern struct sock *crypto_nlsk; - struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact); #ifdef CONFIG_CRYPTO_STATS diff --git a/include/net/net_namespace.h b/include/net/net_namespace.h index 4a9da951a794..85bc1de5dece 100644 --- a/include/net/net_namespace.h +++ b/include/net/net_namespace.h @@ -170,6 +170,9 @@ struct net { #endif #ifdef CONFIG_XDP_SOCKETS struct netns_xdp xdp; +#endif +#if IS_ENABLED(CONFIG_CRYPTO_USER) + struct sock *crypto_nlsk; #endif struct sock *diag_nlsk; atomic_t fnhe_genid; -- cgit v1.2.3 From 6fc4dbcf0276279d488c5fbbfabe94734134f4fa Mon Sep 17 00:00:00 2001 From: Herbert Xu Date: Thu, 18 Jul 2019 23:01:46 +0800 Subject: padata: Replace delayed timer with immediate workqueue in padata_reorder The function padata_reorder will use a timer when it cannot progress while completed jobs are outstanding (pd->reorder_objects > 0). This is suboptimal as if we do end up using the timer then it would have introduced a gratuitous delay of one second. In fact we can easily distinguish between whether completed jobs are outstanding and whether we can make progress. All we have to do is look at the next pqueue list. This patch does that by replacing pd->processed with pd->cpu so that the next pqueue is more accessible. A work queue is used instead of the original try_again to avoid hogging the CPU. Note that we don't bother removing the work queue in padata_flush_queues because the whole premise is broken. You cannot flush async crypto requests so it makes no sense to even try. A subsequent patch will fix it by replacing it with a ref counting scheme. Signed-off-by: Herbert Xu --- include/linux/padata.h | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index 56f09e36f770..8da673861d99 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -12,7 +12,6 @@ #include #include #include -#include #include #include @@ -73,18 +72,14 @@ struct padata_serial_queue { * @serial: List to wait for serialization after reordering. * @pwork: work struct for parallelization. * @swork: work struct for serialization. - * @pd: Backpointer to the internal control structure. * @work: work struct for parallelization. - * @reorder_work: work struct for reordering. * @num_obj: Number of objects that are processed by this cpu. * @cpu_index: Index of the cpu. */ struct padata_parallel_queue { struct padata_list parallel; struct padata_list reorder; - struct parallel_data *pd; struct work_struct work; - struct work_struct reorder_work; atomic_t num_obj; int cpu_index; }; @@ -110,10 +105,10 @@ struct padata_cpumask { * @reorder_objects: Number of objects waiting in the reorder queues. * @refcnt: Number of objects holding a reference on this parallel_data. * @max_seq_nr: Maximal used sequence number. + * @cpu: Next CPU to be processed. * @cpumask: The cpumasks in use for parallel and serial workers. + * @reorder_work: work struct for reordering. * @lock: Reorder lock. - * @processed: Number of already processed objects. - * @timer: Reorder timer. */ struct parallel_data { struct padata_instance *pinst; @@ -122,10 +117,10 @@ struct parallel_data { atomic_t reorder_objects; atomic_t refcnt; atomic_t seq_nr; + int cpu; struct padata_cpumask cpumask; + struct work_struct reorder_work; spinlock_t lock ____cacheline_aligned; - unsigned int processed; - struct timer_list timer; }; /** -- cgit v1.2.3 From 8dfa20fcfbeb245642dfe3a43f8a3735d9aed42a Mon Sep 17 00:00:00 2001 From: Eric Biggers Date: Fri, 19 Jul 2019 23:09:18 -0700 Subject: crypto: ghash - add comment and improve help text To help avoid confusion, add a comment to ghash-generic.c which explains the convention that the kernel's implementation of GHASH uses. Also update the Kconfig help text and module descriptions to call GHASH a "hash function" rather than a "message digest", since the latter normally means a real cryptographic hash function, which GHASH is not. Cc: Pascal Van Leeuwen Signed-off-by: Eric Biggers Reviewed-by: Ard Biesheuvel Acked-by: Pascal Van Leeuwen Signed-off-by: Herbert Xu --- include/crypto/ghash.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/crypto/ghash.h b/include/crypto/ghash.h index 9136301062a5..f832c9f2aca3 100644 --- a/include/crypto/ghash.h +++ b/include/crypto/ghash.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* - * Common values for GHASH algorithms + * Common values for the GHASH hash function */ #ifndef __CRYPTO_GHASH_H__ -- cgit v1.2.3 From 82cb54856874b1b374f18420be013ff4057700a9 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 29 Jul 2019 12:55:21 +0300 Subject: asm-generic: make simd.h a mandatory include/asm header The generic aegis128 software crypto driver recently gained support for using SIMD intrinsics to increase performance, for which it uncondionally #include's the header. Unfortunately, this header does not exist on many architectures, resulting in build failures. Since asm-generic already has a version of simd.h, let's make it a mandatory header so that it gets instantiated on all architectures that don't provide their own version. Signed-off-by: Ard Biesheuvel Acked-by: Arnd Bergmann Signed-off-by: Herbert Xu --- include/asm-generic/Kbuild | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild index 6f4536d70b8e..adff14fcb8e4 100644 --- a/include/asm-generic/Kbuild +++ b/include/asm-generic/Kbuild @@ -3,3 +3,5 @@ # asm headers that all architectures except um should have # (This file is not included when SRCARCH=um since UML borrows several # asm headers from the host architecutre.) + +mandatory-y += simd.h -- cgit v1.2.3 From 68289c634344431d6f97480eb5384516f37e39b8 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 23 Jul 2019 20:43:43 +0900 Subject: crypto: add header include guards Add header include guards in case they are included multiple times. Signed-off-by: Masahiro Yamada Signed-off-by: Herbert Xu --- include/crypto/sha1_base.h | 5 +++++ include/crypto/sha256_base.h | 5 +++++ include/crypto/sha512_base.h | 5 +++++ include/crypto/sm3_base.h | 5 +++++ include/uapi/linux/cryptouser.h | 5 +++++ 5 files changed, 25 insertions(+) (limited to 'include') diff --git a/include/crypto/sha1_base.h b/include/crypto/sha1_base.h index 63c14f2dc7bd..20fd1f7468af 100644 --- a/include/crypto/sha1_base.h +++ b/include/crypto/sha1_base.h @@ -5,6 +5,9 @@ * Copyright (C) 2015 Linaro Ltd */ +#ifndef _CRYPTO_SHA1_BASE_H +#define _CRYPTO_SHA1_BASE_H + #include #include #include @@ -101,3 +104,5 @@ static inline int sha1_base_finish(struct shash_desc *desc, u8 *out) *sctx = (struct sha1_state){}; return 0; } + +#endif /* _CRYPTO_SHA1_BASE_H */ diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index 59159bc944f5..b50a035a2bc7 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -5,6 +5,9 @@ * Copyright (C) 2015 Linaro Ltd */ +#ifndef _CRYPTO_SHA256_BASE_H +#define _CRYPTO_SHA256_BASE_H + #include #include #include @@ -123,3 +126,5 @@ static inline int sha256_base_finish(struct shash_desc *desc, u8 *out) *sctx = (struct sha256_state){}; return 0; } + +#endif /* _CRYPTO_SHA256_BASE_H */ diff --git a/include/crypto/sha512_base.h b/include/crypto/sha512_base.h index 099be8027f3f..fb19c77494dc 100644 --- a/include/crypto/sha512_base.h +++ b/include/crypto/sha512_base.h @@ -5,6 +5,9 @@ * Copyright (C) 2015 Linaro Ltd */ +#ifndef _CRYPTO_SHA512_BASE_H +#define _CRYPTO_SHA512_BASE_H + #include #include #include @@ -126,3 +129,5 @@ static inline int sha512_base_finish(struct shash_desc *desc, u8 *out) *sctx = (struct sha512_state){}; return 0; } + +#endif /* _CRYPTO_SHA512_BASE_H */ diff --git a/include/crypto/sm3_base.h b/include/crypto/sm3_base.h index 31891b0dc7e3..1cbf9aa1fe52 100644 --- a/include/crypto/sm3_base.h +++ b/include/crypto/sm3_base.h @@ -6,6 +6,9 @@ * Written by Gilad Ben-Yossef */ +#ifndef _CRYPTO_SM3_BASE_H +#define _CRYPTO_SM3_BASE_H + #include #include #include @@ -104,3 +107,5 @@ static inline int sm3_base_finish(struct shash_desc *desc, u8 *out) *sctx = (struct sm3_state){}; return 0; } + +#endif /* _CRYPTO_SM3_BASE_H */ diff --git a/include/uapi/linux/cryptouser.h b/include/uapi/linux/cryptouser.h index 4dc1603919ce..5730c67f0617 100644 --- a/include/uapi/linux/cryptouser.h +++ b/include/uapi/linux/cryptouser.h @@ -19,6 +19,9 @@ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. */ +#ifndef _UAPI_LINUX_CRYPTOUSER_H +#define _UAPI_LINUX_CRYPTOUSER_H + #include /* Netlink configuration messages. */ @@ -198,3 +201,5 @@ struct crypto_report_acomp { #define CRYPTO_REPORT_MAXSIZE (sizeof(struct crypto_user_alg) + \ sizeof(struct crypto_report_blkcipher)) + +#endif /* _UAPI_LINUX_CRYPTOUSER_H */ -- cgit v1.2.3 From 939f146b89c22c387d51f69b27ef031664f64cbe Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Tue, 23 Jul 2019 20:43:44 +0900 Subject: crypto: user - fix potential warnings in cryptouser.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Function definitions in headers are usually marked as 'static inline'. Since 'inline' is missing for crypto_reportstat(), if it were not referenced from a .c file that includes this header, it would produce a warning. Also, 'struct crypto_user_alg' is not declared in this header. I included instead of adding the forward declaration as suggested [1]. Detected by compile-testing this header as a standalone unit: ./include/crypto/internal/cryptouser.h:6:44: warning: ‘struct crypto_user_alg’ declared inside parameter list will not be visible outside of this definition or declaration struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact); ^~~~~~~~~~~~~~~ ./include/crypto/internal/cryptouser.h:11:12: warning: ‘crypto_reportstat’ defined but not used [-Wunused-function] static int crypto_reportstat(struct sk_buff *in_skb, struct nlmsghdr *in_nlh, struct nlattr **attrs) ^~~~~~~~~~~~~~~~~ [1] https://lkml.org/lkml/2019/6/13/1121 Signed-off-by: Masahiro Yamada Signed-off-by: Herbert Xu --- include/crypto/internal/cryptouser.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/include/crypto/internal/cryptouser.h b/include/crypto/internal/cryptouser.h index 40623f4457df..fd54074332f5 100644 --- a/include/crypto/internal/cryptouser.h +++ b/include/crypto/internal/cryptouser.h @@ -1,4 +1,5 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#include #include struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact); @@ -6,7 +7,9 @@ struct crypto_alg *crypto_alg_match(struct crypto_user_alg *p, int exact); #ifdef CONFIG_CRYPTO_STATS int crypto_reportstat(struct sk_buff *in_skb, struct nlmsghdr *in_nlh, struct nlattr **attrs); #else -static int crypto_reportstat(struct sk_buff *in_skb, struct nlmsghdr *in_nlh, struct nlattr **attrs) +static inline int crypto_reportstat(struct sk_buff *in_skb, + struct nlmsghdr *in_nlh, + struct nlattr **attrs) { return -ENOTSUPP; } -- cgit v1.2.3 From f9981bc53825de2c001659d35c870786771f1e90 Mon Sep 17 00:00:00 2001 From: Thomas Gleixner Date: Fri, 26 Jul 2019 22:19:07 +0200 Subject: crypto: api - Remove redundant #ifdef in crypto_yield() While looking at CONFIG_PREEMPT dependencies treewide the #ifdef in crypto_yield() matched. CONFIG_PREEMPT and CONFIG_PREEMPT_VOLUNTARY are mutually exclusive so the extra !CONFIG_PREEMPT conditional is redundant. cond_resched() has only an effect when CONFIG_PREEMPT_VOLUNTARY is set, otherwise it's a stub which the compiler optimizes out. Remove the whole conditional. No functional change. Signed-off-by: Thomas Gleixner Cc: linux-crypto@vger.kernel.org Cc: Herbert Xu Cc: "David S. Miller" Signed-off-by: Herbert Xu --- include/crypto/algapi.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include') diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index dc1106af95c3..e5bd302f2c49 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -409,10 +409,8 @@ static inline int crypto_memneq(const void *a, const void *b, size_t size) static inline void crypto_yield(u32 flags) { -#if !defined(CONFIG_PREEMPT) || defined(CONFIG_PREEMPT_VOLUNTARY) if (flags & CRYPTO_TFM_REQ_MAY_SLEEP) cond_resched(); -#endif } int crypto_register_notifier(struct notifier_block *nb); -- cgit v1.2.3 From 2ef540476e0077ca2f9992fde9a014891000c4b5 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 29 Jul 2019 00:32:36 +0900 Subject: hwrng: timeriomem - add include guard to timeriomem-rng.h Add a header include guard just in case. Signed-off-by: Masahiro Yamada Signed-off-by: Herbert Xu --- include/linux/timeriomem-rng.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include') diff --git a/include/linux/timeriomem-rng.h b/include/linux/timeriomem-rng.h index fd4a6e6ec831..672df7fbf6c1 100644 --- a/include/linux/timeriomem-rng.h +++ b/include/linux/timeriomem-rng.h @@ -5,6 +5,9 @@ * Copyright (c) 2009 Alexander Clouter */ +#ifndef _LINUX_TIMERIOMEM_RNG_H +#define _LINUX_TIMERIOMEM_RNG_H + struct timeriomem_rng_data { void __iomem *address; @@ -14,3 +17,5 @@ struct timeriomem_rng_data { /* bits of entropy per 1024 bits read */ unsigned int quality; }; + +#endif /* _LINUX_TIMERIOMEM_RNG_H */ -- cgit v1.2.3 From 65526f638efc6435b9658d45a3ddd07e6fdbbb7d Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Wed, 31 Jul 2019 16:05:54 +0300 Subject: crypto: gcm - helper functions for assoclen/authsize check Added inline helper functions to check authsize and assoclen for gcm, rfc4106 and rfc4543. These are used in the generic implementation of gcm, rfc4106 and rfc4543. Signed-off-by: Iuliana Prodan Signed-off-by: Herbert Xu --- include/crypto/gcm.h | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) (limited to 'include') diff --git a/include/crypto/gcm.h b/include/crypto/gcm.h index c50e057ea17e..9d7eff04f224 100644 --- a/include/crypto/gcm.h +++ b/include/crypto/gcm.h @@ -1,8 +1,63 @@ #ifndef _CRYPTO_GCM_H #define _CRYPTO_GCM_H +#include + #define GCM_AES_IV_SIZE 12 #define GCM_RFC4106_IV_SIZE 8 #define GCM_RFC4543_IV_SIZE 8 +/* + * validate authentication tag for GCM + */ +static inline int crypto_gcm_check_authsize(unsigned int authsize) +{ + switch (authsize) { + case 4: + case 8: + case 12: + case 13: + case 14: + case 15: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +/* + * validate authentication tag for RFC4106 + */ +static inline int crypto_rfc4106_check_authsize(unsigned int authsize) +{ + switch (authsize) { + case 8: + case 12: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +/* + * validate assoclen for RFC4106/RFC4543 + */ +static inline int crypto_ipsec_check_assoclen(unsigned int assoclen) +{ + switch (assoclen) { + case 16: + case 20: + break; + default: + return -EINVAL; + } + + return 0; +} #endif -- cgit v1.2.3 From bc67d04e75260942fb534fb91673103dcad7ca96 Mon Sep 17 00:00:00 2001 From: Iuliana Prodan Date: Wed, 31 Jul 2019 16:05:55 +0300 Subject: crypto: aes - helper function to validate key length for AES algorithms Add inline helper function to check key length for AES algorithms. The key can be 128, 192 or 256 bits size. This function is used in the generic aes implementation. Signed-off-by: Iuliana Prodan Signed-off-by: Herbert Xu --- include/crypto/aes.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'include') diff --git a/include/crypto/aes.h b/include/crypto/aes.h index 8e0f4cf948e5..2090729701ab 100644 --- a/include/crypto/aes.h +++ b/include/crypto/aes.h @@ -31,6 +31,23 @@ struct crypto_aes_ctx { extern const u32 crypto_ft_tab[4][256] ____cacheline_aligned; extern const u32 crypto_it_tab[4][256] ____cacheline_aligned; +/* + * validate key length for AES algorithms + */ +static inline int aes_check_keylen(unsigned int keylen) +{ + switch (keylen) { + case AES_KEYSIZE_128: + case AES_KEYSIZE_192: + case AES_KEYSIZE_256: + break; + default: + return -EINVAL; + } + + return 0; +} + int crypto_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len); -- cgit v1.2.3 From 6ee41e5420d0afa8cddf09aa7384dabe570f8dc7 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 15 Aug 2019 12:00:43 +0300 Subject: crypto: des/3des_ede - add new helpers to verify keys The recently added helper routine to perform key strength validation of triple DES keys is slightly inadequate, since it comes in two versions, neither of which are highly useful for anything other than skciphers (and many drivers still use the older blkcipher interfaces). So let's add a new helper and, considering that this is a helper function that is only intended to be used by crypto code itself, put it in a new des.h header under crypto/internal. While at it, implement a similar helper for single DES, so that we can start replacing the pattern of calling des_ekey() into a temp buffer that occurs in many drivers in drivers/crypto. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/internal/des.h | 141 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 include/crypto/internal/des.h (limited to 'include') diff --git a/include/crypto/internal/des.h b/include/crypto/internal/des.h new file mode 100644 index 000000000000..f5d2e696522e --- /dev/null +++ b/include/crypto/internal/des.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * DES & Triple DES EDE key verification helpers + */ + +#ifndef __CRYPTO_INTERNAL_DES_H +#define __CRYPTO_INTERNAL_DES_H + +#include +#include +#include +#include +#include + +/** + * crypto_des_verify_key - Check whether a DES key is weak + * @tfm: the crypto algo + * @key: the key buffer + * + * Returns -EINVAL if the key is weak and the crypto TFM does not permit weak + * keys. Otherwise, 0 is returned. + * + * It is the job of the caller to ensure that the size of the key equals + * DES_KEY_SIZE. + */ +static inline int crypto_des_verify_key(struct crypto_tfm *tfm, const u8 *key) +{ + u32 tmp[DES_EXPKEY_WORDS]; + int err = 0; + + if (!(crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) + return 0; + + if (!des_ekey(tmp, key)) { + crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); + err = -EINVAL; + } + + memzero_explicit(tmp, sizeof(tmp)); + return err; +} + +/* + * RFC2451: + * + * For DES-EDE3, there is no known need to reject weak or + * complementation keys. Any weakness is obviated by the use of + * multiple keys. + * + * However, if the first two or last two independent 64-bit keys are + * equal (k1 == k2 or k2 == k3), then the DES3 operation is simply the + * same as DES. Implementers MUST reject keys that exhibit this + * property. + * + */ + +/** + * crypto_des3_ede_verify_key - Check whether a DES3-EDE key is weak + * @tfm: the crypto algo + * @key: the key buffer + * + * Returns -EINVAL if the key is weak and the crypto TFM does not permit weak + * keys or when running in FIPS mode. Otherwise, 0 is returned. Note that some + * keys are rejected in FIPS mode even if weak keys are permitted by the TFM + * flags. + * + * It is the job of the caller to ensure that the size of the key equals + * DES3_EDE_KEY_SIZE. + */ +static inline int crypto_des3_ede_verify_key(struct crypto_tfm *tfm, + const u8 *key) +{ + int err = -EINVAL; + u32 K[6]; + + memcpy(K, key, DES3_EDE_KEY_SIZE); + + if ((!((K[0] ^ K[2]) | (K[1] ^ K[3])) || + !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && + (fips_enabled || (crypto_tfm_get_flags(tfm) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS))) + goto bad; + + if ((!((K[0] ^ K[4]) | (K[1] ^ K[5]))) && fips_enabled) + goto bad; + + err = 0; +out: + memzero_explicit(K, DES3_EDE_KEY_SIZE); + return err; + +bad: + crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); + goto out; +} + +static inline int verify_skcipher_des_key(struct crypto_skcipher *tfm, + const u8 *key) +{ + return crypto_des_verify_key(crypto_skcipher_tfm(tfm), key); +} + +static inline int verify_skcipher_des3_key(struct crypto_skcipher *tfm, + const u8 *key) +{ + return crypto_des3_ede_verify_key(crypto_skcipher_tfm(tfm), key); +} + +static inline int verify_ablkcipher_des_key(struct crypto_ablkcipher *tfm, + const u8 *key) +{ + return crypto_des_verify_key(crypto_ablkcipher_tfm(tfm), key); +} + +static inline int verify_ablkcipher_des3_key(struct crypto_ablkcipher *tfm, + const u8 *key) +{ + return crypto_des3_ede_verify_key(crypto_ablkcipher_tfm(tfm), key); +} + +static inline int verify_aead_des_key(struct crypto_aead *tfm, const u8 *key, + int keylen) +{ + if (keylen != DES_KEY_SIZE) { + crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + return crypto_des_verify_key(crypto_aead_tfm(tfm), key); +} + +static inline int verify_aead_des3_key(struct crypto_aead *tfm, const u8 *key, + int keylen) +{ + if (keylen != DES3_EDE_KEY_SIZE) { + crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + return crypto_des3_ede_verify_key(crypto_aead_tfm(tfm), key); +} + +#endif /* __CRYPTO_INTERNAL_DES_H */ -- cgit v1.2.3 From 6b5c4818621fb0d3eda1d4059634b38e09bd7243 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 15 Aug 2019 12:01:08 +0300 Subject: crypto: des - remove unused function Remove the old DES3 verification functions that are no longer used. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/des.h | 41 ----------------------------------------- 1 file changed, 41 deletions(-) (limited to 'include') diff --git a/include/crypto/des.h b/include/crypto/des.h index 72c7c8e5a5a7..31b04ba835b1 100644 --- a/include/crypto/des.h +++ b/include/crypto/des.h @@ -19,47 +19,6 @@ #define DES3_EDE_EXPKEY_WORDS (3 * DES_EXPKEY_WORDS) #define DES3_EDE_BLOCK_SIZE DES_BLOCK_SIZE -static inline int __des3_verify_key(u32 *flags, const u8 *key) -{ - int err = -EINVAL; - u32 K[6]; - - memcpy(K, key, DES3_EDE_KEY_SIZE); - - if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (fips_enabled || - (*flags & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS))) - goto bad; - - if (unlikely(!((K[0] ^ K[4]) | (K[1] ^ K[5]))) && fips_enabled) - goto bad; - - err = 0; - -out: - memzero_explicit(K, DES3_EDE_KEY_SIZE); - - return err; - -bad: - *flags |= CRYPTO_TFM_RES_WEAK_KEY; - goto out; -} - -static inline int des3_verify_key(struct crypto_skcipher *tfm, const u8 *key) -{ - u32 flags; - int err; - - flags = crypto_skcipher_get_flags(tfm); - err = __des3_verify_key(&flags, key); - crypto_skcipher_set_flags(tfm, flags); - return err; -} - -extern unsigned long des_ekey(u32 *pe, const u8 *k); - extern int __des3_ede_setkey(u32 *expkey, u32 *flags, const u8 *key, unsigned int keylen); -- cgit v1.2.3 From 04007b0e6cbbab5836ac891626e91edf10d46341 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 15 Aug 2019 12:01:09 +0300 Subject: crypto: des - split off DES library from generic DES cipher driver Another one for the cipher museum: split off DES core processing into a separate module so other drivers (mostly for crypto accelerators) can reuse the code without pulling in the generic DES cipher itself. This will also permit the cipher interface to be made private to the crypto API itself once we move the only user in the kernel (CIFS) to this library interface. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/des.h | 43 ++++++++++++++++++++++++--- include/crypto/internal/des.h | 69 +++++++++++++++++++++++++------------------ 2 files changed, 79 insertions(+), 33 deletions(-) (limited to 'include') diff --git a/include/crypto/des.h b/include/crypto/des.h index 31b04ba835b1..2c864a4e6707 100644 --- a/include/crypto/des.h +++ b/include/crypto/des.h @@ -6,10 +6,7 @@ #ifndef __CRYPTO_DES_H #define __CRYPTO_DES_H -#include -#include -#include -#include +#include #define DES_KEY_SIZE 8 #define DES_EXPKEY_WORDS 32 @@ -19,6 +16,44 @@ #define DES3_EDE_EXPKEY_WORDS (3 * DES_EXPKEY_WORDS) #define DES3_EDE_BLOCK_SIZE DES_BLOCK_SIZE +struct des_ctx { + u32 expkey[DES_EXPKEY_WORDS]; +}; + +struct des3_ede_ctx { + u32 expkey[DES3_EDE_EXPKEY_WORDS]; +}; + +void des_encrypt(const struct des_ctx *ctx, u8 *dst, const u8 *src); +void des_decrypt(const struct des_ctx *ctx, u8 *dst, const u8 *src); + +void des3_ede_encrypt(const struct des3_ede_ctx *dctx, u8 *dst, const u8 *src); +void des3_ede_decrypt(const struct des3_ede_ctx *dctx, u8 *dst, const u8 *src); + +/** + * des_expand_key - Expand a DES input key into a key schedule + * @ctx: the key schedule + * @key: buffer containing the input key + * @len: size of the buffer contents + * + * Returns 0 on success, -EINVAL if the input key is rejected and -ENOKEY if + * the key is accepted but has been found to be weak. + */ +int des_expand_key(struct des_ctx *ctx, const u8 *key, unsigned int keylen); + +/** + * des3_ede_expand_key - Expand a triple DES input key into a key schedule + * @ctx: the key schedule + * @key: buffer containing the input key + * @len: size of the buffer contents + * + * Returns 0 on success, -EINVAL if the input key is rejected and -ENOKEY if + * the key is accepted but has been found to be weak. Note that weak keys will + * be rejected (and -EINVAL will be returned) when running in FIPS mode. + */ +int des3_ede_expand_key(struct des3_ede_ctx *ctx, const u8 *key, + unsigned int keylen); + extern int __des3_ede_setkey(u32 *expkey, u32 *flags, const u8 *key, unsigned int keylen); diff --git a/include/crypto/internal/des.h b/include/crypto/internal/des.h index f5d2e696522e..81ea1a425e9c 100644 --- a/include/crypto/internal/des.h +++ b/include/crypto/internal/des.h @@ -25,18 +25,21 @@ */ static inline int crypto_des_verify_key(struct crypto_tfm *tfm, const u8 *key) { - u32 tmp[DES_EXPKEY_WORDS]; - int err = 0; - - if (!(crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS)) - return 0; + struct des_ctx tmp; + int err; + + err = des_expand_key(&tmp, key, DES_KEY_SIZE); + if (err == -ENOKEY) { + if (crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_FORBID_WEAK_KEYS) + err = -EINVAL; + else + err = 0; + } - if (!des_ekey(tmp, key)) { + if (err) crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); - err = -EINVAL; - } - memzero_explicit(tmp, sizeof(tmp)); + memzero_explicit(&tmp, sizeof(tmp)); return err; } @@ -53,6 +56,28 @@ static inline int crypto_des_verify_key(struct crypto_tfm *tfm, const u8 *key) * property. * */ +static inline int des3_ede_verify_key(const u8 *key, unsigned int key_len, + bool check_weak) +{ + int ret = fips_enabled ? -EINVAL : -ENOKEY; + u32 K[6]; + + memcpy(K, key, DES3_EDE_KEY_SIZE); + + if ((!((K[0] ^ K[2]) | (K[1] ^ K[3])) || + !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && + (fips_enabled || check_weak)) + goto bad; + + if ((!((K[0] ^ K[4]) | (K[1] ^ K[5]))) && fips_enabled) + goto bad; + + ret = 0; +bad: + memzero_explicit(K, DES3_EDE_KEY_SIZE); + + return ret; +} /** * crypto_des3_ede_verify_key - Check whether a DES3-EDE key is weak @@ -70,28 +95,14 @@ static inline int crypto_des_verify_key(struct crypto_tfm *tfm, const u8 *key) static inline int crypto_des3_ede_verify_key(struct crypto_tfm *tfm, const u8 *key) { - int err = -EINVAL; - u32 K[6]; - - memcpy(K, key, DES3_EDE_KEY_SIZE); - - if ((!((K[0] ^ K[2]) | (K[1] ^ K[3])) || - !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (fips_enabled || (crypto_tfm_get_flags(tfm) & - CRYPTO_TFM_REQ_FORBID_WEAK_KEYS))) - goto bad; - - if ((!((K[0] ^ K[4]) | (K[1] ^ K[5]))) && fips_enabled) - goto bad; + int err; - err = 0; -out: - memzero_explicit(K, DES3_EDE_KEY_SIZE); + err = des3_ede_verify_key(key, DES3_EDE_KEY_SIZE, + crypto_tfm_get_flags(tfm) & + CRYPTO_TFM_REQ_FORBID_WEAK_KEYS); + if (err) + crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); return err; - -bad: - crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_WEAK_KEY); - goto out; } static inline int verify_skcipher_des_key(struct crypto_skcipher *tfm, -- cgit v1.2.3 From 18fbe0da8e98fe167fbfe1757003e2a2a74d24f3 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Thu, 15 Aug 2019 12:01:11 +0300 Subject: crypto: des - remove now unused __des3_ede_setkey() Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/des.h | 3 --- 1 file changed, 3 deletions(-) (limited to 'include') diff --git a/include/crypto/des.h b/include/crypto/des.h index 2c864a4e6707..7812b4331ae4 100644 --- a/include/crypto/des.h +++ b/include/crypto/des.h @@ -54,7 +54,4 @@ int des_expand_key(struct des_ctx *ctx, const u8 *key, unsigned int keylen); int des3_ede_expand_key(struct des3_ede_ctx *ctx, const u8 *key, unsigned int keylen); -extern int __des3_ede_setkey(u32 *expkey, u32 *flags, const u8 *key, - unsigned int keylen); - #endif /* __CRYPTO_DES_H */ -- cgit v1.2.3 From ad767ee858b38af634c957a792cb001d54a7b981 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 17 Aug 2019 16:24:31 +0200 Subject: crypto: sha256 - Move lib/sha256.c to lib/crypto Generic crypto implementations belong under lib/crypto not directly in lib, likewise the header should be in include/crypto, not include/linux. Note that the code in lib/crypto/sha256.c is not yet available for generic use after this commit, it is still only used by the s390 and x86 purgatory code. Making it suitable for generic use is done in further patches in this series. Signed-off-by: Hans de Goede Signed-off-by: Herbert Xu --- include/crypto/sha256.h | 28 ++++++++++++++++++++++++++++ include/linux/sha256.h | 28 ---------------------------- 2 files changed, 28 insertions(+), 28 deletions(-) create mode 100644 include/crypto/sha256.h delete mode 100644 include/linux/sha256.h (limited to 'include') diff --git a/include/crypto/sha256.h b/include/crypto/sha256.h new file mode 100644 index 000000000000..b1f9c6781082 --- /dev/null +++ b/include/crypto/sha256.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2014 Red Hat Inc. + * + * Author: Vivek Goyal + */ + +#ifndef SHA256_H +#define SHA256_H + +#include +#include + +/* + * Stand-alone implementation of the SHA256 algorithm. It is designed to + * have as little dependencies as possible so it can be used in the + * kexec_file purgatory. In other cases you should use the implementation in + * crypto/. + * + * For details see lib/crypto/sha256.c + */ + +extern int sha256_init(struct sha256_state *sctx); +extern int sha256_update(struct sha256_state *sctx, const u8 *input, + unsigned int length); +extern int sha256_final(struct sha256_state *sctx, u8 *hash); + +#endif /* SHA256_H */ diff --git a/include/linux/sha256.h b/include/linux/sha256.h deleted file mode 100644 index 26972b9e92db..000000000000 --- a/include/linux/sha256.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2014 Red Hat Inc. - * - * Author: Vivek Goyal - */ - -#ifndef SHA256_H -#define SHA256_H - -#include -#include - -/* - * Stand-alone implementation of the SHA256 algorithm. It is designed to - * have as little dependencies as possible so it can be used in the - * kexec_file purgatory. In other cases you should use the implementation in - * crypto/. - * - * For details see lib/sha256.c - */ - -extern int sha256_init(struct sha256_state *sctx); -extern int sha256_update(struct sha256_state *sctx, const u8 *input, - unsigned int length); -extern int sha256_final(struct sha256_state *sctx, u8 *hash); - -#endif /* SHA256_H */ -- cgit v1.2.3 From 01d3aee86625bd798a5e69afb92517d5530c7ed1 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 17 Aug 2019 16:24:33 +0200 Subject: crypto: sha256 - Make lib/crypto/sha256.c suitable for generic use Before this commit lib/crypto/sha256.c has only been used in the s390 and x86 purgatory code, make it suitable for generic use: * Export interesting symbols * Add -D__DISABLE_EXPORTS to CFLAGS_sha256.o for purgatory builds to avoid the exports for the purgatory builds * Add to lib/crypto/Makefile and crypto/Kconfig Signed-off-by: Hans de Goede Signed-off-by: Herbert Xu --- include/crypto/sha256.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/crypto/sha256.h b/include/crypto/sha256.h index b1f9c6781082..9cbb3589b8b3 100644 --- a/include/crypto/sha256.h +++ b/include/crypto/sha256.h @@ -14,8 +14,9 @@ /* * Stand-alone implementation of the SHA256 algorithm. It is designed to * have as little dependencies as possible so it can be used in the - * kexec_file purgatory. In other cases you should use the implementation in - * crypto/. + * kexec_file purgatory. In other cases you should generally use the + * hash APIs from include/crypto/hash.h. Especially when hashing large + * amounts of data as those APIs may be hw-accelerated. * * For details see lib/crypto/sha256.c */ -- cgit v1.2.3 From 7d2f5b0c43e0bb346fbf78daefd68cd0bfc56ca3 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sat, 17 Aug 2019 16:24:34 +0200 Subject: crypto: sha256 - Add sha224 support to sha256 library code Add sha224 support to the lib/crypto/sha256 library code. This will allow us to replace both the sha256 and sha224 parts of crypto/sha256_generic.c when we remove the code duplication in further patches in this series. Suggested-by: Eric Biggers Signed-off-by: Hans de Goede Signed-off-by: Herbert Xu --- include/crypto/sha256.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include') diff --git a/include/crypto/sha256.h b/include/crypto/sha256.h index 9cbb3589b8b3..a75998d65a41 100644 --- a/include/crypto/sha256.h +++ b/include/crypto/sha256.h @@ -26,4 +26,9 @@ extern int sha256_update(struct sha256_state *sctx, const u8 *input, unsigned int length); extern int sha256_final(struct sha256_state *sctx, u8 *hash); +extern int sha224_init(struct sha256_state *sctx); +extern int sha224_update(struct sha256_state *sctx, const u8 *input, + unsigned int length); +extern int sha224_final(struct sha256_state *sctx, u8 *hash); + #endif /* SHA256_H */ -- cgit v1.2.3 From 34d6245fbc81e764806a65fceaeb3ab3274a1e63 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sun, 1 Sep 2019 22:35:31 +0200 Subject: crypto: sha256 - Merge crypto/sha256.h into crypto/sha.h The generic sha256 implementation from lib/crypto/sha256.c uses data structs defined in crypto/sha.h, so lets move the function prototypes there too. Signed-off-by: Hans de Goede Signed-off-by: Herbert Xu --- include/crypto/sha.h | 21 +++++++++++++++++++++ include/crypto/sha256.h | 34 ---------------------------------- 2 files changed, 21 insertions(+), 34 deletions(-) delete mode 100644 include/crypto/sha256.h (limited to 'include') diff --git a/include/crypto/sha.h b/include/crypto/sha.h index 8a46202b1857..535955c84187 100644 --- a/include/crypto/sha.h +++ b/include/crypto/sha.h @@ -112,4 +112,25 @@ extern int crypto_sha512_update(struct shash_desc *desc, const u8 *data, extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *hash); + +/* + * Stand-alone implementation of the SHA256 algorithm. It is designed to + * have as little dependencies as possible so it can be used in the + * kexec_file purgatory. In other cases you should generally use the + * hash APIs from include/crypto/hash.h. Especially when hashing large + * amounts of data as those APIs may be hw-accelerated. + * + * For details see lib/crypto/sha256.c + */ + +extern int sha256_init(struct sha256_state *sctx); +extern int sha256_update(struct sha256_state *sctx, const u8 *input, + unsigned int length); +extern int sha256_final(struct sha256_state *sctx, u8 *hash); + +extern int sha224_init(struct sha256_state *sctx); +extern int sha224_update(struct sha256_state *sctx, const u8 *input, + unsigned int length); +extern int sha224_final(struct sha256_state *sctx, u8 *hash); + #endif diff --git a/include/crypto/sha256.h b/include/crypto/sha256.h deleted file mode 100644 index a75998d65a41..000000000000 --- a/include/crypto/sha256.h +++ /dev/null @@ -1,34 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2014 Red Hat Inc. - * - * Author: Vivek Goyal - */ - -#ifndef SHA256_H -#define SHA256_H - -#include -#include - -/* - * Stand-alone implementation of the SHA256 algorithm. It is designed to - * have as little dependencies as possible so it can be used in the - * kexec_file purgatory. In other cases you should generally use the - * hash APIs from include/crypto/hash.h. Especially when hashing large - * amounts of data as those APIs may be hw-accelerated. - * - * For details see lib/crypto/sha256.c - */ - -extern int sha256_init(struct sha256_state *sctx); -extern int sha256_update(struct sha256_state *sctx, const u8 *input, - unsigned int length); -extern int sha256_final(struct sha256_state *sctx, u8 *hash); - -extern int sha224_init(struct sha256_state *sctx); -extern int sha224_update(struct sha256_state *sctx, const u8 *input, - unsigned int length); -extern int sha224_final(struct sha256_state *sctx, u8 *hash); - -#endif /* SHA256_H */ -- cgit v1.2.3 From c75c66bbaa56f130e2be095402422e56f608aa62 Mon Sep 17 00:00:00 2001 From: Hans de Goede Date: Sun, 1 Sep 2019 22:35:32 +0200 Subject: crypto: sha256 - Remove sha256/224_init code duplication lib/crypto/sha256.c and include/crypto/sha256_base.h define 99% identical functions to init a sha256_state struct for sha224 or sha256 use. This commit moves the functions from lib/crypto/sha256.c to include/crypto/sha.h (making them static inline) and makes the sha224/256_base_init static inline functions from include/crypto/sha256_base.h wrappers around the now also static inline include/crypto/sha.h functions. Signed-off-by: Hans de Goede Signed-off-by: Herbert Xu --- include/crypto/sha.h | 30 ++++++++++++++++++++++++++++-- include/crypto/sha256_base.h | 24 ++---------------------- 2 files changed, 30 insertions(+), 24 deletions(-) (limited to 'include') diff --git a/include/crypto/sha.h b/include/crypto/sha.h index 535955c84187..5c2132c71900 100644 --- a/include/crypto/sha.h +++ b/include/crypto/sha.h @@ -123,12 +123,38 @@ extern int crypto_sha512_finup(struct shash_desc *desc, const u8 *data, * For details see lib/crypto/sha256.c */ -extern int sha256_init(struct sha256_state *sctx); +static inline int sha256_init(struct sha256_state *sctx) +{ + sctx->state[0] = SHA256_H0; + sctx->state[1] = SHA256_H1; + sctx->state[2] = SHA256_H2; + sctx->state[3] = SHA256_H3; + sctx->state[4] = SHA256_H4; + sctx->state[5] = SHA256_H5; + sctx->state[6] = SHA256_H6; + sctx->state[7] = SHA256_H7; + sctx->count = 0; + + return 0; +} extern int sha256_update(struct sha256_state *sctx, const u8 *input, unsigned int length); extern int sha256_final(struct sha256_state *sctx, u8 *hash); -extern int sha224_init(struct sha256_state *sctx); +static inline int sha224_init(struct sha256_state *sctx) +{ + sctx->state[0] = SHA224_H0; + sctx->state[1] = SHA224_H1; + sctx->state[2] = SHA224_H2; + sctx->state[3] = SHA224_H3; + sctx->state[4] = SHA224_H4; + sctx->state[5] = SHA224_H5; + sctx->state[6] = SHA224_H6; + sctx->state[7] = SHA224_H7; + sctx->count = 0; + + return 0; +} extern int sha224_update(struct sha256_state *sctx, const u8 *input, unsigned int length); extern int sha224_final(struct sha256_state *sctx, u8 *hash); diff --git a/include/crypto/sha256_base.h b/include/crypto/sha256_base.h index b50a035a2bc7..cea60cff80bd 100644 --- a/include/crypto/sha256_base.h +++ b/include/crypto/sha256_base.h @@ -22,34 +22,14 @@ static inline int sha224_base_init(struct shash_desc *desc) { struct sha256_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA224_H0; - sctx->state[1] = SHA224_H1; - sctx->state[2] = SHA224_H2; - sctx->state[3] = SHA224_H3; - sctx->state[4] = SHA224_H4; - sctx->state[5] = SHA224_H5; - sctx->state[6] = SHA224_H6; - sctx->state[7] = SHA224_H7; - sctx->count = 0; - - return 0; + return sha224_init(sctx); } static inline int sha256_base_init(struct shash_desc *desc) { struct sha256_state *sctx = shash_desc_ctx(desc); - sctx->state[0] = SHA256_H0; - sctx->state[1] = SHA256_H1; - sctx->state[2] = SHA256_H2; - sctx->state[3] = SHA256_H3; - sctx->state[4] = SHA256_H4; - sctx->state[5] = SHA256_H5; - sctx->state[6] = SHA256_H6; - sctx->state[7] = SHA256_H7; - sctx->count = 0; - - return 0; + return sha256_init(sctx); } static inline int sha256_base_do_update(struct shash_desc *desc, -- cgit v1.2.3 From 6017826b494008895632e092cc49c0bdb3019eab Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Tue, 3 Sep 2019 09:43:30 -0700 Subject: crypto: skcipher - add the ability to abort a skcipher walk After starting a skcipher walk, the only way to ensure that all resources it has tied up are released is to complete it. In some cases, it will be useful to be able to abort a walk cleanly after it has started, so add this ability to the skcipher walk API. Signed-off-by: Ard Biesheuvel Signed-off-by: Herbert Xu --- include/crypto/internal/skcipher.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include') diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index d68faa5759ad..734b6f7081b8 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -148,6 +148,11 @@ int skcipher_walk_aead_decrypt(struct skcipher_walk *walk, struct aead_request *req, bool atomic); void skcipher_walk_complete(struct skcipher_walk *walk, int err); +static inline void skcipher_walk_abort(struct skcipher_walk *walk) +{ + skcipher_walk_done(walk, -ECANCELED); +} + static inline void ablkcipher_request_complete(struct ablkcipher_request *req, int err) { -- cgit v1.2.3 From b128a30409356df65f1a51cff3eb986cac8cfedc Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 5 Sep 2019 21:40:21 -0400 Subject: padata: allocate workqueue internally Move workqueue allocation inside of padata to prepare for further changes to how padata uses workqueues. Guarantees the workqueue is created with max_active=1, which padata relies on to work correctly. No functional change. Signed-off-by: Daniel Jordan Acked-by: Steffen Klassert Cc: Herbert Xu Cc: Jonathan Corbet Cc: Lai Jiangshan Cc: Peter Zijlstra Cc: Tejun Heo Cc: linux-crypto@vger.kernel.org Cc: linux-doc@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- include/linux/padata.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index 8da673861d99..839d9319920a 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -151,8 +151,7 @@ struct padata_instance { #define PADATA_INVALID 4 }; -extern struct padata_instance *padata_alloc_possible( - struct workqueue_struct *wq); +extern struct padata_instance *padata_alloc_possible(const char *name); extern void padata_free(struct padata_instance *pinst); extern int padata_do_parallel(struct padata_instance *pinst, struct padata_priv *padata, int cb_cpu); -- cgit v1.2.3 From 513c98d08682957cc9eba20e7e4bb349970711f3 Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 5 Sep 2019 21:40:22 -0400 Subject: workqueue: unconfine alloc/apply/free_workqueue_attrs() padata will use these these interfaces in a later patch, so unconfine them. Signed-off-by: Daniel Jordan Acked-by: Tejun Heo Acked-by: Steffen Klassert Cc: Herbert Xu Cc: Lai Jiangshan Cc: Peter Zijlstra Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- include/linux/workqueue.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'include') diff --git a/include/linux/workqueue.h b/include/linux/workqueue.h index b7c585b5ec1c..4261d1c6e87b 100644 --- a/include/linux/workqueue.h +++ b/include/linux/workqueue.h @@ -435,6 +435,10 @@ struct workqueue_struct *alloc_workqueue(const char *fmt, extern void destroy_workqueue(struct workqueue_struct *wq); +struct workqueue_attrs *alloc_workqueue_attrs(void); +void free_workqueue_attrs(struct workqueue_attrs *attrs); +int apply_workqueue_attrs(struct workqueue_struct *wq, + const struct workqueue_attrs *attrs); int workqueue_set_unbound_cpumask(cpumask_var_t cpumask); extern bool queue_work_on(int cpu, struct workqueue_struct *wq, -- cgit v1.2.3 From e6ce0e0807e90d38a2cefa524ac253d7a85c3f2f Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 5 Sep 2019 21:40:24 -0400 Subject: padata: make padata_do_parallel find alternate callback CPU padata_do_parallel currently returns -EINVAL if the callback CPU isn't in the callback cpumask. pcrypt tries to prevent this situation by keeping its own callback cpumask in sync with padata's and checks that the callback CPU it passes to padata is valid. Make padata handle this instead. padata_do_parallel now takes a pointer to the callback CPU and updates it for the caller if an alternate CPU is used. Overall behavior in terms of which callback CPUs are chosen stays the same. Prepares for removal of the padata cpumask notifier in pcrypt, which will fix a lockdep complaint about nested acquisition of the CPU hotplug lock later in the series. Signed-off-by: Daniel Jordan Acked-by: Steffen Klassert Cc: Herbert Xu Cc: Lai Jiangshan Cc: Peter Zijlstra Cc: Tejun Heo Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- include/linux/padata.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index 839d9319920a..f7851f8e2190 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -154,7 +154,7 @@ struct padata_instance { extern struct padata_instance *padata_alloc_possible(const char *name); extern void padata_free(struct padata_instance *pinst); extern int padata_do_parallel(struct padata_instance *pinst, - struct padata_priv *padata, int cb_cpu); + struct padata_priv *padata, int *cb_cpu); extern void padata_do_serial(struct padata_priv *padata); extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask); -- cgit v1.2.3 From 45d153c08bc73c8ced640dc20d8f2b749a6cb0d0 Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 5 Sep 2019 21:40:27 -0400 Subject: padata: use separate workqueues for parallel and serial work padata currently uses one per-CPU workqueue per instance for all work. Prepare for running parallel jobs on an unbound workqueue by introducing dedicated workqueues for parallel and serial work. Signed-off-by: Daniel Jordan Acked-by: Steffen Klassert Cc: Herbert Xu Cc: Lai Jiangshan Cc: Peter Zijlstra Cc: Tejun Heo Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- include/linux/padata.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index f7851f8e2190..e7978f8942ca 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -127,7 +127,8 @@ struct parallel_data { * struct padata_instance - The overall control structure. * * @cpu_notifier: cpu hotplug notifier. - * @wq: The workqueue in use. + * @parallel_wq: The workqueue used for parallel work. + * @serial_wq: The workqueue used for serial work. * @pd: The internal control structure. * @cpumask: User supplied cpumasks for parallel and serial works. * @cpumask_change_notifier: Notifiers chain for user-defined notify @@ -139,7 +140,8 @@ struct parallel_data { */ struct padata_instance { struct hlist_node node; - struct workqueue_struct *wq; + struct workqueue_struct *parallel_wq; + struct workqueue_struct *serial_wq; struct parallel_data *pd; struct padata_cpumask cpumask; struct blocking_notifier_head cpumask_change_notifier; -- cgit v1.2.3 From bfde23ce200e6d33291d29b9b8b60cc2f30f0805 Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 5 Sep 2019 21:40:28 -0400 Subject: padata: unbind parallel jobs from specific CPUs Padata binds the parallel part of a job to a single CPU and round-robins over all CPUs in the system for each successive job. Though the serial parts rely on per-CPU queues for correct ordering, they're not necessary for parallel work, and it improves performance to run the job locally on NUMA machines and let the scheduler pick the CPU within a node on a busy system. So, make the parallel workqueue unbound. Update the parallel workqueue's cpumask when the instance's parallel cpumask changes. Now that parallel jobs no longer run on max_active=1 workqueues, two or more parallel works that hash to the same CPU may run simultaneously, finish out of order, and so be serialized out of order. Prevent this by keeping the works sorted on the reorder list by sequence number and checking that in the reordering logic. padata_get_next becomes padata_find_next so it can be reused for the end of padata_reorder, where it's used to avoid uselessly queueing work when the next job by sequence number isn't finished yet but a later job that hashed to the same CPU has. The ENODATA case in padata_find_next no longer makes sense because parallel jobs aren't bound to specific CPUs. The EINPROGRESS case takes care of the scenario where a parallel job is potentially running on the same CPU as padata_find_next, and with only one error code left, just use NULL instead. Signed-off-by: Daniel Jordan Cc: Herbert Xu Cc: Lai Jiangshan Cc: Peter Zijlstra Cc: Steffen Klassert Cc: Tejun Heo Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- include/linux/padata.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index e7978f8942ca..43d3fd9d17fc 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -35,6 +35,7 @@ struct padata_priv { struct parallel_data *pd; int cb_cpu; int cpu; + unsigned int seq_nr; int info; void (*parallel)(struct padata_priv *padata); void (*serial)(struct padata_priv *padata); @@ -105,6 +106,7 @@ struct padata_cpumask { * @reorder_objects: Number of objects waiting in the reorder queues. * @refcnt: Number of objects holding a reference on this parallel_data. * @max_seq_nr: Maximal used sequence number. + * @processed: Number of already processed objects. * @cpu: Next CPU to be processed. * @cpumask: The cpumasks in use for parallel and serial workers. * @reorder_work: work struct for reordering. @@ -117,6 +119,7 @@ struct parallel_data { atomic_t reorder_objects; atomic_t refcnt; atomic_t seq_nr; + unsigned int processed; int cpu; struct padata_cpumask cpumask; struct work_struct reorder_work; -- cgit v1.2.3 From c51636a3065491af521187724d14a822548bcfd7 Mon Sep 17 00:00:00 2001 From: Daniel Jordan Date: Thu, 5 Sep 2019 21:40:29 -0400 Subject: padata: remove cpu_index from the parallel_queue With the removal of the ENODATA case from padata_get_next, the cpu_index field is no longer useful, so it can go away. Signed-off-by: Daniel Jordan Acked-by: Steffen Klassert Cc: Herbert Xu Cc: Lai Jiangshan Cc: Peter Zijlstra Cc: Tejun Heo Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu --- include/linux/padata.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include') diff --git a/include/linux/padata.h b/include/linux/padata.h index 43d3fd9d17fc..23717eeaad23 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -75,14 +75,12 @@ struct padata_serial_queue { * @swork: work struct for serialization. * @work: work struct for parallelization. * @num_obj: Number of objects that are processed by this cpu. - * @cpu_index: Index of the cpu. */ struct padata_parallel_queue { struct padata_list parallel; struct padata_list reorder; struct work_struct work; atomic_t num_obj; - int cpu_index; }; /** -- cgit v1.2.3