From 89fec474fa1ab2c754e48d29e1081a2c2bd22dc6 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 10 Jun 2019 21:40:00 -0700 Subject: net/tls: pass record number as a byte array TLS offload code casts record number to a u64. The buffer should be aligned to 8 bytes, but its actually a __be64, and the rest of the TLS code treats it as big int. Make the offload callbacks take a byte array, drivers can make the choice to do the ugly cast if they want to. Prepare for copying the record number onto the stack by defining a constant for max size of the byte array. Signed-off-by: Jakub Kicinski Reviewed-by: Dirk van der Merwe Signed-off-by: David S. Miller --- include/net/tls.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/net/tls.h b/include/net/tls.h index 3ecf45adb707..25641e2f5b96 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -62,6 +62,7 @@ #define TLS_DEVICE_NAME_MAX 32 #define MAX_IV_SIZE 16 +#define TLS_MAX_REC_SEQ_SIZE 8 /* For AES-CCM, the full 16-bytes of IV is made of '4' fields of given sizes. * @@ -299,7 +300,7 @@ struct tlsdev_ops { struct tls_context *ctx, enum tls_offload_ctx_dir direction); void (*tls_dev_resync_rx)(struct net_device *netdev, - struct sock *sk, u32 seq, u64 rcd_sn); + struct sock *sk, u32 seq, u8 *rcd_sn); }; struct tls_offload_context_rx { @@ -607,6 +608,6 @@ int tls_sw_fallback_init(struct sock *sk, int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx); void tls_device_offload_cleanup_rx(struct sock *sk); -void handle_device_resync(struct sock *sk, u32 seq, u64 rcd_sn); +void handle_device_resync(struct sock *sk, u32 seq); #endif /* _TLS_OFFLOAD_H */ -- cgit v1.2.3 From fe58a5a02cd9f49d5868539b4146ec1e5e5176e4 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 10 Jun 2019 21:40:01 -0700 Subject: net/tls: rename handle_device_resync() handle_device_resync() doesn't describe the function very well. The function checks if resync should be issued upon parsing of a new record. Signed-off-by: Jakub Kicinski Reviewed-by: Dirk van der Merwe Signed-off-by: David S. Miller --- include/net/tls.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'include') diff --git a/include/net/tls.h b/include/net/tls.h index 25641e2f5b96..1c512da5e4f4 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -608,6 +608,6 @@ int tls_sw_fallback_init(struct sock *sk, int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx); void tls_device_offload_cleanup_rx(struct sock *sk); -void handle_device_resync(struct sock *sk, u32 seq); +void tls_device_rx_resync_new_rec(struct sock *sk, u32 seq); #endif /* _TLS_OFFLOAD_H */ -- cgit v1.2.3 From f953d33ba1225d68cf8790b4706d8c4410b15926 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 10 Jun 2019 21:40:02 -0700 Subject: net/tls: add kernel-driven TLS RX resync TLS offload device may lose sync with the TCP stream if packets arrive out of order. Drivers can currently request a resync at a specific TCP sequence number. When a record is found starting at that sequence number kernel will inform the device of the corresponding record number. This requires the device to constantly scan the stream for a known pattern (constant bytes of the header) after sync is lost. This patch adds an alternative approach which is entirely under the control of the kernel. Kernel tracks records it had to fully decrypt, even though TLS socket is in TLS_HW mode. If multiple records did not have any decrypted parts - it's a pretty strong indication that the device is out of sync. We choose the min number of fully encrypted records to be 2, which should hopefully be more than will get retransmitted at a time. After kernel decides the device is out of sync it schedules a resync request. If the TCP socket is empty the resync gets performed immediately. If socket is not empty we leave the record parser to resync when next record comes. Before resync in message parser we peek at the TCP socket and don't attempt the sync if the socket already has some of the next record queued. On resync failure (encrypted data continues to flow in) we retry with exponential backoff, up to once every 128 records (with a 16k record thats at most once every 2M of data). Signed-off-by: Jakub Kicinski Reviewed-by: Dirk van der Merwe Signed-off-by: David S. Miller --- include/net/tls.h | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/net/tls.h b/include/net/tls.h index 1c512da5e4f4..28eca6a3b615 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -303,10 +303,33 @@ struct tlsdev_ops { struct sock *sk, u32 seq, u8 *rcd_sn); }; +enum tls_offload_sync_type { + TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ = 0, + TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT = 1, +}; + +#define TLS_DEVICE_RESYNC_NH_START_IVAL 2 +#define TLS_DEVICE_RESYNC_NH_MAX_IVAL 128 + struct tls_offload_context_rx { /* sw must be the first member of tls_offload_context_rx */ struct tls_sw_context_rx sw; - atomic64_t resync_req; + enum tls_offload_sync_type resync_type; + /* this member is set regardless of resync_type, to avoid branches */ + u8 resync_nh_reset:1; + /* CORE_NEXT_HINT-only member, but use the hole here */ + u8 resync_nh_do_now:1; + union { + /* TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ */ + struct { + atomic64_t resync_req; + }; + /* TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT */ + struct { + u32 decrypted_failed; + u32 decrypted_tgt; + } resync_nh; + }; u8 driver_state[] __aligned(8); /* The TLS layer reserves room for driver specific state * Currently the belief is that there is not enough @@ -587,6 +610,13 @@ static inline void tls_offload_rx_resync_request(struct sock *sk, __be32 seq) atomic64_set(&rx_ctx->resync_req, ((u64)ntohl(seq) << 32) | 1); } +static inline void +tls_offload_rx_resync_set_type(struct sock *sk, enum tls_offload_sync_type type) +{ + struct tls_context *tls_ctx = tls_get_ctx(sk); + + tls_offload_ctx_rx(tls_ctx)->resync_type = type; +} int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg, unsigned char *record_type); @@ -608,6 +638,6 @@ int tls_sw_fallback_init(struct sock *sk, int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx); void tls_device_offload_cleanup_rx(struct sock *sk); -void tls_device_rx_resync_new_rec(struct sock *sk, u32 seq); +void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq); #endif /* _TLS_OFFLOAD_H */ -- cgit v1.2.3 From eeb2efaf36c75753f9028de3500669bddfac81a8 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 10 Jun 2019 21:40:08 -0700 Subject: net/tls: generalize the resync callback Currently only RX direction is ever resynced, however, TX may also get out of sequence if packets get dropped on the way to the driver. Rename the resync callback and add a direction parameter. Signed-off-by: Jakub Kicinski Reviewed-by: Dirk van der Merwe Signed-off-by: David S. Miller --- include/net/tls.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/net/tls.h b/include/net/tls.h index 28eca6a3b615..9b49baecc4a8 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -299,8 +299,9 @@ struct tlsdev_ops { void (*tls_dev_del)(struct net_device *netdev, struct tls_context *ctx, enum tls_offload_ctx_dir direction); - void (*tls_dev_resync_rx)(struct net_device *netdev, - struct sock *sk, u32 seq, u8 *rcd_sn); + void (*tls_dev_resync)(struct net_device *netdev, + struct sock *sk, u32 seq, u8 *rcd_sn, + enum tls_offload_ctx_dir direction); }; enum tls_offload_sync_type { -- cgit v1.2.3 From 50180074099fcda752d9d56282d23242b126ebc9 Mon Sep 17 00:00:00 2001 From: Jakub Kicinski Date: Mon, 10 Jun 2019 21:40:09 -0700 Subject: net/tls: add kernel-driven resync mechanism for TX TLS offload drivers keep track of TCP seq numbers to make sure the packets are fed into the HW in order. When packets get dropped on the way through the stack, the driver will get out of sync and have to use fallback encryption, but unless TCP seq number is resynced it will never match the packets correctly (or even worse - use incorrect record sequence number after TCP seq wraps). Existing drivers (mlx5) feed the entire record on every out-of-order event, allowing FW/HW to always be in sync. This patch adds an alternative, more akin to the RX resync. When driver sees a frame which is past its expected sequence number the stream must have gotten out of order (if the sequence number is smaller than expected its likely a retransmission which doesn't require resync). Driver will ask the stack to perform TX sync before it submits the next full record, and fall back to software crypto until stack has performed the sync. Signed-off-by: Jakub Kicinski Reviewed-by: Dirk van der Merwe Signed-off-by: David S. Miller --- include/net/tls.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'include') diff --git a/include/net/tls.h b/include/net/tls.h index 9b49baecc4a8..63e473420b00 100644 --- a/include/net/tls.h +++ b/include/net/tls.h @@ -212,6 +212,11 @@ struct tls_offload_context_tx { enum tls_context_flags { TLS_RX_SYNC_RUNNING = 0, + /* Unlike RX where resync is driven entirely by the core in TX only + * the driver knows when things went out of sync, so we need the flag + * to be atomic. + */ + TLS_TX_SYNC_SCHED = 1, }; struct cipher_context { @@ -619,6 +624,24 @@ tls_offload_rx_resync_set_type(struct sock *sk, enum tls_offload_sync_type type) tls_offload_ctx_rx(tls_ctx)->resync_type = type; } +static inline void tls_offload_tx_resync_request(struct sock *sk) +{ + struct tls_context *tls_ctx = tls_get_ctx(sk); + + WARN_ON(test_and_set_bit(TLS_TX_SYNC_SCHED, &tls_ctx->flags)); +} + +/* Driver's seq tracking has to be disabled until resync succeeded */ +static inline bool tls_offload_tx_resync_pending(struct sock *sk) +{ + struct tls_context *tls_ctx = tls_get_ctx(sk); + bool ret; + + ret = test_bit(TLS_TX_SYNC_SCHED, &tls_ctx->flags); + smp_mb__after_atomic(); + return ret; +} + int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg, unsigned char *record_type); void tls_register_device(struct tls_device *device); -- cgit v1.2.3