summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2026-11-20 10:17:06 -0800
committerLinus Torvalds <torvalds@linux-foundation.org>2026-11-20 10:17:06 -0800
commita904dcf3fdf29cdac99ea354789bc5c82df0d5ad (patch)
treebb7e05226460d2e69d174997ac8e9d6bbfa55d8b
parent78d9aedf2342cac9968392995d8f7fd488caa337 (diff)
parent5a29cfaf51105039d156dd8a61d2783e8b653667 (diff)
Merge tag 'block-7.4-20261119' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux
Pull block fixes from Jens Axboe: - NVMe fixes via Keith: - nvme-tcp fixes for TLS record handling and waiting for an in-flight TLS handshake callback - Fix a deadlock on nvme_subsystems_lock in nvme_init_subsystem() - Fix a gendisk leak in nvme_alloc_ns() - Stop AEN work before ANA teardown - Remove namespaces in parallel - Various nvmet fixes for auth, the file backend, fc, tcp, loop, and connection failure rollback - sysfs input validation and kfree_sensitive() for DH-HMAC-CHAP secrets - Dead code removal - Fix a 32-bit overflow in the blk-iocost donation weight calculation - ublk fixes: validate all parameters for zoned devices, and dispatch the next fetch after a batch failure - Fix a pm_only underflow when a runtime resume is retried after a failed one - nbd: reclassify sockets in the reconnect path too, to avoid lockdep false positives - Take quiesce_lock when removing a queue from the tag set, fixing a use-after-free in blk_mq_unquiesce_tagset() - Allow error injection rules to delay bios, and reject unknown status tags * tag 'block-7.4-20261119' of git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux: (49 commits) Documentation: block: document error injection delay feature block: allow error injection rules to delay bios block: Reject unknown status tags in error injection rules blk-mq: take quiesce_lock when removing a queue from the tag set nbd: Reclassify sockets in the reconnect path block: Keep RPM_ACTIVE in blk_pre_runtime_resume() when already active nvme-fabrics: use kfree_sensitive() for the old DH-HMAC-CHAP secret nvme-sysfs: use kfree_sensitive() for DH-HMAC-CHAP secrets nvmet: roll back queue bindings on connection failure nvmet-fc: reject duplicate queue IDs on Create Connection nvmet-fcloop: free the LS request when delivery fails nvme-tcp: require the R2T offset to match the data sent nvmet-tcp: only wait for in-capsule data for inline SGLs nvme-loop: release the target request when SG setup fails nvmet-auth: store the DH-CHAP key offset instead of hardcoding it nvme-tcp: consume TLS records without application data nvme-tcp: move the TLS record read loop into a helper nvme-tcp: use an aligned buffer for the TLS record type nvmet: use overflow-safe range checks in the file backend nvmet: handle short reads and writes in the file backend ...
-rw-r--r--Documentation/ABI/stable/sysfs-nvme3
-rw-r--r--Documentation/block/error-injection.rst39
-rw-r--r--block/blk-core.c14
-rw-r--r--block/blk-iocost.c3
-rw-r--r--block/blk-mq.c3
-rw-r--r--block/blk-pm.c13
-rw-r--r--block/blk.h2
-rw-r--r--block/error-injection.c150
-rw-r--r--block/error-injection.h1
-rw-r--r--drivers/block/nbd.c1
-rw-r--r--drivers/block/ublk_drv.c35
-rw-r--r--drivers/nvme/common/auth.c6
-rw-r--r--drivers/nvme/host/auth.c1
-rw-r--r--drivers/nvme/host/core.c44
-rw-r--r--drivers/nvme/host/fabrics.c10
-rw-r--r--drivers/nvme/host/fc.c1
-rw-r--r--drivers/nvme/host/multipath.c7
-rw-r--r--drivers/nvme/host/nvme.h13
-rw-r--r--drivers/nvme/host/pci.c35
-rw-r--r--drivers/nvme/host/sysfs.c17
-rw-r--r--drivers/nvme/host/tcp.c175
-rw-r--r--drivers/nvme/target/admin-cmd.c52
-rw-r--r--drivers/nvme/target/auth.c83
-rw-r--r--drivers/nvme/target/configfs.c9
-rw-r--r--drivers/nvme/target/fabrics-cmd.c3
-rw-r--r--drivers/nvme/target/fc.c16
-rw-r--r--drivers/nvme/target/fcloop.c2
-rw-r--r--drivers/nvme/target/io-cmd-file.c47
-rw-r--r--drivers/nvme/target/loop.c1
-rw-r--r--drivers/nvme/target/nvmet.h2
-rw-r--r--drivers/nvme/target/passthru.c2
-rw-r--r--drivers/nvme/target/pci-epf.c2
-rw-r--r--drivers/nvme/target/pr.c10
-rw-r--r--drivers/nvme/target/tcp.c42
-rw-r--r--drivers/nvme/target/zns.c3
-rw-r--r--include/linux/blk-mq.h3
-rw-r--r--include/linux/blk_types.h1
37 files changed, 615 insertions, 236 deletions
diff --git a/Documentation/ABI/stable/sysfs-nvme b/Documentation/ABI/stable/sysfs-nvme
index 9291595f8102..1826df9161d2 100644
--- a/Documentation/ABI/stable/sysfs-nvme
+++ b/Documentation/ABI/stable/sysfs-nvme
@@ -70,8 +70,7 @@ Description:
fabrics controller.
reconnect_delay: Shows or sets the reconnect delay in
- seconds. Reading returns the delay value, or "off" if
- disabled.
+ seconds. Reading returns the delay value.
ctrl_loss_tmo: Shows or sets the controller loss timeout in
seconds. Reading returns the timeout value, or "off" if
diff --git a/Documentation/block/error-injection.rst b/Documentation/block/error-injection.rst
index 81f31af82e65..3c86defddb80 100644
--- a/Documentation/block/error-injection.rst
+++ b/Documentation/block/error-injection.rst
@@ -7,9 +7,9 @@ Configurable Error Injection
Overview
--------
-Configurable error injection allows injecting specific block layer status codes
-for sector ranges of a block device. Errors can be injected unconditionally, or
-with a given probability.
+Configurable error injection allows injecting delays and/or specific block
+layer status codes for sector ranges of a block device by adding rules. The
+rules can be configured to trigger unconditionally, or with a given probability.
To use configurable error injection, CONFIG_BLK_ERROR_INJECTION must be enabled.
@@ -34,15 +34,36 @@ op=<string> block layer operation this rule applies to. This uses
the XYZ for each REQ_OP_XYZ operation, e.g. READ, WRITE
or DISCARD. Mandatory.
status=<string> Status to return. This uses XYZ for each BLK_STS_XYZ
- code, e.g. IOERR or MEDIUM. Mandatory.
+ code, e.g. IOERR or MEDIUM. Mandatory unless delay_us
+ is given.
start=<number> First block layer sector the rule applies to.
Optional, defaults to 0.
nr_sectors=<number> Number of sectors this rule applies.
Optional, defaults to the remainder of the device.
-chance=<number> Only return a failure with a likelihood of 1/chance.
+chance=<number> Only apply the rule with a likelihood of 1/chance.
Optional, defaults to 1 (always).
+delay_us=<number> Hold the bio back for this many microseconds. Without
+ status the bio is then submitted to the device as
+ usual. With status it is failed once the delay has
+ expired. Bios with REQ_NOWAIT set fail with
+ BLK_STS_AGAIN. Optional, defaults to 0 (no delay).
+ Values above 600 seconds are rejected.
=================== =======================================================
+Delays
+------
+
+A delayed bio is held above the driver, so the device never sees a slow I/O.
+A delay does not reach the blk-mq timeout handler or SCSI error handling.
+
+A bio that matched a delay rule is not evaluated against the rules again, even
+after it is split and resubmitted internally. Removing a rule does not release
+bios it is already delaying.
+
+Holding a bio back reorders it against bios submitted later. On zoned devices
+this breaks sequential write ordering and the block layer fails the
+out-of-order writes, so only delay reads there.
+
Example
-------
@@ -54,6 +75,14 @@ Return BLK_STS_MEDIUM for every write to /dev/nvme0n1:
$ echo 'add,op=WRITE,start=0,status=MEDIUM' > /sys/kernel/debug/block/nvme0n1/error_injection
+Delay every read of /dev/nvme0n1 by 10 milliseconds, then issue it normally:
+
+ $ echo 'add,op=READ,delay_us=10000' > /sys/kernel/debug/block/nvme0n1/error_injection
+
+Fail one in 100 writes with BLK_STS_TIMEOUT, but only after 30 seconds:
+
+ $ echo 'add,op=WRITE,status=TIMEOUT,chance=100,delay_us=30000000' > /sys/kernel/debug/block/nvme0n1/error_injection
+
Remove all rules for /dev/nvme0n1:
$ echo 'removeall' > /sys/kernel/debug/block/nvme0n1/error_injection
diff --git a/block/blk-core.c b/block/blk-core.c
index 8836c7028568..2ee079c1ce15 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -226,21 +226,19 @@ const char *blk_status_to_tag(blk_status_t status)
return blk_errors[idx].tag;
}
-blk_status_t tag_to_blk_status(const char *tag)
+bool tag_to_blk_status(const char *tag, blk_status_t *status)
{
int i;
for (i = 0; i < ARRAY_SIZE(blk_errors); i++) {
if (blk_errors[i].tag &&
- !strcmp(blk_errors[i].tag, tag))
- return (__force blk_status_t)i;
+ !strcmp(blk_errors[i].tag, tag)) {
+ *status = (__force blk_status_t)i;
+ return true;
+ }
}
- /*
- * Return BLK_STS_OK for mismatches as this function is intended to
- * parse error status values.
- */
- return BLK_STS_OK;
+ return false;
}
/**
diff --git a/block/blk-iocost.c b/block/blk-iocost.c
index 1c1025157d8e..0ffe34bd95b7 100644
--- a/block/blk-iocost.c
+++ b/block/blk-iocost.c
@@ -1853,7 +1853,8 @@ static u32 hweight_after_donation(struct ioc_gq *iocg, u32 old_hwi, u32 hwm,
delta = div64_s64(WEIGHT_ONE * (now->vnow - vtime),
now->vnow - ioc->period_at_vtime);
target = WEIGHT_ONE * MARGIN_TARGET_PCT / 100;
- new_hwi = div64_s64(WEIGHT_ONE * usage, WEIGHT_ONE - target + delta);
+ new_hwi = div64_s64((s64)WEIGHT_ONE * usage,
+ WEIGHT_ONE - target + delta);
return clamp_t(s64, new_hwi, 1, hwm);
}
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 5ae22d556540..54ccfc43c27c 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -4354,7 +4354,10 @@ static void blk_mq_del_queue_tag_set(struct request_queue *q)
lockdep_assert_held(&set->tag_list_lock);
+ /* blk_mq_unquiesce_tagset() runs the hctxs, which are not RCU freed */
+ spin_lock(&set->quiesce_lock);
list_del_rcu(&q->tag_set_list);
+ spin_unlock(&set->quiesce_lock);
if (list_is_singular(&set->tag_list)) {
/* just transitioned to unshared */
set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
diff --git a/block/blk-pm.c b/block/blk-pm.c
index 8d3e052f91da..34cc7c5ca04e 100644
--- a/block/blk-pm.c
+++ b/block/blk-pm.c
@@ -142,7 +142,8 @@ EXPORT_SYMBOL(blk_post_runtime_suspend);
*
* Description:
* Update the queue's runtime status to RESUMING in preparation for the
- * runtime resume of the device.
+ * runtime resume of the device if the queue is runtime suspended, or keep
+ * RPM_ACTIVE if a previous failed runtime resume already reactivated it.
*
* This function should be called near the start of the device's
* runtime_resume callback.
@@ -153,7 +154,9 @@ void blk_pre_runtime_resume(struct request_queue *q)
return;
spin_lock_irq(&q->queue_lock);
- q->rpm_status = RPM_RESUMING;
+ /* Keep RPM_ACTIVE if a prior failed resume reactivated the queue. */
+ if (q->rpm_status == RPM_SUSPENDED)
+ q->rpm_status = RPM_RESUMING;
spin_unlock_irq(&q->queue_lock);
}
EXPORT_SYMBOL(blk_pre_runtime_resume);
@@ -168,8 +171,8 @@ EXPORT_SYMBOL(blk_pre_runtime_resume);
* driver or error handler will need to communicate with the device.
*
* This function should be called near the end of the device's
- * runtime_resume callback to correct queue runtime PM status and re-enable
- * peeking requests from the queue.
+ * runtime_resume callback to mark the queue runtime active and clear
+ * pm_only if the queue was resuming.
*/
void blk_post_runtime_resume(struct request_queue *q)
{
@@ -185,7 +188,7 @@ void blk_post_runtime_resume(struct request_queue *q)
pm_request_autosuspend(q->dev);
spin_unlock_irq(&q->queue_lock);
- if (old_status != RPM_ACTIVE)
+ if (old_status == RPM_RESUMING)
blk_clear_pm_only(q);
}
EXPORT_SYMBOL(blk_post_runtime_resume);
diff --git a/block/blk.h b/block/blk.h
index 0e38e4b88cf5..105fff157bfb 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -52,7 +52,7 @@ void blk_free_flush_queue(struct blk_flush_queue *q);
const char *blk_status_to_str(blk_status_t status);
const char *blk_status_to_tag(blk_status_t status);
-blk_status_t tag_to_blk_status(const char *tag);
+bool tag_to_blk_status(const char *tag, blk_status_t *status);
enum req_op str_to_blk_op(const char *op);
bool __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic);
diff --git a/block/error-injection.c b/block/error-injection.c
index e14bc4b723ef..a6de52229ef3 100644
--- a/block/error-injection.c
+++ b/block/error-injection.c
@@ -6,9 +6,17 @@
#include <linux/blkdev.h>
#include <linux/parser.h>
#include <linux/seq_file.h>
+#include <linux/workqueue.h>
#include "blk.h"
#include "error-injection.h"
+/*
+ * Cap the delay so that a typo can't wedge a device for good. This is still
+ * well beyond the default hung task timeout, which is one of the things a
+ * delay is useful for triggering.
+ */
+#define BLK_ERROR_INJECT_MAX_DELAY_US (600 * USEC_PER_SEC)
+
struct blk_error_inject {
struct list_head entry;
sector_t start;
@@ -18,14 +26,97 @@ struct blk_error_inject {
/* only inject every 1 / chance times */
unsigned int chance;
+
+ /* hold the bio for this long before submitting or failing it */
+ unsigned int delay_us;
+};
+
+/*
+ * A bio held by a delay rule. It does not point back at the rule, so rules
+ * can be removed while delayed bios are outstanding. The submitter holds the
+ * bdev open until the bio completes, so the disk and queue stay allocated.
+ * A delayed bio holds no queue usage counter reference, so it does not hold
+ * up a queue freeze either.
+ */
+struct blk_error_inject_delay {
+ struct delayed_work dwork;
+ struct bio *bio;
+ blk_status_t status;
};
+static struct workqueue_struct *blk_error_inject_wq;
+
DEFINE_STATIC_KEY_FALSE(blk_error_injection_enabled);
+static void blk_error_inject_fail(struct bio *bio, blk_status_t status)
+{
+ pr_info_ratelimited("%pg: injecting %s error for %s at sector %llu:%u\n",
+ bio->bi_bdev->bd_disk->part0, blk_status_to_str(status),
+ blk_op_str(bio_op(bio)), bio->bi_iter.bi_sector,
+ bio_sectors(bio));
+ bio_endio_status(bio, status);
+}
+
+static void blk_error_inject_delay_work(struct work_struct *work)
+{
+ struct blk_error_inject_delay *d = container_of(to_delayed_work(work),
+ struct blk_error_inject_delay, dwork);
+ struct bio *bio = d->bio;
+ blk_status_t status = d->status;
+
+ kfree(d);
+
+ if (status != BLK_STS_OK) {
+ blk_error_inject_fail(bio, status);
+ return;
+ }
+
+ /*
+ * Resubmit. The BIO_ERROR_INJECTED flag ensures that a bio that was
+ * delayed once skips error injection entirely from here on, including
+ * any other rule that covers it.
+ */
+ submit_bio_noacct_nocheck(bio, false);
+}
+
+static bool blk_error_inject_delay(struct gendisk *disk, struct bio *bio,
+ blk_status_t status, unsigned int delay_us)
+{
+ struct blk_error_inject_delay *d;
+
+ /* fail a bio that can't block, as a real queue would */
+ if (bio->bi_opf & REQ_NOWAIT) {
+ bio_wouldblock_error(bio);
+ return true;
+ }
+
+ d = kmalloc_obj(*d, GFP_NOIO);
+ if (!d) {
+ pr_warn_ratelimited("%pg: failed to delay %s at sector %llu:%u\n",
+ disk->part0, blk_op_str(bio_op(bio)),
+ bio->bi_iter.bi_sector, bio_sectors(bio));
+ return false;
+ }
+
+ pr_info_ratelimited("%pg: delaying %s at sector %llu:%u by %uus\n",
+ disk->part0, blk_op_str(bio_op(bio)),
+ bio->bi_iter.bi_sector, bio_sectors(bio), delay_us);
+
+ d->bio = bio;
+ d->status = status;
+ INIT_DELAYED_WORK(&d->dwork, blk_error_inject_delay_work);
+
+ queue_delayed_work(blk_error_inject_wq, &d->dwork,
+ usecs_to_jiffies(delay_us));
+ return true;
+}
+
bool __blk_error_inject(struct bio *bio)
{
struct gendisk *disk = bio->bi_bdev->bd_disk;
struct blk_error_inject *inj;
+ blk_status_t status = BLK_STS_OK;
+ unsigned int delay_us = 0;
rcu_read_lock();
list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) {
@@ -45,29 +136,34 @@ bool __blk_error_inject(struct bio *bio)
if (inj->chance > 1 && (get_random_u32() % inj->chance) != 0)
continue;
- pr_info_ratelimited("%pg: injecting %s error for %s at sector %llu:%u\n",
- disk->part0, blk_status_to_str(inj->status),
- blk_op_str(inj->op), bio->bi_iter.bi_sector,
- bio_sectors(bio));
- bio->bi_status = inj->status;
- rcu_read_unlock();
- bio_endio(bio);
- return true;
+ status = inj->status;
+ delay_us = inj->delay_us;
+ bio_set_flag(bio, BIO_ERROR_INJECTED);
+ break;
}
rcu_read_unlock();
- return false;
+
+ if (delay_us && blk_error_inject_delay(disk, bio, status, delay_us))
+ return true;
+ if (status == BLK_STS_OK)
+ return false;
+
+ blk_error_inject_fail(bio, status);
+ return true;
}
static int error_inject_add(struct gendisk *disk, enum req_op op,
sector_t start, u64 nr_sectors, blk_status_t status,
- unsigned int chance)
+ unsigned int chance, unsigned int delay_us)
{
struct blk_error_inject *inj;
int error = -EINVAL;
if (op == REQ_OP_LAST)
return -EINVAL;
- if (status == BLK_STS_OK)
+ if (status == BLK_STS_OK && !delay_us)
+ return -EINVAL;
+ if (delay_us > BLK_ERROR_INJECT_MAX_DELAY_US)
return -EINVAL;
inj = kzalloc_obj(*inj);
@@ -86,6 +182,7 @@ static int error_inject_add(struct gendisk *disk, enum req_op op,
inj->start = start;
inj->status = status;
inj->chance = chance;
+ inj->delay_us = delay_us;
pr_debug_ratelimited("%pg: adding %s injection for %s at sector %llu:%llu\n",
disk->part0, blk_status_to_str(status),
@@ -139,6 +236,7 @@ enum options {
Opt_nr_sectors = (1u << 18),
Opt_status = (1u << 19),
Opt_chance = (1u << 20),
+ Opt_delay_us = (1u << 21),
Opt_invalid,
};
@@ -151,6 +249,7 @@ static const match_table_t opt_tokens = {
{ Opt_nr_sectors, "nr_sectors=%u" },
{ Opt_status, "status=%s" },
{ Opt_chance, "chance=%u" },
+ { Opt_delay_us, "delay_us=%u" },
{ Opt_invalid, NULL, },
};
@@ -171,22 +270,23 @@ static int match_op(substring_t *args, enum req_op *op)
static int match_status(substring_t *args, blk_status_t *status)
{
const char *tag;
+ bool found;
tag = match_strdup(args);
if (!tag)
return -ENOMEM;
- *status = tag_to_blk_status(tag);
- if (!*status)
+ found = tag_to_blk_status(tag, status);
+ if (!found)
pr_warn("invalid status '%s'\n", tag);
kfree(tag);
- return 0;
+ return found ? 0 : -EINVAL;
}
static ssize_t blk_error_injection_parse_options(struct gendisk *disk,
char *options)
{
enum { Unset, Add, Removeall } action = Unset;
- unsigned int option_mask = 0, chance = 1;
+ unsigned int option_mask = 0, chance = 1, delay_us = 0;
enum req_op op = REQ_OP_LAST;
u64 start = 0, nr_sectors = 0;
blk_status_t status = BLK_STS_OK;
@@ -229,6 +329,9 @@ static ssize_t blk_error_injection_parse_options(struct gendisk *disk,
if (!error && chance == 0)
error = -EINVAL;
break;
+ case Opt_delay_us:
+ error = match_uint(args, &delay_us);
+ break;
default:
pr_warn("unknown parameter or missing value '%s'\n", p);
error = -EINVAL;
@@ -240,7 +343,7 @@ static ssize_t blk_error_injection_parse_options(struct gendisk *disk,
switch (action) {
case Add:
return error_inject_add(disk, op, start, nr_sectors, status,
- chance);
+ chance, delay_us);
case Removeall:
if (option_mask & ~Opt_removeall)
return -EINVAL;
@@ -276,10 +379,11 @@ static int blk_error_injection_show(struct seq_file *s, void *private)
rcu_read_lock();
list_for_each_entry_rcu(inj, &disk->error_injection_list, entry) {
- seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u",
+ seq_printf(s, "%llu:%llu op=%s,status=%s,chance=%u,delay_us=%u",
inj->start, inj->end,
blk_op_str(inj->op),
- blk_status_to_tag(inj->status), inj->chance);
+ blk_status_to_tag(inj->status), inj->chance,
+ inj->delay_us);
seq_putc(s, '\n');
}
rcu_read_unlock();
@@ -314,3 +418,13 @@ void blk_error_injection_exit(struct gendisk *disk)
{
error_inject_removeall(disk);
}
+
+static int __init blk_error_injection_init_wq(void)
+{
+ blk_error_inject_wq = alloc_workqueue("blk_error_inject",
+ WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
+ if (!blk_error_inject_wq)
+ panic("Failed to create blk_error_inject wq\n");
+ return 0;
+}
+subsys_initcall(blk_error_injection_init_wq);
diff --git a/block/error-injection.h b/block/error-injection.h
index 9821d773abab..8b3809d85e83 100644
--- a/block/error-injection.h
+++ b/block/error-injection.h
@@ -13,6 +13,7 @@ static inline bool blk_error_inject(struct bio *bio)
{
if (IS_ENABLED(CONFIG_BLK_ERROR_INJECTION) &&
static_branch_unlikely(&blk_error_injection_enabled) &&
+ !bio_flagged(bio, BIO_ERROR_INJECTED) &&
test_bit(GD_ERROR_INJECT, &bio->bi_bdev->bd_disk->state))
return __blk_error_inject(bio);
return false;
diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c
index ffce519bf008..b2b27f90a7ed 100644
--- a/drivers/block/nbd.c
+++ b/drivers/block/nbd.c
@@ -1356,6 +1356,7 @@ static int nbd_genl_reconnect_sock_cb(struct nbd_device *nbd, unsigned long arg)
sock = nbd_get_socket(nbd, arg, &err);
if (!sock)
return err;
+ nbd_reclassify_socket(sock);
args = kzalloc_obj(*args);
if (!args) {
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index d415be37d5ed..0699f22714fd 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -835,23 +835,27 @@ static void __ublk_release_fcmd(struct ublk_queue *ubq)
WRITE_ONCE(ubq->active_fcmd, NULL);
}
-/*
- * Nothing can move on, so clear ->active_fcmd, and the caller should stop
- * dispatching
- */
-static void ublk_batch_deinit_fetch_buf(struct ublk_queue *ubq,
- const struct ublk_batch_io_data *data,
- struct ublk_batch_fetch_cmd *fcmd,
- int res)
+static struct ublk_batch_fetch_cmd *__ublk_acquire_fcmd(struct ublk_queue *ubq);
+
+/* Remove the failed fetch and claim the next one before dropping its reference. */
+static struct ublk_batch_fetch_cmd *
+ublk_batch_deinit_fetch_buf(struct ublk_queue *ubq,
+ const struct ublk_batch_io_data *data,
+ struct ublk_batch_fetch_cmd *fcmd,
+ int res)
{
+ struct ublk_batch_fetch_cmd *next_fcmd;
+
spin_lock(&ubq->evts_lock);
list_del_init(&fcmd->node);
WARN_ON_ONCE(fcmd != ubq->active_fcmd);
__ublk_release_fcmd(ubq);
+ next_fcmd = __ublk_acquire_fcmd(ubq);
spin_unlock(&ubq->evts_lock);
io_uring_cmd_done(fcmd->cmd, res, data->issue_flags);
ublk_batch_free_fcmd(fcmd);
+ return next_fcmd;
}
static int ublk_batch_fetch_post_cqe(struct ublk_batch_fetch_cmd *fcmd,
@@ -1012,10 +1016,14 @@ static int ublk_validate_params(const struct ublk_device *ub)
if (ub->params.types & UBLK_PARAM_TYPE_DEVT)
return -EINVAL;
- if (ub->params.types & UBLK_PARAM_TYPE_ZONED)
- return ublk_dev_param_zoned_validate(ub);
- else if (ublk_dev_is_zoned(ub))
+ if (ub->params.types & UBLK_PARAM_TYPE_ZONED) {
+ int ret = ublk_dev_param_zoned_validate(ub);
+
+ if (ret)
+ return ret;
+ } else if (ublk_dev_is_zoned(ub)) {
return -EINVAL;
+ }
if (ub->params.types & UBLK_PARAM_TYPE_DMA_ALIGN) {
const struct ublk_param_dma_align *p = &ub->params.dma;
@@ -2041,7 +2049,10 @@ again:
}
if (ret < 0) {
- ublk_batch_deinit_fetch_buf(ubq, data, fcmd, ret);
+ new_fcmd = ublk_batch_deinit_fetch_buf(ubq, data, fcmd, ret);
+ if (new_fcmd)
+ io_uring_cmd_complete_in_task(new_fcmd->cmd,
+ ublk_batch_tw_cb);
return;
}
diff --git a/drivers/nvme/common/auth.c b/drivers/nvme/common/auth.c
index e2e0c736540a..1fdc4e4ed0b9 100644
--- a/drivers/nvme/common/auth.c
+++ b/drivers/nvme/common/auth.c
@@ -489,17 +489,19 @@ int nvme_auth_parse_key(const char *secret, struct nvme_dhchap_key **ret_key)
{
struct nvme_dhchap_key *key;
u8 key_hash;
+ int offset = 0;
if (!secret) {
*ret_key = NULL;
return 0;
}
- if (sscanf(secret, "DHHC-1:%hhd:%*s:", &key_hash) != 1)
+ if (sscanf(secret, "DHHC-1:%hhd:%n", &key_hash, &offset) != 1 ||
+ !offset)
return -EINVAL;
/* Pass in the secret without the 'DHHC-1:XX:' prefix */
- key = nvme_auth_extract_key(secret + 10, key_hash);
+ key = nvme_auth_extract_key(secret + offset, key_hash);
if (IS_ERR(key)) {
*ret_key = NULL;
return PTR_ERR(key);
diff --git a/drivers/nvme/host/auth.c b/drivers/nvme/host/auth.c
index 85f005a5efd0..9a3da74dec5a 100644
--- a/drivers/nvme/host/auth.c
+++ b/drivers/nvme/host/auth.c
@@ -19,7 +19,6 @@ static struct kmem_cache *nvme_chap_buf_cache;
static mempool_t *nvme_chap_buf_pool;
struct nvme_dhchap_queue_context {
- struct list_head entry;
struct work_struct auth_work;
struct nvme_ctrl *ctrl;
struct crypto_kpp *dh_tfm;
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
index f4aa02869b79..f43582e5f6c4 100644
--- a/drivers/nvme/host/core.c
+++ b/drivers/nvme/host/core.c
@@ -3413,7 +3413,11 @@ static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
return 0;
out_put_subsystem:
+ if (!found)
+ list_del_init(&subsys->entry);
+ mutex_unlock(&nvme_subsystems_lock);
nvme_put_subsystem(subsys);
+ return ret;
out_unlock:
mutex_unlock(&nvme_subsystems_lock);
return ret;
@@ -4378,24 +4382,16 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
mutex_lock(&ctrl->subsys->lock);
list_del_rcu(&ns->siblings);
if (list_empty(&ns->head->list)) {
- list_del_init(&ns->head->entry);
- /*
- * If multipath is not configured, we still create a namespace
- * head (nshead), but head->disk is not initialized in that
- * case. As a result, only a single reference to nshead is held
- * (via kref_init()) when it is created. Therefore, ensure that
- * we do not release the reference to nshead twice if head->disk
- * is not present.
- */
- if (ns->head->disk)
- last_path = true;
+ if (!nvme_mpath_queue_if_no_path(ns->head))
+ list_del_init(&ns->head->entry);
+ last_path = true;
}
mutex_unlock(&ctrl->subsys->lock);
/* guarantee not available in head->list */
synchronize_srcu(&ns->head->srcu);
if (last_path)
- nvme_put_ns_head(ns->head);
+ nvme_mpath_remove_disk(ns->head);
nvme_put_ns_head(ns->head);
out_cleanup_disk:
put_disk(disk);
@@ -4730,6 +4726,11 @@ static void nvme_scan_work(struct work_struct *work)
#endif
}
+static void nvme_ns_remove_async(void *data, async_cookie_t cookie)
+{
+ nvme_ns_remove(data);
+}
+
/*
* This function iterates the namespace list unlocked to allow recovery from
* controller failure. It is up to the caller to ensure the namespace list is
@@ -4739,6 +4740,7 @@ void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
{
struct nvme_ns *ns, *next;
LIST_HEAD(ns_list);
+ ASYNC_DOMAIN(domain);
/*
* make sure to requeue I/O to all namespaces as these
@@ -4772,8 +4774,13 @@ void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
list_splice_init_srcu(&ctrl->namespaces, &ns_list, &ctrl->srcu);
mutex_unlock(&ctrl->namespaces_lock);
+ /*
+ * Remove the namespaces in parallel, as each removal waits for several
+ * RCU grace periods.
+ */
list_for_each_entry_safe(ns, next, &ns_list, list)
- nvme_ns_remove(ns);
+ async_schedule_domain(nvme_ns_remove_async, ns, &domain);
+ async_synchronize_full_domain(&domain);
}
EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
@@ -4903,7 +4910,7 @@ static void nvme_fw_act_work(struct work_struct *work)
if (ctrl->mtfa)
fw_act_timeout = jiffies + msecs_to_jiffies(ctrl->mtfa * 100);
else
- fw_act_timeout = jiffies + secs_to_jiffies(admin_timeout);
+ fw_act_timeout = jiffies + NVME_ADMIN_TIMEOUT;
nvme_quiesce_io_queues(ctrl);
while (nvme_ctrl_pp_status(ctrl)) {
@@ -5195,6 +5202,15 @@ EXPORT_SYMBOL_GPL(nvme_start_ctrl);
void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
{
nvme_stop_keep_alive(ctrl);
+
+ /*
+ * An AER completion can queue async_event_work; drain it before the
+ * controller's final release. Note that ana_work has additional
+ * producers (namespace scanning via nvme_scan_work, and the failover
+ * path in nvme_failover_req), which is why nvme_mpath_uninit()
+ * separately cancels ana_work after this point.
+ */
+ cancel_work_sync(&ctrl->async_event_work);
nvme_hwmon_exit(ctrl);
nvme_fault_inject_fini(&ctrl->fault_inject);
dev_pm_qos_hide_latency_tolerance(ctrl->device);
diff --git a/drivers/nvme/host/fabrics.c b/drivers/nvme/host/fabrics.c
index 3d7ad86c1955..5ed91557f070 100644
--- a/drivers/nvme/host/fabrics.c
+++ b/drivers/nvme/host/fabrics.c
@@ -872,12 +872,6 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
opts->fast_io_fail_tmo = token;
break;
case NVMF_OPT_HOSTNQN:
- if (opts->host) {
- pr_err("hostnqn already user-assigned: %s\n",
- opts->host->nqn);
- ret = -EADDRINUSE;
- goto out;
- }
p = match_strdup(args);
if (!p) {
ret = -ENOMEM;
@@ -1032,7 +1026,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
ret = -EINVAL;
goto out;
}
- kfree(opts->dhchap_secret);
+ kfree_sensitive(opts->dhchap_secret);
opts->dhchap_secret = p;
break;
case NVMF_OPT_DHCHAP_CTRL_SECRET:
@@ -1047,7 +1041,7 @@ static int nvmf_parse_options(struct nvmf_ctrl_options *opts,
ret = -EINVAL;
goto out;
}
- kfree(opts->dhchap_ctrl_secret);
+ kfree_sensitive(opts->dhchap_ctrl_secret);
opts->dhchap_ctrl_secret = p;
break;
case NVMF_OPT_TLS:
diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index 0b9ff50d26ff..2cbd744ffd22 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -39,7 +39,6 @@ struct nvme_fc_queue {
size_t cmnd_capsule_len;
u32 qnum;
u32 rqcnt;
- u32 seqno;
u64 connection_id;
atomic_t csn;
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index ae8059366b57..27facdbb3d4c 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -1608,6 +1608,13 @@ out_uninit:
void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
{
+ /*
+ * ANA work can be queued by nvme_handle_aen_notice, nvme_scan_work,
+ * and nvme_failover_req. Drain remaining work before releasing the ANA
+ * resources.
+ */
+ cancel_work_sync(&ctrl->ana_work);
+
kvfree(ctrl->ana_log_buf);
ctrl->ana_log_buf = NULL;
ctrl->ana_log_size = 0;
diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 8833a463f1c8..2543e52d6358 100644
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -22,11 +22,20 @@
extern const struct pr_ops nvme_pr_ops;
+/* Seconds to jiffies, capped for the u32 timeouts and timer delays */
+static inline unsigned int nvme_timeout_to_jiffies(unsigned int timeout)
+{
+ unsigned long max_secs;
+
+ max_secs = min_t(unsigned long, UINT_MAX, MAX_JIFFY_OFFSET) / HZ;
+ return min_t(unsigned long, timeout, max_secs) * HZ;
+}
+
extern unsigned int nvme_io_timeout;
-#define NVME_IO_TIMEOUT (nvme_io_timeout * HZ)
+#define NVME_IO_TIMEOUT nvme_timeout_to_jiffies(nvme_io_timeout)
extern unsigned int admin_timeout;
-#define NVME_ADMIN_TIMEOUT (admin_timeout * HZ)
+#define NVME_ADMIN_TIMEOUT nvme_timeout_to_jiffies(admin_timeout)
#define NVME_DEFAULT_KATO 5
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index f8088d5aa724..076603df01ab 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -390,7 +390,6 @@ struct nvme_queue {
unsigned long flags;
#define NVMEQ_ENABLED 0
#define NVMEQ_SQ_CMB 1
-#define NVMEQ_DELETE_ERROR 2
#define NVMEQ_POLLED 3
__le32 *dbbuf_sq_db;
__le32 *dbbuf_cq_db;
@@ -2459,8 +2458,6 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid, bool polled)
int result;
u16 vector = 0;
- clear_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
-
/*
* A queue's vector matches the queue identifier unless the controller
* has only one vector available.
@@ -3322,18 +3319,6 @@ static enum rq_end_io_ret nvme_del_queue_end(struct request *req,
return RQ_END_IO_NONE;
}
-static enum rq_end_io_ret nvme_del_cq_end(struct request *req,
- blk_status_t error,
- const struct io_comp_batch *iob)
-{
- struct nvme_queue *nvmeq = req->end_io_data;
-
- if (error)
- set_bit(NVMEQ_DELETE_ERROR, &nvmeq->flags);
-
- return nvme_del_queue_end(req, error, iob);
-}
-
static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
{
struct request_queue *q = nvmeq->dev->ctrl.admin_q;
@@ -3348,10 +3333,7 @@ static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
return PTR_ERR(req);
nvme_init_request(req, &cmd);
- if (opcode == nvme_admin_delete_cq)
- req->end_io = nvme_del_cq_end;
- else
- req->end_io = nvme_del_queue_end;
+ req->end_io = nvme_del_queue_end;
req->end_io_data = nvmeq;
init_completion(&nvmeq->delete_done);
@@ -4229,6 +4211,14 @@ static int nvme_suspend(struct device *dev)
ret = nvme_set_host_mem(ndev, 0);
if (ret < 0)
goto unfreeze;
+ if (ret) {
+ /*
+ * The controller may still access the HMB. Shut it
+ * down instead, which forces a reset on resume.
+ */
+ ret = nvme_disable_prepare_reset(ndev, true);
+ goto unfreeze;
+ }
}
ret = nvme_get_power_state(ctrl, &ndev->last_ps);
@@ -4440,8 +4430,9 @@ static const struct pci_device_id nvme_id_table[] = {
.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
{ PCI_DEVICE(0x15b7, 0x2001), /* Sandisk Skyhawk */
.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
- { PCI_DEVICE(0x1d97, 0x2263), /* SPCC */
- .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES, },
+ { PCI_DEVICE(0x1d97, 0x2263), /* SPCC / Lexar NM610 */
+ .driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES |
+ NVME_QUIRK_BOGUS_NID, },
{ PCI_DEVICE(0x144d, 0xa80b), /* Samsung PM9B1 256G and 512G */
.driver_data = NVME_QUIRK_DISABLE_WRITE_ZEROES |
NVME_QUIRK_BOGUS_NID, },
@@ -4501,8 +4492,6 @@ static const struct pci_device_id nvme_id_table[] = {
.driver_data = NVME_QUIRK_NO_DEEPEST_PS, },
{ PCI_DEVICE(0xc0a9, 0x540a), /* Crucial P2 */
.driver_data = NVME_QUIRK_BOGUS_NID, },
- { PCI_DEVICE(0x1d97, 0x2263), /* Lexar NM610 */
- .driver_data = NVME_QUIRK_BOGUS_NID, },
{ PCI_DEVICE(0x1d97, 0x1d97), /* Lexar NM620 */
.driver_data = NVME_QUIRK_BOGUS_NID, },
{ PCI_DEVICE(0x1d97, 0x2269), /* Lexar NM760 */
diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
index 10018c6356b6..35f871e3e0c0 100644
--- a/drivers/nvme/host/sysfs.c
+++ b/drivers/nvme/host/sysfs.c
@@ -618,8 +618,6 @@ static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
{
struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
- if (ctrl->opts->reconnect_delay == -1)
- return sysfs_emit(buf, "off\n");
return sysfs_emit(buf, "%d\n", ctrl->opts->reconnect_delay);
}
@@ -634,6 +632,9 @@ static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
if (err)
return err;
+ if (!v || v > INT_MAX)
+ return -EINVAL;
+
ctrl->opts->reconnect_delay = v;
return count;
}
@@ -849,10 +850,10 @@ static ssize_t dhchap_secret_store(struct device *dev,
ret = nvme_auth_parse_key(dhchap_secret, &key);
if (ret) {
- kfree(dhchap_secret);
+ kfree_sensitive(dhchap_secret);
return ret;
}
- kfree(opts->dhchap_secret);
+ kfree_sensitive(opts->dhchap_secret);
opts->dhchap_secret = dhchap_secret;
host_key = ctrl->host_key;
mutex_lock(&ctrl->dhchap_auth_mutex);
@@ -860,7 +861,7 @@ static ssize_t dhchap_secret_store(struct device *dev,
mutex_unlock(&ctrl->dhchap_auth_mutex);
nvme_auth_free_key(host_key);
} else
- kfree(dhchap_secret);
+ kfree_sensitive(dhchap_secret);
/* Start re-authentication */
dev_info(ctrl->device, "re-authenticating controller\n");
queue_work(nvme_wq, &ctrl->dhchap_auth_work);
@@ -906,10 +907,10 @@ static ssize_t dhchap_ctrl_secret_store(struct device *dev,
ret = nvme_auth_parse_key(dhchap_secret, &key);
if (ret) {
- kfree(dhchap_secret);
+ kfree_sensitive(dhchap_secret);
return ret;
}
- kfree(opts->dhchap_ctrl_secret);
+ kfree_sensitive(opts->dhchap_ctrl_secret);
opts->dhchap_ctrl_secret = dhchap_secret;
ctrl_key = ctrl->ctrl_key;
mutex_lock(&ctrl->dhchap_auth_mutex);
@@ -917,7 +918,7 @@ static ssize_t dhchap_ctrl_secret_store(struct device *dev,
mutex_unlock(&ctrl->dhchap_auth_mutex);
nvme_auth_free_key(ctrl_key);
} else
- kfree(dhchap_secret);
+ kfree_sensitive(dhchap_secret);
/* Start re-authentication */
dev_info(ctrl->device, "re-authenticating controller\n");
queue_work(nvme_wq, &ctrl->dhchap_auth_work);
diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c
index 7ecc6c007f8d..62a594b8ab9b 100644
--- a/drivers/nvme/host/tcp.c
+++ b/drivers/nvme/host/tcp.c
@@ -77,6 +77,12 @@ MODULE_PARM_DESC(tls_handshake_timeout,
/* TLS records read while waiting for the ICResp, the ICResp included */
#define NVME_TCP_ICRESP_MAX_RECORDS 8
+/* control message buffer for the TLS record type */
+union nvme_tcp_tls_cmsg {
+ struct cmsghdr cmsg;
+ u8 buf[CMSG_SPACE(sizeof(u8))];
+};
+
static int nvme_tcp_cpu_queues[NR_CPUS];
static DEFINE_SPINLOCK(nvme_tcp_cpu_queues_lock);
@@ -117,6 +123,7 @@ enum nvme_tcp_queue_flags {
NVME_TCP_Q_LIVE = 1,
NVME_TCP_Q_POLLING = 2,
NVME_TCP_Q_IO_CPU_SET = 3,
+ NVME_TCP_Q_TLS_RECORD = 4,
};
#define NVME_TCP_OPT_NR_QUEUES(opts) ((opts)->nr_io_queues + \
@@ -847,7 +854,7 @@ static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
return -EPROTO;
}
- if (unlikely(r2t_offset < req->data_sent)) {
+ if (unlikely(r2t_offset != req->data_sent)) {
dev_err(queue->ctrl->ctrl.device,
"req %d unexpected r2t offset %u (expected %zu)\n",
rq->tag, r2t_offset, req->data_sent);
@@ -1444,6 +1451,8 @@ out:
return ret;
}
+static int nvme_tcp_recv_tls_record(struct nvme_tcp_queue *queue);
+
static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
{
struct socket *sock = queue->sock;
@@ -1457,9 +1466,30 @@ static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
queue->nr_cqe = 0;
consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
/*
+ * kTLS cannot pass a TLS record without application data to
+ * nvme_tcp_recv_skb(). It returns -EINVAL and leaves the record at
+ * the head of the receive queue. recvmsg() takes the socket lock,
+ * so read the record without it.
+ */
+ if (unlikely(consumed == -EINVAL && nvme_tcp_queue_tls(queue) &&
+ queue->rd_enabled)) {
+ /*
+ * On a poll queue, nvme_tcp_poll() and nvme_tcp_io_work() can
+ * both get here for the same record. Only one of them reads it.
+ */
+ if (test_and_set_bit(NVME_TCP_Q_TLS_RECORD, &queue->flags)) {
+ consumed = 0;
+ } else {
+ release_sock(sk);
+ consumed = nvme_tcp_recv_tls_record(queue);
+ lock_sock(sk);
+ clear_bit(NVME_TCP_Q_TLS_RECORD, &queue->flags);
+ }
+ }
+ /*
* nvme_tcp_recv_skb() handles its own errors, so a queue that is
- * still enabled failed inside read_sock(). Handle it before
- * release_sock() processes the backlog.
+ * still enabled failed inside read_sock() or while reading a TLS
+ * record. Handle it before release_sock() processes the backlog.
*/
if (unlikely(consumed < 0 && consumed != -EAGAIN &&
queue->rd_enabled)) {
@@ -1623,54 +1653,125 @@ static int nvme_tcp_recv_tls_alert(struct nvme_tcp_queue *queue,
}
/*
- * Discard a TLS record that arrived in place of the ICResp. A peer may send
- * a NewSessionTicket as soon as the handshake completes.
+ * Read the rest of a TLS record whose first part was read into @msg.
*
- * Returns 0 when the record was discarded and the ICResp can be read, or a
- * negative errno if the connection cannot continue.
+ * Returns the bytes read, or a negative errno.
*/
-static int nvme_tcp_skip_tls_record(struct nvme_tcp_queue *queue,
- struct msghdr *msg, u8 ctype, int len)
+static int nvme_tcp_drain_tls_record(struct nvme_tcp_queue *queue,
+ struct msghdr *msg, int flags)
{
u8 buf[NVME_TCP_TLS_REC_CHUNK];
- int ret;
+ int ret, drained = 0;
- switch (ctype) {
- case TLS_RECORD_TYPE_HANDSHAKE:
- break;
- case TLS_RECORD_TYPE_ALERT:
- ret = nvme_tcp_recv_tls_alert(queue, msg, len);
- return ret ? : -ENOTCONN;
- default:
- pr_err("queue %d: unhandled TLS record %u\n",
- nvme_tcp_queue_id(queue), ctype);
- return -ENOTCONN;
- }
-
- /* the ICResp is in a later record, drain this one first */
while (!(msg->msg_flags & MSG_EOR)) {
struct kvec iov = {
.iov_base = buf,
.iov_len = sizeof(buf),
};
- char cbuf[CMSG_LEN(sizeof(char))] = {};
+ union nvme_tcp_tls_cmsg u = {};
/* kTLS fails a control record read without a cmsg buffer */
struct msghdr drain = {
- .msg_control = cbuf,
- .msg_controllen = sizeof(cbuf),
+ .msg_control = &u,
+ .msg_controllen = sizeof(u),
};
ret = kernel_recvmsg(queue->sock, &drain, &iov, 1, sizeof(buf),
- 0);
+ flags);
if (ret <= 0) {
pr_warn("queue %d: failed to drain TLS record: %d\n",
nvme_tcp_queue_id(queue), ret);
return ret ? : -ECONNRESET;
}
+ drained += ret;
msg->msg_flags = drain.msg_flags;
}
- return 0;
+ return drained;
+}
+
+/*
+ * Consume a TLS record without application data. The first @len bytes of
+ * the record were read into @msg and @data.
+ *
+ * A KeyUpdate needs new keys that nothing here can supply. Other handshake
+ * messages, such as a NewSessionTicket, are discarded.
+ *
+ * Returns the bytes read after the first @len, or a negative errno if the
+ * connection cannot continue.
+ */
+static int nvme_tcp_consume_tls_record(struct nvme_tcp_queue *queue,
+ struct msghdr *msg, u8 ctype,
+ const u8 *data, int len, int flags)
+{
+ int ret;
+
+ switch (ctype) {
+ case TLS_RECORD_TYPE_HANDSHAKE:
+ if (data[0] == TLS_HANDSHAKE_KEYUPDATE) {
+ dev_err(queue->ctrl->ctrl.device,
+ "queue %d: TLS KeyUpdate is not supported\n",
+ nvme_tcp_queue_id(queue));
+ return -EKEYEXPIRED;
+ }
+ break;
+ case TLS_RECORD_TYPE_ALERT:
+ ret = nvme_tcp_recv_tls_alert(queue, msg, len);
+ if (ret)
+ return ret;
+ break;
+ default:
+ dev_err(queue->ctrl->ctrl.device,
+ "queue %d: unexpected TLS record %u\n",
+ nvme_tcp_queue_id(queue), ctype);
+ return -EPROTO;
+ }
+
+ return nvme_tcp_drain_tls_record(queue, msg, flags);
+}
+
+/*
+ * Read the TLS record that read_sock() could not pass to
+ * nvme_tcp_recv_skb(). The caller must not hold the socket lock.
+ *
+ * Returns the bytes consumed, or a negative errno if the connection cannot
+ * continue.
+ */
+static int nvme_tcp_recv_tls_record(struct nvme_tcp_queue *queue)
+{
+ u8 buf[NVME_TCP_TLS_REC_CHUNK];
+ union nvme_tcp_tls_cmsg u = {};
+ struct kvec iov = {
+ .iov_base = buf,
+ .iov_len = sizeof(buf),
+ };
+ struct msghdr msg = {
+ .msg_control = &u,
+ .msg_controllen = sizeof(u),
+ };
+ int len, ret;
+ u8 ctype;
+
+ len = kernel_recvmsg(queue->sock, &msg, &iov, 1, sizeof(buf),
+ MSG_DONTWAIT);
+ if (len <= 0)
+ goto fail;
+
+ ctype = tls_get_record_type(queue->sock->sk, &u.cmsg);
+ ret = nvme_tcp_consume_tls_record(queue, &msg, ctype, buf, len,
+ MSG_DONTWAIT);
+ if (ret < 0) {
+ len = ret;
+ goto fail;
+ }
+
+ return len + ret;
+
+fail:
+ /* read_sock() left a whole record, so none of it can be missing */
+ if (len == -EAGAIN)
+ return -EPROTO;
+
+ return len ? : -ECONNRESET;
}
/*
@@ -1682,7 +1783,7 @@ static int nvme_tcp_skip_tls_record(struct nvme_tcp_queue *queue,
static int nvme_tcp_recv_icresp(struct nvme_tcp_queue *queue,
struct nvme_tcp_icresp_pdu *icresp)
{
- char cbuf[CMSG_LEN(sizeof(char))];
+ union nvme_tcp_tls_cmsg u;
struct msghdr msg;
struct kvec iov;
u8 ctype;
@@ -1690,12 +1791,12 @@ static int nvme_tcp_recv_icresp(struct nvme_tcp_queue *queue,
for (i = 0; i < NVME_TCP_ICRESP_MAX_RECORDS; i++) {
memset(&msg, 0, sizeof(msg));
- memset(cbuf, 0, sizeof(cbuf));
+ memset(&u, 0, sizeof(u));
iov.iov_base = icresp;
iov.iov_len = sizeof(*icresp);
if (nvme_tcp_queue_tls(queue)) {
- msg.msg_control = cbuf;
- msg.msg_controllen = sizeof(cbuf);
+ msg.msg_control = &u;
+ msg.msg_controllen = sizeof(u);
}
msg.msg_flags = MSG_WAITALL;
ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
@@ -1703,12 +1804,13 @@ static int nvme_tcp_recv_icresp(struct nvme_tcp_queue *queue,
if (ret <= 0 || !nvme_tcp_queue_tls(queue))
return ret;
- ctype = tls_get_record_type(queue->sock->sk,
- (struct cmsghdr *)cbuf);
+ ctype = tls_get_record_type(queue->sock->sk, &u.cmsg);
if (ctype == TLS_RECORD_TYPE_DATA)
return ret;
- ret = nvme_tcp_skip_tls_record(queue, &msg, ctype, ret);
+ /* the ICResp is in a later record */
+ ret = nvme_tcp_consume_tls_record(queue, &msg, ctype,
+ (const u8 *)icresp, ret, 0);
if (ret < 0)
return ret;
}
@@ -1992,7 +2094,8 @@ static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
dev_err(nctrl->device,
"queue %d: TLS handshake failed, error %d\n",
qid, ret);
- tls_handshake_cancel(queue->sock->sk);
+ if (!tls_handshake_cancel(queue->sock->sk))
+ wait_for_completion(&queue->tls_complete);
} else {
if (queue->tls_err) {
dev_err(nctrl->device,
diff --git a/drivers/nvme/target/admin-cmd.c b/drivers/nvme/target/admin-cmd.c
index f12d83098644..e0be6eb0fa04 100644
--- a/drivers/nvme/target/admin-cmd.c
+++ b/drivers/nvme/target/admin-cmd.c
@@ -176,16 +176,35 @@ static void nvmet_execute_get_log_page_error(struct nvmet_req *req)
{
struct nvmet_ctrl *ctrl = req->sq->ctrl;
unsigned long flags;
- off_t offset = 0;
+ u32 offset = 0;
+ u32 len;
u64 slot;
u64 i;
+ u16 status = 0;
+
+ status = nvmet_zero_sgl(req, 0, req->transfer_len);
+ if (status) {
+ nvmet_req_complete(req, status);
+ return;
+ }
+
+ len = min_t(u32, req->transfer_len,
+ NVMET_ERROR_LOG_SLOTS * sizeof(struct nvme_error_slot));
spin_lock_irqsave(&ctrl->error_lock, flags);
+
+ /*
+ * The NVMe base specification requires the most recent error to be
+ * the first log entry, so walk the ring backwards from the slot of
+ * the last error and stop at the end of the requested buffer.
+ */
slot = ctrl->err_counter % NVMET_ERROR_LOG_SLOTS;
- for (i = 0; i < NVMET_ERROR_LOG_SLOTS; i++) {
- if (nvmet_copy_to_sgl(req, offset, &ctrl->slots[slot],
- sizeof(struct nvme_error_slot)))
+ for (i = 0; i < NVMET_ERROR_LOG_SLOTS &&
+ offset + sizeof(struct nvme_error_slot) <= len; i++) {
+ status = nvmet_copy_to_sgl(req, offset, &ctrl->slots[slot],
+ sizeof(struct nvme_error_slot));
+ if (status)
break;
if (slot == 0)
@@ -195,7 +214,7 @@ static void nvmet_execute_get_log_page_error(struct nvmet_req *req)
offset += sizeof(struct nvme_error_slot);
}
spin_unlock_irqrestore(&ctrl->error_lock, flags);
- nvmet_req_complete(req, 0);
+ nvmet_req_complete(req, status);
}
static void nvmet_execute_get_supported_log_pages(struct nvmet_req *req)
@@ -221,7 +240,9 @@ static void nvmet_execute_get_supported_log_pages(struct nvmet_req *req)
logs->lids[NVME_LOG_RMI] = cpu_to_le32(NVME_LIDS_LSUPP);
logs->lids[NVME_LOG_RESERVATION] = cpu_to_le32(NVME_LIDS_LSUPP);
- status = nvmet_copy_to_sgl(req, 0, logs, sizeof(*logs));
+ status = nvmet_zero_sgl(req, 0, req->transfer_len);
+ if (!status)
+ status = nvmet_copy_to_sgl(req, 0, logs, sizeof(*logs));
kfree(logs);
out:
nvmet_req_complete(req, status);
@@ -303,7 +324,7 @@ static void nvmet_execute_get_log_page_rmi(struct nvmet_req *req)
goto out;
}
- if (req->transfer_len != sizeof(*log)) {
+ if (req->transfer_len < sizeof(*log)) {
status = NVME_SC_SGL_INVALID_DATA | NVME_STATUS_DNR;
goto out;
}
@@ -321,7 +342,9 @@ static void nvmet_execute_get_log_page_rmi(struct nvmet_req *req)
else
log->numa = cpu_to_le16(1);
- status = nvmet_copy_to_sgl(req, 0, log, sizeof(*log));
+ status = nvmet_zero_sgl(req, 0, req->transfer_len);
+ if (!status)
+ status = nvmet_copy_to_sgl(req, 0, log, sizeof(*log));
kfree(log);
out:
nvmet_req_complete(req, status);
@@ -436,7 +459,9 @@ static void nvmet_execute_get_log_cmd_effects_ns(struct nvmet_req *req)
goto free;
}
- status = nvmet_copy_to_sgl(req, 0, log, sizeof(*log));
+ status = nvmet_zero_sgl(req, 0, req->transfer_len);
+ if (!status)
+ status = nvmet_copy_to_sgl(req, 0, log, sizeof(*log));
free:
kfree(log);
out:
@@ -527,7 +552,9 @@ static void nvmet_execute_get_log_page_endgrp(struct nvmet_req *req)
put_unaligned_le64(host_writes, &log->hwc[0]);
put_unaligned_le64(data_units_written, &log->duw[0]);
copy:
- status = nvmet_copy_to_sgl(req, 0, log, sizeof(*log));
+ status = nvmet_zero_sgl(req, 0, req->transfer_len);
+ if (!status)
+ status = nvmet_copy_to_sgl(req, 0, log, sizeof(*log));
kfree(log);
out:
nvmet_req_complete(req, status);
@@ -610,7 +637,9 @@ static void nvmet_execute_get_log_page_features(struct nvmet_req *req)
features->fis[NVME_FEAT_RESV_MASK] =
cpu_to_le32(NVME_FIS_FSUPP | NVME_FIS_NSCPE);
- status = nvmet_copy_to_sgl(req, 0, features, sizeof(*features));
+ status = nvmet_zero_sgl(req, 0, req->transfer_len);
+ if (!status)
+ status = nvmet_copy_to_sgl(req, 0, features, sizeof(*features));
kfree(features);
out:
nvmet_req_complete(req, status);
@@ -1223,6 +1252,7 @@ static u16 nvmet_set_feat_write_protect(struct nvmet_req *req)
status = 0;
break;
default:
+ status = NVME_SC_INVALID_FIELD | NVME_STATUS_DNR;
break;
}
diff --git a/drivers/nvme/target/auth.c b/drivers/nvme/target/auth.c
index ccba84b1344e..a4a2a5cfd823 100644
--- a/drivers/nvme/target/auth.c
+++ b/drivers/nvme/target/auth.c
@@ -24,20 +24,26 @@ int nvmet_auth_set_key(struct nvmet_host *host, const char *secret,
{
unsigned char key_hash;
char *dhchap_secret;
+ int offset = 0;
if (!strlen(secret)) {
+ down_write(&nvmet_config_sem);
if (set_ctrl) {
kfree(host->dhchap_ctrl_secret);
host->dhchap_ctrl_secret = NULL;
host->dhchap_ctrl_key_hash = 0;
+ host->dhchap_ctrl_key_offset = 0;
} else {
kfree(host->dhchap_secret);
host->dhchap_secret = NULL;
host->dhchap_key_hash = 0;
+ host->dhchap_key_offset = 0;
}
+ up_write(&nvmet_config_sem);
return 0;
}
- if (sscanf(secret, "DHHC-1:%hhd:%*s", &key_hash) != 1)
+ if (sscanf(secret, "DHHC-1:%hhd:%n", &key_hash, &offset) != 1 ||
+ !offset)
return -EINVAL;
if (key_hash > 3) {
pr_warn("Invalid DH-HMAC-CHAP hash id %d\n",
@@ -52,10 +58,12 @@ int nvmet_auth_set_key(struct nvmet_host *host, const char *secret,
kfree(host->dhchap_ctrl_secret);
host->dhchap_ctrl_secret = strim(dhchap_secret);
host->dhchap_ctrl_key_hash = key_hash;
+ host->dhchap_ctrl_key_offset = offset;
} else {
kfree(host->dhchap_secret);
host->dhchap_secret = strim(dhchap_secret);
host->dhchap_key_hash = key_hash;
+ host->dhchap_key_offset = offset;
}
up_write(&nvmet_config_sem);
return 0;
@@ -76,8 +84,11 @@ int nvmet_setup_dhgroup(struct nvmet_ctrl *ctrl, u8 dhgroup_id)
return 0;
}
crypto_free_kpp(ctrl->dh_tfm);
+ kfree_sensitive(ctrl->dh_key);
ctrl->dh_tfm = NULL;
ctrl->dh_gid = 0;
+ ctrl->dh_key = NULL;
+ ctrl->dh_keysize = 0;
}
if (dhgroup_id == NVME_AUTH_DHGROUP_NULL)
@@ -95,38 +106,42 @@ int nvmet_setup_dhgroup(struct nvmet_ctrl *ctrl, u8 dhgroup_id)
__func__, ctrl->cntlid, dhgroup_id,
PTR_ERR(ctrl->dh_tfm));
ret = PTR_ERR(ctrl->dh_tfm);
- ctrl->dh_tfm = NULL;
- ctrl->dh_gid = 0;
- } else {
- ctrl->dh_gid = dhgroup_id;
- pr_debug("%s: ctrl %d setup DH group %d\n",
- __func__, ctrl->cntlid, ctrl->dh_gid);
- ret = nvme_auth_gen_privkey(ctrl->dh_tfm, ctrl->dh_gid);
- if (ret < 0) {
- pr_debug("%s: ctrl %d failed to generate private key, err %d\n",
- __func__, ctrl->cntlid, ret);
- kfree_sensitive(ctrl->dh_key);
- ctrl->dh_key = NULL;
- return ret;
- }
- ctrl->dh_keysize = crypto_kpp_maxsize(ctrl->dh_tfm);
- kfree_sensitive(ctrl->dh_key);
- ctrl->dh_key = kzalloc(ctrl->dh_keysize, GFP_KERNEL);
- if (!ctrl->dh_key) {
- pr_warn("ctrl %d failed to allocate public key\n",
- ctrl->cntlid);
- return -ENOMEM;
- }
- ret = nvme_auth_gen_pubkey(ctrl->dh_tfm, ctrl->dh_key,
- ctrl->dh_keysize);
- if (ret < 0) {
- pr_warn("ctrl %d failed to generate public key\n",
- ctrl->cntlid);
- kfree(ctrl->dh_key);
- ctrl->dh_key = NULL;
- }
+ goto err;
+ }
+
+ ctrl->dh_gid = dhgroup_id;
+ pr_debug("%s: ctrl %d setup DH group %d\n",
+ __func__, ctrl->cntlid, ctrl->dh_gid);
+ ret = nvme_auth_gen_privkey(ctrl->dh_tfm, ctrl->dh_gid);
+ if (ret < 0) {
+ pr_debug("%s: ctrl %d failed to generate private key, err %d\n",
+ __func__, ctrl->cntlid, ret);
+ goto err;
+ }
+ ctrl->dh_keysize = crypto_kpp_maxsize(ctrl->dh_tfm);
+ ctrl->dh_key = kzalloc(ctrl->dh_keysize, GFP_KERNEL);
+ if (!ctrl->dh_key) {
+ pr_warn("ctrl %d failed to allocate public key\n",
+ ctrl->cntlid);
+ ret = -ENOMEM;
+ goto err;
+ }
+ ret = nvme_auth_gen_pubkey(ctrl->dh_tfm, ctrl->dh_key,
+ ctrl->dh_keysize);
+ if (ret < 0) {
+ pr_warn("ctrl %d failed to generate public key\n",
+ ctrl->cntlid);
+ goto err;
}
+ return 0;
+err:
+ kfree_sensitive(ctrl->dh_key);
+ ctrl->dh_key = NULL;
+ crypto_free_kpp(ctrl->dh_tfm);
+ ctrl->dh_tfm = NULL;
+ ctrl->dh_gid = 0;
+ ctrl->dh_keysize = 0;
return ret;
}
@@ -177,14 +192,16 @@ u8 nvmet_setup_auth(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, bool reset)
* Skip the 'DHHC-1:XX:' prefix. Keep the old keys until the new ones
* are there: a controller without a host key needs no authentication.
*/
- host_key = nvme_auth_extract_key(host->dhchap_secret + 10,
+ host_key = nvme_auth_extract_key(host->dhchap_secret +
+ host->dhchap_key_offset,
host->dhchap_key_hash);
if (IS_ERR(host_key)) {
ret = NVME_AUTH_DHCHAP_FAILURE_NOT_USABLE;
goto out_unlock;
}
if (host->dhchap_ctrl_secret) {
- ctrl_key = nvme_auth_extract_key(host->dhchap_ctrl_secret + 10,
+ ctrl_key = nvme_auth_extract_key(host->dhchap_ctrl_secret +
+ host->dhchap_ctrl_key_offset,
host->dhchap_ctrl_key_hash);
if (IS_ERR(ctrl_key)) {
nvme_auth_free_key(host_key);
diff --git a/drivers/nvme/target/configfs.c b/drivers/nvme/target/configfs.c
index bde94656e4b7..27c50fa533e7 100644
--- a/drivers/nvme/target/configfs.c
+++ b/drivers/nvme/target/configfs.c
@@ -42,7 +42,6 @@ static struct nvmet_type_name_map nvmet_transport[] = {
};
static const struct nvmet_type_name_map nvmet_addr_family[] = {
- { NVMF_ADDR_FAMILY_PCI, "pcie" },
{ NVMF_ADDR_FAMILY_IP4, "ipv4" },
{ NVMF_ADDR_FAMILY_IP6, "ipv6" },
{ NVMF_ADDR_FAMILY_IB, "ib" },
@@ -68,7 +67,7 @@ static ssize_t nvmet_addr_adrfam_show(struct config_item *item, char *page)
u8 adrfam = to_nvmet_port(item)->disc_addr.adrfam;
int i;
- for (i = 1; i < ARRAY_SIZE(nvmet_addr_family); i++) {
+ for (i = 0; i < ARRAY_SIZE(nvmet_addr_family); i++) {
if (nvmet_addr_family[i].type == adrfam)
return snprintf(page, PAGE_SIZE, "%s\n",
nvmet_addr_family[i].name);
@@ -86,7 +85,7 @@ static ssize_t nvmet_addr_adrfam_store(struct config_item *item,
if (nvmet_is_port_enabled(port, __func__))
return -EACCES;
- for (i = 1; i < ARRAY_SIZE(nvmet_addr_family); i++) {
+ for (i = 0; i < ARRAY_SIZE(nvmet_addr_family); i++) {
if (sysfs_streq(page, nvmet_addr_family[i].name))
goto found;
}
@@ -709,7 +708,7 @@ static ssize_t nvmet_ns_ana_grpid_store(struct config_item *item,
down_write(&nvmet_ana_sem);
oldgrpid = ns->anagrpid;
- newgrpid = array_index_nospec(newgrpid, NVMET_MAX_ANAGRPS);
+ newgrpid = array_index_nospec(newgrpid, NVMET_MAX_ANAGRPS + 1);
nvmet_ana_group_enabled[newgrpid]++;
ns->anagrpid = newgrpid;
nvmet_ana_group_enabled[oldgrpid]--;
@@ -2086,7 +2085,7 @@ static struct config_group *nvmet_ana_groups_make_group(
grp->grpid = grpid;
down_write(&nvmet_ana_sem);
- grpid = array_index_nospec(grpid, NVMET_MAX_ANAGRPS);
+ grpid = array_index_nospec(grpid, NVMET_MAX_ANAGRPS + 1);
nvmet_ana_group_enabled[grpid]++;
up_write(&nvmet_ana_sem);
diff --git a/drivers/nvme/target/fabrics-cmd.c b/drivers/nvme/target/fabrics-cmd.c
index fc626c2fd2c9..6f0fc695e649 100644
--- a/drivers/nvme/target/fabrics-cmd.c
+++ b/drivers/nvme/target/fabrics-cmd.c
@@ -216,6 +216,8 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req)
if (old) {
pr_warn("queue already connected!\n");
req->error_loc = offsetof(struct nvmf_connect_command, opcode);
+ req->sq->ctrl = NULL;
+ nvmet_ctrl_put(ctrl);
return NVME_SC_CONNECT_CTRL_BUSY | NVME_STATUS_DNR;
}
@@ -242,6 +244,7 @@ static u16 nvmet_install_queue(struct nvmet_ctrl *ctrl, struct nvmet_req *req)
err:
req->sq->ctrl = NULL;
+ nvmet_cq_destroy(req->cq);
return ret;
}
diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c
index 2b7dc3f03912..5d6c1c79508a 100644
--- a/drivers/nvme/target/fc.c
+++ b/drivers/nvme/target/fc.c
@@ -27,7 +27,6 @@ struct nvmet_fc_tgt_assoc;
struct nvmet_fc_ls_iod { /* for an LS RQST RCV */
struct nvmefc_ls_rsp *lsrsp;
- struct nvmefc_tgt_fcp_req *fcpreq; /* only if RS */
struct list_head ls_rcv_list; /* tgtport->ls_rcv_list */
@@ -129,7 +128,6 @@ struct nvmet_fc_defer_fcp_req {
};
struct nvmet_fc_tgt_queue {
- bool ninetypercent;
u16 qid;
u16 sqsize;
u16 ersp_ratio;
@@ -818,8 +816,17 @@ nvmet_fc_alloc_target_queue(struct nvmet_fc_tgt_assoc *assoc,
if (ret)
goto out_fail_iodlist;
- WARN_ON(assoc->queues[qid]);
- assoc->queues[qid] = queue;
+ /*
+ * Publish the queue only if the slot is still free. A Create
+ * Connection may arrive twice for the same queue ID, or with the
+ * admin queue ID that was created by Create Association; replacing
+ * the old pointer would leak that queue and its controller
+ * reference forever.
+ */
+ if (cmpxchg(&assoc->queues[qid], NULL, queue)) {
+ nvmet_sq_destroy(&queue->nvme_sq);
+ goto out_fail_iodlist;
+ }
return queue;
@@ -2076,7 +2083,6 @@ nvmet_fc_rcv_ls_req(struct nvmet_fc_target_port *target_port,
}
iod->lsrsp = lsrsp;
- iod->fcpreq = NULL;
memcpy(iod->rqstbuf, lsreqbuf, lsreqbuf_len);
iod->rqstdatalen = lsreqbuf_len;
iod->hosthandle = hosthandle;
diff --git a/drivers/nvme/target/fcloop.c b/drivers/nvme/target/fcloop.c
index b63af3b643a6..a9cde536c33b 100644
--- a/drivers/nvme/target/fcloop.c
+++ b/drivers/nvme/target/fcloop.c
@@ -379,6 +379,8 @@ fcloop_h2t_ls_req(struct nvme_fc_local_port *localport,
ret = nvmet_fc_rcv_ls_req(rport->targetport, rport,
&tls_req->ls_rsp,
lsreq->rqstaddr, lsreq->rqstlen);
+ if (ret)
+ kmem_cache_free(lsreq_cache, tls_req);
return ret;
}
diff --git a/drivers/nvme/target/io-cmd-file.c b/drivers/nvme/target/io-cmd-file.c
index 695963dd932e..3098b902f519 100644
--- a/drivers/nvme/target/io-cmd-file.c
+++ b/drivers/nvme/target/io-cmd-file.c
@@ -113,10 +113,23 @@ static void nvmet_file_io_done(struct kiocb *iocb, long ret)
}
if (unlikely(ret != req->transfer_len))
- status = errno_to_nvme_status(req, ret);
+ status = errno_to_nvme_status(req, ret < 0 ? ret : -EIO);
nvmet_req_complete(req, status);
}
+static bool nvmet_file_check_lba_range(struct nvmet_req *req, u64 slba,
+ u64 len, loff_t *pos)
+{
+ u64 pos64;
+
+ if (check_shl_overflow(slba, req->ns->blksize_shift, &pos64) ||
+ pos64 > req->ns->size || len > req->ns->size - pos64)
+ return false;
+
+ *pos = pos64;
+ return true;
+}
+
static bool nvmet_file_execute_io(struct nvmet_req *req, int ki_flags)
{
ssize_t nr_bvec = req->sg_cnt;
@@ -131,8 +144,9 @@ static bool nvmet_file_execute_io(struct nvmet_req *req, int ki_flags)
if (req->f.mpool_alloc && nr_bvec > NVMET_MAX_MPOOL_BVEC)
is_sync = true;
- pos = le64_to_cpu(req->cmd->rw.slba) << req->ns->blksize_shift;
- if (unlikely(pos + req->transfer_len > req->ns->size)) {
+ if (unlikely(!nvmet_file_check_lba_range(req,
+ le64_to_cpu(req->cmd->rw.slba),
+ req->transfer_len, &pos))) {
ret = -ENOSPC;
goto complete;
}
@@ -150,8 +164,11 @@ static bool nvmet_file_execute_io(struct nvmet_req *req, int ki_flags)
if (unlikely(is_sync) &&
(nr_bvec - 1 == 0 || bv_cnt == NVMET_MAX_MPOOL_BVEC)) {
ret = nvmet_file_submit_bvec(req, pos, bv_cnt, len, 0);
- if (ret < 0)
+ if (ret != len) {
+ if (ret >= 0)
+ ret = -EIO;
goto complete;
+ }
pos += len;
bv_cnt = 0;
@@ -279,19 +296,22 @@ static void nvmet_file_execute_discard(struct nvmet_req *req)
int i;
for (i = 0; i <= le32_to_cpu(req->cmd->dsm.nr); i++) {
+ u64 len64;
+
status = nvmet_copy_from_sgl(req, i * sizeof(range), &range,
sizeof(range));
if (status)
break;
- offset = le64_to_cpu(range.slba) << req->ns->blksize_shift;
- len = le32_to_cpu(range.nlb);
- len <<= req->ns->blksize_shift;
- if (offset + len > req->ns->size) {
+ if (check_shl_overflow((u64)le32_to_cpu(range.nlb),
+ req->ns->blksize_shift, &len64) ||
+ !nvmet_file_check_lba_range(req, le64_to_cpu(range.slba),
+ len64, &offset)) {
req->error_slba = le64_to_cpu(range.slba);
status = errno_to_nvme_status(req, -ENOSPC);
break;
}
+ len = len64;
ret = vfs_fallocate(req->ns->file, mode, offset, len);
if (ret && ret != -EOPNOTSUPP) {
@@ -336,16 +356,17 @@ static void nvmet_file_write_zeroes_work(struct work_struct *w)
int mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE;
loff_t offset;
loff_t len;
+ u64 len64;
int ret;
- offset = le64_to_cpu(write_zeroes->slba) << req->ns->blksize_shift;
- len = (((sector_t)le16_to_cpu(write_zeroes->length) + 1) <<
- req->ns->blksize_shift);
-
- if (unlikely(offset + len > req->ns->size)) {
+ if (check_shl_overflow((u64)le16_to_cpu(write_zeroes->length) + 1,
+ req->ns->blksize_shift, &len64) ||
+ !nvmet_file_check_lba_range(req, le64_to_cpu(write_zeroes->slba),
+ len64, &offset)) {
nvmet_req_complete(req, errno_to_nvme_status(req, -ENOSPC));
return;
}
+ len = len64;
ret = vfs_fallocate(req->ns->file, mode, offset, len);
nvmet_req_complete(req, ret < 0 ? errno_to_nvme_status(req, ret) : 0);
diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c
index 80b67608eb15..d04be1f41d52 100644
--- a/drivers/nvme/target/loop.c
+++ b/drivers/nvme/target/loop.c
@@ -159,6 +159,7 @@ static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
blk_rq_nr_phys_segments(req),
iod->sg_table.sgl, NVME_INLINE_SG_CNT)) {
nvme_cleanup_cmd(req);
+ nvmet_req_uninit(&iod->req);
return BLK_STS_RESOURCE;
}
diff --git a/drivers/nvme/target/nvmet.h b/drivers/nvme/target/nvmet.h
index 2c2460ef5e2b..e33884a81efc 100644
--- a/drivers/nvme/target/nvmet.h
+++ b/drivers/nvme/target/nvmet.h
@@ -401,6 +401,8 @@ struct nvmet_host {
struct config_group group;
u8 *dhchap_secret;
u8 *dhchap_ctrl_secret;
+ int dhchap_key_offset;
+ int dhchap_ctrl_key_offset;
u8 dhchap_key_hash;
u8 dhchap_ctrl_key_hash;
u8 dhchap_hash_id;
diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c
index 6b67880c7bb2..32ae4b6b59bf 100644
--- a/drivers/nvme/target/passthru.c
+++ b/drivers/nvme/target/passthru.c
@@ -550,7 +550,6 @@ u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req)
default:
return nvmet_passthru_get_set_features(req);
}
- break;
case nvme_admin_get_features:
switch (le32_to_cpu(req->cmd->features.fid)) {
case NVME_FEAT_ASYNC_EVENT:
@@ -564,7 +563,6 @@ u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req)
default:
return nvmet_passthru_get_set_features(req);
}
- break;
case nvme_admin_identify:
switch (req->cmd->identify.cns) {
case NVME_ID_CNS_CS_CTRL:
diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c
index 670068d1ee3e..4199fead04ff 100644
--- a/drivers/nvme/target/pci-epf.c
+++ b/drivers/nvme/target/pci-epf.c
@@ -1432,8 +1432,6 @@ static u16 nvmet_pci_epf_delete_sq(struct nvmet_ctrl *tctrl, u16 sqid)
destroy_workqueue(sq->iod_wq);
sq->iod_wq = NULL;
- nvmet_pci_epf_drain_queue(sq);
-
if (sq->nvme_sq.ctrl)
nvmet_sq_destroy(&sq->nvme_sq);
diff --git a/drivers/nvme/target/pr.c b/drivers/nvme/target/pr.c
index 14eaa99f98fa..8c818f24815b 100644
--- a/drivers/nvme/target/pr.c
+++ b/drivers/nvme/target/pr.c
@@ -99,6 +99,12 @@ void nvmet_execute_get_log_page_resv(struct nvmet_req *req)
u64 cur_count;
u64 next_count;
+ status = nvmet_zero_sgl(req, 0, req->transfer_len);
+ if (status) {
+ nvmet_req_complete(req, status);
+ return;
+ }
+
mutex_lock(&log_mgr->lock);
if (!kfifo_get(&log_mgr->log_queue, &log))
goto out;
@@ -335,7 +341,7 @@ static u16 nvmet_pr_update_reg_attr(struct nvmet_pr *pr,
return NVME_SC_SUCCESS;
}
- new = kmalloc_obj(*new, GFP_ATOMIC);
+ new = kmalloc_obj(*new, GFP_KERNEL);
if (!new)
return NVME_SC_INTERNAL;
@@ -1040,7 +1046,7 @@ static int nvmet_pr_alloc_and_insert_pc_ref(struct nvmet_ns *ns,
struct nvmet_pr_per_ctrl_ref *pc_ref;
int ret;
- pc_ref = kmalloc_obj(*pc_ref, GFP_ATOMIC);
+ pc_ref = kmalloc_obj(*pc_ref, GFP_KERNEL);
if (!pc_ref)
return -ENOMEM;
diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c
index 5105317cc61e..8a2e0e056131 100644
--- a/drivers/nvme/target/tcp.c
+++ b/drivers/nvme/target/tcp.c
@@ -270,6 +270,30 @@ static inline bool nvmet_tcp_has_inline_data(struct nvmet_tcp_cmd *cmd)
!cmd->rbytes_done;
}
+/*
+ * Check whether the SGL of @cmd carries in-capsule data that the host
+ * will send right after the command PDU. Only the offset form of the
+ * data block descriptor is in-capsule; a regular transport SGL is
+ * fetched through an R2T instead.
+ */
+static inline bool nvmet_tcp_sgl_is_inline(struct nvmet_tcp_cmd *cmd)
+{
+ struct nvme_sgl_desc *sgl = &cmd->req.cmd->common.dptr.sgl;
+ u32 len = le32_to_cpu(sgl->length);
+
+ if (sgl->type != ((NVME_SGL_FMT_DATA_DESC << 4) |
+ NVME_SGL_FMT_OFFSET))
+ return false;
+
+ if (!nvme_is_write(cmd->req.cmd))
+ return false;
+
+ if (!len)
+ return false;
+
+ return len <= cmd->req.port->inline_data_size;
+}
+
static inline struct nvmet_tcp_cmd *
nvmet_tcp_get_cmd(struct nvmet_tcp_queue *queue)
{
@@ -626,25 +650,19 @@ static void nvmet_tcp_queue_response(struct nvmet_req *req)
struct nvmet_tcp_queue *queue = cmd->queue;
enum nvmet_tcp_recv_state queue_state;
struct nvmet_tcp_cmd *queue_cmd;
- struct nvme_sgl_desc *sgl;
- u32 len;
/* Pairs with store_release in nvmet_prepare_receive_pdu() */
queue_state = smp_load_acquire(&queue->rcv_state);
queue_cmd = READ_ONCE(queue->cmd);
if (unlikely(cmd == queue_cmd)) {
- sgl = &cmd->req.cmd->common.dptr.sgl;
- len = le32_to_cpu(sgl->length);
-
/*
- * Wait for inline data before processing the response.
- * Avoid using helpers, this might happen before
- * nvmet_req_init is completed.
+ * Wait for in-capsule data before processing the response.
+ * This can happen before nvmet_req_init() has completed, so
+ * only look at the SGL descriptor itself.
*/
if (queue_state == NVMET_TCP_RECV_PDU &&
- len && len <= cmd->req.port->inline_data_size &&
- nvme_is_write(cmd->req.cmd))
+ nvmet_tcp_sgl_is_inline(cmd))
return;
}
@@ -982,7 +1000,6 @@ static int nvmet_tcp_handle_icreq(struct nvmet_tcp_queue *queue)
static int nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue,
struct nvmet_tcp_cmd *cmd, struct nvmet_req *req)
{
- size_t data_len = le32_to_cpu(req->cmd->common.dptr.sgl.length);
int ret;
/*
@@ -992,8 +1009,7 @@ static int nvmet_tcp_handle_req_failure(struct nvmet_tcp_queue *queue,
* otherwise we will need to prepare a buffer and receive the
* stale data before continuing forward.
*/
- if (!nvme_is_write(cmd->req.cmd) || !data_len ||
- data_len > cmd->req.port->inline_data_size) {
+ if (!nvmet_tcp_sgl_is_inline(cmd)) {
nvmet_prepare_receive_pdu(queue);
return 0;
}
diff --git a/drivers/nvme/target/zns.c b/drivers/nvme/target/zns.c
index 92f5c1435582..cea05b4bc740 100644
--- a/drivers/nvme/target/zns.c
+++ b/drivers/nvme/target/zns.c
@@ -108,7 +108,7 @@ void nvmet_execute_identify_ns_zns(struct nvmet_req *req)
status = nvmet_req_find_ns(req);
if (status)
- goto done;
+ goto out;
if (nvmet_ns_revalidate(req->ns)) {
mutex_lock(&req->ns->subsys->lock);
@@ -140,7 +140,6 @@ void nvmet_execute_identify_ns_zns(struct nvmet_req *req)
mar--;
id_zns->mar = cpu_to_le32(mar);
-done:
status = nvmet_copy_to_sgl(req, 0, id_zns, sizeof(*id_zns));
out:
kfree(id_zns);
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 1a2f8d48adc4..3c9667854c46 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -524,7 +524,8 @@ enum hctx_type {
* @tag_list: List of the request queues that use this tag set. See also
* request_queue.tag_set_list.
* @quiesce_lock: Serializes blk_mq_[un]quiesce_tagset() against each other
- * and against a request queue being added to @tag_list.
+ * and against a request queue being added to or removed from
+ * @tag_list.
* @quiesce_depth: How often blk_mq_quiesce_tagset() has been called without
* blk_mq_unquiesce_tagset().
* @srcu: Use as lock when type of the request queue is blocking
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index 0cc09b975d8f..1a5812fa4052 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -328,6 +328,7 @@ enum {
BIO_ZONE_WRITE_PLUGGING, /* bio handled through zone write plugging */
BIO_EMULATES_ZONE_APPEND, /* bio emulates a zone append operation */
BIO_COMPLETE_IN_TASK, /* complete bi_end_io() in task context */
+ BIO_ERROR_INJECTED, /* error injection rules already applied */
BIO_FLAG_LAST
};