Skip to content

Commit 1521dc2

Browse files
axboeaxboe
committed
Merge tag 'nvme-6.10-2024-05-29' of git://git.infradead.org/nvme into block-6.10
Pull NVMe fixes from Keith: "nvme fixes for Linux 6.10 - Removing unused fields (Kanchan) - Large folio offsets support (Kundan) - Multipath NUMA node initialiazation fix (Nilay) - Multipath IO stats accounting fixes (Keith) - Circular lockdep fix (Keith) - Target race condition fix (Sagi) - Target memory leak fix (Sagi)" * tag 'nvme-6.10-2024-05-29' of git://git.infradead.org/nvme: nvmet: fix a possible leak when destroy a ctrl during qp establishment nvme: use srcu for iterating namespace list nvme: adjust multiples of NVME_CTRL_PAGE_SIZE in offset nvme: remove sgs and sws nvmet: fix ns enable/disable possible hang nvme-multipath: fix io accounting on failover nvme: fix multipath batched completion accounting nvme-multipath: find NUMA path only for online numa-node
2 parents 74d4ce9 + c758b77 commit 1521dc2

File tree

7 files changed

+117
-67
lines changed

7 files changed

+117
-67
lines changed

‎drivers/nvme/host/core.c‎

Lines changed: 70 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -414,7 +414,15 @@ static inline void nvme_end_req_zoned(struct request *req)
414414
}
415415
}
416416

417-
static inline void nvme_end_req(struct request *req)
417+
static inline void __nvme_end_req(struct request *req)
418+
{
419+
nvme_end_req_zoned(req);
420+
nvme_trace_bio_complete(req);
421+
if (req->cmd_flags & REQ_NVME_MPATH)
422+
nvme_mpath_end_request(req);
423+
}
424+
425+
void nvme_end_req(struct request *req)
418426
{
419427
blk_status_t status = nvme_error_status(nvme_req(req)->status);
420428

@@ -424,10 +432,7 @@ static inline void nvme_end_req(struct request *req)
424432
else
425433
nvme_log_error(req);
426434
}
427-
nvme_end_req_zoned(req);
428-
nvme_trace_bio_complete(req);
429-
if (req->cmd_flags & REQ_NVME_MPATH)
430-
nvme_mpath_end_request(req);
435+
__nvme_end_req(req);
431436
blk_mq_end_request(req, status);
432437
}
433438

@@ -476,7 +481,7 @@ void nvme_complete_batch_req(struct request *req)
476481
{
477482
trace_nvme_complete_rq(req);
478483
nvme_cleanup_cmd(req);
479-
nvme_end_req_zoned(req);
484+
__nvme_end_req(req);
480485
}
481486
EXPORT_SYMBOL_GPL(nvme_complete_batch_req);
482487

@@ -673,7 +678,7 @@ static void nvme_free_ns(struct kref *kref)
673678
kfree(ns);
674679
}
675680

676-
static inline bool nvme_get_ns(struct nvme_ns *ns)
681+
bool nvme_get_ns(struct nvme_ns *ns)
677682
{
678683
return kref_get_unless_zero(&ns->kref);
679684
}
@@ -3679,9 +3684,10 @@ static int nvme_init_ns_head(struct nvme_ns *ns, struct nvme_ns_info *info)
36793684
struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
36803685
{
36813686
struct nvme_ns *ns, *ret = NULL;
3687+
int srcu_idx;
36823688

3683-
down_read(&ctrl->namespaces_rwsem);
3684-
list_for_each_entry(ns, &ctrl->namespaces, list) {
3689+
srcu_idx = srcu_read_lock(&ctrl->srcu);
3690+
list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
36853691
if (ns->head->ns_id == nsid) {
36863692
if (!nvme_get_ns(ns))
36873693
continue;
@@ -3691,7 +3697,7 @@ struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
36913697
if (ns->head->ns_id > nsid)
36923698
break;
36933699
}
3694-
up_read(&ctrl->namespaces_rwsem);
3700+
srcu_read_unlock(&ctrl->srcu, srcu_idx);
36953701
return ret;
36963702
}
36973703
EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
@@ -3705,7 +3711,7 @@ static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
37053711

37063712
list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
37073713
if (tmp->head->ns_id < ns->head->ns_id) {
3708-
list_add(&ns->list, &tmp->list);
3714+
list_add_rcu(&ns->list, &tmp->list);
37093715
return;
37103716
}
37113717
}
@@ -3771,17 +3777,18 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
37713777
if (nvme_update_ns_info(ns, info))
37723778
goto out_unlink_ns;
37733779

3774-
down_write(&ctrl->namespaces_rwsem);
3780+
mutex_lock(&ctrl->namespaces_lock);
37753781
/*
37763782
* Ensure that no namespaces are added to the ctrl list after the queues
37773783
* are frozen, thereby avoiding a deadlock between scan and reset.
37783784
*/
37793785
if (test_bit(NVME_CTRL_FROZEN, &ctrl->flags)) {
3780-
up_write(&ctrl->namespaces_rwsem);
3786+
mutex_unlock(&ctrl->namespaces_lock);
37813787
goto out_unlink_ns;
37823788
}
37833789
nvme_ns_add_to_ctrl_list(ns);
3784-
up_write(&ctrl->namespaces_rwsem);
3790+
mutex_unlock(&ctrl->namespaces_lock);
3791+
synchronize_srcu(&ctrl->srcu);
37853792
nvme_get_ctrl(ctrl);
37863793

37873794
if (device_add_disk(ctrl->device, ns->disk, nvme_ns_attr_groups))
@@ -3804,9 +3811,10 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, struct nvme_ns_info *info)
38043811

38053812
out_cleanup_ns_from_list:
38063813
nvme_put_ctrl(ctrl);
3807-
down_write(&ctrl->namespaces_rwsem);
3808-
list_del_init(&ns->list);
3809-
up_write(&ctrl->namespaces_rwsem);
3814+
mutex_lock(&ctrl->namespaces_lock);
3815+
list_del_rcu(&ns->list);
3816+
mutex_unlock(&ctrl->namespaces_lock);
3817+
synchronize_srcu(&ctrl->srcu);
38103818
out_unlink_ns:
38113819
mutex_lock(&ctrl->subsys->lock);
38123820
list_del_rcu(&ns->siblings);
@@ -3856,9 +3864,10 @@ static void nvme_ns_remove(struct nvme_ns *ns)
38563864
nvme_cdev_del(&ns->cdev, &ns->cdev_device);
38573865
del_gendisk(ns->disk);
38583866

3859-
down_write(&ns->ctrl->namespaces_rwsem);
3860-
list_del_init(&ns->list);
3861-
up_write(&ns->ctrl->namespaces_rwsem);
3867+
mutex_lock(&ns->ctrl->namespaces_lock);
3868+
list_del_rcu(&ns->list);
3869+
mutex_unlock(&ns->ctrl->namespaces_lock);
3870+
synchronize_srcu(&ns->ctrl->srcu);
38623871

38633872
if (last_path)
38643873
nvme_mpath_shutdown_disk(ns->head);
@@ -3948,16 +3957,17 @@ static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
39483957
struct nvme_ns *ns, *next;
39493958
LIST_HEAD(rm_list);
39503959

3951-
down_write(&ctrl->namespaces_rwsem);
3960+
mutex_lock(&ctrl->namespaces_lock);
39523961
list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
39533962
if (ns->head->ns_id > nsid)
3954-
list_move_tail(&ns->list, &rm_list);
3963+
list_splice_init_rcu(&ns->list, &rm_list,
3964+
synchronize_rcu);
39553965
}
3956-
up_write(&ctrl->namespaces_rwsem);
3966+
mutex_unlock(&ctrl->namespaces_lock);
3967+
synchronize_srcu(&ctrl->srcu);
39573968

39583969
list_for_each_entry_safe(ns, next, &rm_list, list)
39593970
nvme_ns_remove(ns);
3960-
39613971
}
39623972

39633973
static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
@@ -4127,9 +4137,10 @@ void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
41274137
/* this is a no-op when called from the controller reset handler */
41284138
nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
41294139

4130-
down_write(&ctrl->namespaces_rwsem);
4131-
list_splice_init(&ctrl->namespaces, &ns_list);
4132-
up_write(&ctrl->namespaces_rwsem);
4140+
mutex_lock(&ctrl->namespaces_lock);
4141+
list_splice_init_rcu(&ctrl->namespaces, &ns_list, synchronize_rcu);
4142+
mutex_unlock(&ctrl->namespaces_lock);
4143+
synchronize_srcu(&ctrl->srcu);
41334144

41344145
list_for_each_entry_safe(ns, next, &ns_list, list)
41354146
nvme_ns_remove(ns);
@@ -4577,6 +4588,7 @@ static void nvme_free_ctrl(struct device *dev)
45774588
key_put(ctrl->tls_key);
45784589
nvme_free_cels(ctrl);
45794590
nvme_mpath_uninit(ctrl);
4591+
cleanup_srcu_struct(&ctrl->srcu);
45804592
nvme_auth_stop(ctrl);
45814593
nvme_auth_free(ctrl);
45824594
__free_page(ctrl->discard_page);
@@ -4609,10 +4621,15 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
46094621
ctrl->passthru_err_log_enabled = false;
46104622
clear_bit(NVME_CTRL_FAILFAST_EXPIRED, &ctrl->flags);
46114623
spin_lock_init(&ctrl->lock);
4624+
mutex_init(&ctrl->namespaces_lock);
4625+
4626+
ret = init_srcu_struct(&ctrl->srcu);
4627+
if (ret)
4628+
return ret;
4629+
46124630
mutex_init(&ctrl->scan_lock);
46134631
INIT_LIST_HEAD(&ctrl->namespaces);
46144632
xa_init(&ctrl->cels);
4615-
init_rwsem(&ctrl->namespaces_rwsem);
46164633
ctrl->dev = dev;
46174634
ctrl->ops = ops;
46184635
ctrl->quirks = quirks;
@@ -4692,6 +4709,7 @@ int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
46924709
out:
46934710
if (ctrl->discard_page)
46944711
__free_page(ctrl->discard_page);
4712+
cleanup_srcu_struct(&ctrl->srcu);
46954713
return ret;
46964714
}
46974715
EXPORT_SYMBOL_GPL(nvme_init_ctrl);
@@ -4700,61 +4718,66 @@ EXPORT_SYMBOL_GPL(nvme_init_ctrl);
47004718
void nvme_mark_namespaces_dead(struct nvme_ctrl *ctrl)
47014719
{
47024720
struct nvme_ns *ns;
4721+
int srcu_idx;
47034722

4704-
down_read(&ctrl->namespaces_rwsem);
4705-
list_for_each_entry(ns, &ctrl->namespaces, list)
4723+
srcu_idx = srcu_read_lock(&ctrl->srcu);
4724+
list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
47064725
blk_mark_disk_dead(ns->disk);
4707-
up_read(&ctrl->namespaces_rwsem);
4726+
srcu_read_unlock(&ctrl->srcu, srcu_idx);
47084727
}
47094728
EXPORT_SYMBOL_GPL(nvme_mark_namespaces_dead);
47104729

47114730
void nvme_unfreeze(struct nvme_ctrl *ctrl)
47124731
{
47134732
struct nvme_ns *ns;
4733+
int srcu_idx;
47144734

4715-
down_read(&ctrl->namespaces_rwsem);
4716-
list_for_each_entry(ns, &ctrl->namespaces, list)
4735+
srcu_idx = srcu_read_lock(&ctrl->srcu);
4736+
list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
47174737
blk_mq_unfreeze_queue(ns->queue);
4718-
up_read(&ctrl->namespaces_rwsem);
4738+
srcu_read_unlock(&ctrl->srcu, srcu_idx);
47194739
clear_bit(NVME_CTRL_FROZEN, &ctrl->flags);
47204740
}
47214741
EXPORT_SYMBOL_GPL(nvme_unfreeze);
47224742

47234743
int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
47244744
{
47254745
struct nvme_ns *ns;
4746+
int srcu_idx;
47264747

4727-
down_read(&ctrl->namespaces_rwsem);
4728-
list_for_each_entry(ns, &ctrl->namespaces, list) {
4748+
srcu_idx = srcu_read_lock(&ctrl->srcu);
4749+
list_for_each_entry_rcu(ns, &ctrl->namespaces, list) {
47294750
timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
47304751
if (timeout <= 0)
47314752
break;
47324753
}
4733-
up_read(&ctrl->namespaces_rwsem);
4754+
srcu_read_unlock(&ctrl->srcu, srcu_idx);
47344755
return timeout;
47354756
}
47364757
EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
47374758

47384759
void nvme_wait_freeze(struct nvme_ctrl *ctrl)
47394760
{
47404761
struct nvme_ns *ns;
4762+
int srcu_idx;
47414763

4742-
down_read(&ctrl->namespaces_rwsem);
4743-
list_for_each_entry(ns, &ctrl->namespaces, list)
4764+
srcu_idx = srcu_read_lock(&ctrl->srcu);
4765+
list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
47444766
blk_mq_freeze_queue_wait(ns->queue);
4745-
up_read(&ctrl->namespaces_rwsem);
4767+
srcu_read_unlock(&ctrl->srcu, srcu_idx);
47464768
}
47474769
EXPORT_SYMBOL_GPL(nvme_wait_freeze);
47484770

47494771
void nvme_start_freeze(struct nvme_ctrl *ctrl)
47504772
{
47514773
struct nvme_ns *ns;
4774+
int srcu_idx;
47524775

47534776
set_bit(NVME_CTRL_FROZEN, &ctrl->flags);
4754-
down_read(&ctrl->namespaces_rwsem);
4755-
list_for_each_entry(ns, &ctrl->namespaces, list)
4777+
srcu_idx = srcu_read_lock(&ctrl->srcu);
4778+
list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
47564779
blk_freeze_queue_start(ns->queue);
4757-
up_read(&ctrl->namespaces_rwsem);
4780+
srcu_read_unlock(&ctrl->srcu, srcu_idx);
47584781
}
47594782
EXPORT_SYMBOL_GPL(nvme_start_freeze);
47604783

@@ -4797,11 +4820,12 @@ EXPORT_SYMBOL_GPL(nvme_unquiesce_admin_queue);
47974820
void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
47984821
{
47994822
struct nvme_ns *ns;
4823+
int srcu_idx;
48004824

4801-
down_read(&ctrl->namespaces_rwsem);
4802-
list_for_each_entry(ns, &ctrl->namespaces, list)
4825+
srcu_idx = srcu_read_lock(&ctrl->srcu);
4826+
list_for_each_entry_rcu(ns, &ctrl->namespaces, list)
48034827
blk_sync_queue(ns->queue);
4804-
up_read(&ctrl->namespaces_rwsem);
4828+
srcu_read_unlock(&ctrl->srcu, srcu_idx);
48054829
}
48064830
EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
48074831

‎drivers/nvme/host/ioctl.c‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -789,15 +789,15 @@ static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp,
789789
bool open_for_write)
790790
{
791791
struct nvme_ns *ns;
792-
int ret;
792+
int ret, srcu_idx;
793793

794-
down_read(&ctrl->namespaces_rwsem);
794+
srcu_idx = srcu_read_lock(&ctrl->srcu);
795795
if (list_empty(&ctrl->namespaces)) {
796796
ret = -ENOTTY;
797797
goto out_unlock;
798798
}
799799

800-
ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
800+
ns = list_first_or_null_rcu(&ctrl->namespaces, struct nvme_ns, list);
801801
if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
802802
dev_warn(ctrl->device,
803803
"NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
@@ -807,15 +807,18 @@ static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp,
807807

808808
dev_warn(ctrl->device,
809809
"using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
810-
kref_get(&ns->kref);
811-
up_read(&ctrl->namespaces_rwsem);
810+
if (!nvme_get_ns(ns)) {
811+
ret = -ENXIO;
812+
goto out_unlock;
813+
}
814+
srcu_read_unlock(&ctrl->srcu, srcu_idx);
812815

813816
ret = nvme_user_cmd(ctrl, ns, argp, 0, open_for_write);
814817
nvme_put_ns(ns);
815818
return ret;
816819

817820
out_unlock:
818-
up_read(&ctrl->namespaces_rwsem);
821+
srcu_read_unlock(&ctrl->srcu, srcu_idx);
819822
return ret;
820823
}
821824

0 commit comments

Comments
 (0)