@@ -335,6 +335,7 @@ static void put_probe_ref(void)
335
335
336
336
static void blk_trace_cleanup (struct blk_trace * bt )
337
337
{
338
+ synchronize_rcu ();
338
339
blk_trace_free (bt );
339
340
put_probe_ref ();
340
341
}
@@ -629,8 +630,10 @@ static int compat_blk_trace_setup(struct request_queue *q, char *name,
629
630
static int __blk_trace_startstop (struct request_queue * q , int start )
630
631
{
631
632
int ret ;
632
- struct blk_trace * bt = q -> blk_trace ;
633
+ struct blk_trace * bt ;
633
634
635
+ bt = rcu_dereference_protected (q -> blk_trace ,
636
+ lockdep_is_held (& q -> blk_trace_mutex ));
634
637
if (bt == NULL )
635
638
return - EINVAL ;
636
639
@@ -740,8 +743,8 @@ int blk_trace_ioctl(struct block_device *bdev, unsigned cmd, char __user *arg)
740
743
void blk_trace_shutdown (struct request_queue * q )
741
744
{
742
745
mutex_lock (& q -> blk_trace_mutex );
743
-
744
- if ( q -> blk_trace ) {
746
+ if ( rcu_dereference_protected ( q -> blk_trace ,
747
+ lockdep_is_held ( & q -> blk_trace_mutex )) ) {
745
748
__blk_trace_startstop (q , 0 );
746
749
__blk_trace_remove (q );
747
750
}
@@ -752,8 +755,10 @@ void blk_trace_shutdown(struct request_queue *q)
752
755
#ifdef CONFIG_BLK_CGROUP
753
756
static u64 blk_trace_bio_get_cgid (struct request_queue * q , struct bio * bio )
754
757
{
755
- struct blk_trace * bt = q -> blk_trace ;
758
+ struct blk_trace * bt ;
756
759
760
+ /* We don't use the 'bt' value here except as an optimization... */
761
+ bt = rcu_dereference_protected (q -> blk_trace , 1 );
757
762
if (!bt || !(blk_tracer_flags .val & TRACE_BLK_OPT_CGROUP ))
758
763
return 0 ;
759
764
@@ -796,10 +801,14 @@ blk_trace_request_get_cgid(struct request_queue *q, struct request *rq)
796
801
static void blk_add_trace_rq (struct request * rq , int error ,
797
802
unsigned int nr_bytes , u32 what , u64 cgid )
798
803
{
799
- struct blk_trace * bt = rq -> q -> blk_trace ;
804
+ struct blk_trace * bt ;
800
805
801
- if (likely (!bt ))
806
+ rcu_read_lock ();
807
+ bt = rcu_dereference (rq -> q -> blk_trace );
808
+ if (likely (!bt )) {
809
+ rcu_read_unlock ();
802
810
return ;
811
+ }
803
812
804
813
if (blk_rq_is_passthrough (rq ))
805
814
what |= BLK_TC_ACT (BLK_TC_PC );
@@ -808,6 +817,7 @@ static void blk_add_trace_rq(struct request *rq, int error,
808
817
809
818
__blk_add_trace (bt , blk_rq_trace_sector (rq ), nr_bytes , req_op (rq ),
810
819
rq -> cmd_flags , what , error , 0 , NULL , cgid );
820
+ rcu_read_unlock ();
811
821
}
812
822
813
823
static void blk_add_trace_rq_insert (void * ignore ,
@@ -853,14 +863,19 @@ static void blk_add_trace_rq_complete(void *ignore, struct request *rq,
853
863
static void blk_add_trace_bio (struct request_queue * q , struct bio * bio ,
854
864
u32 what , int error )
855
865
{
856
- struct blk_trace * bt = q -> blk_trace ;
866
+ struct blk_trace * bt ;
857
867
858
- if (likely (!bt ))
868
+ rcu_read_lock ();
869
+ bt = rcu_dereference (q -> blk_trace );
870
+ if (likely (!bt )) {
871
+ rcu_read_unlock ();
859
872
return ;
873
+ }
860
874
861
875
__blk_add_trace (bt , bio -> bi_iter .bi_sector , bio -> bi_iter .bi_size ,
862
876
bio_op (bio ), bio -> bi_opf , what , error , 0 , NULL ,
863
877
blk_trace_bio_get_cgid (q , bio ));
878
+ rcu_read_unlock ();
864
879
}
865
880
866
881
static void blk_add_trace_bio_bounce (void * ignore ,
@@ -905,11 +920,14 @@ static void blk_add_trace_getrq(void *ignore,
905
920
if (bio )
906
921
blk_add_trace_bio (q , bio , BLK_TA_GETRQ , 0 );
907
922
else {
908
- struct blk_trace * bt = q -> blk_trace ;
923
+ struct blk_trace * bt ;
909
924
925
+ rcu_read_lock ();
926
+ bt = rcu_dereference (q -> blk_trace );
910
927
if (bt )
911
928
__blk_add_trace (bt , 0 , 0 , rw , 0 , BLK_TA_GETRQ , 0 , 0 ,
912
929
NULL , 0 );
930
+ rcu_read_unlock ();
913
931
}
914
932
}
915
933
@@ -921,27 +939,35 @@ static void blk_add_trace_sleeprq(void *ignore,
921
939
if (bio )
922
940
blk_add_trace_bio (q , bio , BLK_TA_SLEEPRQ , 0 );
923
941
else {
924
- struct blk_trace * bt = q -> blk_trace ;
942
+ struct blk_trace * bt ;
925
943
944
+ rcu_read_lock ();
945
+ bt = rcu_dereference (q -> blk_trace );
926
946
if (bt )
927
947
__blk_add_trace (bt , 0 , 0 , rw , 0 , BLK_TA_SLEEPRQ ,
928
948
0 , 0 , NULL , 0 );
949
+ rcu_read_unlock ();
929
950
}
930
951
}
931
952
932
953
static void blk_add_trace_plug (void * ignore , struct request_queue * q )
933
954
{
934
- struct blk_trace * bt = q -> blk_trace ;
955
+ struct blk_trace * bt ;
935
956
957
+ rcu_read_lock ();
958
+ bt = rcu_dereference (q -> blk_trace );
936
959
if (bt )
937
960
__blk_add_trace (bt , 0 , 0 , 0 , 0 , BLK_TA_PLUG , 0 , 0 , NULL , 0 );
961
+ rcu_read_unlock ();
938
962
}
939
963
940
964
static void blk_add_trace_unplug (void * ignore , struct request_queue * q ,
941
965
unsigned int depth , bool explicit )
942
966
{
943
- struct blk_trace * bt = q -> blk_trace ;
967
+ struct blk_trace * bt ;
944
968
969
+ rcu_read_lock ();
970
+ bt = rcu_dereference (q -> blk_trace );
945
971
if (bt ) {
946
972
__be64 rpdu = cpu_to_be64 (depth );
947
973
u32 what ;
@@ -953,14 +979,17 @@ static void blk_add_trace_unplug(void *ignore, struct request_queue *q,
953
979
954
980
__blk_add_trace (bt , 0 , 0 , 0 , 0 , what , 0 , sizeof (rpdu ), & rpdu , 0 );
955
981
}
982
+ rcu_read_unlock ();
956
983
}
957
984
958
985
static void blk_add_trace_split (void * ignore ,
959
986
struct request_queue * q , struct bio * bio ,
960
987
unsigned int pdu )
961
988
{
962
- struct blk_trace * bt = q -> blk_trace ;
989
+ struct blk_trace * bt ;
963
990
991
+ rcu_read_lock ();
992
+ bt = rcu_dereference (q -> blk_trace );
964
993
if (bt ) {
965
994
__be64 rpdu = cpu_to_be64 (pdu );
966
995
@@ -969,6 +998,7 @@ static void blk_add_trace_split(void *ignore,
969
998
BLK_TA_SPLIT , bio -> bi_status , sizeof (rpdu ),
970
999
& rpdu , blk_trace_bio_get_cgid (q , bio ));
971
1000
}
1001
+ rcu_read_unlock ();
972
1002
}
973
1003
974
1004
/**
@@ -988,11 +1018,15 @@ static void blk_add_trace_bio_remap(void *ignore,
988
1018
struct request_queue * q , struct bio * bio ,
989
1019
dev_t dev , sector_t from )
990
1020
{
991
- struct blk_trace * bt = q -> blk_trace ;
1021
+ struct blk_trace * bt ;
992
1022
struct blk_io_trace_remap r ;
993
1023
994
- if (likely (!bt ))
1024
+ rcu_read_lock ();
1025
+ bt = rcu_dereference (q -> blk_trace );
1026
+ if (likely (!bt )) {
1027
+ rcu_read_unlock ();
995
1028
return ;
1029
+ }
996
1030
997
1031
r .device_from = cpu_to_be32 (dev );
998
1032
r .device_to = cpu_to_be32 (bio_dev (bio ));
@@ -1001,6 +1035,7 @@ static void blk_add_trace_bio_remap(void *ignore,
1001
1035
__blk_add_trace (bt , bio -> bi_iter .bi_sector , bio -> bi_iter .bi_size ,
1002
1036
bio_op (bio ), bio -> bi_opf , BLK_TA_REMAP , bio -> bi_status ,
1003
1037
sizeof (r ), & r , blk_trace_bio_get_cgid (q , bio ));
1038
+ rcu_read_unlock ();
1004
1039
}
1005
1040
1006
1041
/**
@@ -1021,11 +1056,15 @@ static void blk_add_trace_rq_remap(void *ignore,
1021
1056
struct request * rq , dev_t dev ,
1022
1057
sector_t from )
1023
1058
{
1024
- struct blk_trace * bt = q -> blk_trace ;
1059
+ struct blk_trace * bt ;
1025
1060
struct blk_io_trace_remap r ;
1026
1061
1027
- if (likely (!bt ))
1062
+ rcu_read_lock ();
1063
+ bt = rcu_dereference (q -> blk_trace );
1064
+ if (likely (!bt )) {
1065
+ rcu_read_unlock ();
1028
1066
return ;
1067
+ }
1029
1068
1030
1069
r .device_from = cpu_to_be32 (dev );
1031
1070
r .device_to = cpu_to_be32 (disk_devt (rq -> rq_disk ));
@@ -1034,6 +1073,7 @@ static void blk_add_trace_rq_remap(void *ignore,
1034
1073
__blk_add_trace (bt , blk_rq_pos (rq ), blk_rq_bytes (rq ),
1035
1074
rq_data_dir (rq ), 0 , BLK_TA_REMAP , 0 ,
1036
1075
sizeof (r ), & r , blk_trace_request_get_cgid (q , rq ));
1076
+ rcu_read_unlock ();
1037
1077
}
1038
1078
1039
1079
/**
@@ -1051,14 +1091,19 @@ void blk_add_driver_data(struct request_queue *q,
1051
1091
struct request * rq ,
1052
1092
void * data , size_t len )
1053
1093
{
1054
- struct blk_trace * bt = q -> blk_trace ;
1094
+ struct blk_trace * bt ;
1055
1095
1056
- if (likely (!bt ))
1096
+ rcu_read_lock ();
1097
+ bt = rcu_dereference (q -> blk_trace );
1098
+ if (likely (!bt )) {
1099
+ rcu_read_unlock ();
1057
1100
return ;
1101
+ }
1058
1102
1059
1103
__blk_add_trace (bt , blk_rq_trace_sector (rq ), blk_rq_bytes (rq ), 0 , 0 ,
1060
1104
BLK_TA_DRV_DATA , 0 , len , data ,
1061
1105
blk_trace_request_get_cgid (q , rq ));
1106
+ rcu_read_unlock ();
1062
1107
}
1063
1108
EXPORT_SYMBOL_GPL (blk_add_driver_data );
1064
1109
@@ -1597,6 +1642,7 @@ static int blk_trace_remove_queue(struct request_queue *q)
1597
1642
return - EINVAL ;
1598
1643
1599
1644
put_probe_ref ();
1645
+ synchronize_rcu ();
1600
1646
blk_trace_free (bt );
1601
1647
return 0 ;
1602
1648
}
@@ -1758,6 +1804,7 @@ static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1758
1804
struct hd_struct * p = dev_to_part (dev );
1759
1805
struct request_queue * q ;
1760
1806
struct block_device * bdev ;
1807
+ struct blk_trace * bt ;
1761
1808
ssize_t ret = - ENXIO ;
1762
1809
1763
1810
bdev = bdget (part_devt (p ));
@@ -1770,21 +1817,23 @@ static ssize_t sysfs_blk_trace_attr_show(struct device *dev,
1770
1817
1771
1818
mutex_lock (& q -> blk_trace_mutex );
1772
1819
1820
+ bt = rcu_dereference_protected (q -> blk_trace ,
1821
+ lockdep_is_held (& q -> blk_trace_mutex ));
1773
1822
if (attr == & dev_attr_enable ) {
1774
- ret = sprintf (buf , "%u\n" , !!q -> blk_trace );
1823
+ ret = sprintf (buf , "%u\n" , !!bt );
1775
1824
goto out_unlock_bdev ;
1776
1825
}
1777
1826
1778
- if (q -> blk_trace == NULL )
1827
+ if (bt == NULL )
1779
1828
ret = sprintf (buf , "disabled\n" );
1780
1829
else if (attr == & dev_attr_act_mask )
1781
- ret = blk_trace_mask2str (buf , q -> blk_trace -> act_mask );
1830
+ ret = blk_trace_mask2str (buf , bt -> act_mask );
1782
1831
else if (attr == & dev_attr_pid )
1783
- ret = sprintf (buf , "%u\n" , q -> blk_trace -> pid );
1832
+ ret = sprintf (buf , "%u\n" , bt -> pid );
1784
1833
else if (attr == & dev_attr_start_lba )
1785
- ret = sprintf (buf , "%llu\n" , q -> blk_trace -> start_lba );
1834
+ ret = sprintf (buf , "%llu\n" , bt -> start_lba );
1786
1835
else if (attr == & dev_attr_end_lba )
1787
- ret = sprintf (buf , "%llu\n" , q -> blk_trace -> end_lba );
1836
+ ret = sprintf (buf , "%llu\n" , bt -> end_lba );
1788
1837
1789
1838
out_unlock_bdev :
1790
1839
mutex_unlock (& q -> blk_trace_mutex );
@@ -1801,6 +1850,7 @@ static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1801
1850
struct block_device * bdev ;
1802
1851
struct request_queue * q ;
1803
1852
struct hd_struct * p ;
1853
+ struct blk_trace * bt ;
1804
1854
u64 value ;
1805
1855
ssize_t ret = - EINVAL ;
1806
1856
@@ -1831,8 +1881,10 @@ static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1831
1881
1832
1882
mutex_lock (& q -> blk_trace_mutex );
1833
1883
1884
+ bt = rcu_dereference_protected (q -> blk_trace ,
1885
+ lockdep_is_held (& q -> blk_trace_mutex ));
1834
1886
if (attr == & dev_attr_enable ) {
1835
- if (!!value == !!q -> blk_trace ) {
1887
+ if (!!value == !!bt ) {
1836
1888
ret = 0 ;
1837
1889
goto out_unlock_bdev ;
1838
1890
}
@@ -1844,18 +1896,18 @@ static ssize_t sysfs_blk_trace_attr_store(struct device *dev,
1844
1896
}
1845
1897
1846
1898
ret = 0 ;
1847
- if (q -> blk_trace == NULL )
1899
+ if (bt == NULL )
1848
1900
ret = blk_trace_setup_queue (q , bdev );
1849
1901
1850
1902
if (ret == 0 ) {
1851
1903
if (attr == & dev_attr_act_mask )
1852
- q -> blk_trace -> act_mask = value ;
1904
+ bt -> act_mask = value ;
1853
1905
else if (attr == & dev_attr_pid )
1854
- q -> blk_trace -> pid = value ;
1906
+ bt -> pid = value ;
1855
1907
else if (attr == & dev_attr_start_lba )
1856
- q -> blk_trace -> start_lba = value ;
1908
+ bt -> start_lba = value ;
1857
1909
else if (attr == & dev_attr_end_lba )
1858
- q -> blk_trace -> end_lba = value ;
1910
+ bt -> end_lba = value ;
1859
1911
}
1860
1912
1861
1913
out_unlock_bdev :
0 commit comments