@@ -472,8 +472,11 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
472
472
cntr = & hdev -> aggregated_cs_counters ;
473
473
474
474
cs = kzalloc (sizeof (* cs ), GFP_ATOMIC );
475
- if (!cs )
475
+ if (!cs ) {
476
+ atomic64_inc (& ctx -> cs_counters .out_of_mem_drop_cnt );
477
+ atomic64_inc (& cntr -> out_of_mem_drop_cnt );
476
478
return - ENOMEM ;
479
+ }
477
480
478
481
cs -> ctx = ctx ;
479
482
cs -> submitted = false;
@@ -486,6 +489,8 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
486
489
487
490
cs_cmpl = kmalloc (sizeof (* cs_cmpl ), GFP_ATOMIC );
488
491
if (!cs_cmpl ) {
492
+ atomic64_inc (& ctx -> cs_counters .out_of_mem_drop_cnt );
493
+ atomic64_inc (& cntr -> out_of_mem_drop_cnt );
489
494
rc = - ENOMEM ;
490
495
goto free_cs ;
491
496
}
@@ -513,6 +518,8 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx,
513
518
cs -> jobs_in_queue_cnt = kcalloc (hdev -> asic_prop .max_queues ,
514
519
sizeof (* cs -> jobs_in_queue_cnt ), GFP_ATOMIC );
515
520
if (!cs -> jobs_in_queue_cnt ) {
521
+ atomic64_inc (& ctx -> cs_counters .out_of_mem_drop_cnt );
522
+ atomic64_inc (& cntr -> out_of_mem_drop_cnt );
516
523
rc = - ENOMEM ;
517
524
goto free_fence ;
518
525
}
@@ -562,7 +569,7 @@ void hl_cs_rollback_all(struct hl_device *hdev)
562
569
for (i = 0 ; i < hdev -> asic_prop .completion_queues_count ; i ++ )
563
570
flush_workqueue (hdev -> cq_wq [i ]);
564
571
565
- /* Make sure we don't have leftovers in the H/W queues mirror list */
572
+ /* Make sure we don't have leftovers in the CS mirror list */
566
573
list_for_each_entry_safe (cs , tmp , & hdev -> cs_mirror_list , mirror_node ) {
567
574
cs_get (cs );
568
575
cs -> aborted = true;
@@ -764,11 +771,14 @@ static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args)
764
771
765
772
static int hl_cs_copy_chunk_array (struct hl_device * hdev ,
766
773
struct hl_cs_chunk * * cs_chunk_array ,
767
- void __user * chunks , u32 num_chunks )
774
+ void __user * chunks , u32 num_chunks ,
775
+ struct hl_ctx * ctx )
768
776
{
769
777
u32 size_to_copy ;
770
778
771
779
if (num_chunks > HL_MAX_JOBS_PER_CS ) {
780
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
781
+ atomic64_inc (& hdev -> aggregated_cs_counters .validation_drop_cnt );
772
782
dev_err (hdev -> dev ,
773
783
"Number of chunks can NOT be larger than %d\n" ,
774
784
HL_MAX_JOBS_PER_CS );
@@ -777,11 +787,16 @@ static int hl_cs_copy_chunk_array(struct hl_device *hdev,
777
787
778
788
* cs_chunk_array = kmalloc_array (num_chunks , sizeof (* * cs_chunk_array ),
779
789
GFP_ATOMIC );
780
- if (!* cs_chunk_array )
790
+ if (!* cs_chunk_array ) {
791
+ atomic64_inc (& ctx -> cs_counters .out_of_mem_drop_cnt );
792
+ atomic64_inc (& hdev -> aggregated_cs_counters .out_of_mem_drop_cnt );
781
793
return - ENOMEM ;
794
+ }
782
795
783
796
size_to_copy = num_chunks * sizeof (struct hl_cs_chunk );
784
797
if (copy_from_user (* cs_chunk_array , chunks , size_to_copy )) {
798
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
799
+ atomic64_inc (& hdev -> aggregated_cs_counters .validation_drop_cnt );
785
800
dev_err (hdev -> dev , "Failed to copy cs chunk array from user\n" );
786
801
kfree (* cs_chunk_array );
787
802
return - EFAULT ;
@@ -797,6 +812,7 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
797
812
struct hl_device * hdev = hpriv -> hdev ;
798
813
struct hl_cs_chunk * cs_chunk_array ;
799
814
struct hl_cs_counters_atomic * cntr ;
815
+ struct hl_ctx * ctx = hpriv -> ctx ;
800
816
struct hl_cs_job * job ;
801
817
struct hl_cs * cs ;
802
818
struct hl_cb * cb ;
@@ -805,7 +821,8 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
805
821
cntr = & hdev -> aggregated_cs_counters ;
806
822
* cs_seq = ULLONG_MAX ;
807
823
808
- rc = hl_cs_copy_chunk_array (hdev , & cs_chunk_array , chunks , num_chunks );
824
+ rc = hl_cs_copy_chunk_array (hdev , & cs_chunk_array , chunks , num_chunks ,
825
+ hpriv -> ctx );
809
826
if (rc )
810
827
goto out ;
811
828
@@ -832,17 +849,17 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
832
849
rc = validate_queue_index (hdev , chunk , & queue_type ,
833
850
& is_kernel_allocated_cb );
834
851
if (rc ) {
835
- atomic64_inc (& hpriv -> ctx -> cs_counters .parsing_drop_cnt );
836
- atomic64_inc (& cntr -> parsing_drop_cnt );
852
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
853
+ atomic64_inc (& cntr -> validation_drop_cnt );
837
854
goto free_cs_object ;
838
855
}
839
856
840
857
if (is_kernel_allocated_cb ) {
841
858
cb = get_cb_from_cs_chunk (hdev , & hpriv -> cb_mgr , chunk );
842
859
if (!cb ) {
843
860
atomic64_inc (
844
- & hpriv -> ctx -> cs_counters .parsing_drop_cnt );
845
- atomic64_inc (& cntr -> parsing_drop_cnt );
861
+ & ctx -> cs_counters .validation_drop_cnt );
862
+ atomic64_inc (& cntr -> validation_drop_cnt );
846
863
rc = - EINVAL ;
847
864
goto free_cs_object ;
848
865
}
@@ -856,8 +873,7 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
856
873
job = hl_cs_allocate_job (hdev , queue_type ,
857
874
is_kernel_allocated_cb );
858
875
if (!job ) {
859
- atomic64_inc (
860
- & hpriv -> ctx -> cs_counters .out_of_mem_drop_cnt );
876
+ atomic64_inc (& ctx -> cs_counters .out_of_mem_drop_cnt );
861
877
atomic64_inc (& cntr -> out_of_mem_drop_cnt );
862
878
dev_err (hdev -> dev , "Failed to allocate a new job\n" );
863
879
rc = - ENOMEM ;
@@ -891,7 +907,7 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
891
907
892
908
rc = cs_parser (hpriv , job );
893
909
if (rc ) {
894
- atomic64_inc (& hpriv -> ctx -> cs_counters .parsing_drop_cnt );
910
+ atomic64_inc (& ctx -> cs_counters .parsing_drop_cnt );
895
911
atomic64_inc (& cntr -> parsing_drop_cnt );
896
912
dev_err (hdev -> dev ,
897
913
"Failed to parse JOB %d.%llu.%d, err %d, rejecting the CS\n" ,
@@ -901,8 +917,8 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks,
901
917
}
902
918
903
919
if (int_queues_only ) {
904
- atomic64_inc (& hpriv -> ctx -> cs_counters .parsing_drop_cnt );
905
- atomic64_inc (& cntr -> parsing_drop_cnt );
920
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
921
+ atomic64_inc (& cntr -> validation_drop_cnt );
906
922
dev_err (hdev -> dev ,
907
923
"Reject CS %d.%llu because only internal queues jobs are present\n" ,
908
924
cs -> ctx -> asid , cs -> sequence );
@@ -1042,7 +1058,7 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args,
1042
1058
}
1043
1059
1044
1060
static int cs_ioctl_extract_signal_seq (struct hl_device * hdev ,
1045
- struct hl_cs_chunk * chunk , u64 * signal_seq )
1061
+ struct hl_cs_chunk * chunk , u64 * signal_seq , struct hl_ctx * ctx )
1046
1062
{
1047
1063
u64 * signal_seq_arr = NULL ;
1048
1064
u32 size_to_copy , signal_seq_arr_len ;
@@ -1052,6 +1068,8 @@ static int cs_ioctl_extract_signal_seq(struct hl_device *hdev,
1052
1068
1053
1069
/* currently only one signal seq is supported */
1054
1070
if (signal_seq_arr_len != 1 ) {
1071
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
1072
+ atomic64_inc (& hdev -> aggregated_cs_counters .validation_drop_cnt );
1055
1073
dev_err (hdev -> dev ,
1056
1074
"Wait for signal CS supports only one signal CS seq\n" );
1057
1075
return - EINVAL ;
@@ -1060,13 +1078,18 @@ static int cs_ioctl_extract_signal_seq(struct hl_device *hdev,
1060
1078
signal_seq_arr = kmalloc_array (signal_seq_arr_len ,
1061
1079
sizeof (* signal_seq_arr ),
1062
1080
GFP_ATOMIC );
1063
- if (!signal_seq_arr )
1081
+ if (!signal_seq_arr ) {
1082
+ atomic64_inc (& ctx -> cs_counters .out_of_mem_drop_cnt );
1083
+ atomic64_inc (& hdev -> aggregated_cs_counters .out_of_mem_drop_cnt );
1064
1084
return - ENOMEM ;
1085
+ }
1065
1086
1066
1087
size_to_copy = chunk -> num_signal_seq_arr * sizeof (* signal_seq_arr );
1067
1088
if (copy_from_user (signal_seq_arr ,
1068
1089
u64_to_user_ptr (chunk -> signal_seq_arr ),
1069
1090
size_to_copy )) {
1091
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
1092
+ atomic64_inc (& hdev -> aggregated_cs_counters .validation_drop_cnt );
1070
1093
dev_err (hdev -> dev ,
1071
1094
"Failed to copy signal seq array from user\n" );
1072
1095
rc = - EFAULT ;
@@ -1153,23 +1176,28 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1153
1176
struct hl_device * hdev = hpriv -> hdev ;
1154
1177
struct hl_cs_compl * sig_waitcs_cmpl ;
1155
1178
u32 q_idx , collective_engine_id = 0 ;
1179
+ struct hl_cs_counters_atomic * cntr ;
1156
1180
struct hl_fence * sig_fence = NULL ;
1157
1181
struct hl_ctx * ctx = hpriv -> ctx ;
1158
1182
enum hl_queue_type q_type ;
1159
1183
struct hl_cs * cs ;
1160
1184
u64 signal_seq ;
1161
1185
int rc ;
1162
1186
1187
+ cntr = & hdev -> aggregated_cs_counters ;
1163
1188
* cs_seq = ULLONG_MAX ;
1164
1189
1165
- rc = hl_cs_copy_chunk_array (hdev , & cs_chunk_array , chunks , num_chunks );
1190
+ rc = hl_cs_copy_chunk_array (hdev , & cs_chunk_array , chunks , num_chunks ,
1191
+ ctx );
1166
1192
if (rc )
1167
1193
goto out ;
1168
1194
1169
1195
/* currently it is guaranteed to have only one chunk */
1170
1196
chunk = & cs_chunk_array [0 ];
1171
1197
1172
1198
if (chunk -> queue_index >= hdev -> asic_prop .max_queues ) {
1199
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
1200
+ atomic64_inc (& cntr -> validation_drop_cnt );
1173
1201
dev_err (hdev -> dev , "Queue index %d is invalid\n" ,
1174
1202
chunk -> queue_index );
1175
1203
rc = - EINVAL ;
@@ -1181,6 +1209,8 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1181
1209
q_type = hw_queue_prop -> type ;
1182
1210
1183
1211
if (!hw_queue_prop -> supports_sync_stream ) {
1212
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
1213
+ atomic64_inc (& cntr -> validation_drop_cnt );
1184
1214
dev_err (hdev -> dev ,
1185
1215
"Queue index %d does not support sync stream operations\n" ,
1186
1216
q_idx );
@@ -1190,6 +1220,8 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1190
1220
1191
1221
if (cs_type == CS_TYPE_COLLECTIVE_WAIT ) {
1192
1222
if (!(hw_queue_prop -> collective_mode == HL_COLLECTIVE_MASTER )) {
1223
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
1224
+ atomic64_inc (& cntr -> validation_drop_cnt );
1193
1225
dev_err (hdev -> dev ,
1194
1226
"Queue index %d is invalid\n" , q_idx );
1195
1227
rc = - EINVAL ;
@@ -1200,12 +1232,14 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1200
1232
}
1201
1233
1202
1234
if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT ) {
1203
- rc = cs_ioctl_extract_signal_seq (hdev , chunk , & signal_seq );
1235
+ rc = cs_ioctl_extract_signal_seq (hdev , chunk , & signal_seq , ctx );
1204
1236
if (rc )
1205
1237
goto free_cs_chunk_array ;
1206
1238
1207
1239
sig_fence = hl_ctx_get_fence (ctx , signal_seq );
1208
1240
if (IS_ERR (sig_fence )) {
1241
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
1242
+ atomic64_inc (& cntr -> validation_drop_cnt );
1209
1243
dev_err (hdev -> dev ,
1210
1244
"Failed to get signal CS with seq 0x%llx\n" ,
1211
1245
signal_seq );
@@ -1223,6 +1257,8 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1223
1257
container_of (sig_fence , struct hl_cs_compl , base_fence );
1224
1258
1225
1259
if (sig_waitcs_cmpl -> type != CS_TYPE_SIGNAL ) {
1260
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
1261
+ atomic64_inc (& cntr -> validation_drop_cnt );
1226
1262
dev_err (hdev -> dev ,
1227
1263
"CS seq 0x%llx is not of a signal CS\n" ,
1228
1264
signal_seq );
@@ -1270,8 +1306,11 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type,
1270
1306
else if (cs_type == CS_TYPE_COLLECTIVE_WAIT )
1271
1307
rc = hdev -> asic_funcs -> collective_wait_create_jobs (hdev , ctx ,
1272
1308
cs , q_idx , collective_engine_id );
1273
- else
1309
+ else {
1310
+ atomic64_inc (& ctx -> cs_counters .validation_drop_cnt );
1311
+ atomic64_inc (& cntr -> validation_drop_cnt );
1274
1312
rc = - EINVAL ;
1313
+ }
1275
1314
1276
1315
if (rc )
1277
1316
goto free_cs_object ;
0 commit comments