@@ -530,21 +530,34 @@ define SetupRunGtestTestBody
530530 $$(call LogWarn, Test report is stored in $$(strip \
531531 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
532532 $$(if $$(wildcard $$($1_RESULT_FILE)), \
533- $$(eval $1_TOTAL := $$(shell $$(AWK) '/==========.* tests? from .* \
534- test (cases?|suites?) ran/ { print $$$$2 }' $$($1_RESULT_FILE))) \
535- $$(if $$($1_TOTAL), , $$(eval $1_TOTAL := 0)) \
533+ $$(eval $1_RUN := $$(shell $$(AWK) \
534+ '/==========.* tests? from .* test (cases?|suites?) ran/ { print $$$$2 }' \
535+ $$($1_RESULT_FILE))) \
536+ $$(if $$($1_RUN), , $$(eval $1_RUN := 0)) \
536537 $$(eval $1_PASSED := $$(shell $$(AWK) '/\[ PASSED \] .* tests?./ \
537538 { print $$$$4 }' $$($1_RESULT_FILE))) \
538539 $$(if $$($1_PASSED), , $$(eval $1_PASSED := 0)) \
540+ $$(eval $1_SKIPPED := $$(shell $$(AWK) \
541+ '/YOU HAVE [0-9]+ DISABLED TEST/ { \
542+ if (match($$$$0, /[0-9]+/, arr)) { \
543+ print arr[0]; \
544+ found=1; \
545+ } \
546+ if (!found) { print 0; } \
547+ }' \
548+ $$($1_RESULT_FILE))) \
539549 $$(eval $1_FAILED := $$(shell $$(AWK) '/\[ FAILED \] .* tests?, \
540550 listed below/ { print $$$$4 }' $$($1_RESULT_FILE))) \
541551 $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \
542552 $$(eval $1_ERROR := $$(shell \
543- $$(EXPR) $$($1_TOTAL) - $$($1_PASSED) - $$($1_FAILED))) \
553+ $$(EXPR) $$($1_RUN) - $$($1_PASSED) - $$($1_FAILED))) \
554+ $$(eval $1_TOTAL := $$(shell \
555+ $$(EXPR) $$($1_RUN) + $$($1_SKIPPED))) \
544556 , \
545557 $$(eval $1_PASSED := 0) \
546558 $$(eval $1_FAILED := 0) \
547559 $$(eval $1_ERROR := 1) \
560+ $$(eval $1_SKIPPED := 0) \
548561 $$(eval $1_TOTAL := 1) \
549562 )
550563
@@ -668,6 +681,7 @@ define SetupRunMicroTestBody
668681 $$(eval $1_ERROR := 1) \
669682 $$(eval $1_TOTAL := 1) \
670683 )
684+ $$(eval $1_SKIPPED := 0)
671685
672686 $1: run-test-$1 parse-test-$1
673687
@@ -1036,23 +1050,64 @@ define SetupRunJtregTestBody
10361050 $$(call LogWarn, Finished running test '$$($1_TEST)')
10371051 $$(call LogWarn, Test report is stored in $$(strip \
10381052 $$(subst $$(TOPDIR)/, , $$($1_TEST_RESULTS_DIR))))
1053+
1054+ # Read jtreg documentation to learn on the test stats categories:
1055+ # https://github.com/openjdk/jtreg/blob/master/src/share/doc/javatest/regtest/faq.md#what-do-all-those-numbers-in-the-test-results-line-mean
1056+ # In jtreg, "skipped:" category accounts for tests that threw jtreg.SkippedException at runtime.
1057+ # At the same time these tests contribute to "passed:" tests.
1058+ # In here we don't want that and so we substract number of "skipped:" from "passed:".
1059+
10391060 $$(if $$(wildcard $$($1_RESULT_FILE)), \
1040- $$(eval $1_PASSED := $$(shell $$(AWK) '{ gsub(/[,;]/, ""); \
1061+ $$(eval $1_PASSED_AND_RUNTIME_SKIPPED := $$(shell $$(AWK) '{ gsub(/[,;]/, ""); \
10411062 for (i=1; i<=NF; i++) { if ($$$$i == "passed:") \
10421063 print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
1043- $$(if $$($1_PASSED ), , $$(eval $1_PASSED := 0)) \
1064+ $$(if $$($1_PASSED_AND_RUNTIME_SKIPPED ), , $$(eval $1_PASSED_AND_RUNTIME_SKIPPED := 0)) \
10441065 $$(eval $1_FAILED := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
10451066 for (i=1; i<=NF; i++) { if ($$$$i == "failed:") \
10461067 print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
10471068 $$(if $$($1_FAILED), , $$(eval $1_FAILED := 0)) \
1069+ $$(eval $1_RUNTIME_SKIPPED := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
1070+ for (i=1; i<=NF; i++) { if ($$$$i == "skipped:") \
1071+ print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
1072+ $$(if $$($1_RUNTIME_SKIPPED), , $$(eval $1_RUNTIME_SKIPPED := 0)) \
1073+ $$(eval $1_SKIPPED := $$(shell \
1074+ $$(AWK) \
1075+ 'BEGIN { \
1076+ overall_skipped = 0; \
1077+ patterns[1] = "skipped"; \
1078+ patterns[2] = "excluded"; \
1079+ patterns[3] = "not in match-list"; \
1080+ patterns[4] = "did not match keywords"; \
1081+ patterns[5] = "did not meet module requirements"; \
1082+ patterns[6] = "did not meet platform requirements"; \
1083+ patterns[7] = "did not match prior status"; \
1084+ patterns[8] = "did not meet time-limit requirements"; \
1085+ } { \
1086+ split($$$$0, arr, ";"); \
1087+ for (item in arr) { \
1088+ for (p in patterns) { \
1089+ if (match(arr[item], patterns[p] ": [0-9]+")) { \
1090+ overall_skipped += substr(arr[item], RSTART + length(patterns[p]) + 2, RLENGTH); \
1091+ } \
1092+ } \
1093+ } \
1094+ print overall_skipped; \
1095+ }' \
1096+ $$($1_RESULT_FILE) \
1097+ )) \
10481098 $$(eval $1_ERROR := $$(shell $$(AWK) '{gsub(/[,;]/, ""); \
10491099 for (i=1; i<=NF; i++) { if ($$$$i == "error:") \
10501100 print $$$$(i+1) } }' $$($1_RESULT_FILE))) \
10511101 $$(if $$($1_ERROR), , $$(eval $1_ERROR := 0)) \
1102+ \
1103+ $$(eval $1_PASSED := $$(shell \
1104+ $$(EXPR) $$($1_PASSED_AND_RUNTIME_SKIPPED) - $$($1_RUNTIME_SKIPPED))) \
10521105 $$(eval $1_TOTAL := $$(shell \
1053- $$(EXPR) $$($1_PASSED) + $$($1_FAILED) + $$($1_ERROR))) \
1106+ $$(EXPR) $$($1_PASSED) + $$($1_FAILED) + $$($1_ERROR) + $$($1_SKIPPED) )) \
10541107 , \
1055- $$(eval $1_PASSED := 0) \
1108+ $$(eval $1_PASSED_AND_RUNTIME_SKIPPED := 0) \
1109+ $$(eval $1_RUNTIME_SKIPPED := 0) \
1110+ $$(eval $1_SKIPPED := 0) \
10561111 $$(eval $1_FAILED := 0) \
10571112 $$(eval $1_ERROR := 1) \
10581113 $$(eval $1_TOTAL := 1) \
@@ -1111,8 +1166,6 @@ define SetupRunSpecialTestBody
11111166 || $$(ECHO) $$$$? > $$($1_EXITCODE) \
11121167 ))
11131168
1114- $1_RESULT_FILE := $$($1_TEST_RESULTS_DIR)/gtest.txt
1115-
11161169 # We can not parse the various "special" tests.
11171170 parse-test-$1: run-test-$1
11181171 $$(call LogWarn, Finished running test '$$($1_TEST)')
@@ -1122,6 +1175,7 @@ define SetupRunSpecialTestBody
11221175 $$(eval $1_PASSED := $$(shell \
11231176 if [ `$(CAT) $$($1_EXITCODE)` = "0" ]; then $(ECHO) 1; else $(ECHO) 0; fi \
11241177 ))
1178+ $$(eval $1_SKIPPED := 0)
11251179 $$(eval $1_FAILED := $$(shell \
11261180 if [ `$(CAT) $$($1_EXITCODE)` = "0" ]; then $(ECHO) 0; else $(ECHO) 1; fi \
11271181 ))
@@ -1231,8 +1285,8 @@ run-test-report: post-run-test
12311285 $(ECHO) >> $(TEST_SUMMARY) ==============================
12321286 $(ECHO) >> $(TEST_SUMMARY) Test summary
12331287 $(ECHO) >> $(TEST_SUMMARY) ==============================
1234- $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5s %5s %5s %5s %2s\n" " " \
1235- TEST TOTAL PASS FAIL ERROR " "
1288+ $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5s %5s %5s %5s %5s % 2s\n" " " \
1289+ TEST TOTAL PASS FAIL ERROR SKIP " "
12361290 $(foreach test, $(TESTS_TO_RUN), \
12371291 $(eval TEST_ID := $(shell $(ECHO) $(strip $(test)) | \
12381292 $(TR) -cs '[a-z][A-Z][0-9]\n' '[_*1000]')) \
@@ -1244,15 +1298,15 @@ run-test-report: post-run-test
12441298 , \
12451299 $(eval TEST_NAME := $(test)) \
12461300 ) \
1247- $(if $(filter $($(TEST_ID)_PASSED), $($(TEST_ID)_TOTAL)), \
1248- $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %2s\n" \
1249- " " "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \
1250- $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) " " $(NEWLINE) \
1251- , \
1252- $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %2s\n" \
1301+ $(if $(filter-out 0, $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR)), \
1302+ $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %5d %2s\n" \
12531303 ">>" "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \
1254- $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) "<<" $(NEWLINE) \
1304+ $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) $($(TEST_ID)_SKIPPED) "<<" $(NEWLINE) \
12551305 $(eval TEST_FAILURE := true) \
1306+ , \
1307+ $(PRINTF) >> $(TEST_SUMMARY) "%2s %-49s %5d %5d %5d %5d %5d %2s\n" \
1308+ " " "$(TEST_NAME)" $($(TEST_ID)_TOTAL) $($(TEST_ID)_PASSED) \
1309+ $($(TEST_ID)_FAILED) $($(TEST_ID)_ERROR) $($(TEST_ID)_SKIPPED) " " $(NEWLINE) \
12561310 ) \
12571311 )
12581312 $(ECHO) >> $(TEST_SUMMARY) ==============================
0 commit comments