Update check regression script

This commit updates the regression check script to skip the initial
value in all BenchmarkTimes array of the benchmark results json file to
calculate a new p90. We use this new p90 to identify regression, this
change was made to combat the skewed p90 metrics we were seeing due to
the slow starts of the benchmark pull times which were affecting the
overall regression comparison. Skipping the first value allows us to
have a more uniform comparison, remove github environment noise and we'd be able to identify true regression in our code

Signed-off-by: Arjun Raja Yogidas <[email protected]>

Author: coderbirju

scripts/check_regression.sh

@@ -22,12 +22,12 @@ compare_stat_p90() {
     local current_value="$2"
     local stat_name="$3"
 
-    # Calculate 110% of the past value
+    # Calculate 125% of the past value
     local threshold=$(calculate_threshold "$past_value")
 
     # Compare the current value with the threshold
-    if (( $(awk 'BEGIN {print ("'"$current_value"'" > "'"$threshold"'")}') )); then
-        echo "ERROR: $stat_name - Current P90 value ($current_value) exceeds the 110% threshold ($threshold) of the past P90 value ($past_value)"
+    if (( $(echo "$current_value > $threshold" |bc -l) )); then
+        echo "ERROR: $stat_name - Current P90 value ($current_value) exceeds the 125% threshold ($threshold) of the past P90 value ($past_value)"
         return 1
     fi
 
@@ -36,7 +36,18 @@ compare_stat_p90() {
 
 calculate_threshold() {
     local past_value="$1"
-    awk -v past="$past_value" 'BEGIN { print past * 1.1 }'
+    awk -v past="$past_value" 'BEGIN { print past * 1.25 }'
+}
+
+calculate_p90_after_skip() {
+    local times_array="$1"
+    local num_entries=$(echo "$times_array" | jq 'length')
+    local times=$(echo "$times_array" | jq -r '.[1:] | .[]')
+    local sorted_times=$(echo "$times" | tr '\n' ' ' | xargs -n1 | sort -g)
+    local index=$((num_entries * 90 / 100))
+
+    local p90=$(echo "$sorted_times" | sed -n "${index}p")
+    echo "$p90"
 }
 
 # Loop through each object in past.json and compare P90 values with current.json for all statistics
@@ -52,8 +63,10 @@ compare_p90_values() {
     for test_name in $test_names; do
         echo "Checking for regression in '$test_name'"
         for stat_name in "fullRunStats" "pullStats" "lazyTaskStats" "localTaskStats"; do
-            local past_p90=$(echo "$past_json" | jq -r --arg test "$test_name" '.benchmarkTests[] | select(.testName == $test) | .'"$stat_name"'.pct90')
-            local current_p90=$(echo "$current_json" | jq -r --arg test "$test_name" '.benchmarkTests[] | select(.testName == $test) | .'"$stat_name"'.pct90')
+            local past_p90_array=$(echo "$past_json" | jq -r --arg test "$test_name" '.benchmarkTests[] | select(.testName == $test) | .'"$stat_name"'.BenchmarkTimes')
+            local past_p90=$(calculate_p90_after_skip "$past_p90_array")
+            local current_p90_array=$(echo "$current_json" | jq -r --arg test "$test_name" '.benchmarkTests[] | select(.testName == $test) | .'"$stat_name"'.BenchmarkTimes')
+            local current_p90=$(calculate_p90_after_skip "$current_p90_array")
 
             # Call the compare_stat_p90 function
             compare_stat_p90 "$past_p90" "$current_p90" "$stat_name" || regression_detected=1
@@ -64,8 +77,6 @@ compare_p90_values() {
     return $regression_detected
 }
 
-# ... (remaining code)
-
 # Call compare_p90_values and store the exit code in a variable
 compare_p90_values "$past_data" "$current_data"
 exit_code=$?
@@ -78,4 +89,4 @@ else
 fi
 
 # Set the final exit code to indicate if any regression occurred
-exit $exit_code
+exit $exit_code