Merge pull request #67 from coderbirju/update_regression_check

Update check regression script

Author: coderbirju

scripts/check_regression.sh

@@ -22,12 +22,12 @@ compare_stat_p90() {
     local current_value="$2"
     local stat_name="$3"
 
-    # Calculate 110% of the past value
+    # Calculate 125% of the past value
     local threshold=$(calculate_threshold "$past_value")
 
     # Compare the current value with the threshold
-    if (( $(awk 'BEGIN {print ("'"$current_value"'" > "'"$threshold"'")}') )); then
-        echo "ERROR: $stat_name - Current P90 value ($current_value) exceeds the 110% threshold ($threshold) of the past P90 value ($past_value)"
+    if (( $(echo "$current_value > $threshold" |bc -l) )); then
+        echo "ERROR: $stat_name - Current P90 value ($current_value) exceeds the 125% threshold ($threshold) of the past P90 value ($past_value)"
         return 1
     fi
 
@@ -36,7 +36,18 @@ compare_stat_p90() {
 
 calculate_threshold() {
     local past_value="$1"
-    awk -v past="$past_value" 'BEGIN { print past * 1.1 }'
+    awk -v past="$past_value" 'BEGIN { print past * 1.25 }'
+}
+
+calculate_p90_after_skip() {
+    local times_array="$1"
+    local num_entries=$(echo "$times_array" | jq 'length')
+    local times=$(echo "$times_array" | jq -r '.[1:] | .[]')
+    local sorted_times=$(echo "$times" | tr '\n' ' ' | xargs -n1 | sort -g)
+    local index=$((num_entries * 90 / 100))
+
+    local p90=$(echo "$sorted_times" | sed -n "${index}p")
+    echo "$p90"
 }
 
 # Loop through each object in past.json and compare P90 values with current.json for all statistics
@@ -52,8 +63,10 @@ compare_p90_values() {
     for test_name in $test_names; do
         echo "Checking for regression in '$test_name'"
         for stat_name in "fullRunStats" "pullStats" "lazyTaskStats" "localTaskStats"; do
-            local past_p90=$(echo "$past_json" | jq -r --arg test "$test_name" '.benchmarkTests[] | select(.testName == $test) | .'"$stat_name"'.pct90')
-            local current_p90=$(echo "$current_json" | jq -r --arg test "$test_name" '.benchmarkTests[] | select(.testName == $test) | .'"$stat_name"'.pct90')
+            local past_p90_array=$(echo "$past_json" | jq -r --arg test "$test_name" '.benchmarkTests[] | select(.testName == $test) | .'"$stat_name"'.BenchmarkTimes')
+            local past_p90=$(calculate_p90_after_skip "$past_p90_array")
+            local current_p90_array=$(echo "$current_json" | jq -r --arg test "$test_name" '.benchmarkTests[] | select(.testName == $test) | .'"$stat_name"'.BenchmarkTimes')
+            local current_p90=$(calculate_p90_after_skip "$current_p90_array")
 
             # Call the compare_stat_p90 function
             compare_stat_p90 "$past_p90" "$current_p90" "$stat_name" || regression_detected=1
@@ -64,8 +77,6 @@ compare_p90_values() {
     return $regression_detected
 }
 
-# ... (remaining code)
-
 # Call compare_p90_values and store the exit code in a variable
 compare_p90_values "$past_data" "$current_data"
 exit_code=$?
@@ -78,4 +89,4 @@ else
 fi
 
 # Set the final exit code to indicate if any regression occurred
-exit $exit_code
+exit $exit_code