ci: fix when benchmarks run (#2759)

Author: miparnisari

.github/workflows/build-test.yaml

@@ -423,7 +423,8 @@ jobs:
     # Benchmarks on Main are ran through a different workflow
     if: |
       needs.paths-filter.outputs.codechange == 'true' &&
-      github.base_ref != 'main'
+      github.event_name == 'pull_request' &&
+      github.base_ref == 'main'
     steps:
       - uses: "actions/checkout@ff7abcd0c3c05ccf6adc123a8cd1fd4fb30fb493" # v4.2.2
         with: