Add Arm machines support to benchamrks [MOD-8531]

.github/workflows/benchmark-runner.yml

@@ -0,0 +1,112 @@
+on:
+  workflow_call:
+    inputs:
+      setup:
+        required: true
+        type: string
+      architecture:
+        required: true
+        type: string
+      instance-type:
+        required: true
+        type: string
+      ami-id:
+        required: true
+        type: string
+      github-runner-label:
+        required: true
+        type: string
+
+jobs:
+  start-runner:
+    name: Start self-hosted EC2 runner
+    runs-on: ubuntu-latest
+    outputs:
+      runner_label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2_instance_id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION_BENCHMARK }}
+      - name: Start EC2 runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: start
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          ec2-image-id: ${{ inputs.ami-id }}
+          ec2-instance-type: ${{ inputs.instance-type }}
+          subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID_BENCHMARK }}
+          security-group-id: ${{ secrets.AWS_EC2_SG_ID_BENCHMARK }}
+          label: ${{ inputs.github-runner-label }}
+
+  benchmark:
+    name: Run benchmarks on runner
+    needs: start-runner
+    runs-on: ${{ needs.start-runner.outputs.runner_label }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.number && format('refs/pull/{0}/merge', github.event.number) || github.head_ref }}
+      - name: Print runner info
+        run: |
+          printf "Runner lscpu:\n$(lscpu)\n"
+          printf "Runner lsmem:\n$(lsmem)\n"
+          printf "Runner nproc:\n$(nproc)\n"
+          printf "Runner uname:\n$(uname -a)\n"
+          printf "Runner arch:\n$(arch)\n"
+      - name: Install benchmark dependencies
+        run: |
+          sudo .install/install_script.sh
+          sudo apt install python3-pip -y
+          pip3 install --upgrade pip PyYAML setuptools redisbench-admin
+          pip3 install -r requirements.txt
+      - name: Download pre-generated indices
+        timeout-minutes: 20
+        run: ./tests/benchmark/bm_files.sh ${{ inputs.setup }}
+      - name: Run Benchmark
+        env:
+          ARCH: ${{ inputs.architecture }}
+        timeout-minutes: 300
+        run: |
+          make benchmark BM_FILTER=${{ inputs.setup }}
+      - name: Collect results
+        run: |
+          ./tests/benchmark/benchmarks.sh ${{ inputs.setup }} | xargs -P 0 -I {} redisbench-admin export \
+          --redistimeseries_host ${{ secrets.PERFORMANCE_RTS_HOST }} \
+          --redistimeseries_port ${{ secrets.PERFORMANCE_RTS_PORT }} \
+          --redistimeseries_user default \
+          --redistimeseries_pass '${{ secrets.PERFORMANCE_RTS_AUTH }}' \
+          --github_repo ${{ github.event.repository.name }} \
+          --github_org ${{ github.repository_owner }} \
+          --github_branch ${{ github.head_ref || github.ref_name }} \
+          --github_actor ${{ github.triggering_actor }} \
+          --results-format google.benchmark \
+          --benchmark-result-file {}_results.json
+
+  stop-runner:
+    name: Stop self-hosted EC2 runner
+    needs:
+      - start-runner # required to get output from the start-runner job
+      - benchmark # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ secrets.AWS_REGION_BENCHMARK }}
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2
+        with:
+          mode: stop
+          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          label: ${{ inputs.github-runner-label }}
+          ec2-instance-id: ${{ needs.start-runner.outputs.ec2_instance_id }}
+

.github/workflows/benchmark.yml

@@ -4,6 +4,11 @@ on:
       setup:
         type: string
         required: true
+      architecture:
+        type: string
+        required: false
+        default: 'all'
+        description: 'Run only on specific architecture'
   workflow_dispatch:
     inputs:
       setup:
@@ -32,107 +37,66 @@ on:
           - bm-spaces
         description: 'Benchmarks set to run'
         default: benchmarks-all
+      architecture:
+          type: choice
+          options:
+            - all
+            - arm64
+            - x86_64
+          description: 'Run only on specific architecture'
+          default: 'all'
 
 jobs:
-  start-runner:
-    name: Start self-hosted EC2 runner
+  prepare_runner_configurations:
     runs-on: ubuntu-latest
     outputs:
-      label: ${{ steps.start-ec2-runner.outputs.label }}
-      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-    steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ secrets.AWS_REGION }}
-      - name: Start EC2 runner
-        id: start-ec2-runner
-        uses: machulav/ec2-github-runner@v2
-        with:
-          mode: start
-          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          # Ubuntu 22.04 128GB Storage AMI
-          ec2-image-id: ami-0ba430d4b7b64de57
-          ec2-instance-type: r7i.xlarge
-          subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
-          security-group-id: ${{ secrets.AWS_EC2_SG_ID }}
-
-  benchmark:
-    name: Run the benchmarks on the runner
-    needs: start-runner # required to start the main job when the runner is ready
-    runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
-      - name: checkout
-        uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.number && format('refs/pull/{0}/merge', github.event.number) || github.head_ref }}
-      - name: Print runner info
+      - name: Set matrix
+        id: set-matrix
         run: |
-          printf "Runner lscpu:\n$(lscpu)\n"
-          printf "Runner lsmem:\n$(lsmem)\n"
-          printf "Runner nproc:\n$(nproc)\n"
-          printf "Runner uname:\n$(uname -a)\n"
-
-      - name: Install benchmark dependencies
-        run: |
-             sudo .install/install_script.sh
-             sudo apt install python3-pip -y
-             pip3 install --upgrade pip PyYAML setuptools redisbench-admin
-             pip3 install -r requirements.txt
-
-      # - name: stress test
-      #   run: |
-      #     sudo apt install stress-ng -qqy
-      #     uptime
-      #     stress-ng -c 1 --timeout 60s --metrics-brief
-      #     uptime
-      #     stress-ng --stream 1 -t 60 --metrics-brief
-      #     uptime
-      #     stress-ng --ipsec-mb=1 -t 60 --metrics-brief
-      #     uptime
+          # Define the full matrix as a JSON string
+          FULL_MATRIX='
+          {
+            "include": [
+              {
+                "architecture": "arm64",
+                "instance-type": "i8g.xlarge",
+                "ami-id": "ami-0d6c92b636b74f843"
+              },
+              {
+                "architecture": "x86_64",
+                "instance-type": "r7i.xlarge",
+                "ami-id": "ami-09fabd03bb09b3704"
+              }
+            ]
+          }
+          '
+
+          # Filter the matrix based on architecture
+          if [ "${{ inputs.architecture }}" = "all" ]; then
+            # Use the full matrix
+            FILTERED_MATRIX="$FULL_MATRIX"
+          else
+            # Filter to only the selected architecture
+            FILTERED_MATRIX=$(echo "$FULL_MATRIX" | jq -c '{include: [.include[] | select(.architecture | contains("${{ inputs.architecture }}"))]}')
+          fi
+
+          # Use multiline output delimiter syntax for GitHub Actions
+          echo "matrix<<EOF" >> $GITHUB_OUTPUT
+          echo "$FILTERED_MATRIX" >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT
 
-      # TODO: remove "--no-check-certificate" when possible
-      - name: Download pre-generated indices
-        timeout-minutes: 20
-        run: ./tests/benchmark/bm_files.sh ${{ inputs.setup }}
-      - name: Benchmark
-        timeout-minutes: 120
-        run: make benchmark BM_FILTER=${{ inputs.setup }}
-
-      - name: Collect results
-        run: |
-          ./tests/benchmark/benchmarks.sh ${{ inputs.setup }} | xargs -P 0 -I {} redisbench-admin export     \
-          --redistimeseries_host      ${{ secrets.PERFORMANCE_RTS_HOST }}           \
-          --redistimeseries_port      ${{ secrets.PERFORMANCE_RTS_PORT }}           \
-          --redistimeseries_user      default                                       \
-          --redistimeseries_pass      '${{ secrets.PERFORMANCE_RTS_AUTH }}'         \
-          --github_repo               ${{ github.event.repository.name }}           \
-          --github_org                ${{ github.repository_owner }}                \
-          --github_branch             ${{ github.head_ref || github.ref_name }}     \
-          --github_actor              ${{ github.triggering_actor }}                \
-          --results-format            google.benchmark                              \
-          --benchmark-result-file     {}_results.json
-
-  stop-runner:
-    name: Stop self-hosted EC2 runner
-    needs:
-      - start-runner # required to get output from the start-runner job
-      - benchmark # required to wait when the main job is done
-    runs-on: ubuntu-latest
-    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
-    steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: ${{ secrets.AWS_REGION }}
-      - name: Stop EC2 runner
-        uses: machulav/ec2-github-runner@v2
-        with:
-          mode: stop
-          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-          label: ${{ needs.start-runner.outputs.label }}
-          ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}
+  run_benchmarks:
+    name: Run ${{ matrix.architecture }} benchmarks
+    needs: prepare_runner_configurations
+    uses: ./.github/workflows/benchmark-runner.yml
+    secrets: inherit
+    strategy:
+      matrix: ${{ fromJson(needs.prepare_runner_configurations.outputs.matrix) }}
+    with:
+      setup: ${{ inputs.setup }}
+      architecture: ${{ matrix.architecture }}
+      instance-type: ${{ matrix.instance-type }}
+      ami-id: ${{ matrix.ami-id }}
+      github-runner-label: ${{ matrix.architecture }}-${{ matrix.instance-type }}-${{ github.run_id }}

cmake/aarch64InstructionFlags.cmake

@@ -0,0 +1,21 @@
+include(CheckCXXCompilerFlag)
+
+
+message(STATUS "Building for ARM aarch64")
+
+# Check what compiler flags are supported
+CHECK_CXX_COMPILER_FLAG("-march=armv8-a" CXX_ARMV8A)
+CHECK_CXX_COMPILER_FLAG("-march=armv8-a+sve" CXX_SVE)
+CHECK_CXX_COMPILER_FLAG("-march=armv9-a+sve2" CXX_ARMV9)
+
+# Only use ARMv9 if both compiler and CPU support it
+if(CXX_ARMV9)
+  message(STATUS "Using ARMv9.0-a with SVE2 (supported by CPU)")
+  add_compile_definitions(OPT_ARMV9)
+endif()
+if (CXX_ARMV8A)
+  add_compile_definitions(OPT_NEON)
+endif()
+if (CXX_SVE)
+  add_compile_definitions(OPT_SVE)
+endif()

tests/benchmark/CMakeLists.txt

@@ -1,4 +1,3 @@
-
 message("# VectorSimilarity_Benchmark root: " ${root})
 message("# VectorSimilarity_Benchmark binroot: " ${binroot})
 
@@ -9,6 +8,7 @@ set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_FLAGS_DEBUG "-g")
 set(CMAKE_CXX_FLAGS_RELEASE "-O3")
 include_directories(../)
+include_directories(./)
 
 enable_testing()
 
@@ -25,12 +25,20 @@ foreach(benchmark IN ITEMS ${BENCHMARKS})
 	target_link_libraries(bm_${benchmark} VectorSimilarity benchmark::benchmark)
 endforeach()
 
+if (CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
+	message(STATUS "Enabling Arm Performance Libraries integration")
+	include(${root}/cmake/aarch64InstructionFlags.cmake)
+	add_compile_definitions(BENCHMARK_ARCH=arm64)
+
+else()
+	include(${root}/cmake/x86_64InstructionFlags.cmake)
+	add_compile_definitions(BENCHMARK_ARCH=x86_64)
+endif()
+
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 # Spaces benchmarks								                                          #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
-include(${root}/cmake/x86_64InstructionFlags.cmake)
-
 set(DATA_TYPE fp32 fp64 bf16 fp16 int8 uint8)
 foreach(data_type IN LISTS DATA_TYPE)
 	add_executable(bm_spaces_${data_type} spaces_benchmarks/bm_spaces_${data_type}.cpp)

tests/benchmark/bm_macros.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "cpu_features_macros.h"
+
+#define EXPAND(x)  x
+#define EXPAND2(x) EXPAND(x)
+// Helper for raw concatenation with varying arguments
+#define BM_FUNC_NAME_HELPER1_2(a, b)          a##_##b
+#define BM_FUNC_NAME_HELPER1_3(a, b, c)       a##_##b##_##c
+#define BM_FUNC_NAME_HELPER1_4(a, b, c, d)    a##_##b##_##c##_##d
+#define BM_FUNC_NAME_HELPER1_5(a, b, c, d, e) a##_##b##_##c##_##d##_##e
+
+// Force expansion of macro arguments
+#define BM_FUNC_NAME_HELPER_2(a, b)          BM_FUNC_NAME_HELPER1_2(a, b)
+#define BM_FUNC_NAME_HELPER_3(a, b, c)       BM_FUNC_NAME_HELPER1_3(a, b, c)
+#define BM_FUNC_NAME_HELPER_4(a, b, c, d)    BM_FUNC_NAME_HELPER1_4(a, b, c, d)
+#define BM_FUNC_NAME_HELPER_5(a, b, c, d, e) BM_FUNC_NAME_HELPER1_5(a, b, c, d, e)
+
+// Determine the number of arguments and select the appropriate helper
+#define COUNT_ARGS(...)                             COUNT_ARGS_(__VA_ARGS__, 6, 5, 4, 3, 2, 1)
+#define COUNT_ARGS_(_1, _2, _3, _4, _5, _6, N, ...) N
+
+// Concatenate BM_FUNC_NAME_HELPER with the number of arguments
+#define CONCAT_HELPER(a, b) a##_##b
+#define CONCAT(a, b)        CONCAT_HELPER(a, b)
+
+// Main macro that selects the appropriate helper based on argument count
+#define CONCAT_WITH_UNDERSCORE(...)                                                                \
+    EXPAND2(CONCAT(BM_FUNC_NAME_HELPER, EXPAND2(COUNT_ARGS(__VA_ARGS__)))(__VA_ARGS__))
+
+// Modify this macro to account for the extra BENCHMARK_ARCH parameter
+#define CONCAT_WITH_UNDERSCORE_ARCH(...) CONCAT_WITH_UNDERSCORE(__VA_ARGS__, BENCHMARK_ARCH)

tests/benchmark/bm_vecsim_general.h

@@ -20,6 +20,7 @@
 #include "VecSim/algorithms/hnsw/hnsw.h"
 #include "VecSim/index_factories/hnsw_factory.h"
 #include "bm_definitions.h"
+#include "bm_macros.h"
 #include "utils/mock_thread_pool.h"
 
 // This class includes every static data member that is: