Fix/reconcile/merge cpp files after merge

Author: ldematte

libs/simdvec/native/publish_vec_binaries.sh

@@ -20,7 +20,7 @@ if [ -z "$ARTIFACTORY_API_KEY" ]; then
   exit 1;
 fi
 
-VERSION="1.0.17"
+VERSION="1.0.18"
 ARTIFACTORY_REPOSITORY="${ARTIFACTORY_REPOSITORY:-https://artifactory.elastic.dev/artifactory/elasticsearch-native/}"
 TEMP=$(mktemp -d)
 

libs/simdvec/native/src/vec/c/aarch64/vec_1.cpp

@@ -87,7 +87,7 @@ static inline int32_t dot7u_inner(const int8_t* a, const int8_t* b, const int32_
     return vaddvq_s32(vaddq_s32(acc5, acc6));
 }
 
-EXPORT int32_t vec_dot7u(int8_t* a, int8_t* b, const int32_t dims) {
+EXPORT int32_t vec_dot7u(const int8_t* a, const int8_t* b, const int32_t dims) {
     int32_t res = 0;
     int i = 0;
     if (dims > DOT7U_STRIDE_BYTES_LEN) {
@@ -100,6 +100,103 @@ EXPORT int32_t vec_dot7u(int8_t* a, int8_t* b, const int32_t dims) {
     return res;
 }
 
+template <int32_t(*mapper)(const int32_t, const int32_t*)>
+static inline void dot7u_inner_bulk(const int8_t* a, const int8_t* b, const int32_t dims, const int32_t* offsets, const int32_t count, f32_t* results) {
+    size_t blk = dims & ~15;
+    size_t c = 0;
+
+    // Process 4 vectors at a time
+    for (; c + 3 < count; c += 4) {
+        const int8_t* a0 = a + mapper(c, offsets) * dims;
+        const int8_t* a1 = a + mapper(c + 1, offsets) * dims;
+        const int8_t* a2 = a + mapper(c + 2, offsets) * dims;
+        const int8_t* a3 = a + mapper(c + 3, offsets) * dims;
+
+        int32x4_t acc0 = vdupq_n_s32(0);
+        int32x4_t acc1 = vdupq_n_s32(0);
+        int32x4_t acc2 = vdupq_n_s32(0);
+        int32x4_t acc3 = vdupq_n_s32(0);
+        int32x4_t acc4 = vdupq_n_s32(0);
+        int32x4_t acc5 = vdupq_n_s32(0);
+        int32x4_t acc6 = vdupq_n_s32(0);
+        int32x4_t acc7 = vdupq_n_s32(0);
+
+        for (size_t i = 0; i < blk; i += 16) {
+            int8x16_t vb = vld1q_s8(b + i);
+
+            int8x16_t v0 = vld1q_s8(a0 + i);
+            int16x8_t lo0 = vmull_s8(vget_low_s8(v0), vget_low_s8(vb));
+            int16x8_t hi0 = vmull_s8(vget_high_s8(v0), vget_high_s8(vb));
+            acc0 = vpadalq_s16(acc0, lo0);
+            acc1 = vpadalq_s16(acc1, hi0);
+
+            int8x16_t v1 = vld1q_s8(a1 + i);
+            int16x8_t lo1 = vmull_s8(vget_low_s8(v1), vget_low_s8(vb));
+            int16x8_t hi1 = vmull_s8(vget_high_s8(v1), vget_high_s8(vb));
+            acc2 = vpadalq_s16(acc2, lo1);
+            acc3 = vpadalq_s16(acc3, hi1);
+
+            int8x16_t v2 = vld1q_s8(a2 + i);
+            int16x8_t lo2 = vmull_s8(vget_low_s8(v2), vget_low_s8(vb));
+            int16x8_t hi2 = vmull_s8(vget_high_s8(v2), vget_high_s8(vb));
+            acc4 = vpadalq_s16(acc4, lo2);
+            acc5 = vpadalq_s16(acc5, hi2);
+
+            int8x16_t v3 = vld1q_s8(a3 + i);
+            int16x8_t lo3 = vmull_s8(vget_low_s8(v3), vget_low_s8(vb));
+            int16x8_t hi3 = vmull_s8(vget_high_s8(v3), vget_high_s8(vb));
+            acc6 = vpadalq_s16(acc6, lo3);
+            acc7 = vpadalq_s16(acc7, hi3);
+        }
+        int32x4_t acc01 = vaddq_s32(acc0, acc1);
+        int32x4_t acc23 = vaddq_s32(acc2, acc3);
+        int32x4_t acc45 = vaddq_s32(acc4, acc5);
+        int32x4_t acc67 = vaddq_s32(acc6, acc7);
+
+        int32_t acc_scalar0 = vaddvq_s32(acc01);
+        int32_t acc_scalar1 = vaddvq_s32(acc23);
+        int32_t acc_scalar2 = vaddvq_s32(acc45);
+        int32_t acc_scalar3 = vaddvq_s32(acc67);
+        if (blk != dims) {
+            // scalar tail
+            for (size_t t = blk; t < dims; t++) {
+                const int8_t bb = b[t];
+                acc_scalar0 += a0[t] * bb;
+                acc_scalar1 += a1[t] * bb;
+                acc_scalar2 += a2[t] * bb;
+                acc_scalar3 += a3[t] * bb;
+            }
+        }
+        results[c + 0] = (f32_t)acc_scalar0;
+        results[c + 1] = (f32_t)acc_scalar1;
+        results[c + 2] = (f32_t)acc_scalar2;
+        results[c + 3] = (f32_t)acc_scalar3;
+    }
+
+    // Tail-handling: remaining 0..3 vectors
+    for (; c < count; c++) {
+        const int8_t* a0 = a + mapper(c, offsets) * dims;
+        results[c] = (f32_t)vec_dot7u(a0, b, dims);
+    }
+}
+
+static inline int identity(const int32_t i, const int32_t* offsets) {
+   return i;
+}
+
+static inline int index(const int32_t i, const int32_t* offsets) {
+   return offsets[i];
+}
+
+EXPORT void vec_dot7u_bulk(const int8_t* a, const int8_t* b, const int32_t dims, const int32_t count, f32_t* results) {
+    dot7u_inner_bulk<identity>(a, b, dims, NULL, count, results);
+}
+
+
+EXPORT void vec_dot7u_bulk_offsets(const int8_t* a, const int8_t* b, const int32_t dims, const int32_t* offsets, const int32_t count, f32_t* results) {
+    dot7u_inner_bulk<index>(a, b, dims, offsets, count, results);
+}
+
 static inline int32_t sqr7u_inner(int8_t *a, int8_t *b, const int32_t dims) {
     int32x4_t acc1 = vdupq_n_s32(0);
     int32x4_t acc2 = vdupq_n_s32(0);

libs/simdvec/native/src/vec/c/aarch64/vec_bulk.cpp

@@ -144,7 +144,7 @@ static inline int32_t dot7u_inner(const int8_t* a, const int8_t* b, const int32_
     return hsum_i32_8(acc1);
 }
 
-EXPORT int32_t vec_dot7u(int8_t* a, int8_t* b, const int32_t dims) {
+EXPORT int32_t vec_dot7u(const int8_t* a, const int8_t* b, const int32_t dims) {
     int32_t res = 0;
     int i = 0;
     if (dims > STRIDE_BYTES_LEN) {
@@ -157,6 +157,49 @@ EXPORT int32_t vec_dot7u(int8_t* a, int8_t* b, const int32_t dims) {
     return res;
 }
 
+template <int32_t(*mapper)(int32_t, const int32_t*)>
+static inline void dot7u_inner_bulk(const int8_t* a, const int8_t* b, const int32_t dims, const int32_t* offsets, const int32_t count, f32_t* results) {
+    int32_t res = 0;
+    if (dims > STRIDE_BYTES_LEN) {
+        const int limit = dims & ~(STRIDE_BYTES_LEN - 1);
+        for (int32_t c = 0; c < count; c++) {
+            const int8_t* a0 = a + mapper(c, offsets) * dims;
+            int i = limit;
+            res = dot7u_inner(a, b, i);
+            for (; i < dims; i++) {
+                res += a0[i] * b[i];
+            }
+            results[c] = (f32_t)res;
+        }
+    } else {
+        for (int32_t c = 0; c < count; c++) {
+            const int8_t* a0 = a + mapper(c, offsets) * dims;
+            res = 0;
+            for (int32_t i = 0; i < dims; i++) {
+                res += a0[i] * b[i];
+            }
+            results[c] = (f32_t)res;
+        }
+    }
+}
+
+static inline int identity(const int32_t i, const int32_t* offsets) {
+   return i;
+}
+
+static inline int index(const int32_t i, const int32_t* offsets) {
+   return offsets[i];
+}
+
+EXPORT void vec_dot7u_bulk(const int8_t* a, const int8_t* b, const int32_t dims, const int32_t count, f32_t* results) {
+    dot7u_inner_bulk<identity>(a, b, dims, NULL, count, results);
+}
+
+
+EXPORT void vec_dot7u_bulk_offsets(const int8_t* a, const int8_t* b, const int32_t dims, const int32_t* offsets, const int32_t count, f32_t* results) {
+    dot7u_inner_bulk<index>(a, b, dims, offsets, count, results);
+}
+
 static inline int32_t sqr7u_inner(int8_t *a, int8_t *b, const int32_t dims) {
     // Init accumulator(s) with 0
     __m256i acc1 = _mm256_setzero_si256();

libs/simdvec/native/src/vec/c/amd64/vec_1.cpp

@@ -57,7 +57,7 @@ inline __m512i fma8(__m512i acc, const int8_t* p1, const int8_t* p2) {
     return _mm512_add_epi32(_mm512_madd_epi16(ones, dot), acc);
 }
 
-static inline int32_t dot7u_inner_avx512(int8_t* a, const int8_t* b, const int32_t dims) {
+static inline int32_t dot7u_inner_avx512(const int8_t* a, const int8_t* b, const int32_t dims) {
     constexpr int stride8 = 8 * STRIDE_BYTES_LEN;
     constexpr int stride4 = 4 * STRIDE_BYTES_LEN;
     const int8_t* p1 = a;
@@ -110,7 +110,7 @@ static inline int32_t dot7u_inner_avx512(int8_t* a, const int8_t* b, const int32
     return _mm512_reduce_add_epi32(_mm512_add_epi32(acc0, acc4));
 }
 
-EXPORT int32_t vec_dot7u_2(int8_t* a, int8_t* b, const int32_t dims) {
+EXPORT int32_t vec_dot7u_2(const int8_t* a, const int8_t* b, const int32_t dims) {
     int32_t res = 0;
     int i = 0;
     if (dims > STRIDE_BYTES_LEN) {
@@ -123,8 +123,8 @@ EXPORT int32_t vec_dot7u_2(int8_t* a, int8_t* b, const int32_t dims) {
     return res;
 }
 
-template <int(*mapper)(int, int32_t*)>
-static inline void dot7u_inner_bulk(int8_t* a, int8_t* b, int dims, int32_t* offsets, int count, f32_t* results) {
+template <int32_t(*mapper)(int32_t, const int32_t*)>
+static inline void dot7u_inner_bulk(const int8_t* a, const int8_t* b, const int32_t dims, const int32_t* offsets, const int32_t count, f32_t* results) {
     int32_t res = 0;
     if (dims > STRIDE_BYTES_LEN) {
         const int limit = dims & ~(STRIDE_BYTES_LEN - 1);
@@ -149,21 +149,20 @@ static inline void dot7u_inner_bulk(int8_t* a, int8_t* b, int dims, int32_t* off
     }
 }
 
-static inline int identity(int i, int32_t* offsets) {
+static inline int identity(const int32_t i, const int32_t* offsets) {
    return i;
 }
 
-static inline int index(int i, int32_t* offsets) {
+static inline int index(const int32_t i, const int32_t* offsets) {
    return offsets[i];
 }
 
-extern "C"
-EXPORT void dot7u_bulk(int8_t* a, int8_t* b, int dims, int count, f32_t* results) {
+EXPORT void vec_dot7u_bulk_2(const int8_t* a, const int8_t* b, const int32_t dims, const int32_t count, f32_t* results) {
     dot7u_inner_bulk<identity>(a, b, dims, NULL, count, results);
 }
 
-extern "C"
-EXPORT void dot7u_bulk_offsets(int8_t* a, int8_t* b, int dims, int32_t* offsets, int count, f32_t* results) {
+
+EXPORT void vec_dot7u_bulk_offsets_2(const int8_t* a, const int8_t* b, const int32_t dims, const int32_t* offsets, const int32_t count, f32_t* results) {
     dot7u_inner_bulk<index>(a, b, dims, offsets, count, results);
 }