Merge pull request ceph#58594 from jamiepryde/isa-xor-raid

erasure-code/isa: Use isa/raid's xor_gen() instead of the region_xor(…

Reviewed-by: Mark Nelson <[email protected]>

Author: yuriw

src/erasure-code/isa/CMakeLists.txt

@@ -54,10 +54,19 @@ if(HAVE_NASM_X64_AVX2)
     ${isal_src_dir}/erasure_code/gf_4vect_mad_avx512.asm
     ${isal_src_dir}/erasure_code/gf_vect_dot_prod_avx512.asm
     ${isal_src_dir}/erasure_code/gf_vect_mad_avx512.asm
+    ${isal_src_dir}/raid/raid_base.c
+    ${isal_src_dir}/raid/raid_multibinary.asm
+    ${isal_src_dir}/raid/xor_check_sse.asm
+    ${isal_src_dir}/raid/xor_gen_sse.asm
+    ${isal_src_dir}/raid/xor_gen_avx.asm
+    ${isal_src_dir}/raid/xor_gen_avx512.asm
+    ${isal_src_dir}/raid/pq_check_sse.asm
+    ${isal_src_dir}/raid/pq_gen_sse.asm
+    ${isal_src_dir}/raid/pq_gen_avx.asm
+    ${isal_src_dir}/raid/pq_gen_avx2.asm
     ErasureCodeIsa.cc
     ErasureCodeIsaTableCache.cc
     ErasureCodePluginIsa.cc
-    xor_op.cc
   )
 elseif(HAVE_ARMV8_SIMD)
   set(isa_srcs
@@ -77,13 +86,20 @@ elseif(HAVE_ARMV8_SIMD)
     ${isal_src_dir}/erasure_code/aarch64/gf_vect_mad_neon.S
     ${isal_src_dir}/erasure_code/aarch64/gf_vect_mul_neon.S
     ${isal_src_dir}/erasure_code/aarch64/ec_multibinary_arm.S
+    ${isal_src_dir}/raid/raid_base.c
+    ${isal_src_dir}/raid/aarch64/raid_aarch64_dispatcher.c
+    ${isal_src_dir}/raid/aarch64/raid_multibinary_arm.S
+    ${isal_src_dir}/raid/aarch64/xor_check_neon.S
+    ${isal_src_dir}/raid/aarch64/xor_gen_neon.S
+    ${isal_src_dir}/raid/aarch64/pq_check_neon.S
+    ${isal_src_dir}/raid/aarch64/pq_gen_neon.S
     ErasureCodeIsa.cc
     ErasureCodeIsaTableCache.cc
     ErasureCodePluginIsa.cc
-    xor_op.cc
   )
   set_source_files_properties(
     ${isal_src_dir}/erasure_code/aarch64/ec_multibinary_arm.S
+    ${isal_src_dir}/raid/aarch64/raid_multibinary_arm.S
     PROPERTIES COMPILE_FLAGS "-D__ASSEMBLY__"
   )
 endif()

src/erasure-code/isa/ErasureCodeIsa.cc

@@ -18,14 +18,14 @@
 // -----------------------------------------------------------------------------
 #include "common/debug.h"
 #include "ErasureCodeIsa.h"
-#include "xor_op.h"
 #include "include/ceph_assert.h"
 using namespace std;
 using namespace ceph;
 
 // -----------------------------------------------------------------------------
 extern "C" {
 #include "isa-l/include/erasure_code.h"
+#include "isa-l/include/raid.h"
 }
 // -----------------------------------------------------------------------------
 #define dout_context g_ceph_context
@@ -121,10 +121,9 @@ ErasureCodeIsaDefault::isa_encode(char **data,
                                   char **coding,
                                   int blocksize)
 {
-
   if (m == 1)
     // single parity stripe
-    region_xor((unsigned char**) data, (unsigned char*) coding[0], k, blocksize);
+    xor_gen(k+m, blocksize, (void**) data);
   else
     ec_encode_data(blocksize, k, m, encode_tbls,
                    (unsigned char**) data, (unsigned char**) coding);
@@ -157,61 +156,81 @@ ErasureCodeIsaDefault::isa_decode(int *erasures,
   int nerrs = 0;
   int i, r, s;
 
+  unsigned char *recover_source[k];
+  unsigned char *recover_target[m];
+  unsigned char *recover_buf[k+1];
+
   // count the errors
   for (int l = 0; erasures[l] != -1; l++) {
     nerrs++;
   }
 
-  unsigned char *recover_source[k];
-  unsigned char *recover_target[m];
-
-  memset(recover_source, 0, sizeof (recover_source));
-  memset(recover_target, 0, sizeof (recover_target));
+  if (nerrs > m)
+    return -1;
 
-  // ---------------------------------------------
-  // Assign source and target buffers
-  // ---------------------------------------------
-  for (i = 0, s = 0, r = 0; ((r < k) || (s < nerrs)) && (i < (k + m)); i++) {
-    if (!erasure_contains(erasures, i)) {
-      if (r < k) {
+  // -----------------------------------
+  // Assign source and target buffers.
+  // -----------------------------------
+  if ((m == 1) || 
+      ((matrixtype == kVandermonde) && (nerrs == 1) && (erasures[0] < (k + 1)))) {
+    // We need a single buffer to use the xor_gen() optimisation.
+    // The last index must point to the erasure, and index that contained
+    // the erasure must point to the parity.
+    memset(recover_buf, 0, sizeof (recover_buf));
+    bool parity_set = false;
+    for (i = 0; i < (k + 1); i++) {
+      if (erasure_contains(erasures, i)) {
+          if (i < k) {
+            recover_buf[i] = (unsigned char*) coding[0];
+            recover_buf[k] = (unsigned char*) data[i];
+            parity_set = true;
+          } else {
+            recover_buf[i] = (unsigned char*) coding[0];
+          }
+      } else {
         if (i < k) {
-          recover_source[r] = (unsigned char*) data[i];
+          recover_buf[i] = (unsigned char*) data[i];
         } else {
-          recover_source[r] = (unsigned char*) coding[i - k];
+          if (!parity_set) {
+            recover_buf[i] = (unsigned char*) coding[0];
+          }
         }
-        r++;
       }
-    } else {
-      if (s < m) {
-        if (i < k) {
-          recover_target[s] = (unsigned char*) data[i];
-        } else {
-          recover_target[s] = (unsigned char*) coding[i - k];
+    }
+  }
+  else {
+    // We need source and target buffers to use ec_encode_data().
+    // The erasure must be moved to the target buffer.
+    memset(recover_source, 0, sizeof (recover_source));
+    memset(recover_target, 0, sizeof (recover_target));
+    for (i = 0, s = 0, r = 0; ((r < k) || (s < nerrs)) && (i < (k + m)); i++) {
+      if (!erasure_contains(erasures, i)) {
+        if (r < k) {
+          if (i < k) {
+            recover_source[r] = (unsigned char*) data[i];
+          } else {
+            recover_source[r] = (unsigned char*) coding[i - k];
+          }
+          r++;
+        }
+      } else {
+        if (s < m) {
+          if (i < k) {
+            recover_target[s] = (unsigned char*) data[i];
+          } else {
+            recover_target[s] = (unsigned char*) coding[i - k];
+          }
+          s++;
         }
-        s++;
       }
     }
   }
 
-  if (m == 1) {
+  if ((m == 1) || 
+      ((matrixtype == kVandermonde) && (nerrs == 1) && (erasures[0] < (k + 1)))) {
     // single parity decoding
-    ceph_assert(1 == nerrs);
-    dout(20) << "isa_decode: reconstruct using region xor [" <<
-      erasures[0] << "]" << dendl;
-    region_xor(recover_source, recover_target[0], k, blocksize);
-    return 0;
-  }
-
-
-  if ((matrixtype == kVandermonde) &&
-      (nerrs == 1) &&
-      (erasures[0] < (k + 1))) {
-    // use xor decoding if a data chunk is missing or the first coding chunk
-    dout(20) << "isa_decode: reconstruct using region xor [" <<
-      erasures[0] << "]" << dendl;
-    ceph_assert(1 == s);
-    ceph_assert(k == r);
-    region_xor(recover_source, recover_target[0], k, blocksize);
+    dout(20) << "isa_decode: reconstruct using xor_gen [" << erasures[0] << "]" << dendl;
+    xor_gen(k+1, blocksize, (void **) recover_buf);
     return 0;
   }
 
@@ -221,9 +240,6 @@ ErasureCodeIsaDefault::isa_decode(int *erasures,
 
   int decode_index[k];
 
-  if (nerrs > m)
-    return -1;
-
   std::string erasure_signature; // describes a matrix configuration for caching
 
   // ---------------------------------------------

src/erasure-code/isa/ErasureCodeIsa.h

@@ -30,6 +30,8 @@
 #include "ErasureCodeIsaTableCache.h"
 // -----------------------------------------------------------------------------
 
+#define EC_ISA_ADDRESS_ALIGNMENT 32u
+
 class ErasureCodeIsa : public ceph::ErasureCode {
 public: