Merge pull request ceph#57271 from tj90241/a-better-crc32c

common: Leverage a better CRC32C implementation

Reviewed-by: Samuel Just <[email protected]>
Reviewed-by: Mark Nelson <[email protected]>

Author: yuriw

src/common/CMakeLists.txt

@@ -215,6 +215,7 @@ if(HAVE_INTEL)
     set(CMAKE_ASM_FLAGS "-i ${PROJECT_SOURCE_DIR}/src/isa-l/include/ ${CMAKE_ASM_FLAGS}")
     list(APPEND crc32_srcs
       ${PROJECT_SOURCE_DIR}/src/isa-l/crc/crc32_iscsi_00.asm
+      ${PROJECT_SOURCE_DIR}/src/isa-l/crc/crc32_iscsi_01.asm
       crc32c_intel_fast_zero_asm.s)
   endif(HAVE_NASM_X64)
 elseif(HAVE_POWER8)

src/common/crc32c.cc

@@ -24,6 +24,9 @@ ceph_crc32c_func_t ceph_choose_crc32(void)
   // use that.
 #if defined(__i386__) || defined(__x86_64__)
   if (ceph_arch_intel_sse42 && ceph_crc32c_intel_fast_exists()) {
+    if (ceph_arch_intel_pclmul) {
+      return ceph_crc32c_intel_fast_pclmul;
+    }
     return ceph_crc32c_intel_fast;
   }
 #elif defined(__arm__) || defined(__aarch64__)

src/common/crc32c_intel_fast.c

@@ -2,10 +2,25 @@
 #include "common/crc32c_intel_baseline.h"
 
 extern unsigned int crc32_iscsi_00(unsigned char const *buffer, uint64_t len, uint64_t crc) asm("crc32_iscsi_00");
+extern unsigned int crc32_iscsi_01(unsigned char const *buffer, uint64_t len, uint64_t crc) asm("crc32_iscsi_01");
 extern unsigned int crc32_iscsi_zero_00(unsigned char const *buffer, uint64_t len, uint64_t crc) asm("crc32_iscsi_zero_00");
 
 #ifdef HAVE_NASM_X64
 
+uint32_t ceph_crc32c_intel_fast_pclmul(uint32_t crc, unsigned char const *buffer, unsigned len)
+{
+	if (!buffer)
+	{
+	  return crc32_iscsi_zero_00(buffer, len, crc);
+	}
+
+	/* Unlike crc32_iscsi_00, crc32_iscsi_01 handles the case where the
+	 * input buffer is less than 8 bytes in its prelude, and does not
+	 * prefetch beyond said buffer.
+	 */
+	return crc32_iscsi_01(buffer, len, crc);
+}
+
 uint32_t ceph_crc32c_intel_fast(uint32_t crc, unsigned char const *buffer, unsigned len)
 {
 	uint32_t v;
@@ -43,6 +58,11 @@ int ceph_crc32c_intel_fast_exists(void)
 	return 0;
 }
 
+uint32_t ceph_crc32c_intel_fast_pclmul(uint32_t crc, unsigned char const *buffer, unsigned len)
+{
+	return 0;
+}
+
 uint32_t ceph_crc32c_intel_fast(uint32_t crc, unsigned char const *buffer, unsigned len)
 {
 	return 0;

src/common/crc32c_intel_fast.h

@@ -10,10 +10,16 @@ extern int ceph_crc32c_intel_fast_exists(void);
 
 #ifdef __x86_64__
 
+extern uint32_t ceph_crc32c_intel_fast_pclmul(uint32_t crc, unsigned char const *buffer, unsigned len);
 extern uint32_t ceph_crc32c_intel_fast(uint32_t crc, unsigned char const *buffer, unsigned len);
 
 #else
 
+static inline uint32_t ceph_crc32c_intel_fast_pclmul(uint32_t crc, unsigned char const *buffer, unsigned len)
+{
+	return 0;
+}
+
 static inline uint32_t ceph_crc32c_intel_fast(uint32_t crc, unsigned char const *buffer, unsigned len)
 {
 	return 0;