optimize crc32c for riscv64 with Zbc carry-less multiplication (apache#3312)

* optimize crc32c for riscv64 with Zbc carry-less multiplication

Implement hardware-accelerated CRC32C for RISC-V using the Zbc
(carry-less multiplication) extension. The implementation uses
128-bit folding with 4-way parallelism and Barrett reduction,
following the approach from Hadoop PR #8371.

Key changes:
- Add rv_clmul/rv_clmulh inline assembly wrappers
- Implement 128-bit fold with 4-way parallel processing (64 bytes/iter)
- Add Barrett reduction for final 128-bit to 32-bit conversion
- Runtime CPU feature detection via /proc/cpuinfo
- Compile-time guard: #ifdef __riscv_zbc
- CMake option: WITH_RISCV_ZBC (default OFF)

Performance: 3-4x speedup over table-based 8-byte unrolled baseline,
~1.1 GB/s throughput on 1MB data.
Correctness: Verified against RFC 3720 B.4 and bitwise reference.

Signed-off-by: Felix-Gong <gongxiaofei24@iscas.ac.cn>

* fix: wrap isSSE42() in #ifdef __SSE4_2__ to fix -Wunused-function

isSSE42() is only called from within #ifdef __SSE4_2__ blocks,
but the function definition was unconditional, causing
-Wunused-function errors on non-x86 builds with -Werror.

Signed-off-by: Felix-Gong <gongxiaofei24@iscas.ac.cn>

* Potential fix for pull request finding

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

* Fix CRC32C Zbc: add missing ^ 0xFFFFFFFF pre/post processing

rv_crc32c_clmul was missing the standard CRC32C pre/post XOR
conversion. ExtendImpl does crc ^ 0xFFFFFFFF at entry and
result ^ 0xFFFFFFFF at exit, but rv_crc32c_clmul did neither,
causing wrong CRC values on RISC-V with Zbc extension.

Signed-off-by: Felix-Gong <gongxiaofei24@iscas.ac.cn>

---------

Signed-off-by: Felix-Gong <gongxiaofei24@iscas.ac.cn>
Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

Author: Felix-Gong

CMakeLists.txt

@@ -172,7 +172,12 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-gcse")
     elseif((CMAKE_SYSTEM_PROCESSOR MATCHES "riscv64"))
         # RISC-V specific optimizations
-        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=rv64gc")
+        option(WITH_RISCV_ZBC "Enable RISC-V Zbc carry-less multiplication for CRC32C acceleration" OFF)
+        if(WITH_RISCV_ZBC)
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=rv64gc_zbc")
+        else()
+            set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=rv64gc")
+        endif()
     endif()
     if(NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0))
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-aligned-new")

src/butil/crc32c.cc

@@ -421,7 +421,194 @@ uint32_t ExtendImpl(uint32_t crc, const char* buf, size_t size) {
   return static_cast<uint32_t>(l ^ 0xffffffffu);
 }
 
-// Detect if SS42 or not.
+#if defined(__riscv) && (__riscv_xlen == 64) && defined(__riscv_zbc)
+#include <stdio.h>
+
+// RISC-V Zbc carry-less multiplication inline helpers
+static inline uint64_t rv_clmul(uint64_t a, uint64_t b) {
+  uint64_t result;
+  __asm__ volatile ("clmul %0, %1, %2" : "=r"(result) : "r"(a), "r"(b));
+  return result;
+}
+
+static inline uint64_t rv_clmulh(uint64_t a, uint64_t b) {
+  uint64_t result;
+  __asm__ volatile ("clmulh %0, %1, %2" : "=r"(result) : "r"(a), "r"(b));
+  return result;
+}
+
+// Bitwise CRC32C fallback for small chunks
+static inline uint32_t rv_crc32c_bitwise(uint32_t crc, const uint8_t* buf,
+                                         size_t len) {
+  uint32_t c = crc;
+  for (size_t i = 0; i < len; ++i) {
+    c ^= buf[i];
+    for (int k = 0; k < 8; ++k) {
+      c = (c >> 1) ^ ((c & 1) ? 0x82F63B78U : 0);
+    }
+  }
+  return c;
+}
+
+// Fold a 128-bit CRC state (lo:hi) with fold constants and XOR in new data
+static inline void rv_fold_pair_xor_data(uint64_t* lo, uint64_t* hi,
+                                         uint64_t k0, uint64_t k1,
+                                         uint64_t d0, uint64_t d1) {
+  uint64_t l = rv_clmul(*lo, k0) ^ rv_clmul(*hi, k1);
+  uint64_t h = rv_clmulh(*lo, k0) ^ rv_clmulh(*hi, k1);
+  *lo = l ^ d0;
+  *hi = h ^ d1;
+}
+
+// Fold a 128-bit CRC state with fold constants and XOR in another state
+static inline void rv_fold_pair_xor_state(uint64_t* lo, uint64_t* hi,
+                                          uint64_t k0, uint64_t k1,
+                                          uint64_t s0, uint64_t s1) {
+  uint64_t l = rv_clmul(*lo, k0) ^ rv_clmul(*hi, k1);
+  uint64_t h = rv_clmulh(*lo, k0) ^ rv_clmulh(*hi, k1);
+  *lo = l ^ s0;
+  *hi = h ^ s1;
+}
+
+// Folding constants for CRC32C (Castagnoli polynomial 0x1EDC6F41)
+// x^(64*i+64) mod P(x) for i=1..4, in bit-reflected form
+static const uint64_t crc32c_fold_const[4] __attribute__((aligned(16))) = {
+  0x00000000740eef02ULL,  // k1: fold 512->256
+  0x000000009e4addf8ULL,  // k2: fold 512->256
+  0x00000000f20c0dfeULL,  // k3: fold 256->128
+  0x00000000493c7d27ULL   // k4: fold 256->128
+};
+
+// Barrett reduction constants for CRC32C finalization
+#define RV_CRC32C_CONST_0    0x00000000dd45aab8ULL  // x^64 mod P
+#define RV_CRC32C_CONST_1    0x00000000493c7d27ULL  // x^96 mod P
+#define RV_CRC32C_CONST_QUO  0x0000000dea713f1ULL   // floor(x^64 / P)
+#define RV_CRC32C_CONST_POLY 0x0000000105ec76f1ULL  // P(x) true LE full
+#define RV_CRC32_MASK32      0x00000000FFFFFFFFULL
+
+// Hardware-accelerated CRC32C using RISC-V Zbc carry-less multiplication.
+// Processes data in 64-byte chunks with 128-bit folding, then Barrett reduces.
+static uint32_t rv_crc32c_clmul(uint32_t crc, const char* buf, size_t len) {
+  // Convert external CRC to internal register state
+  crc ^= 0xFFFFFFFF;
+
+  const uint8_t* p = reinterpret_cast<const uint8_t*>(buf);
+  size_t n = len;
+
+  // Small data: use bitwise fallback
+  if (n < 64) {
+    return rv_crc32c_bitwise(crc, p, n) ^ 0xFFFFFFFF;
+  }
+
+  // Align to 16-byte boundary
+  uintptr_t mis = (uintptr_t)p & 0xF;
+  if (mis) {
+    size_t pre = 16 - mis;
+    if (pre > n) pre = n;
+    crc = rv_crc32c_bitwise(crc, p, pre);
+    p += pre;
+    n -= pre;
+    if (n < 64) {
+      return rv_crc32c_bitwise(crc, p, n) ^ 0xFFFFFFFF;
+    }
+  }
+
+  // Load first 64 bytes and XOR CRC into the first 8 bytes
+  uint64_t x0, x1, y0, y1, z0, z1, w0, w1;
+  memcpy(&x0, p + 0, 8);
+  memcpy(&x1, p + 8, 8);
+  memcpy(&y0, p + 16, 8);
+  memcpy(&y1, p + 24, 8);
+  memcpy(&z0, p + 32, 8);
+  memcpy(&z1, p + 40, 8);
+  memcpy(&w0, p + 48, 8);
+  memcpy(&w1, p + 56, 8);
+
+  x0 ^= (uint64_t)crc;
+  p += 64;
+  n -= 64;
+
+  const uint64_t k1 = crc32c_fold_const[0];
+  const uint64_t k2 = crc32c_fold_const[1];
+  const uint64_t k3 = crc32c_fold_const[2];
+  const uint64_t k4 = crc32c_fold_const[3];
+
+  // Main loop: fold 64 bytes per iteration using 128-bit folding
+  while (n >= 64) {
+    uint64_t d0, d1;
+    memcpy(&d0, p + 0, 8);
+    memcpy(&d1, p + 8, 8);
+    rv_fold_pair_xor_data(&x0, &x1, k1, k2, d0, d1);
+    memcpy(&d0, p + 16, 8);
+    memcpy(&d1, p + 24, 8);
+    rv_fold_pair_xor_data(&y0, &y1, k1, k2, d0, d1);
+    memcpy(&d0, p + 32, 8);
+    memcpy(&d1, p + 40, 8);
+    rv_fold_pair_xor_data(&z0, &z1, k1, k2, d0, d1);
+    memcpy(&d0, p + 48, 8);
+    memcpy(&d1, p + 56, 8);
+    rv_fold_pair_xor_data(&w0, &w1, k1, k2, d0, d1);
+    p += 64;
+    n -= 64;
+  }
+
+  // Reduce 4x128-bit to 1x128-bit
+  rv_fold_pair_xor_state(&x0, &x1, k3, k4, y0, y1);
+  rv_fold_pair_xor_state(&x0, &x1, k3, k4, z0, z1);
+  rv_fold_pair_xor_state(&x0, &x1, k3, k4, w0, w1);
+
+  // Barrett reduction: 128-bit -> 32-bit CRC
+  uint64_t t4 = rv_clmul(x0, RV_CRC32C_CONST_1);
+  uint64_t t3 = rv_clmulh(x0, RV_CRC32C_CONST_1);
+  uint64_t t1 = x1 ^ t4;
+  t4 = t1 & RV_CRC32_MASK32;
+  t1 >>= 32;
+  uint64_t t0 = rv_clmul(t4, RV_CRC32C_CONST_0);
+  t3 = (t3 << 32) ^ t1 ^ t0;
+
+  t4 = t3 & RV_CRC32_MASK32;
+  t4 = rv_clmul(t4, RV_CRC32C_CONST_QUO);
+  t4 &= RV_CRC32_MASK32;
+  t4 = rv_clmul(t4, RV_CRC32C_CONST_POLY);
+  t4 ^= t3;
+
+  uint32_t c = (uint32_t)((t4 >> 32) & RV_CRC32_MASK32);
+  // Handle remaining bytes
+  if (n) {
+    c = rv_crc32c_bitwise(c, p, n);
+  }
+  // Convert internal register state to external CRC
+  return c ^ 0xFFFFFFFF;
+}
+
+// Runtime detection: check if RISC-V CPU supports Zbc extension
+static bool isZbc() {
+  static const bool zbc_supported = []() {
+    FILE* f = fopen("/proc/cpuinfo", "r");
+    if (!f) return false;
+    bool supported = false;
+    char line[1024];
+    while (fgets(line, sizeof(line), f)) {
+      if (strstr(line, "isa") || strstr(line, "hart isa")) {
+        char* colon = strchr(line, ':');
+        if (colon) {
+          if (strstr(colon, "_zbc") || strstr(colon, "zbc")) {
+            supported = true;
+            break;
+          }
+        }
+      }
+    }
+    fclose(f);
+    return supported;
+  }();
+  return zbc_supported;
+}
+}
+#endif  // __riscv && __riscv_xlen == 64
+
+// Detect if SSE4.2 or not.
+#ifdef __SSE4_2__
 static bool isSSE42() {
 #if defined(__GNUC__) && defined(__x86_64__) && !defined(IOS_CROSS_COMPILE)
   uint32_t c_;
@@ -432,20 +619,32 @@ static bool isSSE42() {
   return false;
 #endif
 }
+#endif
 
 typedef uint32_t (*Function)(uint32_t, const char*, size_t);
 
 static inline Function Choose_Extend() {
-  return isSSE42() ? (Function)ExtendImpl<FastCRC32Functor> : 
-                    (Function)ExtendImpl<SlowCRC32Functor>;
+#ifdef __SSE4_2__
+  if (isSSE42()) {
+    return (Function)ExtendImpl<FastCRC32Functor>;
+  }
+#endif
+#if defined(__riscv) && (__riscv_xlen == 64) && defined(__riscv_zbc)
+  if (isZbc()) {
+    return (Function)rv_crc32c_clmul;
+  }
+#endif
+  return (Function)ExtendImpl<SlowCRC32Functor>;
 }
 
 bool IsFastCrc32Supported() {
 #ifdef __SSE4_2__
-  return isSSE42();
-#else
-  return false;
+  if (isSSE42()) return true;
+#endif
+#if defined(__riscv) && (__riscv_xlen == 64) && defined(__riscv_zbc)
+  if (isZbc()) return true;
 #endif
+  return false;
 }
 
 uint32_t Extend(uint32_t crc, const char* buf, size_t size) {