SIMD optimization RaBitQ (facebookresearch#4515)

Summary:

This diff introduces a new file rabitq_simd.h with multiple SIMD-optimized implementations of the dot product calculation using population count (popcnt) operations:

 1. AVX-512 implementation with AVX512VPOPCNTDQ: Processes data in 512-bit (64-byte) chunks using dedicated AVX-512 popcnt instructions, with fallbacks to smaller vector sizes for remaining data.
2. AVX-512 fallback implementation without AVX512VPOPCNTDQ: Uses AVX512F instructions with a lookup-based popcount method for 512-bit vectors, falling back to smaller vectors for remaining data.

3.  AVX2 implementation: Uses a lookup-based popcount method with 256-bit (32-byte) AVX2 instructions, handling leftovers with 128-bit SSE operations and scalar processing.

4.  Scalar fallback: Processes data in 64-bit chunks with builtin popcount operations for systems without SIMD support.

Differential Revision: D79301607

Author: alibeklfc

faiss/impl/RaBitQuantizer.cpp

@@ -7,16 +7,16 @@
 
 #include <faiss/impl/RaBitQuantizer.h>
 
+#include <faiss/impl/FaissAssert.h>
+#include <faiss/utils/distances.h>
+#include <faiss/utils/rabitq_simd.h>
 #include <algorithm>
 #include <cmath>
 #include <cstring>
 #include <limits>
 #include <memory>
 #include <vector>
 
-#include <faiss/impl/FaissAssert.h>
-#include <faiss/utils/distances.h>
-
 namespace faiss {
 
 struct FactorsData {
@@ -351,27 +351,9 @@ float RaBitDistanceComputerQ::distance_to_code(const uint8_t* code) {
     const size_t di_8b = (d + 7) / 8;
     const size_t di_64b = (di_8b / 8) * 8;
 
-    uint64_t dot_qo = 0;
-    for (size_t j = 0; j < qb; j++) {
-        const uint8_t* query_j = rearranged_rotated_qq.data() + j * di_8b;
-
-        // process 64-bit popcounts
-        uint64_t count_dot = 0;
-        for (size_t i = 0; i < di_64b; i += 8) {
-            const auto qv = *(const uint64_t*)(query_j + i);
-            const auto yv = *(const uint64_t*)(binary_data + i);
-            count_dot += __builtin_popcountll(qv & yv);
-        }
-
-        // process leftovers
-        for (size_t i = di_64b; i < di_8b; i++) {
-            const auto qv = *(query_j + i);
-            const auto yv = *(binary_data + i);
-            count_dot += __builtin_popcount(qv & yv);
-        }
-
-        dot_qo += (count_dot << j);
-    }
+    // Use the optimized popcount function from rabitq_simd.h
+    float dot_qo =
+            rabitq_dp_popcnt(rearranged_rotated_qq.data(), binary_data, d, qb);
 
     // It was a willful decision (after the discussion) to not to pre-cache
     //   the sum of all bits, just in order to reduce the overhead per vector.