Index serialization backward compatibility test

.github/actions/build_cmake/action.yml

@@ -16,10 +16,19 @@ inputs:
     description: 'Enable ROCm support.'
     required: false
     default: OFF
+  setup_conda:
+    description: 'Setup miniconda environment.'
+    required: false
+    default: 'true'
+  upload_artifacts:
+    description: 'Upload test artifacts. Prevents collisions when multiple jobs need to run build_cmake.'
+    required: false
+    default: 'true'
 runs:
   using: composite
   steps:
     - name: Setup miniconda
+      if: inputs.setup_conda == 'true'
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.11'
@@ -186,7 +195,7 @@ runs:
         FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs $CONDA/bin/python -c "import faiss" 2>&1 | grep faiss.so
         LD_DEBUG=libs $CONDA/bin/python -c "import faiss" 2>&1 | grep faiss_avx2.so
     - name: Upload test results
-      if: always()
+      if: inputs.upload_artifacts == 'true'
       uses: actions/upload-artifact@v4
       with:
         name: test-results-arch=${{ runner.arch }}-opt=${{ inputs.opt_level }}-gpu=${{ inputs.gpu }}-cuvs=${{ inputs.cuvs }}-rocm=${{ inputs.rocm }}

.github/workflows/build-pull-request.yml

@@ -144,3 +144,7 @@ jobs:
           fetch-tags: true
       - name: Build and Package (conda)
         uses: ./.github/actions/build_conda
+  index-io-backward-compatibility:
+    needs: linux-x86_64-cmake
+    name: Index serialization backward compatibility
+    uses: ./.github/workflows/index-io-backward-compatibility.yml

.github/workflows/index-io-backward-compatibility.yml

@@ -0,0 +1,155 @@
+name: Index serialization backward compatibility test
+
+on:
+  workflow_call:
+
+env:
+  OMP_NUM_THREADS: '10'
+  MKL_THREADING_LAYER: GNU
+
+jobs:
+  # Scenario 1: CMake writes files, then Conda reads them
+  cmake-write-conda-read:
+    name: CMake Write -> Conda Read
+    runs-on: ubuntu-latest
+    env:
+      # run_id does not change if we re-run the job, so this is safe for
+      # the partway-failure-rerun scenario.
+      SHARED_DATA_DIR: /tmp/faiss-serialization-backward-compatibility-${{ github.run_id }}-cmake-write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      # Step 1: Build with CMake and write files
+      - name: Build with CMake
+        uses: ./.github/actions/build_cmake
+        with:
+          upload_artifacts: 'false'
+
+      - name: Create shared data directory
+        shell: bash
+        run: |
+          mkdir -p ${{ env.SHARED_DATA_DIR }}
+          chmod 777 ${{ env.SHARED_DATA_DIR }}
+
+      - name: Run CMake writer (write Faiss index and metadata)
+        shell: bash
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate
+          $CONDA/bin/python tests/index_io_backward_compatibility/cmake_writer.py ${{ env.SHARED_DATA_DIR }}
+
+      - name: Verify files were written
+        shell: bash
+        run: |
+          echo "Files created by CMake build:"
+          ls -lh ${{ env.SHARED_DATA_DIR }}
+
+      # Step 2: Install conda faiss-cpu and read files
+      - name: Clean cmake-built packages
+        shell: bash
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate
+          # Remove packages that conflict with faiss-cpu
+          conda remove -y numpy scipy pytest gflags swig cmake make mkl mkl-devel || true
+
+      - name: Install faiss-cpu from pytorch channel
+        shell: bash
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate
+          conda list
+          conda install -y -c pytorch faiss-cpu=1.13.1
+          conda list
+
+      - name: Run Conda reader (read Faiss index and verify)
+        shell: bash
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate
+          python tests/index_io_backward_compatibility/conda_reader.py ${{ env.SHARED_DATA_DIR }}
+
+      - name: Upload artifacts from cmake->conda test
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: cmake-write-conda-read-data
+          path: ${{ env.SHARED_DATA_DIR }}
+
+  # Scenario 2: Conda writes files, then CMake reads them
+  conda-write-cmake-read:
+    name: Conda Write -> CMake Read
+    runs-on: ubuntu-latest
+    env:
+      SHARED_DATA_DIR: /tmp/faiss-serialization-backward-compatibility-${{ github.run_id }}-conda-write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          fetch-tags: true
+
+      # Step 1: Install conda faiss-cpu package and write files
+      - name: Setup miniconda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: '3.11'
+          miniforge-version: latest
+
+      - name: Install faiss-cpu from pytorch channel
+        shell: bash
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate
+          # Install pre-built faiss-cpu
+          conda install -y -c pytorch faiss-cpu=1.13.1
+          conda list
+
+      - name: Create shared data directory
+        shell: bash
+        run: |
+          mkdir -p ${{ env.SHARED_DATA_DIR }}
+          chmod 777 ${{ env.SHARED_DATA_DIR }}
+
+      - name: Run Conda writer (write Faiss index and metadata)
+        shell: bash
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate
+          python tests/index_io_backward_compatibility/conda_writer.py ${{ env.SHARED_DATA_DIR }}
+
+      - name: Verify files were written
+        shell: bash
+        run: |
+          echo "Files created by Conda build:"
+          ls -lh ${{ env.SHARED_DATA_DIR }}
+
+      # Step 2: Rebuild with CMake and read files
+      - name: Clean conda artifacts
+        shell: bash
+        run: |
+          # Uninstall conda-built faiss to avoid conflicts
+          eval "$(conda shell.bash hook)"
+          conda activate
+          conda uninstall -y faiss-cpu || true
+
+      - name: Build with CMake
+        uses: ./.github/actions/build_cmake
+        with:
+          setup_conda: 'false'
+          upload_artifacts: 'false'
+
+      - name: Run CMake reader (read Faiss index and verify)
+        shell: bash
+        run: |
+          eval "$(conda shell.bash hook)"
+          conda activate
+          $CONDA/bin/python tests/index_io_backward_compatibility/cmake_reader.py ${{ env.SHARED_DATA_DIR }}
+
+      - name: Upload artifacts from conda->cmake test
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: conda-write-cmake-read-data
+          path: ${{ env.SHARED_DATA_DIR }}

tests/index_io_backward_compatibility/__init__.py

@@ -0,0 +1,11 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+"""
+Index serialization testing package for Faiss.
+
+This package contains test scripts for validating file I/O compatibility
+between cmake-built and conda-built Faiss installations.
+"""

tests/index_io_backward_compatibility/cmake_reader.py

@@ -0,0 +1,27 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+#!/usr/bin/env python3
+"""
+CMake Reader: Tests deserialization of conda-written index files using
+cmake-built Faiss.
+Validates that all index types written by conda can be loaded and searched
+without crashing.
+"""
+
+import sys
+from common_io import read_test_all_files
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python cmake_reader.py <input_directory>")
+        sys.exit(1)
+
+    input_dir = sys.argv[1]
+    exit_code = read_test_all_files(
+        reader="cmake", writer="conda", input_dir=input_dir
+    )
+    sys.exit(exit_code)

tests/index_io_backward_compatibility/cmake_writer.py

@@ -0,0 +1,27 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+#!/usr/bin/env python3
+"""
+CMake Writer: Tests serialization of all Faiss index types using cmake-built
+Faiss.
+Creates index files that will be read by conda-built Faiss to test index
+serialization backward compatibility.
+"""
+
+import sys
+from common_io import write_test_all_files
+
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: python cmake_writer.py <output_directory>")
+        sys.exit(1)
+
+    output_dir = sys.argv[1]
+    exit_code = write_test_all_files(
+        writer="cmake", output_dir=output_dir, seed=1234
+    )
+    sys.exit(exit_code)