Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions faiss/impl/index_read.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2522,13 +2522,14 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
}
#ifdef FAISS_ENABLE_SVS
else if (
h == fourcc("ILVQ") || h == fourcc("ISVL") || h == fourcc("ISVD")) {
h == fourcc("ILVQ") || h == fourcc("ISVL") || h == fourcc("ISVD") ||
h == fourcc("ISV2")) {
std::unique_ptr<IndexSVSVamana> svs;
if (h == fourcc("ILVQ")) {
svs = std::make_unique<IndexSVSVamanaLVQ>();
} else if (h == fourcc("ISVL")) {
svs = std::make_unique<IndexSVSVamanaLeanVec>();
} else if (h == fourcc("ISVD")) {
} else if (h == fourcc("ISVD") || h == fourcc("ISV2")) {
svs = std::make_unique<IndexSVSVamana>();
}

Expand Down Expand Up @@ -2580,6 +2581,11 @@ std::unique_ptr<Index> read_index_up(IOReader* f, int io_flags) {
leanvec->deserialize_training_data(is);
}
}
if (h == fourcc("ISV2")) {
READVECTOR(svs->stored_vectors);
} else if (h == fourcc("ISVD")) {
svs->stored_vectors_valid = false;
}
idx = std::move(svs);
} else if (h == fourcc("ISVF")) {
auto svs = std::make_unique<IndexSVSFlat>();
Expand Down
6 changes: 6 additions & 0 deletions faiss/impl/index_write.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1027,6 +1027,8 @@ void write_index(const Index* idx, IOWriter* f, int io_flags) {
h = fourcc("ILVQ"); // LVQ
} else if (lean != nullptr) {
h = fourcc("ISVL"); // LeanVec
} else if (svs->stored_vectors_valid && !svs->stored_vectors.empty()) {
h = fourcc("ISV2"); // uncompressed + stored_vectors
} else {
h = fourcc("ISVD"); // uncompressed
}
Expand Down Expand Up @@ -1069,6 +1071,10 @@ void write_index(const Index* idx, IOWriter* f, int io_flags) {
os.flush();
}
}

if (h == fourcc("ISV2")) {
WRITEVECTOR(svs->stored_vectors);
}
} else if (
const IndexSVSFlat* svs = dynamic_cast<const IndexSVSFlat*>(idx)) {
uint32_t h = fourcc("ISVF");
Expand Down
12 changes: 12 additions & 0 deletions faiss/index_factory.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,11 @@ char get_trains_alone(const Index* coarse_quantizer) {
if (dynamic_cast<const IndexHNSWFlat*>(coarse_quantizer)) {
return 2;
}
#ifdef FAISS_ENABLE_SVS
if (dynamic_cast<const IndexSVSVamana*>(coarse_quantizer)) {
return 2;
}
#endif
return 2; // for complicated indexes, we assume they can't be used as a
// kmeans index
}
Expand Down Expand Up @@ -299,6 +304,13 @@ Index* parse_coarse_quantizer(
int R = std::stoi(sm[2]);
return new IndexNSGFlat(d, R, mt);
}
#ifdef FAISS_ENABLE_SVS
if (match("IVF([0-9]+[kM]?)_SVSVamana([0-9]*)")) {
nlist = parse_nlist(sm[1].str());
int degree = sm[2].length() > 0 ? std::stoi(sm[2]) : 32;
return new IndexSVSVamana(d, degree, mt);
}
#endif
if (match("IVF([0-9]+[kM]?)\\(Index([0-9])\\)")) {
nlist = parse_nlist(sm[1].str());
int no = std::stoi(sm[2].str());
Expand Down
22 changes: 22 additions & 0 deletions faiss/svs/IndexSVSVamana.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
#include <svs/runtime/vamana_index.h>

#include <cstddef>
#include <cstring>
#include <numeric>
#include <span>
#include <type_traits>
Expand Down Expand Up @@ -114,13 +115,32 @@ void IndexSVSVamana::add(idx_t n, const float* x) {
if (!status.ok()) {
FAISS_THROW_MSG(status.message());
}

if (stored_vectors_valid) {
size_t prev = static_cast<size_t>(ntotal) * d;
stored_vectors.resize(prev + static_cast<size_t>(n) * d);
std::memcpy(stored_vectors.data() + prev, x, sizeof(float) * n * d);
}
ntotal += n;
}

void IndexSVSVamana::reconstruct(idx_t key, float* recons) const {
FAISS_THROW_IF_NOT_MSG(
key >= 0 && key < ntotal,
"IndexSVSVamana::reconstruct: key out of range");
FAISS_THROW_IF_NOT_MSG(
stored_vectors_valid && !stored_vectors.empty(),
"IndexSVSVamana::reconstruct: stored_vectors unavailable "
"(invalidated by remove_ids or not restored after deserialization)");
std::memcpy(recons, stored_vectors.data() + key * d, sizeof(float) * d);
}

void IndexSVSVamana::reset() {
if (impl) {
impl->reset();
}
stored_vectors.clear();
stored_vectors_valid = true;
is_trained = false;
ntotal = 0;
}
Expand Down Expand Up @@ -189,6 +209,8 @@ size_t IndexSVSVamana::remove_ids(const IDSelector& sel) {
size_t removed = 0;
auto Status = impl->remove_selected(&removed, id_filter);
ntotal -= removed;
stored_vectors.clear();
stored_vectors_valid = false;
return removed;
}

Expand Down
9 changes: 9 additions & 0 deletions faiss/svs/IndexSVSVamana.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include <svs/runtime/dynamic_vamana_index.h>

#include <iostream>
#include <vector>

namespace faiss {

Expand Down Expand Up @@ -108,6 +109,8 @@ struct IndexSVSVamana : Index {

void add(idx_t n, const float* x) override;

void reconstruct(idx_t key, float* recons) const override;

void search(
idx_t n,
const float* x,
Expand All @@ -134,6 +137,12 @@ struct IndexSVSVamana : Index {
/* The actual SVS implementation */
svs_runtime::DynamicVamanaIndex* impl{nullptr};

// The SVS runtime API does not expose vector retrieval, so we keep a copy
// of added vectors to support reconstruct(). When used as a coarse
// quantizer this holds only nlist centroids.
std::vector<float> stored_vectors;
bool stored_vectors_valid{true};

protected:
/* Initializes the implementation*/
virtual void create_impl();
Expand Down
Loading