Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions c/src/neighbors/cagra.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ static void merge_indices_for_layout(
cuvs::neighbors::cagra::detail::merged_dataset_size<T, uint32_t, DatasetViewT>(
*res_ptr, index_ptrs, row_filter);
auto const dim = static_cast<uint32_t>(index_ptrs.front()->dim());
auto const stride = static_cast<int64_t>(index_ptrs.front()->dataset().stride());
auto const stride = static_cast<int64_t>(index_ptrs.front()->dataset().data_view().stride());

try {
auto matrix = raft::make_device_matrix<T, int64_t>(*res_ptr, final_row_count, stride);
Expand Down Expand Up @@ -192,8 +192,8 @@ static void merge_indices_for_layout(
auto const& input = index->dataset();
raft::copy_matrix(matrix.data_handle() + row_offset * static_cast<std::size_t>(stride),
static_cast<std::size_t>(stride),
input.view().data_handle(),
static_cast<std::size_t>(input.stride()),
input.data_view().data_handle(),
static_cast<std::size_t>(input.data_view().stride()),
static_cast<std::size_t>(dim),
static_cast<std::size_t>(input.n_rows()),
stream);
Expand Down Expand Up @@ -1159,7 +1159,7 @@ void get_dataset_view(cuvsCagraIndex_t index, DLManagedTensor* dataset)
box,
"cuvsCagraIndexGetDataset: null index handle",
"cuvsCagraIndexGetDataset: host indices are allowed",
[&](auto& idx) { cuvs::core::to_dlpack(idx.dataset().view(), dataset); });
[&](auto& idx) { cuvs::core::to_dlpack(idx.dataset().data_view(), dataset); });
}

template <typename T, typename IdxT>
Expand Down
8 changes: 5 additions & 3 deletions c/src/preprocessing/quantize/pq.cpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION.
* SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand Down Expand Up @@ -244,7 +244,8 @@ extern "C" cuvsError_t cuvsProductQuantizerGetPqCodebook(cuvsProductQuantizer_t
if (quantizer->dtype.code == kDLFloat && quantizer->dtype.bits == 32) {
auto pq_mdspan =
(reinterpret_cast<cuvs::preprocessing::quantize::pq::quantizer<float>*>(quant_addr))
->vpq_codebooks.pq_code_book.view();
->vpq_codebooks.dictionary_view()
.pq_code_book;
cuvs::core::to_dlpack(pq_mdspan, pq_codebook);
} else {
RAFT_FAIL("Unsupported quantizer dtype: %d and bits: %d",
Expand All @@ -266,7 +267,8 @@ extern "C" cuvsError_t cuvsProductQuantizerGetVqCodebook(cuvsProductQuantizer_t
if (quantizer->dtype.code == kDLFloat && quantizer->dtype.bits == 32) {
auto pq_mdspan =
(reinterpret_cast<cuvs::preprocessing::quantize::pq::quantizer<float>*>(quant_addr))
->vpq_codebooks.vq_code_book.view();
->vpq_codebooks.dictionary_view()
.vq_code_book;
cuvs::core::to_dlpack(pq_mdspan, vq_codebook);
} else {
RAFT_FAIL("Unsupported quantizer dtype: %d and bits: %d",
Expand Down
1,030 changes: 562 additions & 468 deletions cpp/include/cuvs/neighbors/common.hpp

Large diffs are not rendered by default.

10 changes: 5 additions & 5 deletions cpp/include/cuvs/preprocessing/quantize/pq.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -302,13 +302,13 @@ template <typename SrcT>
SrcT const& src)
-> cuvs::neighbors::device_vpq_dataset<half, int64_t>
{
// A cuVS dataset keeps its logical width in `dim()` while `view()` spans the full row pitch.
// A cuVS dataset keeps its logical width in `dim()` while `data_view()` spans the full row
// pitch.
if constexpr (requires {
src.view();
src.data_view();
src.dim();
src.stride();
}) {
auto const rows = src.view();
auto const rows = src.data_view();
using value_type = typename decltype(rows)::value_type;
using extents_type = raft::matrix_extent<int64_t>;
return make_vpq_dataset(
Expand All @@ -317,7 +317,7 @@ template <typename SrcT>
raft::mdspan<const value_type, extents_type, raft::layout_stride>{
rows.data_handle(),
raft::make_strided_layout(extents_type{rows.extent(0), int64_t{src.dim()}},
cuda::std::array<int64_t, 2>{int64_t{src.stride()}, 1})});
cuda::std::array<int64_t, 2>{int64_t{rows.stride()}, 1})});
} else {
using value_type = typename SrcT::value_type;
static_assert(std::is_same_v<value_type, float> || std::is_same_v<value_type, half> ||
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/neighbors/cagra.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ CUVS_EXPORT void index<T, IdxT, DatasetViewT>::compute_dataset_norms_(raft::reso

if constexpr (nb::is_padded_dataset_view_v<DatasetViewT> ||
nb::is_standard_dataset_view_v<DatasetViewT>) {
rm_dataset = dataset_.view();
rm_dataset = dataset_.data_view().view();
} else if constexpr (nb::is_vpq_dataset_view_v<DatasetViewT>) {
skip_norms = true;
}
Expand Down
2 changes: 1 addition & 1 deletion cpp/src/neighbors/detail/cagra/add_nodes.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ void extend_core(raft::resources const& handle,
"cuvs::neighbors::hnsw::deserialize() before calling extend().");

const std::size_t initial_dataset_size = index.size();
const auto extended_view = extended_dataset.view();
const auto extended_view = extended_dataset.data_view();
const std::size_t new_dataset_size = static_cast<std::size_t>(extended_view.extent(0));
const std::size_t degree = index.graph_degree();
const std::size_t dim = index.dim();
Expand Down
10 changes: 5 additions & 5 deletions cpp/src/neighbors/detail/cagra/cagra_build.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -1341,7 +1341,7 @@ auto build_ace(raft::resources const& res, const index_params& params, DatasetVi
params.graph_degree,
npartitions);

auto dataset_view = dataset.view();
auto dataset_view = dataset.data_view();
size_t dataset_size = dataset.n_rows();
size_t dataset_dim = dataset.dim();

Expand Down Expand Up @@ -2290,7 +2290,7 @@ auto ensure_device_padded_for_iterative_search(
if constexpr (cuvs::neighbors::is_device_padded_dataset_view_v<DatasetViewT>) {
return dataset;
} else {
padded_own = cuvs::neighbors::make_device_padded_dataset(res, dataset.view());
padded_own = cuvs::neighbors::make_device_padded_dataset(res, dataset.data_view());
return padded_own->as_dataset_view();
}
}
Expand All @@ -2315,7 +2315,7 @@ auto iterative_build_graph(raft::resources const& res,
std::unique_ptr<cuvs::neighbors::device_padded_dataset<T, int64_t>> padded_own;
auto search_dataset = ensure_device_padded_for_iterative_search<T>(res, dataset, padded_own);

auto dev_dataset = search_dataset.view();
auto dev_dataset = search_dataset.data_view();
uint32_t logical_dim = search_dataset.dim();

// Determine initial graph size.
Expand Down Expand Up @@ -2636,7 +2636,7 @@ auto build_from_host_matrix(raft::resources const& res,
static_cast<int64_t>(n_rows),
intermediate_degree,
graph_degree,
dataset.view());
dataset.data_view());
}();

RAFT_LOG_TRACE("Graph optimized, creating index");
Expand Down Expand Up @@ -2688,7 +2688,7 @@ auto build_from_device_matrix(raft::resources const& res,
device_dataset.n_rows(),
intermediate_degree,
graph_degree,
device_dataset.view());
device_dataset.data_view());
}();

RAFT_LOG_TRACE("Graph optimized, creating index");
Expand Down
24 changes: 12 additions & 12 deletions cpp/src/neighbors/detail/cagra/cagra_merge.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -91,10 +91,10 @@ cuvs::neighbors::cagra::index<T, IdxT, DatasetViewT> merge_rebuild(
"before merge.");
if (dim == 0) {
dim = index->dim();
stride = static_cast<int64_t>(dataset.stride());
stride = static_cast<int64_t>(dataset.data_view().stride());
} else {
RAFT_EXPECTS(dim == index->dim(), "Dimension of datasets in indices must be equal.");
RAFT_EXPECTS(stride == static_cast<int64_t>(dataset.stride()),
RAFT_EXPECTS(stride == static_cast<int64_t>(dataset.data_view().stride()),
"Row stride of datasets in indices must be equal.");
}
merged_rows += static_cast<int64_t>(index->size());
Expand All @@ -116,12 +116,12 @@ cuvs::neighbors::cagra::index<T, IdxT, DatasetViewT> merge_rebuild(
"merged_dataset dimension (%u) must equal the input dimension (%u)",
unsigned(merged_dataset.dim()),
unsigned(dim));
RAFT_EXPECTS(merged_dataset.stride() == stride,
RAFT_EXPECTS(merged_dataset.data_view().stride() == stride,
"merged_dataset stride (%u) must equal the input stride (%ld)",
unsigned(merged_dataset.stride()),
unsigned(merged_dataset.data_view().stride()),
long(stride));

auto output_const_view = merged_dataset.view();
auto output_const_view = merged_dataset.data_view();
auto output_view = raft::make_device_matrix_view<T, int64_t>(
const_cast<T*>(output_const_view.data_handle()), final_rows, stride);

Expand All @@ -132,7 +132,7 @@ cuvs::neighbors::cagra::index<T, IdxT, DatasetViewT> merge_rebuild(
std::size_t n_rows = 0;
auto const& v = index->dataset();
if constexpr (cuvs::neighbors::is_dense_row_major_dataset_view_v<std::decay_t<decltype(v)>>) {
src_ptr = v.view().data_handle();
src_ptr = v.data_view().data_handle();
n_rows = static_cast<std::size_t>(v.n_rows());
} else {
RAFT_FAIL("cagra::merge: unexpected dataset type while copying rows");
Expand Down Expand Up @@ -300,14 +300,14 @@ auto preflight_fastener(
}
if (result.offsets.size() == 1) {
result.dim = static_cast<int64_t>(index->dim());
result.stride = static_cast<int64_t>(dataset.stride());
result.stride = static_cast<int64_t>(dataset.data_view().stride());
} else {
if (result.dim != static_cast<int64_t>(index->dim())) {
return reject("all input dimensions must match");
}
// The merged dataset has a single row pitch, so mixed input strides cannot be consolidated
// without re-padding each input separately.
if (result.stride != static_cast<int64_t>(dataset.stride())) {
if (result.stride != static_cast<int64_t>(dataset.data_view().stride())) {
return reject("all input row strides must match");
}
}
Expand Down Expand Up @@ -382,8 +382,8 @@ void copy_input_datasets(
auto const& source = indices[i]->dataset();
raft::copy_matrix(destination + offsets[i] * destination_stride,
static_cast<std::size_t>(destination_stride),
source.view().data_handle(),
static_cast<std::size_t>(source.stride()),
source.data_view().data_handle(),
static_cast<std::size_t>(source.data_view().stride()),
static_cast<std::size_t>(dim),
static_cast<std::size_t>(source.n_rows()),
raft::resource::get_cuda_stream(handle));
Expand All @@ -399,7 +399,7 @@ auto merge_fastener(raft::resources const& handle,
fastener_preflight_result const& preflight)
-> cuvs::neighbors::cagra::index<T, IdxT, DatasetViewT>
{
auto const stride = static_cast<int64_t>(merged_dataset.stride());
auto const stride = static_cast<int64_t>(merged_dataset.data_view().stride());
RAFT_EXPECTS(merged_dataset.n_rows() == preflight.rows,
"merged_dataset rows (%ld) must equal the merged row count (%ld)",
long(merged_dataset.n_rows()),
Expand All @@ -409,7 +409,7 @@ auto merge_fastener(raft::resources const& handle,
unsigned(merged_dataset.dim()),
long(preflight.dim));

auto const output_const_view = merged_dataset.view();
auto const output_const_view = merged_dataset.data_view();
auto* destination = const_cast<T*>(output_const_view.data_handle());
{
raft::common::nvtx::range<cuvs::common::nvtx::domain::cuvs> scope("cagra::merge/consolidate");
Expand Down
10 changes: 5 additions & 5 deletions cpp/src/neighbors/detail/cagra/cagra_search.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,13 @@ void search_main_core(
std::unique_ptr<cuvs::neighbors::device_padded_dataset<DataT, int64_t>> queries_padded_own;
if (cuvs::neighbors::matrix_row_width_matches_cagra_required(queries)) {
auto v = cuvs::neighbors::make_device_padded_dataset_view(res, queries);
queries_buf = v.view().data_handle();
query_row_stride = v.stride();
queries_buf = v.data_view().data_handle();
query_row_stride = v.data_view().stride();
} else {
queries_padded_own = cuvs::neighbors::make_device_padded_dataset(res, queries);
auto v = queries_padded_own->as_dataset_view();
queries_buf = v.view().data_handle();
query_row_stride = v.stride();
queries_buf = v.data_view().data_handle();
query_row_stride = v.data_view().stride();
}
const bool can_batch_n_queries = (query_row_stride == query_dim);

Expand Down Expand Up @@ -245,7 +245,7 @@ void search_main(raft::resources const& res,
params.smem_dtype = cuvs::neighbors::cagra::internal_dtype::F16;
}
auto desc = dataset_descriptor_init_with_cache<T, graph_idx_type, DistanceT>(
res, params, vv.dset(), index.metric(), nullptr);
res, params, vv, index.metric(), nullptr);
search_main_core<T, graph_idx_type, DistanceT, CagraSampleFilterT, IdxT, OutputIdxT>(
res,
params,
Expand Down
24 changes: 12 additions & 12 deletions cpp/src/neighbors/detail/cagra/cagra_serialize.cuh
Original file line number Diff line number Diff line change
Expand Up @@ -293,8 +293,8 @@ void write_hnswlib_rows_host(
"CAGRA dataset rows (%zu) do not match index size (%zu)",
static_cast<size_t>(dataset_view.n_rows()),
n_rows);
dataset_data = dataset_view.view().data_handle();
dataset_stride = dataset_view.stride();
dataset_data = dataset_view.data_view().data_handle();
dataset_stride = dataset_view.data_view().stride();
dataset_is_device = is_device_cagra_hnsw_serialize_index_v<T, IdxT, CagraIndexT>;
}

Expand Down Expand Up @@ -427,16 +427,16 @@ void write_hnswlib_rows_device(raft::resources const& res,
for (size_t first_row = 0; first_row < n_rows; first_row += batch_rows) {
auto const rows = std::min(batch_rows, n_rows - first_row);
auto const blocks = (rows + warps_per_block - 1) / warps_per_block;
pack_hnswlib_rows<T, IdxT>
<<<static_cast<unsigned int>(blocks), block_size, 0, stream>>>(output.data_handle(),
row_size,
graph.data_handle(),
dataset.view().data_handle(),
first_row,
rows,
graph_degree,
dim,
dataset.stride());
pack_hnswlib_rows<T, IdxT><<<static_cast<unsigned int>(blocks), block_size, 0, stream>>>(
output.data_handle(),
row_size,
graph.data_handle(),
dataset.data_view().data_handle(),
first_row,
rows,
graph_degree,
dim,
dataset.data_view().stride());
RAFT_CUDA_TRY(cudaPeekAtLastError());
raft::resource::sync_stream(res);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ struct standard_descriptor_spec : public instance_spec<DataT, IndexT, DistanceT>
template <typename DatasetT>
constexpr static inline bool accepts_dataset()
{
return is_padded_dataset_v<DatasetT>;
return cuvs::neighbors::is_padded_dataset_view_v<DatasetT>;
}

template <typename DatasetT>
Expand All @@ -37,11 +37,12 @@ struct standard_descriptor_spec : public instance_spec<DataT, IndexT, DistanceT>
cuvs::distance::DistanceType metric,
const DistanceT* dataset_norms = nullptr) -> host_type
{
auto const data_view = dataset.data_view();
return init_(params,
dataset.view().data_handle(),
data_view.data_handle(),
IndexT(dataset.n_rows()),
dataset.dim(),
dataset.stride(),
data_view.stride(),
dataset_norms);
}

Expand Down
25 changes: 16 additions & 9 deletions cpp/src/neighbors/detail/cagra/compute_distance_vpq.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,16 +41,20 @@ struct vpq_descriptor_spec : public instance_spec<DataT, IndexT, DistanceT> {
using typename base_type::host_type;
using typename base_type::index_type;

// `DatasetT` here is the non-owning dataset_view (not the owning dataset) -- callers pass the
// view directly rather than reaching back through a `.dset()`-style owner pointer, so
// classification and the codebook element type are read off the view via its own dictionary
// state (`is_vpq_dataset_view_v`, `DatasetT::value_type`), not an owning-only trait/typedef.
template <typename DatasetT>
constexpr static inline auto accepts_dataset()
-> std::enable_if_t<is_vpq_dataset_v<DatasetT>, bool>
-> std::enable_if_t<cuvs::neighbors::is_vpq_dataset_view_v<DatasetT>, bool>
{
return std::is_same_v<typename DatasetT::math_type, CodebookT>;
return std::is_same_v<typename DatasetT::value_type, CodebookT>;
}

template <typename DatasetT>
constexpr static inline auto accepts_dataset()
-> std::enable_if_t<!is_vpq_dataset_v<DatasetT>, bool>
-> std::enable_if_t<!cuvs::neighbors::is_vpq_dataset_view_v<DatasetT>, bool>
{
return false;
}
Expand All @@ -61,11 +65,13 @@ struct vpq_descriptor_spec : public instance_spec<DataT, IndexT, DistanceT> {
cuvs::distance::DistanceType metric,
const DistanceT* dataset_norms = nullptr) -> host_type
{
auto const data_view = dataset.data_view();
auto const dict_view = dataset.dictionary_view();
return init_(params,
dataset.data.data_handle(),
dataset.encoded_row_length(),
dataset.vq_code_book.data_handle(),
dataset.pq_code_book.data_handle(),
data_view.data_handle(),
static_cast<uint32_t>(data_view.extent(1)),
dict_view.vq_code_book.data_handle(),
dict_view.pq_code_book.data_handle(),
IndexT(dataset.n_rows()),
dataset.dim());
}
Expand All @@ -79,8 +85,9 @@ struct vpq_descriptor_spec : public instance_spec<DataT, IndexT, DistanceT> {
if (params.team_size != 0 && TeamSize != params.team_size) { return -1.0; }
if (cuvs::distance::DistanceType::L2Expanded != metric) { return -1.0; }
// Match codebook params
if (dataset.pq_bits() != PqBits) { return -1.0; }
if (dataset.pq_len() != PqLen) { return -1.0; }
auto const dict_view = dataset.dictionary_view();
if (dict_view.pq_bits() != PqBits) { return -1.0; }
if (dict_view.pq_len() != PqLen) { return -1.0; }
if (select_supported_vpq_smem_dtype(params) != SmemDType) { return -1.0; }
// Keep auto-selection on the tuned VPQ diagonal while allowing explicit team_size requests to
// use the expanded team_size / dataset_block_dim grid.
Expand Down
Loading
Loading