|
| 1 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +# or more contributor license agreements. See the NOTICE file |
| 3 | +# distributed with this work for additional information |
| 4 | +# regarding copyright ownership. The ASF licenses this file |
| 5 | +# to you under the Apache License, Version 2.0 (the |
| 6 | +# "License"); you may not use this file except in compliance |
| 7 | +# with the License. You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, |
| 12 | +# software distributed under the License is distributed on an |
| 13 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +# KIND, either express or implied. See the License for the |
| 15 | +# specific language governing permissions and limitations |
| 16 | +# under the License. |
| 17 | + |
| 18 | +"""Vector global index reader using paimon-vindex.""" |
| 19 | + |
| 20 | +import os |
| 21 | +import threading |
| 22 | + |
| 23 | +import numpy as np |
| 24 | + |
| 25 | +from pypaimon.common.file_io import pread, supports_pread |
| 26 | +from pypaimon.globalindex.global_index_reader import GlobalIndexReader, _completed_future |
| 27 | +from pypaimon.globalindex.vector_search_result import DictBasedScoredIndexResult |
| 28 | + |
| 29 | +VINDEX_IDENTIFIERS = ("ivf-flat", "ivf-pq", "ivf-hnsw-flat", "ivf-hnsw-sq") |
| 30 | + |
| 31 | +NPROBE_PARAMETER = "ivf.nprobe" |
| 32 | +EF_SEARCH_PARAMETER = "hnsw.ef_search" |
| 33 | +DEFAULT_NPROBE = 16 |
| 34 | +DEFAULT_EF_SEARCH = 0 |
| 35 | + |
| 36 | + |
| 37 | +class PaimonVindexInput: |
| 38 | + """Input adapter required by paimon_vindex.VectorIndexReader.""" |
| 39 | + |
| 40 | + def __init__(self, stream): |
| 41 | + self._stream = stream |
| 42 | + self._supports_pread = supports_pread(stream) |
| 43 | + self._lock = threading.Lock() |
| 44 | + |
| 45 | + def pread_many(self, ranges): |
| 46 | + if self._supports_pread: |
| 47 | + return [pread(self._stream, length, offset) for offset, length in ranges] |
| 48 | + |
| 49 | + chunks = [] |
| 50 | + with self._lock: |
| 51 | + for offset, length in ranges: |
| 52 | + self._stream.seek(offset) |
| 53 | + chunks.append(self._stream.read(length)) |
| 54 | + return chunks |
| 55 | + |
| 56 | + |
| 57 | +class VindexVectorGlobalIndexReader(GlobalIndexReader): |
| 58 | + """Vector global index reader using paimon-vindex.""" |
| 59 | + |
| 60 | + def __init__(self, file_io, index_path, io_metas, options=None): |
| 61 | + assert len(io_metas) == 1, "Expected exactly one index file per shard" |
| 62 | + self._file_io = file_io |
| 63 | + self._index_path = index_path |
| 64 | + self._io_meta = io_metas[0] |
| 65 | + self._options = dict(options or {}) |
| 66 | + self._stream = None |
| 67 | + self._index_input = None |
| 68 | + self._reader = None |
| 69 | + self._metadata = None |
| 70 | + self._load_lock = threading.Lock() |
| 71 | + |
| 72 | + def visit_vector_search(self, vector_search): |
| 73 | + self._ensure_loaded() |
| 74 | + |
| 75 | + query = np.asarray(vector_search.vector, dtype=np.float32) |
| 76 | + if query.ndim != 1: |
| 77 | + raise ValueError("Query vector must be a one-dimensional float32 array") |
| 78 | + expected_dim = self._metadata.dimension |
| 79 | + if query.shape[0] != expected_dim: |
| 80 | + raise ValueError( |
| 81 | + "Query vector dimension mismatch: expected %d, got %d" |
| 82 | + % (expected_dim, query.shape[0])) |
| 83 | + |
| 84 | + effective_k = self._effective_k(vector_search) |
| 85 | + if effective_k <= 0: |
| 86 | + return _completed_future(None) |
| 87 | + |
| 88 | + options = vector_search.options or {} |
| 89 | + nprobe = _int_parameter(options, NPROBE_PARAMETER, DEFAULT_NPROBE) |
| 90 | + ef_search = _int_parameter(options, EF_SEARCH_PARAMETER, DEFAULT_EF_SEARCH) |
| 91 | + filter_bytes = _filter_bytes(vector_search.include_row_ids) |
| 92 | + |
| 93 | + ids, distances = self._reader.search( |
| 94 | + query, effective_k, nprobe, ef_search, filter_bytes=filter_bytes) |
| 95 | + id_to_scores = _build_scores(ids, distances, self._metadata.metric) |
| 96 | + if not id_to_scores: |
| 97 | + return _completed_future(None) |
| 98 | + return _completed_future(DictBasedScoredIndexResult(id_to_scores)) |
| 99 | + |
| 100 | + def vector_metric(self): |
| 101 | + self._ensure_loaded() |
| 102 | + return self._metadata.metric |
| 103 | + |
| 104 | + def _effective_k(self, vector_search): |
| 105 | + limit = vector_search.limit |
| 106 | + total_vectors = getattr(self._metadata, "total_vectors", limit) |
| 107 | + effective_k = min(limit, int(total_vectors)) |
| 108 | + include_row_ids = vector_search.include_row_ids |
| 109 | + if include_row_ids is not None: |
| 110 | + cardinality = include_row_ids.cardinality() |
| 111 | + if cardinality == 0: |
| 112 | + return 0 |
| 113 | + effective_k = min(effective_k, cardinality) |
| 114 | + return effective_k |
| 115 | + |
| 116 | + def _ensure_loaded(self): |
| 117 | + if self._reader is not None: |
| 118 | + return |
| 119 | + |
| 120 | + with self._load_lock: |
| 121 | + if self._reader is not None: |
| 122 | + return |
| 123 | + |
| 124 | + try: |
| 125 | + from paimon_vindex import VectorIndexReader |
| 126 | + except ImportError as e: |
| 127 | + raise ImportError( |
| 128 | + "paimon-vindex is required to read vindex vector indexes. " |
| 129 | + "Install paimon-vindex==0.1.0 or pypaimon[vindex].") from e |
| 130 | + |
| 131 | + file_path = (self._io_meta.external_path |
| 132 | + if self._io_meta.external_path |
| 133 | + else os.path.join(self._index_path, self._io_meta.file_name)) |
| 134 | + stream = self._file_io.new_input_stream(file_path) |
| 135 | + try: |
| 136 | + index_input = PaimonVindexInput(stream) |
| 137 | + reader = VectorIndexReader(index_input) |
| 138 | + self._metadata = reader.metadata() |
| 139 | + self._index_input = index_input |
| 140 | + self._reader = reader |
| 141 | + self._stream = stream |
| 142 | + except Exception: |
| 143 | + stream.close() |
| 144 | + raise |
| 145 | + |
| 146 | + def __enter__(self): |
| 147 | + return self |
| 148 | + |
| 149 | + def __exit__(self, exc_type, exc_val, exc_tb): |
| 150 | + self.close() |
| 151 | + return False |
| 152 | + |
| 153 | + def close(self): |
| 154 | + if self._reader is not None: |
| 155 | + self._reader.close() |
| 156 | + self._reader = None |
| 157 | + if self._stream is not None: |
| 158 | + self._stream.close() |
| 159 | + self._stream = None |
| 160 | + |
| 161 | + |
| 162 | +def _filter_bytes(include_row_ids): |
| 163 | + if include_row_ids is None: |
| 164 | + return None |
| 165 | + if include_row_ids.cardinality() == 0: |
| 166 | + return None |
| 167 | + return include_row_ids.serialize() |
| 168 | + |
| 169 | + |
| 170 | +def _build_scores(ids, distances, metric): |
| 171 | + id_to_scores = {} |
| 172 | + for row_id, distance in zip(ids, distances): |
| 173 | + row_id = int(row_id) |
| 174 | + if row_id < 0: |
| 175 | + continue |
| 176 | + id_to_scores[row_id] = _convert_distance_to_score(float(distance), metric) |
| 177 | + return id_to_scores |
| 178 | + |
| 179 | + |
| 180 | +def _convert_distance_to_score(distance, metric): |
| 181 | + if metric == "l2": |
| 182 | + return 1.0 / (1.0 + distance) |
| 183 | + if metric == "cosine": |
| 184 | + return 1.0 - distance |
| 185 | + if metric == "inner_product": |
| 186 | + return distance |
| 187 | + raise ValueError("Unknown vector search metric: %s" % metric) |
| 188 | + |
| 189 | + |
| 190 | +def _int_parameter(options, key, default_value): |
| 191 | + value = options.get(key) |
| 192 | + if value is None: |
| 193 | + return default_value |
| 194 | + try: |
| 195 | + return int(value) |
| 196 | + except ValueError as e: |
| 197 | + raise ValueError( |
| 198 | + "Invalid value for '%s': %s. Must be an integer." % (key, value)) from e |
0 commit comments