From 920a91ac6ac3057e695d5998c0c58a63d1a30691 Mon Sep 17 00:00:00 2001 From: gergo Date: Sun, 7 Dec 2025 20:43:53 +0100 Subject: [PATCH 1/5] removed obsolete timeout --- .../io/gitlab/qfoundation/partition/PartitionManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/src/main/java/io/gitlab/qfoundation/partition/PartitionManagerImpl.java b/runtime/src/main/java/io/gitlab/qfoundation/partition/PartitionManagerImpl.java index 4fbb2113..d2719efe 100644 --- a/runtime/src/main/java/io/gitlab/qfoundation/partition/PartitionManagerImpl.java +++ b/runtime/src/main/java/io/gitlab/qfoundation/partition/PartitionManagerImpl.java @@ -125,7 +125,7 @@ public class PartitionManagerImpl implements PartitionManager { persistence.persist(new PartitionRegistration( new PartitionRegistrationId(subject.subsystem(), subject.topic(), nodeId), weight, Instant.now().plus(timeout))); - try (var scope = forker.createScope(Duration.ofSeconds(2))) { + try (var scope = forker.createScope()) { // Read current allocation table var oldFuture = scope.fork(() -> { -- GitLab From 241cde35a059de5e8032953ad064840f7487143b Mon Sep 17 00:00:00 2001 From: gergo Date: Mon, 8 Dec 2025 22:56:44 +0100 Subject: [PATCH 2/5] Tab based navigation on FDB dashboard --- .../QFoundationAdminResource/fdb.txt | 67 ++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/runtime/src/main/resources/templates/QFoundationAdminResource/fdb.txt b/runtime/src/main/resources/templates/QFoundationAdminResource/fdb.txt index 3078b337..2896e9b4 100644 --- a/runtime/src/main/resources/templates/QFoundationAdminResource/fdb.txt +++ b/runtime/src/main/resources/templates/QFoundationAdminResource/fdb.txt @@ -34,22 +34,59 @@

FoundationDB cluster dashboard

- +
+ + + +
-
+
{#include QFoundationAdminResource/fdb-cluster dbStatus=dbStatus /}
-- GitLab From edf1e3f784f709f706f2f74c2133276f32da6faa Mon Sep 17 00:00:00 2001 From: gergo Date: Tue, 9 Dec 2025 22:12:50 +0100 Subject: [PATCH 3/5] Automatic beam estimation and code cleanup --- .../vector/AutomatedRecallTest.java | 2 +- .../vector/recall/RecallTester.java | 2 +- .../qfoundation/vector/BeamSearcher.java | 28 ++++++++++- .../qfoundation/vector/SearchSettings.java | 50 +++++++++++++++++-- .../qfoundation/vector/VectorSearcher.java | 16 ++++++ .../vector/compression/MultiRaBitQ.java | 2 +- .../vector/maintenance/LireImpl.java | 4 +- .../vector/maintenance/Reassigner.java | 11 ++++ .../vector/maintenance/Reviser.java | 10 ++-- .../vector/maintenance/Splitter.java | 12 +++++ 10 files changed, 122 insertions(+), 15 deletions(-) diff --git a/deployment/src/test/java/io/gitlab/qfoundation/deployment/vector/AutomatedRecallTest.java b/deployment/src/test/java/io/gitlab/qfoundation/deployment/vector/AutomatedRecallTest.java index d5c39172..60d666d5 100644 --- a/deployment/src/test/java/io/gitlab/qfoundation/deployment/vector/AutomatedRecallTest.java +++ b/deployment/src/test/java/io/gitlab/qfoundation/deployment/vector/AutomatedRecallTest.java @@ -299,7 +299,7 @@ public class AutomatedRecallTest { int totalMatches = 0; int totalExpected = queryVectors.size() * k; - SearchSettings settings = new SearchSettings(k, 32, 20.0, true, reRanking); + SearchSettings settings = new SearchSettings(k, 32, 20.0, true, reRanking, Double.NaN); for (Vector query : queryVectors) { List results = QuarkusTransaction.requiringNew() diff --git a/deployment/src/test/java/io/gitlab/qfoundation/deployment/vector/recall/RecallTester.java b/deployment/src/test/java/io/gitlab/qfoundation/deployment/vector/recall/RecallTester.java index 6aa61774..14475e76 100644 --- a/deployment/src/test/java/io/gitlab/qfoundation/deployment/vector/recall/RecallTester.java +++ b/deployment/src/test/java/io/gitlab/qfoundation/deployment/vector/recall/RecallTester.java @@ -39,7 +39,7 @@ public class RecallTester { var searchResults = RetryUtil.withRetry(() -> QuarkusTransaction.requiringNew() .call(() -> { var index = persistence.getReference(VectorIndex.class, indexId); - return searcher.nearestNeighbors(index, sample.getVector(), new SearchSettings(k, beams, oversampling, true, true)); + return searcher.nearestNeighbors(index, sample.getVector(), new SearchSettings(k, beams, oversampling, true, true, Double.NaN)); }), 1000); long elapsed = System.currentTimeMillis() - started; diff --git a/runtime/src/main/java/io/gitlab/qfoundation/vector/BeamSearcher.java b/runtime/src/main/java/io/gitlab/qfoundation/vector/BeamSearcher.java index 68ee3cf5..0da2f8a0 100644 --- a/runtime/src/main/java/io/gitlab/qfoundation/vector/BeamSearcher.java +++ b/runtime/src/main/java/io/gitlab/qfoundation/vector/BeamSearcher.java @@ -28,6 +28,30 @@ public class BeamSearcher extends HierarchicalVectorSearcher { @Override protected List closestLeaves(VectorIndex index, VectorPartition rootNode, float[] queryVector, SearchSettings settings) { - return vectorIndexDao.findClosestLeaves(index, queryVector, settings.beams()); + + double targetRecall = settings.targetRecall(); + + int effectiveBeams; + + if (Double.isNaN(targetRecall) || targetRecall <= 0 || targetRecall > 1) { + effectiveBeams = settings.beams(); + } else { + int dimensions = index.getDimensions(); + int approximateVectorCount = rootNode.getVectorCount(); + double lnN = Math.log(approximateVectorCount); + double lnD = Math.log(dimensions); + double lnA = -1.2 + 0.45 * lnN - 0.6 * lnD; + double a = Math.exp(lnA); + double oneMinusR = 1.0 - targetRecall; + if (oneMinusR <= 0.0) { + effectiveBeams = 512; + } else { + double linearBeams = a / oneMinusR; + double conservativeBeams = linearBeams * 1.4; + effectiveBeams = (int) Math.round(Math.max(1.0, Math.min(512.0, conservativeBeams))); + } + } + + return vectorIndexDao.findClosestLeaves(index, queryVector, effectiveBeams); } -} \ No newline at end of file +} diff --git a/runtime/src/main/java/io/gitlab/qfoundation/vector/SearchSettings.java b/runtime/src/main/java/io/gitlab/qfoundation/vector/SearchSettings.java index 3cb0bee5..1ee5952d 100644 --- a/runtime/src/main/java/io/gitlab/qfoundation/vector/SearchSettings.java +++ b/runtime/src/main/java/io/gitlab/qfoundation/vector/SearchSettings.java @@ -1,9 +1,53 @@ package io.gitlab.qfoundation.vector; -public record SearchSettings(int k, int beams, double oversamplingFactor, boolean cache, boolean reRanking) { - public static SearchSettings DEFAULT = new SearchSettings(10, 64, 10.0d, true, true); +/** + * Configuration for approximate nearest neighbor search in hierarchical vector indexes. + * Used by {@link VectorSearcher#nearestNeighbors(VectorIndex, float[], SearchSettings)}. + *

+ * Key parameters: + *

    + *
  • {@link #k}: Number of neighbors to retrieve. + *
  • {@link #beams}/{@link #targetRecall}: Controls leaf probing in beam search. + *
  • {@link #oversamplingFactor}: Candidate multiplier for reranking. + *
  • {@link #cache}: Enable partition/leaf caching. + *
  • {@link #reRanking}: Enable bound-based filtering. + *
+ */ +public record SearchSettings(int k, int beams, double oversamplingFactor, boolean cache, boolean reRanking, + double targetRecall) { + /** + * Default search settings: k=10, beams=64, oversamplingFactor=10.0, cache=true, reRanking=true, targetRecall=NaN. + */ + public static SearchSettings DEFAULT = new SearchSettings(10, 64, 10.0d, true, true, Double.NaN); + + /** + * Create settings with custom k, using defaults for other parameters. + * @param k number of neighbors + * @return new SearchSettings + */ public static SearchSettings defaults(int k) { - return new SearchSettings(k, DEFAULT.beams(), DEFAULT.oversamplingFactor(), DEFAULT.cache(), DEFAULT.reRanking()); + return new SearchSettings(k, DEFAULT.beams(), DEFAULT.oversamplingFactor(), DEFAULT.cache(), DEFAULT.reRanking(), DEFAULT.targetRecall()); + } + + /** + * Create settings with custom k and beams, using defaults for other parameters. + * @param k number of neighbors + * @param beams number of beams for leaf selection + * @return new SearchSettings + */ + public static SearchSettings forBeams(int k, int beams) { + return new SearchSettings(k, beams, DEFAULT.oversamplingFactor(), DEFAULT.cache(), DEFAULT.reRanking(), DEFAULT.targetRecall()); } + + /** + * Create settings with custom k and target recall, using defaults for other parameters. + * @param k number of neighbors + * @param targetRecall target recall (0-1) to auto-tune beams + * @return new SearchSettings + */ + public static SearchSettings forRecall(int k, double targetRecall) { + return new SearchSettings(k, DEFAULT.beams(), DEFAULT.oversamplingFactor(), DEFAULT.cache(), DEFAULT.reRanking(), targetRecall); + } + } diff --git a/runtime/src/main/java/io/gitlab/qfoundation/vector/VectorSearcher.java b/runtime/src/main/java/io/gitlab/qfoundation/vector/VectorSearcher.java index 35181c41..7a0741fe 100644 --- a/runtime/src/main/java/io/gitlab/qfoundation/vector/VectorSearcher.java +++ b/runtime/src/main/java/io/gitlab/qfoundation/vector/VectorSearcher.java @@ -2,6 +2,22 @@ package io.gitlab.qfoundation.vector; import java.util.List; +/** + * Interface for approximate nearest neighbor search in hierarchical vector indexes. + *

+ * Implementations traverse a hierarchical partition structure to select candidate leaf partitions, + * then perform coarse search using quantized vector representations (MultiRaBitQ) with optional + * re-ranking, followed by exact distance computation on top-K candidates. + */ public interface VectorSearcher { + /** + * Searches for the nearest neighbors of a query vector in a hierarchical vector index. + * + * @param index the hierarchical vector index to search + * @param queryVector the input vector to find neighbors for; must match the index dimensions + * @param settings search configuration including max results (k), beam width, caching, and re-ranking + * @return a list of search results sorted by ascending distance, limited to k results + * @throws IllegalArgumentException if the query vector dimensions do not match the index dimensions + */ List nearestNeighbors(VectorIndex index, float[] queryVector, SearchSettings settings); } diff --git a/runtime/src/main/java/io/gitlab/qfoundation/vector/compression/MultiRaBitQ.java b/runtime/src/main/java/io/gitlab/qfoundation/vector/compression/MultiRaBitQ.java index 75357ecb..bae5efb5 100644 --- a/runtime/src/main/java/io/gitlab/qfoundation/vector/compression/MultiRaBitQ.java +++ b/runtime/src/main/java/io/gitlab/qfoundation/vector/compression/MultiRaBitQ.java @@ -13,7 +13,7 @@ import java.util.*; *

* Key features: * - 1-bit quantization: Binary codes based on residual vector signs - * - Multi-bit quantization: Additional precision bits (1-9 bits total) + * - Multi-bit quantization: Additional precision bits (2-9 bits total including the signs bit) * - Fast distance estimation without full reconstruction * - Rotation matrix support for improved quantization */ diff --git a/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/LireImpl.java b/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/LireImpl.java index 503df4fb..ed6901f2 100644 --- a/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/LireImpl.java +++ b/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/LireImpl.java @@ -128,11 +128,11 @@ public class LireImpl implements Lire { scope.fork(() -> meterRegistry.timer("qfoundation.vector.maintain.write.split", "dimensions", Integer.toString(index.getDimensions()), "type", "LEAF") .record(() -> saveSplits(index, partition, splitType, revised.revisedSplitContent()))); - // Insert violating vectors to neighbouring partitions + // Insert violating vectors to neighboring partitions scope.fork(() -> meterRegistry.timer("qfoundation.vector.maintain.write.insert", "dimensions", Integer.toString(index.getDimensions())) .record(() -> insertToNeighbours(index, revised.neighbouringInsertions()))); - // Delete violating vectors from neighbouring partitions + // Delete violating vectors from neighboring partitions scope.fork(() -> meterRegistry.timer("qfoundation.vector.maintain.write.delete", "dimensions", Integer.toString(index.getDimensions())) .record(() -> deleteFromNeighbours(index, revised.neighbouringDeletions()))); diff --git a/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Reassigner.java b/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Reassigner.java index 0e9f54de..5982769b 100644 --- a/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Reassigner.java +++ b/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Reassigner.java @@ -23,5 +23,16 @@ public interface Reassigner { */ List lireReassign(VectorIndex index, IdentifierAndCentroid oldPartition, Map> splitContent, List potentialNearbyPartitions); + /** + * Reclusters child partitions in the local area around a split internal partition. + * 1. Loads children from nearby partitions. + * 2. Combines with split content into seed assignments. + * 3. Performs k-means reclustering using WeightedKMeans for the configured number of rounds. + * + * @param index the vector index + * @param splitContent the initial assignments from the split (one list per new partition) + * @param potentialNearbyPartitions nearby partitions (same level) to include in reclustering + * @return the reclustered assignments for local partitions (split + nearby) + */ Map> reclusterReassign(VectorIndex index, Map> splitContent, List potentialNearbyPartitions); } diff --git a/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Reviser.java b/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Reviser.java index 4731ff45..fe1df4d0 100644 --- a/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Reviser.java +++ b/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Reviser.java @@ -8,11 +8,11 @@ import java.util.Map; public interface Reviser { /** - * Calculated the revised split and neighbouring changes. Processes the reassignment results. - * - If there is a move from a neighbouring partition into a split affected partition, we register a delete for the - * source and reflect the new item in the appropriate list of revisedSplitContent. - * - If there is a move from one of the affected split partitions to one of the neighbouring partitions, we remove - * the item form the revisedSplitContent and register an insert for the neighbouring partitions. + * Calculated the revised split and neighboring changes. Processes the reassignment results. + * - If there is a move from a neighboring partition into a split affected partition, we register a delete intent + * for the source and reflect the new item in the appropriate list of revisedSplitContent. + * - If there is a move from one of the affected split partitions to one of the neighboring partitions, we remove + * the item form the revisedSplitContent and register an insert for the neighboring partitions. * * @param reassignments reassignments to process * @param splitContent the allocations of the split to process diff --git a/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Splitter.java b/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Splitter.java index d16977eb..c496e422 100644 --- a/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Splitter.java +++ b/runtime/src/main/java/io/gitlab/qfoundation/vector/maintenance/Splitter.java @@ -4,6 +4,18 @@ import io.gitlab.qfoundation.vector.IdentifierAndCentroid; import java.util.List; +/** + * Strategy interface for splitting child partitions into subclusters during vector index maintenance. + * Implementations cluster the provided {@link IdentifierAndCentroid} children based on their vectors, + * typically using k-means. + */ public interface Splitter { + /** + * Splits the given list of child partitions into k clusters. + * + * @param childrenToSplit the child partitions to split, each with a centroid vector + * @param k the number of clusters to create (must be > 1 for splitting) + * @return list of k lists, each containing children assigned to a cluster; if k <= 1 or insufficient data, returns single list with all children + */ List> split(List childrenToSplit, int k); } -- GitLab From 9ce67df655146ac45a97791299a3cbee3b944634 Mon Sep 17 00:00:00 2001 From: gergo Date: Tue, 9 Dec 2025 23:00:06 +0100 Subject: [PATCH 4/5] Change vector dashboard chart size --- .../resources/templates/VectorDashboardResource/vectors.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/runtime/src/main/resources/templates/VectorDashboardResource/vectors.txt b/runtime/src/main/resources/templates/VectorDashboardResource/vectors.txt index 6bc0a999..82ba191a 100644 --- a/runtime/src/main/resources/templates/VectorDashboardResource/vectors.txt +++ b/runtime/src/main/resources/templates/VectorDashboardResource/vectors.txt @@ -7,7 +7,7 @@

Search Operations
-
+
Total Search Time
@@ -17,7 +17,7 @@ width="100%" height="90px" interval="PT1M" - items=120 + items=60 summing=false value=true rate=true @@ -154,7 +154,7 @@
-
+
Full Precision Reranking
-- GitLab From 56bd1c75b759458415e939f1897a9b393460356b Mon Sep 17 00:00:00 2001 From: gergo Date: Wed, 10 Dec 2025 23:21:48 +0100 Subject: [PATCH 5/5] Fix test timeout --- sample/src/main/resources/application.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/sample/src/main/resources/application.properties b/sample/src/main/resources/application.properties index 2ba4c03f..84146ebd 100644 --- a/sample/src/main/resources/application.properties +++ b/sample/src/main/resources/application.properties @@ -5,6 +5,7 @@ %demo.demo.items=3 quarkus.native.additional-build-args=-march=x86-64-v3,-Ob,--add-modules=jdk.incubator.vector,-H:+VectorAPISupport +quarkus.http.test-timeout=120s # PostgreSQL and XA config so FoundationDB can work together with other DataSources quarkus.datasource.db-kind=postgresql -- GitLab