Skip to content

Commit 263c3a4

Browse files
costincostin
authored
Apply GCD bound transform to sorted numeric rangeIntoBitSet (#16285)
GCD- and delta-encoded multi-value SortedNumericDocValues decode every packed value during range evaluation. This transforms query bounds into the encoded domain and compares raw values directly, matching the recipe used for single-value NumericDocValues.
1 parent 4b982ff commit 263c3a4

4 files changed

Lines changed: 527 additions & 29 deletions

File tree

‎lucene/CHANGES.txt‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -341,6 +341,9 @@ Optimizations
341341

342342
* GITHUB#16352: Better cost() estimation for SkipBlockRangeIterator. (Alan Woodward)
343343

344+
* GITHUB#16285: Apply GCD bound transform to sorted numeric rangeIntoBitSet, comparing raw
345+
encoded values directly instead of decoding every packed value. (Costin Leau)
346+
344347
* GITHUB#16307: ReqExclBulkScorer now skips runs of excluded docs via
345348
TwoPhaseIterator#docIDRunEnd for two-phase excluded clauses (e.g. a doc-values
346349
not-equals filter), instead of advancing one doc at a time. (Jim Ferenczi)
@@ -358,18 +361,16 @@ Bug Fixes
358361
---------------------
359362
* GITHUB#16350: Disable bulk-scoring in monitor queries. (Alan Woodward)
360363

361-
* GITHUB#16295: SingletonSortedNumericDocValues now delegates rangeIntoBitSet
362-
to the wrapped NumericDocValues, enabling optimized range evaluation for
363-
single-valued sorted numeric fields. (Costin Leau)
364-
365-
<<<<<<< join_util_scores
366364
* GITHUB#16378: Accumulate join Total/Avg scores in double precision in
367365
TermsWithScoreCollector and JoinUtil's numeric point join, fixing
368366
intermittent failures caused by non-associative float addition. (Luca Cavanna)
369-
=======
367+
370368
* GITHUB#16296: Fix missing null check in RamUsageEstimator.sizeOf(Accountable)
371369
to be consistent with sizeOf(String) and sizeOf(Accountable[]). (Tim Grein)
372-
>>>>>>> main
370+
371+
* GITHUB#16295: SingletonSortedNumericDocValues now delegates rangeIntoBitSet
372+
to the wrapped NumericDocValues, enabling optimized range evaluation for
373+
single-valued sorted numeric fields. (Costin Leau)
373374

374375
Other
375376
---------------------

‎lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/SortedNumericGcdRangeIntoBitSetBenchmark.java‎

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.lucene.benchmark.jmh;
18+
19+
import java.io.IOException;
20+
import java.nio.file.Files;
21+
import java.nio.file.Path;
22+
import java.util.Comparator;
23+
import java.util.Random;
24+
import java.util.concurrent.TimeUnit;
25+
import java.util.stream.Stream;
26+
import org.apache.lucene.document.Document;
27+
import org.apache.lucene.document.Field;
28+
import org.apache.lucene.document.SortedNumericDocValuesField;
29+
import org.apache.lucene.document.StringField;
30+
import org.apache.lucene.index.DirectoryReader;
31+
import org.apache.lucene.index.IndexWriter;
32+
import org.apache.lucene.index.IndexWriterConfig;
33+
import org.apache.lucene.index.Term;
34+
import org.apache.lucene.search.BooleanClause.Occur;
35+
import org.apache.lucene.search.BooleanQuery;
36+
import org.apache.lucene.search.IndexSearcher;
37+
import org.apache.lucene.search.Query;
38+
import org.apache.lucene.search.TermQuery;
39+
import org.apache.lucene.store.Directory;
40+
import org.apache.lucene.store.MMapDirectory;
41+
import org.openjdk.jmh.annotations.Benchmark;
42+
import org.openjdk.jmh.annotations.BenchmarkMode;
43+
import org.openjdk.jmh.annotations.Fork;
44+
import org.openjdk.jmh.annotations.Level;
45+
import org.openjdk.jmh.annotations.Measurement;
46+
import org.openjdk.jmh.annotations.Mode;
47+
import org.openjdk.jmh.annotations.OutputTimeUnit;
48+
import org.openjdk.jmh.annotations.Param;
49+
import org.openjdk.jmh.annotations.Scope;
50+
import org.openjdk.jmh.annotations.Setup;
51+
import org.openjdk.jmh.annotations.State;
52+
import org.openjdk.jmh.annotations.TearDown;
53+
import org.openjdk.jmh.annotations.Warmup;
54+
55+
/**
56+
* Benchmarks range queries over GCD/delta-encoded sorted numeric doc values with multiple values
57+
* per doc.
58+
*/
59+
@State(Scope.Thread)
60+
@BenchmarkMode(Mode.Throughput)
61+
@OutputTimeUnit(TimeUnit.SECONDS)
62+
@Warmup(iterations = 3, time = 3)
63+
@Measurement(iterations = 5, time = 5)
64+
public class SortedNumericGcdRangeIntoBitSetBenchmark {
65+
66+
private static final String FIELD = "val";
67+
private static final String LEAD_FIELD = "lead";
68+
private static final String LEAD_VALUE = "yes";
69+
private static final long DOMAIN = 10_000_000L;
70+
private static final long DELTA = 1_700_000_000_000L;
71+
72+
private Directory dir;
73+
private DirectoryReader reader;
74+
private IndexSearcher searcher;
75+
private Path path;
76+
private Query query;
77+
78+
@Param({"1000000"})
79+
public int numDocs;
80+
81+
@Param({"delta_only", "gcd_1000", "gcd_100_delta"})
82+
public String encoding;
83+
84+
@Param({"1", "3", "5"})
85+
public int cardinality;
86+
87+
@Param({"0.01", "0.1", "0.5"})
88+
public double selectivity;
89+
90+
@Setup(Level.Trial)
91+
public void setup() throws Exception {
92+
path = Files.createTempDirectory("sortedNumericGcdRange");
93+
dir = MMapDirectory.open(path);
94+
95+
Random random = new Random(0);
96+
try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig())) {
97+
for (int i = 0; i < numDocs; i++) {
98+
Document doc = new Document();
99+
long base = valueForDoc(i, random);
100+
for (int c = 0; c < cardinality; c++) {
101+
doc.add(SortedNumericDocValuesField.indexedField(FIELD, base + c * step()));
102+
}
103+
doc.add(new StringField(LEAD_FIELD, LEAD_VALUE, Field.Store.NO));
104+
writer.addDocument(doc);
105+
}
106+
writer.forceMerge(1);
107+
}
108+
109+
reader = DirectoryReader.open(dir);
110+
searcher = new IndexSearcher(reader);
111+
query = rangeQuery();
112+
}
113+
114+
private long valueForDoc(int doc, Random random) {
115+
long value = random.nextLong(0, DOMAIN);
116+
return switch (encoding) {
117+
case "delta_only" -> DELTA + value;
118+
case "gcd_1000" -> value * 1_000L;
119+
case "gcd_100_delta" -> DELTA + value * 100L;
120+
default -> throw new IllegalArgumentException("Unknown encoding: " + encoding);
121+
};
122+
}
123+
124+
private long step() {
125+
return switch (encoding) {
126+
case "delta_only" -> 1;
127+
case "gcd_1000" -> 1_000L;
128+
case "gcd_100_delta" -> 100L;
129+
default -> throw new IllegalArgumentException("Unknown encoding: " + encoding);
130+
};
131+
}
132+
133+
private Query rangeQuery() {
134+
long range = Math.max(1, (long) (DOMAIN * selectivity));
135+
long min = (DOMAIN - range) / 2;
136+
long max = min + range;
137+
long actualMin = actualValue(min);
138+
long actualMax = actualValue(max);
139+
Query rangeQuery = SortedNumericDocValuesField.newSlowRangeQuery(FIELD, actualMin, actualMax);
140+
return new BooleanQuery.Builder()
141+
.add(new TermQuery(new Term(LEAD_FIELD, LEAD_VALUE)), Occur.FILTER)
142+
.add(rangeQuery, Occur.FILTER)
143+
.build();
144+
}
145+
146+
private long actualValue(long value) {
147+
return switch (encoding) {
148+
case "delta_only" -> DELTA + value;
149+
case "gcd_1000" -> value * 1_000L;
150+
case "gcd_100_delta" -> DELTA + value * 100L;
151+
default -> throw new IllegalArgumentException("Unknown encoding: " + encoding);
152+
};
153+
}
154+
155+
@TearDown(Level.Trial)
156+
public void tearDown() throws Exception {
157+
reader.close();
158+
dir.close();
159+
if (Files.exists(path)) {
160+
try (Stream<Path> walk = Files.walk(path)) {
161+
walk.sorted(Comparator.reverseOrder())
162+
.forEach(
163+
p -> {
164+
try {
165+
Files.delete(p);
166+
} catch (IOException _) {
167+
}
168+
});
169+
}
170+
}
171+
}
172+
173+
@Benchmark
174+
@Fork(
175+
value = 1,
176+
jvmArgsAppend = {"-Xmx2g", "-Xms2g", "-XX:+AlwaysPreTouch"})
177+
public int rangeQueryDefaultProvider() throws IOException {
178+
return searcher.count(query);
179+
}
180+
181+
@Benchmark
182+
@Fork(
183+
value = 1,
184+
jvmArgsAppend = {
185+
"--add-modules",
186+
"jdk.incubator.vector",
187+
"-Xmx2g",
188+
"-Xms2g",
189+
"-XX:+AlwaysPreTouch"
190+
})
191+
public int rangeQueryPanamaProvider() throws IOException {
192+
return searcher.count(query);
193+
}
194+
}

‎lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java‎

Lines changed: 81 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -482,12 +482,28 @@ static void rangeIntoBitSet(
482482
}
483483

484484
/**
485-
* Maps the raw query bounds {@code [minValue, maxValue]} into the encoded domain so the SIMD
486-
* kernel in {@link org.apache.lucene.internal.vectorization.DocValuesRangeSupport} can run
487-
* directly on packed values: {@code [encodedMin, encodedMax] = [ceil((min - delta) / mul),
488-
* floor((max - delta) / mul)]}. Open bounds (e.g. {@code Long.MIN_VALUE} or {@code
489-
* Long.MAX_VALUE}) are saturated to the encoded domain so they keep the SIMD path even when
490-
* {@code min - delta} or {@code max - delta} would overflow.
485+
* Transforms query bounds {@code [minValue, maxValue]} into the encoded domain where stored
486+
* values satisfy {@code stored = raw * mul + delta}. Returns {@code {encodedMin, encodedMax}} or
487+
* {@code null} if the range is empty (no raw value can match).
488+
*/
489+
private static long[] transformGcdBounds(long minValue, long maxValue, long mul, long delta) {
490+
assert mul > 0;
491+
long encodedMin = saturatingShiftLower(minValue, delta);
492+
long encodedMax = saturatingShiftUpper(maxValue, delta);
493+
if (mul != 1) {
494+
encodedMin = Math.ceilDiv(encodedMin, mul);
495+
encodedMax = Math.floorDiv(encodedMax, mul);
496+
}
497+
encodedMin = Math.max(0, encodedMin);
498+
if (encodedMin > encodedMax) {
499+
return null;
500+
}
501+
return new long[] {encodedMin, encodedMax};
502+
}
503+
504+
/**
505+
* Maps the raw query bounds {@code [minValue, maxValue]} into the encoded domain and runs the
506+
* SIMD range scan directly on packed values.
491507
*/
492508
private static void rangeGcdDeltaIntoBitSet(
493509
LongValues values,
@@ -499,18 +515,9 @@ private static void rangeGcdDeltaIntoBitSet(
499515
long delta,
500516
FixedBitSet bitSet,
501517
int offset) {
502-
assert mul > 0;
503-
long encodedMin = saturatingShiftLower(minValue, delta);
504-
long encodedMax = saturatingShiftUpper(maxValue, delta);
505-
if (mul != 1) {
506-
// Math.ceilDiv / Math.floorDiv never overflow for mul > 0 (only Long.MIN_VALUE / -1 does),
507-
// so the SIMD path is always taken; no fallback to the per-doc decoded loop is required.
508-
encodedMin = Math.ceilDiv(encodedMin, mul);
509-
encodedMax = Math.floorDiv(encodedMax, mul);
510-
}
511-
encodedMin = Math.max(0, encodedMin);
512-
if (encodedMin <= encodedMax) {
513-
rangeIntoBitSet(values, fromDoc, toDoc, encodedMin, encodedMax, bitSet, offset);
518+
long[] bounds = transformGcdBounds(minValue, maxValue, mul, delta);
519+
if (bounds != null) {
520+
rangeIntoBitSet(values, fromDoc, toDoc, bounds[0], bounds[1], bitSet, offset);
514521
}
515522
}
516523

@@ -1933,13 +1940,20 @@ private SortedNumericDocValues getSortedNumeric(
19331940
final LongValues values = getNumericValues(entry);
19341941
final int denseFixedCardinality = fixedCardinality(entry, skipperEntry);
19351942

1943+
final boolean hasGcdEncoding =
1944+
entry.bitsPerValue > 0
1945+
&& entry.blockShift < 0
1946+
&& entry.table == null
1947+
&& (entry.gcd != 1 || entry.minValue != 0);
1948+
19361949
if (entry.docsWithFieldOffset == -1) {
19371950
// dense
19381951
return new SortedNumericDocValues() {
19391952

19401953
int doc = -1;
19411954
long start, end;
19421955
int count;
1956+
LongValues rawValues;
19431957

19441958
@Override
19451959
public int nextDoc() throws IOException {
@@ -1988,7 +2002,8 @@ public int docValueCount() {
19882002

19892003
@Override
19902004
public void rangeIntoBitSet(
1991-
int fromDoc, int toDoc, long minValue, long maxValue, FixedBitSet bitSet, int offset) {
2005+
int fromDoc, int toDoc, long minValue, long maxValue, FixedBitSet bitSet, int offset)
2006+
throws IOException {
19922007
int endDoc = Math.min(toDoc, maxDoc);
19932008
if (fromDoc >= endDoc) {
19942009
return;
@@ -1999,16 +2014,37 @@ public void rangeIntoBitSet(
19992014
}
20002015
return;
20012016
}
2017+
LongValues v;
2018+
long lo, hi;
2019+
if (hasGcdEncoding) {
2020+
long[] bounds = transformGcdBounds(minValue, maxValue, entry.gcd, entry.minValue);
2021+
if (bounds == null) {
2022+
return;
2023+
}
2024+
if (rawValues == null) {
2025+
RandomAccessInput rawSlice =
2026+
data.randomAccessSlice(entry.valuesOffset, entry.valuesLength);
2027+
rawValues =
2028+
getDirectReaderInstance(rawSlice, entry.bitsPerValue, 0L, entry.numValues);
2029+
}
2030+
v = rawValues;
2031+
lo = bounds[0];
2032+
hi = bounds[1];
2033+
} else {
2034+
v = values;
2035+
lo = minValue;
2036+
hi = maxValue;
2037+
}
20022038
int cardinality = denseFixedCardinality;
20032039
if (cardinality > 1) {
20042040
DOC_VALUES_RANGE_SUPPORT.sortedNumericRangeIntoBitSet(
2005-
values, fromDoc, endDoc, cardinality, minValue, maxValue, bitSet, offset);
2041+
v, fromDoc, endDoc, cardinality, lo, hi, bitSet, offset);
20062042
return;
20072043
}
20082044
for (int currentDoc = fromDoc; currentDoc < endDoc; currentDoc++) {
20092045
long startOffset = addresses.get(currentDoc);
20102046
long endOffset = addresses.get(currentDoc + 1L);
2011-
if (sortedNumericMatchesRange(values, startOffset, endOffset, minValue, maxValue)) {
2047+
if (sortedNumericMatchesRange(v, startOffset, endOffset, lo, hi)) {
20122048
bitSet.set(currentDoc - offset);
20132049
}
20142050
}
@@ -2044,6 +2080,7 @@ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOExcept
20442080
boolean set;
20452081
long start, end;
20462082
int count;
2083+
LongValues rawValues;
20472084

20482085
@Override
20492086
public int nextDoc() throws IOException {
@@ -2108,11 +2145,33 @@ public void rangeIntoBitSet(
21082145
set = false;
21092146
return;
21102147
}
2148+
LongValues v;
2149+
long lo, hi;
2150+
if (hasGcdEncoding) {
2151+
long[] bounds = transformGcdBounds(minValue, maxValue, entry.gcd, entry.minValue);
2152+
if (bounds == null) {
2153+
set = false;
2154+
return;
2155+
}
2156+
if (rawValues == null) {
2157+
RandomAccessInput rawSlice =
2158+
data.randomAccessSlice(entry.valuesOffset, entry.valuesLength);
2159+
rawValues =
2160+
getDirectReaderInstance(rawSlice, entry.bitsPerValue, 0L, entry.numValues);
2161+
}
2162+
v = rawValues;
2163+
lo = bounds[0];
2164+
hi = bounds[1];
2165+
} else {
2166+
v = values;
2167+
lo = minValue;
2168+
hi = maxValue;
2169+
}
21112170
for (; currentDoc < endDoc; currentDoc = disi.nextDoc()) {
21122171
int index = disi.index();
21132172
long startOffset = addresses.get(index);
21142173
long endOffset = addresses.get(index + 1L);
2115-
if (sortedNumericMatchesRange(values, startOffset, endOffset, minValue, maxValue)) {
2174+
if (sortedNumericMatchesRange(v, startOffset, endOffset, lo, hi)) {
21162175
bitSet.set(currentDoc - offset);
21172176
}
21182177
}

0 commit comments

Comments
 (0)