Skip to content

[Approximation Framework] Extending approximation to top-level term queries on numeric fields #18679

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
import org.opensearch.index.fielddata.IndexNumericFieldData;
import org.opensearch.index.fielddata.LeafNumericFieldData;
import org.opensearch.index.fielddata.SortedNumericDoubleValues;
import org.opensearch.search.approximate.ApproximatePointRangeQuery;
import org.opensearch.search.approximate.ApproximateScoreQuery;

import java.io.IOException;
Expand All @@ -68,7 +69,17 @@ public void testTermQuery() {
long scaledValue = Math.round(value * ft.getScalingFactor());
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery("scaled_float", scaledValue);
Query query = new IndexOrDocValuesQuery(LongPoint.newExactQuery("scaled_float", scaledValue), dvQuery);
assertEquals(query, ft.termQuery(value, null));
Query expectedQuery = new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
"scaled_float",
LongPoint.pack(new long[] { scaledValue }).bytes,
LongPoint.pack(new long[] { scaledValue }).bytes,
1,
ApproximatePointRangeQuery.LONG_FORMAT
)
);
assertEquals(expectedQuery, ft.termQuery(value, null));
}

public void testTermsQuery() {
Expand Down Expand Up @@ -128,6 +139,9 @@ public void testRangeQuery() throws IOException {
MOCK_QSC
);
Query scaledFloatQ = ft.rangeQuery(l, u, includeLower, includeUpper, MOCK_QSC);
assertTrue(scaledFloatQ instanceof ApproximateScoreQuery);
Query approximationQuery = ((ApproximateScoreQuery) scaledFloatQ).getApproximationQuery();
assertTrue(approximationQuery instanceof ApproximatePointRangeQuery);
assertEquals(searcher.count(doubleQ), searcher.count(scaledFloatQ));
}
IOUtils.close(reader, dir);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -298,15 +298,34 @@
@Override
public Query termQuery(String field, Object value, boolean hasDocValues, boolean isSearchable) {
float v = parse(value, false);
if (isSearchable && hasDocValues) {
Query query = HalfFloatPoint.newExactQuery(field, v);
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, HalfFloatPoint.halfFloatToSortableShort(v));
return new IndexOrDocValuesQuery(query, dvQuery);
}
Query dvQuery = null;
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowExactQuery(field, HalfFloatPoint.halfFloatToSortableShort(v));
dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, HalfFloatPoint.halfFloatToSortableShort(v));
}
if (isSearchable) {
Query pointQuery = HalfFloatPoint.newExactQuery(field, v);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointQuery, dvQuery);
} else {
query = pointQuery;

Check warning on line 311 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L311

Added line #L311 was not covered by tests
}

// Use this instead of HalfFloatPoint's pack until Lucene 10.3.0
byte[] exactPoint = NumberType.HALF_FLOAT.encodePoint(v);

return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
exactPoint,
exactPoint,
new float[] { v }.length,
ApproximatePointRangeQuery.HALF_FLOAT_FORMAT
)
);
}
return HalfFloatPoint.newExactQuery(field, v);
return dvQuery;

Check warning on line 328 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L328

Added line #L328 was not covered by tests
}

@Override
Expand Down Expand Up @@ -474,15 +493,30 @@
@Override
public Query termQuery(String field, Object value, boolean hasDocValues, boolean isSearchable) {
float v = parse(value, false);
if (isSearchable && hasDocValues) {
Query query = FloatPoint.newExactQuery(field, v);
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(v));
return new IndexOrDocValuesQuery(query, dvQuery);
}
Query dvQuery = null;
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(v));
dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.floatToSortableInt(v));
}
return FloatPoint.newExactQuery(field, v);
if (isSearchable) {
Query pointQuery = FloatPoint.newExactQuery(field, v);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointQuery, dvQuery);
} else {
query = pointQuery;

Check warning on line 506 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L506

Added line #L506 was not covered by tests
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
FloatPoint.pack(new float[] { v }).bytes,
FloatPoint.pack(new float[] { v }).bytes,
new float[] { v }.length,
ApproximatePointRangeQuery.FLOAT_FORMAT
)
);
}
return dvQuery;

Check warning on line 519 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L519

Added line #L519 was not covered by tests
}

@Override
Expand Down Expand Up @@ -641,15 +675,30 @@
@Override
public Query termQuery(String field, Object value, boolean hasDocValues, boolean isSearchable) {
double v = parse(value, false);
if (isSearchable && hasDocValues) {
Query query = DoublePoint.newExactQuery(field, v);
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.doubleToSortableLong(v));
return new IndexOrDocValuesQuery(query, dvQuery);
}
Query dvQuery = null;
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.doubleToSortableLong(v));
dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, NumericUtils.doubleToSortableLong(v));
}
if (isSearchable) {
Query pointQuery = DoublePoint.newExactQuery(field, v);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointQuery, dvQuery);
} else {
query = pointQuery;

Check warning on line 688 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L688

Added line #L688 was not covered by tests
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
DoublePoint.pack(new double[] { v }).bytes,
DoublePoint.pack(new double[] { v }).bytes,
new double[] { v }.length,
ApproximatePointRangeQuery.DOUBLE_FORMAT
)
);
}
return DoublePoint.newExactQuery(field, v);
return dvQuery;

Check warning on line 701 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L701

Added line #L701 was not covered by tests
}

@Override
Expand Down Expand Up @@ -969,15 +1018,30 @@
return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
}
int v = parse(value, true);
if (isSearchable && hasDocValues) {
Query query = IntPoint.newExactQuery(field, v);
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, v);
return new IndexOrDocValuesQuery(query, dvQuery);
}
Query dvQuery = null;
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowExactQuery(field, v);
dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, v);
}
if (isSearchable) {
Query pointQuery = IntPoint.newExactQuery(field, v);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointQuery, dvQuery);
} else {
query = pointQuery;

Check warning on line 1031 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L1031

Added line #L1031 was not covered by tests
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
IntPoint.pack(new int[] { v }).bytes,
IntPoint.pack(new int[] { v }).bytes,
new int[] { v }.length,
ApproximatePointRangeQuery.INT_FORMAT
)
);
}
return IntPoint.newExactQuery(field, v);
return dvQuery;

Check warning on line 1044 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L1044

Added line #L1044 was not covered by tests
}

@Override
Expand Down Expand Up @@ -1155,17 +1219,30 @@
return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
}
long v = parse(value, true);
if (isSearchable && hasDocValues) {
Query query = LongPoint.newExactQuery(field, v);
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, v);
return new IndexOrDocValuesQuery(query, dvQuery);

}
Query dvQuery = null;
if (hasDocValues) {
return SortedNumericDocValuesField.newSlowExactQuery(field, v);

dvQuery = SortedNumericDocValuesField.newSlowExactQuery(field, v);
}
return LongPoint.newExactQuery(field, v);
if (isSearchable) {
Query pointQuery = LongPoint.newExactQuery(field, v);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointQuery, dvQuery);
} else {
query = pointQuery;

Check warning on line 1232 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L1232

Added line #L1232 was not covered by tests
}
return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(
field,
LongPoint.pack(new long[] { v }).bytes,
LongPoint.pack(new long[] { v }).bytes,
new long[] { v }.length,
ApproximatePointRangeQuery.LONG_FORMAT
)
);
}
return dvQuery;

Check warning on line 1245 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L1245

Added line #L1245 was not covered by tests
}

@Override
Expand Down Expand Up @@ -1297,15 +1374,28 @@
return Queries.newMatchNoDocsQuery("Value [" + value + "] has a decimal part");
}
BigInteger v = parse(value, true);
if (isSearchable && hasDocValues) {
Query query = BigIntegerPoint.newExactQuery(field, v);
Query dvQuery = SortedUnsignedLongDocValuesSetQuery.newSlowExactQuery(field, v);
return new IndexOrDocValuesQuery(query, dvQuery);
}
Query dvQuery = null;
if (hasDocValues) {
return SortedUnsignedLongDocValuesSetQuery.newSlowExactQuery(field, v);
dvQuery = SortedUnsignedLongDocValuesSetQuery.newSlowExactQuery(field, v);
}
return BigIntegerPoint.newExactQuery(field, v);
if (isSearchable) {
Query pointQuery = BigIntegerPoint.newExactQuery(field, v);
Query query;
if (dvQuery != null) {
query = new IndexOrDocValuesQuery(pointQuery, dvQuery);
} else {
query = pointQuery;

Check warning on line 1387 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L1387

Added line #L1387 was not covered by tests
}

// Until Lucene 10.3.0
byte[] exactPoint = NumberType.UNSIGNED_LONG.encodePoint(v);

return new ApproximateScoreQuery(
query,
new ApproximatePointRangeQuery(field, exactPoint, exactPoint, 1, ApproximatePointRangeQuery.UNSIGNED_LONG_FORMAT)
);
}
return dvQuery;

Check warning on line 1398 in server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java

View check run for this annotation

Codecov / codecov/patch

server/src/main/java/org/opensearch/index/mapper/NumberFieldMapper.java#L1398

Added line #L1398 was not covered by tests
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -181,8 +181,19 @@ private static MappedFieldType unsearchable() {
public void testTermQuery() {
MappedFieldType ft = new NumberFieldMapper.NumberFieldType("field", NumberFieldMapper.NumberType.LONG);
Query dvQuery = SortedNumericDocValuesField.newSlowExactQuery("field", 42);
Query query = new IndexOrDocValuesQuery(LongPoint.newExactQuery("field", 42), dvQuery);
assertEquals(query, ft.termQuery("42", null));
Query pointQuery = LongPoint.newExactQuery("field", 42);
Query indexOrDocValuesQuery = new IndexOrDocValuesQuery(pointQuery, dvQuery);
Query approximateQuery = new ApproximateScoreQuery(
indexOrDocValuesQuery,
new ApproximatePointRangeQuery(
"field",
LongPoint.pack(new long[] { 42 }).bytes,
LongPoint.pack(new long[] { 42 }).bytes,
1,
ApproximatePointRangeQuery.LONG_FORMAT
)
);
assertEquals(approximateQuery, ft.termQuery("42", null));

MappedFieldType unsearchable = unsearchable();
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("42", null));
Expand Down Expand Up @@ -641,10 +652,26 @@ public void testNegativeZero() {
NumberType.HALF_FLOAT.rangeQuery("field", null, +0f, true, false, false, true, MOCK_QSC)
);

assertFalse(NumberType.DOUBLE.termQuery("field", -0d, true, true).equals(NumberType.DOUBLE.termQuery("field", +0d, true, true)));
assertFalse(NumberType.FLOAT.termQuery("field", -0f, true, true).equals(NumberType.FLOAT.termQuery("field", +0f, true, true)));
// For term queries, we need to extract the original query from the ApproximateScoreQuery
Query negativeZeroDouble = NumberType.DOUBLE.termQuery("field", -0d, true, true);
Query positiveZeroDouble = NumberType.DOUBLE.termQuery("field", +0d, true, true);
assertFalse(
((ApproximateScoreQuery) negativeZeroDouble).getOriginalQuery()
.equals(((ApproximateScoreQuery) positiveZeroDouble).getOriginalQuery())
);

Query negativeZeroFloat = NumberType.FLOAT.termQuery("field", -0f, true, true);
Query positiveZeroFloat = NumberType.FLOAT.termQuery("field", +0f, true, true);
assertFalse(
((ApproximateScoreQuery) negativeZeroFloat).getOriginalQuery()
.equals(((ApproximateScoreQuery) positiveZeroFloat).getOriginalQuery())
);

Query negativeZeroHalfFloat = NumberType.HALF_FLOAT.termQuery("field", -0f, true, true);
Query positiveZeroHalfFloat = NumberType.HALF_FLOAT.termQuery("field", +0f, true, true);
assertFalse(
NumberType.HALF_FLOAT.termQuery("field", -0f, true, true).equals(NumberType.HALF_FLOAT.termQuery("field", +0f, true, true))
((ApproximateScoreQuery) negativeZeroHalfFloat).getOriginalQuery()
.equals(((ApproximateScoreQuery) positiveZeroHalfFloat).getOriginalQuery())
);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import org.apache.lucene.search.TermQuery;
import org.opensearch.core.common.ParsingException;
import org.opensearch.index.mapper.MappedFieldType;
import org.opensearch.search.approximate.ApproximateScoreQuery;

import java.io.IOException;

Expand Down Expand Up @@ -111,6 +112,7 @@ protected void doAssertLuceneQuery(TermQueryBuilder queryBuilder, Query query, Q
assertThat(
query,
either(instanceOf(TermQuery.class)).or(instanceOf(PointRangeQuery.class))
.or(instanceOf(ApproximateScoreQuery.class))
.or(instanceOf(MatchNoDocsQuery.class))
.or(instanceOf(AutomatonQuery.class))
.or(instanceOf(IndexOrDocValuesQuery.class))
Expand Down
Loading