Skip to content

Commit 0e32ea5

Browse files
mkhludnevakolarkunnu
mkhludnev
authored and
akolarkunnu
committed
Search dv only IP masks (opensearch-project#16628)
* search dv only ip masks Signed-off-by: mikhail-khludnev <[email protected]> * search dv only ip masks: changes Signed-off-by: Mikhail Khludnev <[email protected]> * drop fancy closures Signed-off-by: mikhail-khludnev <[email protected]> * drop fancy closures. one more Signed-off-by: mikhail-khludnev <[email protected]> * fix unit tests; add some more dvOnly Signed-off-by: mikhail-khludnev <[email protected]> * drop skipping dvOnly in 2.x Signed-off-by: mikhail-khludnev <[email protected]> * drop redundant brackets Signed-off-by: mikhail-khludnev <[email protected]> * extract conditions Signed-off-by: mikhail-khludnev <[email protected]> * asserts Signed-off-by: mikhail-khludnev <[email protected]> * spotless apply Signed-off-by: mikhail-khludnev <[email protected]> * bring back skip before Signed-off-by: mikhail-khludnev <[email protected]> * combine asserts Signed-off-by: mikhail-khludnev <[email protected]> * inline, copy-paste Signed-off-by: mikhail-khludnev <[email protected]> --------- Signed-off-by: mikhail-khludnev <[email protected]> Signed-off-by: Mikhail Khludnev <[email protected]> Co-authored-by: mikhail-khludnev <[email protected]>
1 parent ee92938 commit 0e32ea5

File tree

4 files changed

+240
-46
lines changed

4 files changed

+240
-46
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4949
- Support retrieving doc values of unsigned long field ([#16543](https://github.com/opensearch-project/OpenSearch/pull/16543))
5050
- Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483))
5151
- Fix permissions error on scripted query against remote snapshot ([#16544](https://github.com/opensearch-project/OpenSearch/pull/16544))
52+
- Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628))
5253

5354
### Security
5455

‎rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml

Lines changed: 90 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -449,6 +449,28 @@
449449

450450
- match: { hits.total: 2 }
451451

452+
- do:
453+
search:
454+
rest_total_hits_as_int: true
455+
index: test-iodvq
456+
body:
457+
query:
458+
term:
459+
ip_field: "192.168.0.1/24"
460+
461+
- match: { hits.total: 3 }
462+
463+
- do:
464+
search:
465+
rest_total_hits_as_int: true
466+
index: test-iodvq
467+
body:
468+
query:
469+
term:
470+
ip_field: "192.168.0.1/31"
471+
472+
- match: { hits.total: 1 }
473+
452474
- do:
453475
search:
454476
rest_total_hits_as_int: true
@@ -987,6 +1009,28 @@
9871009

9881010
- match: { hits.total: 2 }
9891011

1012+
- do:
1013+
search:
1014+
rest_total_hits_as_int: true
1015+
index: test-index
1016+
body:
1017+
query:
1018+
term:
1019+
ip_field: "192.168.0.1/24"
1020+
1021+
- match: { hits.total: 3 }
1022+
1023+
- do:
1024+
search:
1025+
rest_total_hits_as_int: true
1026+
index: test-index
1027+
body:
1028+
query:
1029+
term:
1030+
ip_field: "192.168.0.1/31"
1031+
1032+
- match: { hits.total: 1 }
1033+
9901034
- do:
9911035
search:
9921036
rest_total_hits_as_int: true
@@ -1077,8 +1121,8 @@
10771121
"search on fields with only doc_values enabled":
10781122
- skip:
10791123
features: [ "headers" ]
1080-
version: " - 2.99.99"
1081-
reason: "searching with only doc_values was added in 3.0.0"
1124+
version: " - 2.18.99"
1125+
reason: "searching with only doc_values was finally added in 2.19.0"
10821126
- do:
10831127
indices.create:
10841128
index: test-doc-values
@@ -1372,6 +1416,28 @@
13721416

13731417
- match: { hits.total: 2 }
13741418

1419+
- do:
1420+
search:
1421+
rest_total_hits_as_int: true
1422+
index: test-doc-values
1423+
body:
1424+
query:
1425+
terms:
1426+
ip_field: ["192.168.0.1", "192.168.0.2"]
1427+
1428+
- match: { hits.total: 2 }
1429+
1430+
- do:
1431+
search:
1432+
rest_total_hits_as_int: true
1433+
index: test-doc-values
1434+
body:
1435+
query:
1436+
terms:
1437+
ip_field: ["192.168.0.1/31", "192.168.0.3"]
1438+
1439+
- match: { hits.total: 2 }
1440+
13751441
- do:
13761442
search:
13771443
rest_total_hits_as_int: true
@@ -1516,6 +1582,28 @@
15161582

15171583
- match: { hits.total: 2 }
15181584

1585+
- do:
1586+
search:
1587+
rest_total_hits_as_int: true
1588+
index: test-doc-values
1589+
body:
1590+
query:
1591+
term:
1592+
ip_field: "192.168.0.1/31"
1593+
1594+
- match: { hits.total: 1 }
1595+
1596+
- do:
1597+
search:
1598+
rest_total_hits_as_int: true
1599+
index: test-doc-values
1600+
body:
1601+
query:
1602+
term:
1603+
ip_field: "192.168.0.1/24"
1604+
1605+
- match: { hits.total: 3 }
1606+
15191607
- do:
15201608
search:
15211609
rest_total_hits_as_int: true

‎server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java

Lines changed: 46 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import java.util.Map;
6565
import java.util.function.BiFunction;
6666
import java.util.function.Supplier;
67+
import java.util.stream.Collectors;
6768

6869
/**
6970
* A {@link FieldMapper} for ip addresses.
@@ -225,42 +226,37 @@ protected Object parseSourceValue(Object value) {
225226
@Override
226227
public Query termQuery(Object value, @Nullable QueryShardContext context) {
227228
failIfNotIndexedAndNoDocValues();
228-
Query query;
229+
final PointRangeQuery pointQuery;
229230
if (value instanceof InetAddress) {
230-
query = InetAddressPoint.newExactQuery(name(), (InetAddress) value);
231+
pointQuery = (PointRangeQuery) InetAddressPoint.newExactQuery(name(), (InetAddress) value);
231232
} else {
232233
if (value instanceof BytesRef) {
233234
value = ((BytesRef) value).utf8ToString();
234235
}
235236
String term = value.toString();
236237
if (term.contains("/")) {
237238
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
238-
query = InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
239+
pointQuery = (PointRangeQuery) InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
239240
} else {
240241
InetAddress address = InetAddresses.forString(term);
241-
query = InetAddressPoint.newExactQuery(name(), address);
242+
pointQuery = (PointRangeQuery) InetAddressPoint.newExactQuery(name(), address);
242243
}
243244
}
244-
if (isSearchable() && hasDocValues()) {
245-
String term = value.toString();
246-
if (term.contains("/")) {
247-
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
248-
return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
249-
}
250-
return new IndexOrDocValuesQuery(
251-
query,
252-
SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint()))
245+
Query dvQuery = null;
246+
if (hasDocValues()) {
247+
dvQuery = SortedSetDocValuesField.newSlowRangeQuery(
248+
name(),
249+
new BytesRef(pointQuery.getLowerPoint()),
250+
new BytesRef(pointQuery.getUpperPoint()),
251+
true,
252+
true
253253
);
254254
}
255-
if (hasDocValues()) {
256-
String term = value.toString();
257-
if (term.contains("/")) {
258-
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
259-
return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
260-
}
261-
return SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint()));
255+
if (isSearchable() && hasDocValues()) {
256+
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
257+
} else {
258+
return isSearchable() ? pointQuery : dvQuery;
262259
}
263-
return query;
264260
}
265261

266262
@Override
@@ -285,36 +281,46 @@ public Query termsQuery(List<?> values, QueryShardContext context) {
285281
}
286282
addresses[i++] = address;
287283
}
288-
return InetAddressPoint.newSetQuery(name(), addresses);
284+
Query dvQuery = null;
285+
if (hasDocValues()) {
286+
List<BytesRef> bytesRefs = Arrays.stream(addresses)
287+
.distinct()
288+
.map(InetAddressPoint::encode)
289+
.map(BytesRef::new)
290+
.collect(Collectors.<BytesRef>toList());
291+
dvQuery = SortedSetDocValuesField.newSlowSetQuery(name(), bytesRefs);
292+
}
293+
Query pointQuery = null;
294+
if (isSearchable()) {
295+
pointQuery = InetAddressPoint.newSetQuery(name(), addresses);
296+
}
297+
if (isSearchable() && hasDocValues()) {
298+
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
299+
} else {
300+
return isSearchable() ? pointQuery : dvQuery;
301+
}
289302
}
290303

291304
@Override
292305
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
293306
failIfNotIndexedAndNoDocValues();
294307
return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (lower, upper) -> {
295-
Query query = InetAddressPoint.newRangeQuery(name(), lower, upper);
296-
if (isSearchable() && hasDocValues()) {
297-
return new IndexOrDocValuesQuery(
298-
query,
299-
SortedSetDocValuesField.newSlowRangeQuery(
300-
((PointRangeQuery) query).getField(),
301-
new BytesRef(((PointRangeQuery) query).getLowerPoint()),
302-
new BytesRef(((PointRangeQuery) query).getUpperPoint()),
303-
true,
304-
true
305-
)
306-
);
307-
}
308+
PointRangeQuery pointQuery = (PointRangeQuery) InetAddressPoint.newRangeQuery(name(), lower, upper);
309+
Query dvQuery = null;
308310
if (hasDocValues()) {
309-
return SortedSetDocValuesField.newSlowRangeQuery(
310-
((PointRangeQuery) query).getField(),
311-
new BytesRef(((PointRangeQuery) query).getLowerPoint()),
312-
new BytesRef(((PointRangeQuery) query).getUpperPoint()),
311+
dvQuery = SortedSetDocValuesField.newSlowRangeQuery(
312+
pointQuery.getField(),
313+
new BytesRef(pointQuery.getLowerPoint()),
314+
new BytesRef(pointQuery.getUpperPoint()),
313315
true,
314316
true
315317
);
316318
}
317-
return query;
319+
if (isSearchable() && hasDocValues()) {
320+
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
321+
} else {
322+
return isSearchable() ? pointQuery : dvQuery;
323+
}
318324
});
319325
}
320326

0 commit comments

Comments
 (0)