diff --git a/CHANGELOG.md b/CHANGELOG.md index 66c5533c335ac..063c31fbda08a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Added - Add multi-threaded writer support in pull-based ingestion ([#17912](https://github.com/opensearch-project/OpenSearch/pull/17912)) - Unset discovery nodes for every transport node actions request ([#17682](https://github.com/opensearch-project/OpenSearch/pull/17682)) +- [Star Tree] Support of Boolean Queries in Aggregations ([#17941](https://github.com/opensearch-project/OpenSearch/pull/17941)) - Enabled default throttling for all tasks submitted to cluster manager ([#17711](https://github.com/opensearch-project/OpenSearch/pull/17711)) ### Changed diff --git a/server/src/main/java/org/opensearch/search/startree/filter/DimensionFilter.java b/server/src/main/java/org/opensearch/search/startree/filter/DimensionFilter.java index 64f971a58f216..be4549fc66522 100644 --- a/server/src/main/java/org/opensearch/search/startree/filter/DimensionFilter.java +++ b/server/src/main/java/org/opensearch/search/startree/filter/DimensionFilter.java @@ -74,6 +74,10 @@ public boolean matchDimValue(long ordinal, StarTreeValues starTreeValues) { */ boolean matchDimValue(long ordinal, StarTreeValues starTreeValues); + default String getDimensionName() { + return null; + } + /** * Represents how to match a value when comparing during StarTreeTraversal */ diff --git a/server/src/main/java/org/opensearch/search/startree/filter/DimensionFilterMergerUtils.java b/server/src/main/java/org/opensearch/search/startree/filter/DimensionFilterMergerUtils.java new file mode 100644 index 0000000000000..4ee0be7c05835 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/DimensionFilterMergerUtils.java @@ -0,0 +1,191 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree.filter; + +import org.opensearch.search.startree.filter.provider.DimensionFilterMapper; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Utility class for merging different types of {@link DimensionFilter} + * Handles intersection operations between {@link ExactMatchDimFilter} and {@link RangeMatchDimFilter} + */ +public class DimensionFilterMergerUtils { + + /** + * Gets intersection of two DimensionFilters + * Returns null if intersection results in no possible matches. + */ + public static DimensionFilter intersect(DimensionFilter filter1, DimensionFilter filter2, DimensionFilterMapper mapper) { + if (filter1 == null || filter2 == null) { + return null; + } + + if (filter1.getDimensionName() == null || filter2.getDimensionName() == null) { + throw new IllegalArgumentException("Cannot intersect filters with null dimension name"); + } + + // Verify filters are for same dimension + if (!filter1.getDimensionName().equals(filter2.getDimensionName())) { + throw new IllegalArgumentException( + "Cannot intersect filters for different dimensions: " + filter1.getDimensionName() + " and " + filter2.getDimensionName() + ); + } + + // Handle Range + Range combination + if (filter1 instanceof RangeMatchDimFilter && filter2 instanceof RangeMatchDimFilter) { + return intersectRangeFilters((RangeMatchDimFilter) filter1, (RangeMatchDimFilter) filter2, mapper); + } + + // Handle ExactMatch + ExactMatch combination + if (filter1 instanceof ExactMatchDimFilter && filter2 instanceof ExactMatchDimFilter) { + return intersectExactMatchFilters((ExactMatchDimFilter) filter1, (ExactMatchDimFilter) filter2); + } + + // Handle Range + ExactMatch combination + if (filter1 instanceof RangeMatchDimFilter && filter2 instanceof ExactMatchDimFilter) { + return intersectRangeWithExactMatch((RangeMatchDimFilter) filter1, (ExactMatchDimFilter) filter2, mapper); + } + + // Handle ExactMatch + Range combination + if (filter1 instanceof ExactMatchDimFilter && filter2 instanceof RangeMatchDimFilter) { + return intersectRangeWithExactMatch((RangeMatchDimFilter) filter2, (ExactMatchDimFilter) filter1, mapper); + } + + // throw exception for unsupported exception + throw new IllegalArgumentException( + "Unsupported filter combination: " + filter1.getClass().getSimpleName() + " and " + filter2.getClass().getSimpleName() + ); + } + + /** + * Intersects two range filters + * Returns null if ranges don't overlap + */ + private static DimensionFilter intersectRangeFilters( + RangeMatchDimFilter range1, + RangeMatchDimFilter range2, + DimensionFilterMapper mapper + ) { + Object low1 = range1.getLow(); + Object high1 = range1.getHigh(); + Object low2 = range2.getLow(); + Object high2 = range2.getHigh(); + + // Find the more restrictive bounds + Object newLow; + boolean includeLow; + if (low1 == null) { + newLow = low2; + includeLow = range2.isIncludeLow(); + } else if (low2 == null) { + newLow = low1; + includeLow = range1.isIncludeLow(); + } else { + int comparison = mapper.compareValues(low1, low2); + if (comparison > 0) { + newLow = low1; + includeLow = range1.isIncludeLow(); + } else if (comparison < 0) { + newLow = low2; + includeLow = range2.isIncludeLow(); + } else { + newLow = low1; + includeLow = range1.isIncludeLow() && range2.isIncludeLow(); + } + } + + Object newHigh; + boolean includeHigh; + if (high1 == null) { + newHigh = high2; + includeHigh = range2.isIncludeHigh(); + } else if (high2 == null) { + newHigh = high1; + includeHigh = range1.isIncludeHigh(); + } else { + int comparison = mapper.compareValues(high1, high2); + if (comparison < 0) { + newHigh = high1; + includeHigh = range1.isIncludeHigh(); + } else if (comparison > 0) { + newHigh = high2; + includeHigh = range2.isIncludeHigh(); + } else { + newHigh = high1; + includeHigh = range1.isIncludeHigh() && range2.isIncludeHigh(); + } + } + + // Check if range is valid + if (newLow != null && newHigh != null) { + if (!mapper.isValidRange(newLow, newHigh, includeLow, includeHigh)) { + return null; // No overlap + } + } + + return new RangeMatchDimFilter(range1.getDimensionName(), newLow, newHigh, includeLow, includeHigh); + } + + /** + * Intersects two exact match filters + * Returns null if no common values + */ + private static DimensionFilter intersectExactMatchFilters(ExactMatchDimFilter exact1, ExactMatchDimFilter exact2) { + List values1 = exact1.getRawValues(); + Set values2Set = new HashSet<>(exact2.getRawValues()); + + List intersection = new ArrayList<>(); + for (Object value : values1) { + if (values2Set.contains(value)) { + intersection.add(value); + } + } + + if (intersection.isEmpty()) { + return null; + } + + return new ExactMatchDimFilter(exact1.getDimensionName(), intersection); + } + + /** + * Intersects a range filter with an exact match filter. + * Returns null if no values from exact match are within range. + */ + private static DimensionFilter intersectRangeWithExactMatch( + RangeMatchDimFilter range, + ExactMatchDimFilter exact, + DimensionFilterMapper mapper + ) { + List validValues = new ArrayList<>(); + + for (Object value : exact.getRawValues()) { + if (isValueInRange(value, range, mapper)) { + validValues.add(value); + } + } + + if (validValues.isEmpty()) { + return null; + } + + return new ExactMatchDimFilter(exact.getDimensionName(), validValues); + } + + /** + * Checks if a value falls within a range. + */ + private static boolean isValueInRange(Object value, RangeMatchDimFilter range, DimensionFilterMapper mapper) { + return mapper.isValueInRange(value, range.getLow(), range.getHigh(), range.isIncludeLow(), range.isIncludeHigh()); + } +} diff --git a/server/src/main/java/org/opensearch/search/startree/filter/ExactMatchDimFilter.java b/server/src/main/java/org/opensearch/search/startree/filter/ExactMatchDimFilter.java index 0ea603f18495f..d734bb0690f90 100644 --- a/server/src/main/java/org/opensearch/search/startree/filter/ExactMatchDimFilter.java +++ b/server/src/main/java/org/opensearch/search/startree/filter/ExactMatchDimFilter.java @@ -84,4 +84,13 @@ public void matchStarTreeNodes(StarTreeNode parentNode, StarTreeValues starTreeV public boolean matchDimValue(long ordinal, StarTreeValues starTreeValues) { return convertedOrdinals.contains(ordinal); } + + public List getRawValues() { + return rawValues; + } + + @Override + public String getDimensionName() { + return dimensionName; + } } diff --git a/server/src/main/java/org/opensearch/search/startree/filter/MatchNoneFilter.java b/server/src/main/java/org/opensearch/search/startree/filter/MatchNoneFilter.java index 3066b4d7a8a3f..12550b989a220 100644 --- a/server/src/main/java/org/opensearch/search/startree/filter/MatchNoneFilter.java +++ b/server/src/main/java/org/opensearch/search/startree/filter/MatchNoneFilter.java @@ -19,6 +19,7 @@ */ @ExperimentalApi public class MatchNoneFilter implements DimensionFilter { + @Override public void initialiseForSegment(StarTreeValues starTreeValues, SearchContext searchContext) { // Nothing to do as we won't match anything. diff --git a/server/src/main/java/org/opensearch/search/startree/filter/RangeMatchDimFilter.java b/server/src/main/java/org/opensearch/search/startree/filter/RangeMatchDimFilter.java index d41c7815ad7a7..1a74f2ef7a2f8 100644 --- a/server/src/main/java/org/opensearch/search/startree/filter/RangeMatchDimFilter.java +++ b/server/src/main/java/org/opensearch/search/startree/filter/RangeMatchDimFilter.java @@ -26,7 +26,6 @@ public class RangeMatchDimFilter implements DimensionFilter { private final String dimensionName; - private final Object low; private final Object high; private final boolean includeLow; @@ -35,6 +34,7 @@ public class RangeMatchDimFilter implements DimensionFilter { private Long lowOrdinal; private Long highOrdinal; + // TODO - see if we need to handle this while intersecting private boolean skipRangeCollection = false; private DimensionFilterMapper dimensionFilterMapper; @@ -90,4 +90,25 @@ public boolean matchDimValue(long ordinal, StarTreeValues starTreeValues) { && dimensionFilterMapper.comparator().compare(ordinal, highOrdinal) <= 0; } + @Override + public String getDimensionName() { + return dimensionName; + } + + public Object getLow() { + return low; + } + + public Object getHigh() { + return high; + } + + public boolean isIncludeLow() { + return includeLow; + } + + public boolean isIncludeHigh() { + return includeHigh; + } + } diff --git a/server/src/main/java/org/opensearch/search/startree/filter/provider/BoolStarTreeFilterProvider.java b/server/src/main/java/org/opensearch/search/startree/filter/provider/BoolStarTreeFilterProvider.java new file mode 100644 index 0000000000000..315999b0d78d5 --- /dev/null +++ b/server/src/main/java/org/opensearch/search/startree/filter/provider/BoolStarTreeFilterProvider.java @@ -0,0 +1,212 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.startree.filter.provider; + +import org.opensearch.index.mapper.CompositeDataCubeFieldType; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.QueryBuilder; +import org.opensearch.index.query.RangeQueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.filter.DimensionFilter; +import org.opensearch.search.startree.filter.DimensionFilterMergerUtils; +import org.opensearch.search.startree.filter.StarTreeFilter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Converts {@link BoolQueryBuilder} into {@link StarTreeFilter} + */ +public class BoolStarTreeFilterProvider implements StarTreeFilterProvider { + + private static final Set> SUPPORTED_NON_BOOL_QUERIES = Set.of( + TermQueryBuilder.class, + TermsQueryBuilder.class, + RangeQueryBuilder.class + ); + + @Override + public StarTreeFilter getFilter(SearchContext context, QueryBuilder rawFilter, CompositeDataCubeFieldType compositeFieldType) + throws IOException { + return processBoolQuery((BoolQueryBuilder) rawFilter, context, compositeFieldType); + } + + private StarTreeFilter processBoolQuery( + BoolQueryBuilder boolQuery, + SearchContext context, + CompositeDataCubeFieldType compositeFieldType + ) throws IOException { + if (boolQuery.hasClauses() == false) { + return null; + } + if (boolQuery.minimumShouldMatch() != null) { + return null; // We cannot support this yet and would need special handling while processing SHOULD clause + } + if (boolQuery.must().isEmpty() == false || boolQuery.filter().isEmpty() == false) { + return processMustClauses(getCombinedMustAndFilterClauses(boolQuery), context, compositeFieldType); + } + if (boolQuery.should().isEmpty() == false) { + return processShouldClauses(boolQuery.should(), context, compositeFieldType); + } + return null; + } + + private StarTreeFilter processNonBoolSupportedQueries( + QueryBuilder query, + SearchContext context, + CompositeDataCubeFieldType compositeFieldType + ) throws IOException { + // Only allow other supported QueryBuilders + if (SUPPORTED_NON_BOOL_QUERIES.contains(query.getClass()) == false) { + return null; + } + // Process individual clause + StarTreeFilterProvider provider = SingletonFactory.getProvider(query); + if (provider == null) { + return null; + } + return provider.getFilter(context, query, compositeFieldType); + } + + private StarTreeFilter processMustClauses( + List mustClauses, + SearchContext context, + CompositeDataCubeFieldType compositeFieldType + ) throws IOException { + if (mustClauses.isEmpty()) { + return null; + } + Map> dimensionToFilters = new HashMap<>(); + + for (QueryBuilder clause : mustClauses) { + StarTreeFilter clauseFilter; + + if (clause instanceof BoolQueryBuilder) { + clauseFilter = processBoolQuery((BoolQueryBuilder) clause, context, compositeFieldType); + } else { + clauseFilter = processNonBoolSupportedQueries(clause, context, compositeFieldType); + } + + if (clauseFilter == null) { + return null; + } + + // Merge filters for each dimension + for (String dimension : clauseFilter.getDimensions()) { + List existingFilters = dimensionToFilters.get(dimension); + List newFilters = clauseFilter.getFiltersForDimension(dimension); + + if (existingFilters == null) { + // No existing filters for this dimension + dimensionToFilters.put(dimension, new ArrayList<>(newFilters)); + } else { + // We have existing filters for this dimension + // Get the appropriate mapper for this dimension + DimensionFilterMapper mapper = DimensionFilterMapper.Factory.fromMappedFieldType( + context.mapperService().fieldType(dimension) + ); + if (mapper == null) { + return null; // Unsupported field type + } + + // We have existing filters for this dimension + if (newFilters.size() > 1) { + // New filters are from SHOULD clause (multiple filters = OR condition) + // Need to intersect each SHOULD filter with existing filters + List intersectedFilters = new ArrayList<>(); + for (DimensionFilter shouldFilter : newFilters) { + for (DimensionFilter existingFilter : existingFilters) { + DimensionFilter intersected = DimensionFilterMergerUtils.intersect(existingFilter, shouldFilter, mapper); + if (intersected != null) { + intersectedFilters.add(intersected); + } + } + } + if (intersectedFilters.isEmpty()) { + return null; // No valid intersections + } + dimensionToFilters.put(dimension, intersectedFilters); + } else { + // Here's where we need the DimensionFilter merging logic + // For example: merging range with term, or range with range + // And a single dimension filter coming from should clause is as good as must clause + DimensionFilter mergedFilter = DimensionFilterMergerUtils.intersect( + existingFilters.getFirst(), + newFilters.getFirst(), + mapper + ); + if (mergedFilter == null) { + return null; // No possible matches after merging + } + dimensionToFilters.put(dimension, Collections.singletonList(mergedFilter)); + } + } + } + } + + return new StarTreeFilter(dimensionToFilters); + } + + private StarTreeFilter processShouldClauses( + List shouldClauses, + SearchContext context, + CompositeDataCubeFieldType compositeFieldType + ) throws IOException { + if (shouldClauses.isEmpty()) { + return null; + } + + // First, validate all SHOULD clauses are for same dimension + String commonDimension = null; + Map> dimensionToFilters = new HashMap<>(); + for (QueryBuilder clause : shouldClauses) { + StarTreeFilter clauseFilter; + + if (clause instanceof BoolQueryBuilder) { + clauseFilter = processBoolQuery((BoolQueryBuilder) clause, context, compositeFieldType); + } else { + clauseFilter = processNonBoolSupportedQueries(clause, context, compositeFieldType); + } + + if (clauseFilter == null) { + return null; + } + + // Validate single dimension + if (clauseFilter.getDimensions().size() != 1) { + return null; // SHOULD clause must operate on single dimension + } + + String dimension = clauseFilter.getDimensions().iterator().next(); + if (commonDimension == null) { + commonDimension = dimension; + } else if (commonDimension.equals(dimension) == false) { + return null; // All SHOULD clauses must operate on same dimension + } + + // Simply collect all filters - StarTreeTraversal will handle OR operation + dimensionToFilters.computeIfAbsent(dimension, k -> new ArrayList<>()).addAll(clauseFilter.getFiltersForDimension(dimension)); + } + return new StarTreeFilter(dimensionToFilters); + } + + private List getCombinedMustAndFilterClauses(BoolQueryBuilder boolQuery) { + List mustAndFilterClauses = new ArrayList<>(); + mustAndFilterClauses.addAll(boolQuery.must()); + mustAndFilterClauses.addAll(boolQuery.filter()); + return mustAndFilterClauses; + } +} diff --git a/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java b/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java index 3b1713450e278..fc5d3dd64058e 100644 --- a/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java +++ b/server/src/main/java/org/opensearch/search/startree/filter/provider/DimensionFilterMapper.java @@ -92,6 +92,43 @@ Optional getMatchingOrdinal( DimensionFilter.MatchType matchType ); + /** + * Compares two values of the same type. + * @param v1 first object + * @param v2 second object + * @return : + */ + int compareValues(Object v1, Object v2); + + /** + * Checks if a value falls within a range. + * Default implementation for regular types. + */ + default boolean isValueInRange(Object value, Object low, Object high, boolean includeLow, boolean includeHigh) { + if (low != null) { + int comparison = compareValues(value, low); + if (comparison < 0 || (comparison == 0 && !includeLow)) { + return false; + } + } + + if (high != null) { + int comparison = compareValues(value, high); + if (comparison > 0 || (comparison == 0 && !includeHigh)) { + return false; + } + } + return true; + } + + default boolean isValidRange(Object low, Object high, boolean includeLow, boolean includeHigh) { + if (low == null || high == null) { + return true; + } + int comparison = compareValues(low, high); + return comparison < 0 || (comparison == 0 && includeLow && includeHigh); + } + default Comparator comparator() { return DimensionDataType.LONG::compare; } @@ -144,6 +181,14 @@ public Optional getMatchingOrdinal( // Casting to long ensures that all numeric fields have been converted to equivalent long at request parsing time. return Optional.of((long) value); } + + @Override + public int compareValues(Object v1, Object v2) { + if (!(v1 instanceof Long) || !(v2 instanceof Long)) { + throw new IllegalArgumentException("Expected Long values for numeric comparison"); + } + return Long.compare((Long) v1, (Long) v2); + } } abstract class NumericNonDecimalMapper extends NumericMapper { @@ -235,6 +280,29 @@ public Comparator comparator() { return DimensionDataType.UNSIGNED_LONG::compare; } + @Override + public int compareValues(Object v1, Object v2) { + if (!(v1 instanceof Long) || !(v2 instanceof Long)) { + throw new IllegalArgumentException("Expected Long values for unsigned comparison"); + } + return Long.compareUnsigned((Long) v1, (Long) v2); + } + + @Override + public boolean isValueInRange(Object value, Object low, Object high, boolean includeLow, boolean includeHigh) { + long v = (Long) value; + long l = low != null ? (Long) low : 0L; + long h = high != null ? (Long) high : -1L; // -1L is max unsigned + + if (Long.compareUnsigned(l, h) > 0) { + return (Long.compareUnsigned(v, l) > 0 || (Long.compareUnsigned(v, l) == 0 && includeLow)) + || (Long.compareUnsigned(v, h) < 0 || (Long.compareUnsigned(v, h) == 0 && includeHigh)); + } + + // Normal case + return super.isValueInRange(value, low, high, includeLow, includeHigh); + } + } abstract class NumericDecimalFieldMapper extends NumericMapper { @@ -436,4 +504,12 @@ private Object parseRawKeyword(String field, Object rawValue, KeywordFieldType k return parsedValue; } + @Override + public int compareValues(Object v1, Object v2) { + if (!(v1 instanceof BytesRef) || !(v2 instanceof BytesRef)) { + throw new IllegalArgumentException("Expected BytesRef values for keyword comparison"); + } + return ((BytesRef) v1).compareTo((BytesRef) v2); + } + } diff --git a/server/src/main/java/org/opensearch/search/startree/filter/provider/StarTreeFilterProvider.java b/server/src/main/java/org/opensearch/search/startree/filter/provider/StarTreeFilterProvider.java index 61f44ba3f163e..b58e284323fdf 100644 --- a/server/src/main/java/org/opensearch/search/startree/filter/provider/StarTreeFilterProvider.java +++ b/server/src/main/java/org/opensearch/search/startree/filter/provider/StarTreeFilterProvider.java @@ -12,6 +12,7 @@ import org.opensearch.index.compositeindex.datacube.Dimension; import org.opensearch.index.mapper.CompositeDataCubeFieldType; import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.MatchAllQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.RangeQueryBuilder; @@ -59,7 +60,9 @@ class SingletonFactory { TermsQueryBuilder.NAME, new TermsStarTreeFilterProvider(), RangeQueryBuilder.NAME, - new RangeStarTreeFilterProvider() + new RangeStarTreeFilterProvider(), + BoolQueryBuilder.NAME, + new BoolStarTreeFilterProvider() ); public static StarTreeFilterProvider getProvider(QueryBuilder query) { @@ -153,7 +156,5 @@ public StarTreeFilter getFilter(SearchContext context, QueryBuilder rawFilter, C ); } } - } - } diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/BoolStarTreeFilterProviderTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/BoolStarTreeFilterProviderTests.java new file mode 100644 index 0000000000000..631033a15684e --- /dev/null +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/BoolStarTreeFilterProviderTests.java @@ -0,0 +1,1135 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.startree; + +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; +import org.opensearch.index.compositeindex.datacube.Metric; +import org.opensearch.index.compositeindex.datacube.MetricStat; +import org.opensearch.index.compositeindex.datacube.OrdinalDimension; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; +import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; +import org.opensearch.index.mapper.CompositeDataCubeFieldType; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.mapper.StarTreeMapper; +import org.opensearch.index.mapper.WildcardFieldMapper; +import org.opensearch.index.query.BoolQueryBuilder; +import org.opensearch.index.query.RangeQueryBuilder; +import org.opensearch.index.query.TermQueryBuilder; +import org.opensearch.index.query.TermsQueryBuilder; +import org.opensearch.index.query.WildcardQueryBuilder; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.filter.DimensionFilter; +import org.opensearch.search.startree.filter.ExactMatchDimFilter; +import org.opensearch.search.startree.filter.RangeMatchDimFilter; +import org.opensearch.search.startree.filter.StarTreeFilter; +import org.opensearch.search.startree.filter.provider.StarTreeFilterProvider; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class BoolStarTreeFilterProviderTests extends OpenSearchTestCase { + private SearchContext searchContext; + private MapperService mapperService; + private CompositeDataCubeFieldType compositeFieldType; + + @Before + public void setup() { + // Setup common test dependencies + searchContext = mock(SearchContext.class); + mapperService = mock(MapperService.class); + when(searchContext.mapperService()).thenReturn(mapperService); + + // Setup field types + KeywordFieldMapper.KeywordFieldType methodType = new KeywordFieldMapper.KeywordFieldType("method"); + NumberFieldMapper.NumberFieldType statusType = new NumberFieldMapper.NumberFieldType( + "status", + NumberFieldMapper.NumberType.INTEGER + ); + NumberFieldMapper.NumberFieldType portType = new NumberFieldMapper.NumberFieldType("port", NumberFieldMapper.NumberType.INTEGER); + KeywordFieldMapper.KeywordFieldType zoneType = new KeywordFieldMapper.KeywordFieldType("zone"); + NumberFieldMapper.NumberFieldType responseTimeType = new NumberFieldMapper.NumberFieldType( + "response_time", + NumberFieldMapper.NumberType.INTEGER + ); + NumberFieldMapper.NumberFieldType latencyType = new NumberFieldMapper.NumberFieldType( + "latency", + NumberFieldMapper.NumberType.FLOAT + ); + KeywordFieldMapper.KeywordFieldType regionType = new KeywordFieldMapper.KeywordFieldType("region"); + when(mapperService.fieldType("method")).thenReturn(methodType); + when(mapperService.fieldType("status")).thenReturn(statusType); + when(mapperService.fieldType("port")).thenReturn(portType); + when(mapperService.fieldType("zone")).thenReturn(zoneType); + when(mapperService.fieldType("response_time")).thenReturn(responseTimeType); + when(mapperService.fieldType("latency")).thenReturn(latencyType); + when(mapperService.fieldType("region")).thenReturn(regionType); + + // Create composite field type with dimensions + compositeFieldType = new StarTreeMapper.StarTreeFieldType( + "star_tree", + new StarTreeField( + "star_tree", + List.of( + new OrdinalDimension("method"), + new OrdinalDimension("status"), + new OrdinalDimension("port"), + new OrdinalDimension("zone"), + new OrdinalDimension("response_time"), + new OrdinalDimension("latency"), + new OrdinalDimension("region") + ), + List.of(new Metric("size", List.of(MetricStat.SUM))), + new StarTreeFieldConfiguration( + randomIntBetween(1, 10_000), + Collections.emptySet(), + StarTreeFieldConfiguration.StarTreeBuildMode.ON_HEAP + ) + ) + ); + } + + public void testSimpleMustWithMultipleDimensions() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must(new TermQueryBuilder("status", 200)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have two dimensions", 2, filter.getDimensions().size()); + assertTrue("Should contain method dimension", filter.getDimensions().contains("method")); + assertTrue("Should contain status dimension", filter.getDimensions().contains("status")); + + List methodFilters = filter.getFiltersForDimension("method"); + assertEquals("Should have one filter for method", 1, methodFilters.size()); + assertTrue("Should be ExactMatchDimFilter", methodFilters.getFirst() instanceof ExactMatchDimFilter); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "GET"); + + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Should have one filter for status", 1, statusFilters.size()); + assertTrue("Should be ExactMatchDimFilter", statusFilters.getFirst() instanceof ExactMatchDimFilter); + assertExactMatchValue((ExactMatchDimFilter) statusFilters.getFirst(), 200L); + } + + public void testNestedMustClauses() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must(new BoolQueryBuilder().must(new TermQueryBuilder("status", 200)).must(new TermQueryBuilder("port", 443))); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have three dimensions", 3, filter.getDimensions().size()); + + // Verify method filter + List methodFilters = filter.getFiltersForDimension("method"); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "GET"); + + // Verify status filter + List statusFilters = filter.getFiltersForDimension("status"); + assertExactMatchValue((ExactMatchDimFilter) statusFilters.getFirst(), 200L); + + // Verify port filter + List portFilters = filter.getFiltersForDimension("port"); + assertExactMatchValue((ExactMatchDimFilter) portFilters.getFirst(), 443L); + } + + public void testMustWithDifferentQueryTypes() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must(new TermsQueryBuilder("status", Arrays.asList(200, 201))) + .must(new RangeQueryBuilder("port").gte(80).lte(443)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + + // Verify method filter + List methodFilters = filter.getFiltersForDimension("method"); + assertTrue("Method should be ExactMatchDimFilter", methodFilters.getFirst() instanceof ExactMatchDimFilter); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "GET"); + + // Verify status filter + List statusFilters = filter.getFiltersForDimension("status"); + assertTrue("Status should be ExactMatchDimFilter", statusFilters.getFirst() instanceof ExactMatchDimFilter); + Set expectedStatusValues = Set.of(200L, 201L); + Set actualStatusValues = new HashSet<>(((ExactMatchDimFilter) statusFilters.getFirst()).getRawValues()); + assertEquals("Status should have expected values", expectedStatusValues, actualStatusValues); + + // Verify port filter + List portFilters = filter.getFiltersForDimension("port"); + assertTrue("Port should be RangeMatchDimFilter", portFilters.getFirst() instanceof RangeMatchDimFilter); + RangeMatchDimFilter portRange = (RangeMatchDimFilter) portFilters.getFirst(); + assertEquals("Port lower bound should be 80", 80L, portRange.getLow()); + assertEquals("Port upper bound should be 443", 443L, portRange.getHigh()); + assertTrue("Port bounds should be inclusive", portRange.isIncludeLow() && portRange.isIncludeHigh()); + } + + public void testMustWithSameDimension() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("status", 200)) + .must(new TermQueryBuilder("status", 404)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + // this should return null as same dimension in MUST is logically impossible + assertNull("Filter should be null for same dimension in MUST", filter); + } + + public void testEmptyMustClause() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for empty bool query", filter); + } + + public void testShouldWithSameDimension() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)) + .should(new TermQueryBuilder("status", 404)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have one dimension", 1, filter.getDimensions().size()); + assertTrue("Should contain status dimension", filter.getDimensions().contains("status")); + + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Should have two filters for status", 2, statusFilters.size()); + assertTrue("Both should be ExactMatchDimFilter", statusFilters.stream().allMatch(f -> f instanceof ExactMatchDimFilter)); + + Set expectedValues = Set.of(200L, 404L); + Set actualValues = new HashSet<>(); + for (DimensionFilter dimensionFilter : statusFilters) { + actualValues.addAll(((ExactMatchDimFilter) dimensionFilter).getRawValues()); + } + assertEquals("Should contain expected status values", expectedValues, actualValues); + } + + public void testShouldWithSameDimensionRange() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().should(new RangeQueryBuilder("status").gte(200).lte(300)) + .should(new RangeQueryBuilder("status").gte(400).lte(500)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have one dimension", 1, filter.getDimensions().size()); + + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Should have two filters for status", 2, statusFilters.size()); + assertTrue("Both should be RangeMatchDimFilter", statusFilters.stream().allMatch(f -> f instanceof RangeMatchDimFilter)); + + // Verify first range + RangeMatchDimFilter firstRange = (RangeMatchDimFilter) statusFilters.getFirst(); + assertEquals("First range lower bound should be 200", 200L, firstRange.getLow()); + assertEquals("First range upper bound should be 300", 300L, firstRange.getHigh()); + assertTrue("First range lower bound should be inclusive", firstRange.isIncludeLow()); + assertTrue("First range upper bound should be inclusive", firstRange.isIncludeHigh()); + + // Verify second range + RangeMatchDimFilter secondRange = (RangeMatchDimFilter) statusFilters.get(1); + assertEquals("Second range lower bound should be 400", 400L, secondRange.getLow()); + assertEquals("Second range upper bound should be 500", 500L, secondRange.getHigh()); + assertTrue("Second range lower bound should be inclusive", secondRange.isIncludeLow()); + assertTrue("Second range upper bound should be inclusive", secondRange.isIncludeHigh()); + } + + public void testShouldWithSameDimensionMixed() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)) + .should(new RangeQueryBuilder("status").gte(400).lte(500)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have one dimension", 1, filter.getDimensions().size()); + + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Should have two filters for status", 2, statusFilters.size()); + + // Find and verify exact match filter + Optional exactFilter = statusFilters.stream() + .filter(f -> f instanceof ExactMatchDimFilter) + .map(f -> (ExactMatchDimFilter) f) + .findFirst(); + assertTrue("Should have exact match filter", exactFilter.isPresent()); + assertEquals("Exact match should be 200", 200L, exactFilter.get().getRawValues().get(0)); + + // Find and verify range filter + Optional rangeFilter = statusFilters.stream() + .filter(f -> f instanceof RangeMatchDimFilter) + .map(f -> (RangeMatchDimFilter) f) + .findFirst(); + assertTrue("Should have range filter", rangeFilter.isPresent()); + assertEquals("Range lower bound should be 400", 400L, rangeFilter.get().getLow()); + assertEquals("Range upper bound should be 500", 500L, rangeFilter.get().getHigh()); + assertTrue("Range lower bound should be inclusive", rangeFilter.get().isIncludeLow()); + assertTrue("Range upper bound should be inclusive", rangeFilter.get().isIncludeHigh()); + } + + public void testShouldWithDifferentDimensions() throws IOException { + // SHOULD with different dimensions (should be rejected) + BoolQueryBuilder boolQuery = new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)) + .should(new TermQueryBuilder("method", "GET")); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for SHOULD across different dimensions", filter); + } + + public void testNestedShouldSameDimension() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)) + .should(new BoolQueryBuilder().should(new TermQueryBuilder("status", 404)).should(new TermQueryBuilder("status", 500))); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have one dimension", 1, filter.getDimensions().size()); + + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Should have three filters for status", 3, statusFilters.size()); + assertTrue("All should be ExactMatchDimFilter", statusFilters.stream().allMatch(f -> f instanceof ExactMatchDimFilter)); + + Set expectedValues = Set.of(200L, 404L, 500L); + Set actualValues = new HashSet<>(); + for (DimensionFilter dimensionFilter : statusFilters) { + actualValues.addAll(((ExactMatchDimFilter) dimensionFilter).getRawValues()); + } + assertEquals("Should contain all expected status values", expectedValues, actualValues); + } + + public void testEmptyShouldClause() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().should(new BoolQueryBuilder()); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for empty SHOULD clause", filter); + } + + public void testMustContainingShouldSameDimension() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new RangeQueryBuilder("status").gte(200).lt(500)) + .must(new BoolQueryBuilder().should(new TermQueryBuilder("status", 404)).should(new TermQueryBuilder("status", 403))); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have one dimension", 1, filter.getDimensions().size()); + + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Should have two filters after intersection", 2, statusFilters.size()); + + Set expectedValues = Set.of(403L, 404L); + Set actualValues = new HashSet<>(); + for (DimensionFilter dimFilter : statusFilters) { + assertTrue("Should be ExactMatchDimFilter", dimFilter instanceof ExactMatchDimFilter); + ExactMatchDimFilter exactFilter = (ExactMatchDimFilter) dimFilter; + actualValues.addAll(exactFilter.getRawValues()); + } + assertEquals("Should contain expected status values", expectedValues, actualValues); + } + + public void testMustContainingShouldDifferentDimension() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must(new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)).should(new TermQueryBuilder("status", 404))); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have two dimensions", 2, filter.getDimensions().size()); + + // Verify method filter + List methodFilters = filter.getFiltersForDimension("method"); + assertEquals("Method should have one filter", 1, methodFilters.size()); + assertTrue("Should be ExactMatchDimFilter", methodFilters.getFirst() instanceof ExactMatchDimFilter); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "GET"); + + // Verify status filters + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Status should have two filters", 2, statusFilters.size()); + Set expectedStatusValues = Set.of(200L, 404L); + Set actualStatusValues = new HashSet<>(); + for (DimensionFilter dimFilter : statusFilters) { + assertTrue("Should be ExactMatchDimFilter", dimFilter instanceof ExactMatchDimFilter); + ExactMatchDimFilter exactFilter = (ExactMatchDimFilter) dimFilter; + actualStatusValues.addAll(exactFilter.getRawValues()); + } + assertEquals("Should contain expected status values", expectedStatusValues, actualStatusValues); + } + + public void testMultipleLevelsMustNesting() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must( + new BoolQueryBuilder().must(new RangeQueryBuilder("status").gte(200).lte(300)) + .must(new BoolQueryBuilder().must(new TermQueryBuilder("port", 443))) + ); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + + // Verify method filter + List methodFilters = filter.getFiltersForDimension("method"); + assertEquals("Method should have one filter", 1, methodFilters.size()); + assertTrue("Should be ExactMatchDimFilter", methodFilters.getFirst() instanceof ExactMatchDimFilter); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "GET"); + + // Verify status filter + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Status should have one filter", 1, statusFilters.size()); + assertTrue("Should be RangeMatchDimFilter", statusFilters.getFirst() instanceof RangeMatchDimFilter); + RangeMatchDimFilter rangeFilter = (RangeMatchDimFilter) statusFilters.getFirst(); + assertEquals("Lower bound should be 200", 200L, rangeFilter.getLow()); + assertEquals("Upper bound should be 300", 300L, rangeFilter.getHigh()); + assertTrue("Lower bound should be inclusive", rangeFilter.isIncludeLow()); + assertTrue("Upper bound should be inclusive", rangeFilter.isIncludeHigh()); + + // Verify port filter + List portFilters = filter.getFiltersForDimension("port"); + assertEquals("Port should have one filter", 1, portFilters.size()); + assertTrue("Should be ExactMatchDimFilter", portFilters.getFirst() instanceof ExactMatchDimFilter); + assertExactMatchValue((ExactMatchDimFilter) portFilters.getFirst(), 443L); + } + + public void testShouldInsideShouldSameDimension() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)) + .should(new BoolQueryBuilder().should(new TermQueryBuilder("status", 404)).should(new TermQueryBuilder("status", 500))); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have one dimension", 1, filter.getDimensions().size()); + + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Should have three filters", 3, statusFilters.size()); + + Set expectedValues = Set.of(200L, 404L, 500L); + Set actualValues = new HashSet<>(); + for (DimensionFilter dimFilter : statusFilters) { + assertTrue("Should be ExactMatchDimFilter", dimFilter instanceof ExactMatchDimFilter); + ExactMatchDimFilter exactFilter = (ExactMatchDimFilter) dimFilter; + actualValues.addAll(exactFilter.getRawValues()); + } + assertEquals("Should contain all expected values", expectedValues, actualValues); + } + + public void testMustInsideShouldDifferentDimensionRejected() throws IOException { + // MUST inside SHOULD for different dimension (should be rejected) + BoolQueryBuilder boolQuery = new BoolQueryBuilder().should( + new BoolQueryBuilder().must(new TermQueryBuilder("status", 200)).must(new TermQueryBuilder("method", "GET")) + ); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for MUST inside SHOULD", filter); + } + + public void testComplexNestedStructure() throws IOException { + // Complex nested structure with both MUST and SHOULD + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must( + new BoolQueryBuilder().must(new RangeQueryBuilder("port").gte(80).lte(443)) + .must(new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)).should(new TermQueryBuilder("status", 404))) + ); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have three dimensions", 3, filter.getDimensions().size()); + assertTrue("Should contain all dimensions", filter.getDimensions().containsAll(Set.of("method", "port", "status"))); + } + + public void testMaximumNestingDepth() throws IOException { + // Build a deeply nested bool query + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")); + + BoolQueryBuilder current = boolQuery; + for (int i = 0; i < 10; i++) { // Test with 10 levels of nesting + BoolQueryBuilder nested = new BoolQueryBuilder().must(new TermQueryBuilder("status", 200 + i)); + current.must(nested); + current = nested; + } + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should not be null", filter); + } + + public void testAllClauseTypesCombined() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must( + new BoolQueryBuilder().must(new RangeQueryBuilder("port").gte(80).lte(443)) + .must(new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)).should(new TermQueryBuilder("status", 201))) + ) + .must(new TermsQueryBuilder("method", Arrays.asList("GET", "POST"))); // This should intersect with first method term + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have three dimensions", 3, filter.getDimensions().size()); + + // Verify method filter (should be intersection of term and terms) + List methodFilters = filter.getFiltersForDimension("method"); + assertEquals("Should have one filter for method after intersection", 1, methodFilters.size()); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "GET"); + + // Verify port filter + List portFilters = filter.getFiltersForDimension("port"); + assertTrue("Port should be RangeMatchDimFilter", portFilters.getFirst() instanceof RangeMatchDimFilter); + RangeMatchDimFilter portRange = (RangeMatchDimFilter) portFilters.getFirst(); + assertEquals("Port lower bound should be 80", 80L, portRange.getLow()); + assertEquals("Port upper bound should be 443", 443L, portRange.getHigh()); + + // Verify status filters + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Should have two filters for status", 2, statusFilters.size()); + Set expectedStatusValues = Set.of(200L, 201L); + Set actualStatusValues = new HashSet<>(); + for (DimensionFilter statusFilter : statusFilters) { + actualStatusValues.addAll(((ExactMatchDimFilter) statusFilter).getRawValues()); + } + assertEquals("Status should have expected values", expectedStatusValues, actualStatusValues); + } + + public void testEmptyNestedBools() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new BoolQueryBuilder()) + .must(new BoolQueryBuilder().must(new BoolQueryBuilder())); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for empty nested bool queries", filter); + } + + public void testSingleClauseBoolQueries() throws IOException { + // Test single MUST clause + BoolQueryBuilder mustOnly = new BoolQueryBuilder().must(new TermQueryBuilder("status", 200)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(mustOnly); + StarTreeFilter filter = provider.getFilter(searchContext, mustOnly, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have one dimension", 1, filter.getDimensions().size()); + assertExactMatchValue((ExactMatchDimFilter) filter.getFiltersForDimension("status").get(0), 200L); + + // Test single SHOULD clause + BoolQueryBuilder shouldOnly = new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)); + + filter = provider.getFilter(searchContext, shouldOnly, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have one dimension", 1, filter.getDimensions().size()); + assertExactMatchValue((ExactMatchDimFilter) filter.getFiltersForDimension("status").get(0), 200L); + } + + public void testDuplicateDimensionsAcrossNesting() throws IOException { + // Test duplicate dimensions that should be merged/intersected + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new RangeQueryBuilder("status").gte(200).lte(500)) + .must(new BoolQueryBuilder().must(new RangeQueryBuilder("status").gte(300).lte(400))); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have one dimension", 1, filter.getDimensions().size()); + + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Should have one filter after intersection", 1, statusFilters.size()); + RangeMatchDimFilter rangeFilter = (RangeMatchDimFilter) statusFilters.getFirst(); + assertEquals("Lower bound should be 300", 300L, rangeFilter.getLow()); + assertEquals("Upper bound should be 400", 400L, rangeFilter.getHigh()); + assertTrue("Lower bound should be exclusive", rangeFilter.isIncludeLow()); + assertTrue("Upper bound should be exclusive", rangeFilter.isIncludeHigh()); + } + + public void testKeywordFieldTypeHandling() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermsQueryBuilder("method", Arrays.asList("GET", "POST"))) + .must(new TermQueryBuilder("status", 200)) + .must(new BoolQueryBuilder().should(new TermQueryBuilder("port", 80)).should(new TermQueryBuilder("port", 9200))); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + + // Verify method filter (keyword term query) + List statusFilters = filter.getFiltersForDimension("status"); + assertExactMatchValue((ExactMatchDimFilter) statusFilters.getFirst(), 200L); + + // Verify method filter (keyword terms query) + List methodFilters = filter.getFiltersForDimension("method"); + ExactMatchDimFilter methodFilter = (ExactMatchDimFilter) methodFilters.getFirst(); + Set expectedMethod = new HashSet<>(); + expectedMethod.add(new BytesRef("GET")); + expectedMethod.add(new BytesRef("POST")); + assertEquals(expectedMethod, new HashSet<>(methodFilter.getRawValues())); + + // Verify port filter (keyword SHOULD terms) + List portFilters = filter.getFiltersForDimension("port"); + assertEquals(2, portFilters.size()); + Set expectedPorts = new HashSet<>(); + expectedPorts.add(80L); + expectedPorts.add(9200L); + Set actualZones = new HashSet<>(); + for (DimensionFilter portFilter : portFilters) { + actualZones.addAll(((ExactMatchDimFilter) portFilter).getRawValues()); + } + assertEquals(expectedPorts, actualZones); + } + + public void testInvalidDimensionNames() throws IOException { + // Test dimension that doesn't exist in mapping + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("non_existent_field", "value")) + .must(new TermQueryBuilder("method", "GET")); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for non-existent dimension", filter); + + // Test dimension that exists in mapping but not in star tree dimensions + NumberFieldMapper.NumberFieldType nonStarTreeField = new NumberFieldMapper.NumberFieldType( + "non_star_tree_field", + NumberFieldMapper.NumberType.INTEGER + ); + when(mapperService.fieldType("non_star_tree_field")).thenReturn(nonStarTreeField); + + boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("non_star_tree_field", 100)) + .must(new TermQueryBuilder("method", "GET")); + + filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + assertNull("Filter should be null for non-star-tree dimension", filter); + } + + public void testUnsupportedQueryTypes() throws IOException { + // Test unsupported query type in MUST + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new WildcardQueryBuilder("method", "GET*")) + .must(new TermQueryBuilder("status", 200)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for unsupported query type", filter); + + // Test unsupported query type in SHOULD + boolQuery = new BoolQueryBuilder().should(new WildcardQueryBuilder("status", "2*")).should(new TermQueryBuilder("status", 404)); + + filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + assertNull("Filter should be null for unsupported query type in SHOULD", filter); + } + + public void testInvalidFieldTypes() throws IOException { + // Test with unsupported field type + WildcardFieldMapper.WildcardFieldType wildcardType = new WildcardFieldMapper.WildcardFieldType("wildcard_field"); + when(mapperService.fieldType("wildcard_field")).thenReturn(wildcardType); + + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("wildcard_field", "value")) + .must(new TermQueryBuilder("method", "GET")); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for unsupported field type", filter); + } + + public void testInvalidShouldClauses() throws IOException { + // Test SHOULD clauses with different dimensions + BoolQueryBuilder boolQuery = new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)) + .should(new TermQueryBuilder("method", "GET")); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for SHOULD with different dimensions", filter); + + // Test nested MUST inside SHOULD + boolQuery = new BoolQueryBuilder().should( + new BoolQueryBuilder().must(new TermQueryBuilder("status", 200)).must(new TermQueryBuilder("method", "GET")) + ); + + filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + assertNull("Filter should be null for MUST inside SHOULD", filter); + } + + public void testInvalidMustClauses() throws IOException { + // Test MUST clauses with same dimension + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("status", 200)) + .must(new TermQueryBuilder("status", 404)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for multiple MUST on same dimension", filter); + + // Test incompatible range intersections + boolQuery = new BoolQueryBuilder().must(new RangeQueryBuilder("status").gte(200).lt(300)) + .must(new RangeQueryBuilder("status").gte(400).lt(500)); + + filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + assertNull("Filter should be null for non-overlapping ranges", filter); + } + + public void testMalformedQueries() throws IOException { + // Test empty bool query + BoolQueryBuilder boolQuery = new BoolQueryBuilder(); + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for empty bool query", filter); + + // Test deeply nested empty bool queries + boolQuery = new BoolQueryBuilder().must(new BoolQueryBuilder().must(new BoolQueryBuilder().must(new BoolQueryBuilder()))); + + filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + assertNull("Filter should be null for nested empty bool queries", filter); + } + + public void testComplexMustWithNestedShould() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must(new RangeQueryBuilder("port").gte(80).lte(443)) + .must( + new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)).should(new RangeQueryBuilder("status").gte(500).lt(600)) + ) // Success or 5xx errors + .must( + new BoolQueryBuilder().must( + new BoolQueryBuilder().should(new TermQueryBuilder("zone", "us-east")).should(new TermQueryBuilder("zone", "us-west")) + ) + ); + + // Add field type for zone + KeywordFieldMapper.KeywordFieldType zoneType = new KeywordFieldMapper.KeywordFieldType("zone"); + when(mapperService.fieldType("zone")).thenReturn(zoneType); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have four dimensions", 4, filter.getDimensions().size()); + + // Verify method filter + List methodFilters = filter.getFiltersForDimension("method"); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "GET"); + + // Verify port range + List portFilters = filter.getFiltersForDimension("port"); + RangeMatchDimFilter portRange = (RangeMatchDimFilter) portFilters.getFirst(); + assertEquals(80L, portRange.getLow()); + assertEquals(443L, portRange.getHigh()); + assertTrue(portRange.isIncludeLow() && portRange.isIncludeHigh()); + + // Verify status filters (term OR range) + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals(2, statusFilters.size()); + for (DimensionFilter statusFilter : statusFilters) { + if (statusFilter instanceof ExactMatchDimFilter) { + assertEquals(200L, ((ExactMatchDimFilter) statusFilter).getRawValues().getFirst()); + } else { + RangeMatchDimFilter statusRange = (RangeMatchDimFilter) statusFilter; + assertEquals(500L, statusRange.getLow()); + assertEquals(599L, statusRange.getHigh()); + assertTrue(statusRange.isIncludeLow()); + assertTrue(statusRange.isIncludeHigh()); + } + } + + // Verify zone filters + List zoneFilters = filter.getFiltersForDimension("zone"); + assertEquals(2, zoneFilters.size()); + Set expectedZones = new HashSet<>(); + expectedZones.add(new BytesRef("us-east")); + expectedZones.add(new BytesRef("us-west")); + Set actualZones = new HashSet<>(); + for (DimensionFilter zoneFilter : zoneFilters) { + actualZones.addAll(((ExactMatchDimFilter) zoneFilter).getRawValues()); + } + assertEquals(expectedZones, actualZones); + } + + public void testRangeAndTermCombinations() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new RangeQueryBuilder("status").gte(200).lt(300)) // 2xx status codes + .must(new BoolQueryBuilder().should(new TermQueryBuilder("status", 201)).should(new TermQueryBuilder("status", 204))) // Specific + // success + // codes + .must(new TermQueryBuilder("method", "POST")); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + + // Verify method filter + List methodFilters = filter.getFiltersForDimension("method"); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "POST"); + + // Verify status filters (intersection of range and terms) + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals(2, statusFilters.size()); + Set expectedStatus = Set.of(201L, 204L); + Set actualStatus = new HashSet<>(); + for (DimensionFilter statusFilter : statusFilters) { + assertTrue(statusFilter instanceof ExactMatchDimFilter); + actualStatus.addAll(((ExactMatchDimFilter) statusFilter).getRawValues()); + } + assertEquals(expectedStatus, actualStatus); + } + + public void testDeepNestedShouldClauses() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must( + new BoolQueryBuilder().should( + new BoolQueryBuilder().should(new TermQueryBuilder("response_time", 100)) + .should(new TermQueryBuilder("response_time", 200)) + ) + .should( + new BoolQueryBuilder().should(new TermQueryBuilder("response_time", 300)) + .should(new TermQueryBuilder("response_time", 400)) + ) + ); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + + // Verify method filter + List methodFilters = filter.getFiltersForDimension("method"); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.get(0), "GET"); + + // Verify response_time filters (all SHOULD conditions) + List responseTimeFilters = filter.getFiltersForDimension("response_time"); + assertEquals(4, responseTimeFilters.size()); + Set expectedTimes = Set.of(100L, 200L, 300L, 400L); + Set actualTimes = new HashSet<>(); + for (DimensionFilter timeFilter : responseTimeFilters) { + assertTrue(timeFilter instanceof ExactMatchDimFilter); + actualTimes.addAll(((ExactMatchDimFilter) timeFilter).getRawValues()); + } + assertEquals(expectedTimes, actualTimes); + } + + public void testLargeNumberOfClauses() throws IOException { + // Create a bool query with large number of SHOULD clauses + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")); + + // Add 100 SHOULD clauses for status + BoolQueryBuilder statusShould = new BoolQueryBuilder(); + for (int i = 200; i < 300; i++) { + statusShould.should(new TermQueryBuilder("status", i)); + } + boolQuery.must(statusShould); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + + // Verify filters + List methodFilters = filter.getFiltersForDimension("method"); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "GET"); + + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals(100, statusFilters.size()); + } + + public void testMustInsideShould() throws IOException { + // Test valid case - all clauses on same dimension + BoolQueryBuilder validBoolQuery = new BoolQueryBuilder().should( + new BoolQueryBuilder().must(new RangeQueryBuilder("status").gte(200).lt(300)).must(new TermQueryBuilder("status", 201)) + ).should(new TermQueryBuilder("status", 404)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(validBoolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, validBoolQuery, compositeFieldType); + + assertNotNull("Filter should not be null for same dimension", filter); + assertEquals("Should have one dimension", 1, filter.getDimensions().size()); + List statusFilters = filter.getFiltersForDimension("status"); + assertEquals("Should have two filters", 2, statusFilters.size()); + Set expectedValues = Set.of(201L, 404L); + Set actualValues = new HashSet<>(); + for (DimensionFilter dimFilter : statusFilters) { + assertTrue("Should be ExactMatchDimFilter", dimFilter instanceof ExactMatchDimFilter); + actualValues.addAll(((ExactMatchDimFilter) dimFilter).getRawValues()); + } + assertEquals("Should contain expected values", expectedValues, actualValues); + + // Test invalid case - multiple dimensions in MUST inside SHOULD + BoolQueryBuilder invalidBoolQuery = new BoolQueryBuilder().should( + new BoolQueryBuilder().must(new TermQueryBuilder("status", 200)).must(new TermQueryBuilder("method", "GET")) + ).should(new TermQueryBuilder("status", 404)); + + filter = provider.getFilter(searchContext, invalidBoolQuery, compositeFieldType); + assertNull("Filter should be null for multiple dimensions in MUST inside SHOULD", filter); + } + + public void testCombinedMustAndFilterClauses() throws IOException { + // Test combination of MUST and FILTER clauses + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .filter(new TermQueryBuilder("status", 200)) + .filter(new RangeQueryBuilder("port").gte(80).lte(443)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have three dimensions", 3, filter.getDimensions().size()); + + // Verify method filter (from MUST) + List methodFilters = filter.getFiltersForDimension("method"); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "GET"); + + // Verify status filter (from FILTER) + List statusFilters = filter.getFiltersForDimension("status"); + assertExactMatchValue((ExactMatchDimFilter) statusFilters.getFirst(), 200L); + + // Verify port filter (from FILTER) + List portFilters = filter.getFiltersForDimension("port"); + RangeMatchDimFilter portRange = (RangeMatchDimFilter) portFilters.getFirst(); + assertEquals(80L, portRange.getLow()); + assertEquals(443L, portRange.getHigh()); + assertTrue(portRange.isIncludeLow() && portRange.isIncludeHigh()); + } + + public void testNestedBoolWithMustAndFilter() throws IOException { + // Test nested bool query with both MUST and FILTER clauses + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must(new BoolQueryBuilder().filter(new RangeQueryBuilder("status").gte(200).lt(300)).must(new TermQueryBuilder("status", 201))) // Should + // intersect + // with + // range + .filter(new TermQueryBuilder("port", 443)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + assertEquals("Should have three dimensions", 3, filter.getDimensions().size()); + + // Verify method filter + List methodFilters = filter.getFiltersForDimension("method"); + assertExactMatchValue((ExactMatchDimFilter) methodFilters.getFirst(), "GET"); + + // Verify status filter (intersection of range and term) + List statusFilters = filter.getFiltersForDimension("status"); + assertExactMatchValue((ExactMatchDimFilter) statusFilters.getFirst(), 201L); + + // Verify port filter + List portFilters = filter.getFiltersForDimension("port"); + assertExactMatchValue((ExactMatchDimFilter) portFilters.getFirst(), 443L); + } + + public void testInvalidMustAndFilterCombination() throws IOException { + // Test invalid combination - same dimension in MUST and FILTER + BoolQueryBuilder boolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("status", 200)) + .filter(new TermQueryBuilder("status", 404)); // Different value for same dimension + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null for conflicting conditions", filter); + } + + public void testKeywordRanges() throws IOException { + BoolQueryBuilder keywordRangeQuery = new BoolQueryBuilder().must(new RangeQueryBuilder("region").gte("eu-").lt("eu-z")) // Range of + // region + // codes + .must(new TermQueryBuilder("method", "GET")); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(keywordRangeQuery); + StarTreeFilter filter = provider.getFilter(searchContext, keywordRangeQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + List regionFilters = filter.getFiltersForDimension("region"); + assertTrue(regionFilters.getFirst() instanceof RangeMatchDimFilter); + RangeMatchDimFilter regionRange = (RangeMatchDimFilter) regionFilters.getFirst(); + assertEquals(new BytesRef("eu-"), regionRange.getLow()); + assertEquals(new BytesRef("eu-z"), regionRange.getHigh()); + assertTrue(regionRange.isIncludeLow()); + assertFalse(regionRange.isIncludeHigh()); + } + + public void testFloatRanges() throws IOException { + BoolQueryBuilder floatRangeQuery = new BoolQueryBuilder().must(new RangeQueryBuilder("latency").gte(0.5f).lte(2.0f)) + .must(new TermQueryBuilder("status", 200)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(floatRangeQuery); + StarTreeFilter filter = provider.getFilter(searchContext, floatRangeQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + List latencyFilters = filter.getFiltersForDimension("latency"); + assertTrue(latencyFilters.getFirst() instanceof RangeMatchDimFilter); + RangeMatchDimFilter latencyRange = (RangeMatchDimFilter) latencyFilters.getFirst(); + assertEquals(NumericUtils.floatToSortableInt(0.5f), ((Number) latencyRange.getLow()).floatValue(), 0.0001); + assertEquals(NumericUtils.floatToSortableInt(2.0f), ((Number) latencyRange.getHigh()).floatValue(), 0.0001); + assertTrue(latencyRange.isIncludeLow()); + assertTrue(latencyRange.isIncludeHigh()); + + // Test combined ranges in SHOULD + BoolQueryBuilder combinedRangeQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must( + new BoolQueryBuilder().should(new RangeQueryBuilder("latency").gte(0.0).lt(1.0)) + .should(new RangeQueryBuilder("latency").gte(2.0).lt(3.0)) + ); + + filter = provider.getFilter(searchContext, combinedRangeQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + latencyFilters = filter.getFiltersForDimension("latency"); + assertEquals(2, latencyFilters.size()); + for (DimensionFilter dimFilter : latencyFilters) { + assertTrue(dimFilter instanceof RangeMatchDimFilter); + } + } + + public void testFloatRanges_Exclusive() throws IOException { + // Test float range with different inclusivity combinations + BoolQueryBuilder floatRangeQuery = new BoolQueryBuilder().must(new RangeQueryBuilder("latency").gt(0.5).lt(2.0)) // exclusive + // bounds + .must(new TermQueryBuilder("status", 200)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(floatRangeQuery); + StarTreeFilter filter = provider.getFilter(searchContext, floatRangeQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + List latencyFilters = filter.getFiltersForDimension("latency"); + assertTrue(latencyFilters.getFirst() instanceof RangeMatchDimFilter); + RangeMatchDimFilter latencyRange = (RangeMatchDimFilter) latencyFilters.getFirst(); + + // For exclusive bounds (gt/lt), we need to use next/previous float values + long expectedLow = NumericUtils.floatToSortableInt(FloatPoint.nextUp(0.5f)); + long expectedHigh = NumericUtils.floatToSortableInt(FloatPoint.nextDown(2.0f)); + + assertEquals(expectedLow, ((Number) latencyRange.getLow()).longValue()); + assertEquals(expectedHigh, ((Number) latencyRange.getHigh()).longValue()); + assertTrue(latencyRange.isIncludeLow()); // After using nextUp, bound becomes inclusive + assertTrue(latencyRange.isIncludeHigh()); // After using nextDown, bound becomes inclusive + } + + public void testFloatRanges_Intersection() throws IOException { + // Test float range with different inclusivity combinations + BoolQueryBuilder floatRangeQuery = new BoolQueryBuilder().must(new RangeQueryBuilder("latency").gt(0.5).lt(2.0)) + .must(new RangeQueryBuilder("latency").gte(0.6).lt(1.8)) + .must(new TermQueryBuilder("status", 200)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(floatRangeQuery); + StarTreeFilter filter = provider.getFilter(searchContext, floatRangeQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + List latencyFilters = filter.getFiltersForDimension("latency"); + assertTrue(latencyFilters.getFirst() instanceof RangeMatchDimFilter); + RangeMatchDimFilter latencyRange = (RangeMatchDimFilter) latencyFilters.getFirst(); + + // For exclusive bounds (gt/lt), we need to use next/previous float values + long expectedLow = NumericUtils.floatToSortableInt(0.6f); + long expectedHigh = NumericUtils.floatToSortableInt(FloatPoint.nextDown(1.8f)); + + assertEquals(expectedLow, ((Number) latencyRange.getLow()).longValue()); + assertEquals(expectedHigh, ((Number) latencyRange.getHigh()).longValue()); + assertTrue(latencyRange.isIncludeLow()); + assertTrue(latencyRange.isIncludeHigh()); + } + + public void testKeywordRangeEdgeCases() throws IOException { + // Test unbounded ranges + BoolQueryBuilder unboundedQuery = new BoolQueryBuilder().must(new RangeQueryBuilder("region").gt("eu-")) // No upper bound + .must(new TermQueryBuilder("status", 200)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(unboundedQuery); + StarTreeFilter filter = provider.getFilter(searchContext, unboundedQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + List regionFilters = filter.getFiltersForDimension("region"); + RangeMatchDimFilter regionRange = (RangeMatchDimFilter) regionFilters.get(0); + assertEquals(new BytesRef("eu-"), regionRange.getLow()); + assertNull(regionRange.getHigh()); // Unbounded high + assertFalse(regionRange.isIncludeLow()); + assertTrue(regionRange.isIncludeHigh()); + + // Test range intersection + BoolQueryBuilder intersectionQuery = new BoolQueryBuilder().must(new RangeQueryBuilder("region").gte("eu-").lt("eu-z")) + .must(new RangeQueryBuilder("region").gt("eu-a").lte("eu-m")); + + filter = provider.getFilter(searchContext, intersectionQuery, compositeFieldType); + + assertNotNull("Filter should not be null", filter); + regionFilters = filter.getFiltersForDimension("region"); + regionRange = (RangeMatchDimFilter) regionFilters.get(0); + assertEquals(new BytesRef("eu-a"), regionRange.getLow()); + assertEquals(new BytesRef("eu-m"), regionRange.getHigh()); + assertFalse(regionRange.isIncludeLow()); + assertTrue(regionRange.isIncludeHigh()); + } + + public void testMinimumShouldMatch() throws IOException { + // Test with minimum_should_match set + BoolQueryBuilder boolQuery = new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)) + .should(new TermQueryBuilder("status", 404)) + .minimumShouldMatch(2); // Explicitly set minimum_should_match + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + assertNull("Filter should be null when minimum_should_match is set", filter); + + // Test nested bool with minimum_should_match + BoolQueryBuilder nestedBoolQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must( + new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)) + .should(new TermQueryBuilder("status", 404)) + .minimumShouldMatch(1) + ); // Set in nested bool + + filter = provider.getFilter(searchContext, nestedBoolQuery, compositeFieldType); + assertNull("Filter should be null when minimum_should_match is set in nested query", filter); + } + + public void testMustNotClauseReturnsNull() throws IOException { + BoolQueryBuilder boolQuery = new BoolQueryBuilder().mustNot(new TermQueryBuilder("status", 200)); + + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(boolQuery); + StarTreeFilter filter = provider.getFilter(searchContext, boolQuery, compositeFieldType); + + // this should return null as must not clause is not supported + assertNull("Filter should be null for same dimension in MUST", filter); + } + + // Helper methods for assertions + private void assertExactMatchValue(ExactMatchDimFilter filter, String expectedValue) { + assertEquals(new BytesRef(expectedValue), filter.getRawValues().getFirst()); + } + + private void assertExactMatchValue(ExactMatchDimFilter filter, Long expectedValue) { + assertEquals(expectedValue, filter.getRawValues().getFirst()); + } +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java index e89bc8e60e9da..d10ba190f86f1 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterAndMapperTests.java @@ -20,8 +20,10 @@ import org.opensearch.index.mapper.CompositeDataCubeFieldType; import org.opensearch.index.mapper.KeywordFieldMapper; import org.opensearch.index.mapper.MapperService; +import org.opensearch.index.mapper.NumberFieldMapper; import org.opensearch.index.mapper.StarTreeMapper; import org.opensearch.index.mapper.WildcardFieldMapper; +import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.RangeQueryBuilder; import org.opensearch.index.query.TermQueryBuilder; @@ -30,6 +32,7 @@ import org.opensearch.search.startree.filter.DimensionFilter; import org.opensearch.search.startree.filter.DimensionFilter.MatchType; import org.opensearch.search.startree.filter.MatchNoneFilter; +import org.opensearch.search.startree.filter.StarTreeFilter; import org.opensearch.search.startree.filter.provider.DimensionFilterMapper; import org.opensearch.search.startree.filter.provider.StarTreeFilterProvider; import org.opensearch.test.OpenSearchTestCase; @@ -132,7 +135,7 @@ public void testStarTreeFilterProviders() throws IOException { "star_tree", new StarTreeField( "star_tree", - List.of(new OrdinalDimension("keyword")), + List.of(new OrdinalDimension("keyword"), new OrdinalDimension("status"), new OrdinalDimension("method")), List.of(new Metric("field", List.of(MetricStat.MAX))), new StarTreeFieldConfiguration( randomIntBetween(1, 10_000), @@ -188,6 +191,48 @@ public void testStarTreeFilterProviders() throws IOException { assertFalse(dimensionFilter.matchDimValue(1, null)); dimensionFilter.matchStarTreeNodes(null, null, collector); assertEquals(0, collector.collectedNodeCount()); - } + // Setup common field types + KeywordFieldMapper.KeywordFieldType methodType = new KeywordFieldMapper.KeywordFieldType("method"); + NumberFieldMapper.NumberFieldType statusType = new NumberFieldMapper.NumberFieldType( + "status", + NumberFieldMapper.NumberType.INTEGER + ); + when(mapperService.fieldType("method")).thenReturn(methodType); + when(mapperService.fieldType("status")).thenReturn(statusType); + + // Test simple MUST clause + BoolQueryBuilder simpleMustQuery = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must(new TermQueryBuilder("status", 200)); + StarTreeFilterProvider provider = StarTreeFilterProvider.SingletonFactory.getProvider(simpleMustQuery); + StarTreeFilter filter = provider.getFilter(searchContext, simpleMustQuery, compositeDataCubeFieldType); + assertNotNull(filter); + assertEquals(2, filter.getDimensions().size()); + assertTrue(filter.getDimensions().contains("method")); + assertTrue(filter.getDimensions().contains("status")); + + // Test MUST with nested SHOULD on different dimension + BoolQueryBuilder mustWithShould = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must(new BoolQueryBuilder().should(new TermQueryBuilder("status", 200)).should(new TermQueryBuilder("status", 404))); + filter = provider.getFilter(searchContext, mustWithShould, compositeDataCubeFieldType); + assertNotNull(filter); + assertEquals(2, filter.getDimensions().size()); + assertEquals(1, filter.getFiltersForDimension("method").size()); + assertEquals(2, filter.getFiltersForDimension("status").size()); + + // Test invalid SHOULD across dimensions + BoolQueryBuilder invalidShould = new BoolQueryBuilder().should(new TermQueryBuilder("method", "GET")) + .should(new TermQueryBuilder("status", 200)); + assertNull(provider.getFilter(searchContext, invalidShould, compositeDataCubeFieldType)); + + // Test MUST with invalid nested SHOULD + BoolQueryBuilder invalidNestedShould = new BoolQueryBuilder().must(new TermQueryBuilder("method", "GET")) + .must(new BoolQueryBuilder().should(new TermQueryBuilder("method", "POST")).should(new TermQueryBuilder("status", 200))); + assertNull(provider.getFilter(searchContext, invalidNestedShould, compositeDataCubeFieldType)); + + // Test MUST with SHOULD on same dimension + BoolQueryBuilder mustWithSameDimShould = new BoolQueryBuilder().must(new TermQueryBuilder("status", 200)) + .must(new BoolQueryBuilder().should(new TermQueryBuilder("status", 404)).should(new TermQueryBuilder("status", 500))); + assertNull(provider.getFilter(searchContext, mustWithSameDimShould, compositeDataCubeFieldType)); + } } diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterMergerUtilsTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterMergerUtilsTests.java new file mode 100644 index 0000000000000..cfaf5823428e1 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/DimensionFilterMergerUtilsTests.java @@ -0,0 +1,401 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.search.aggregations.startree; + +import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; +import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.index.mapper.KeywordFieldMapper; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.search.internal.SearchContext; +import org.opensearch.search.startree.StarTreeNodeCollector; +import org.opensearch.search.startree.filter.DimensionFilter; +import org.opensearch.search.startree.filter.DimensionFilterMergerUtils; +import org.opensearch.search.startree.filter.ExactMatchDimFilter; +import org.opensearch.search.startree.filter.RangeMatchDimFilter; +import org.opensearch.search.startree.filter.provider.DimensionFilterMapper; +import org.opensearch.test.OpenSearchTestCase; +import org.junit.Before; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +public class DimensionFilterMergerUtilsTests extends OpenSearchTestCase { + + private DimensionFilterMapper numericMapper; + private DimensionFilterMapper keywordMapper; + + @Before + public void setup() { + numericMapper = DimensionFilterMapper.Factory.fromMappedFieldType( + new NumberFieldMapper.NumberFieldType("status", NumberFieldMapper.NumberType.LONG) + ); + keywordMapper = DimensionFilterMapper.Factory.fromMappedFieldType(new KeywordFieldMapper.KeywordFieldType("method")); + } + + public void testRangeIntersection() { + // Basic range intersection + assertRangeIntersection( + range("status", 200L, 500L, true, true), + range("status", 300L, 400L, true, true), + range("status", 300L, 400L, true, true), + numericMapper + ); + + // Boundary conditions + assertRangeIntersection( + range("status", 200L, 200L, true, true), + range("status", 200L, 200L, true, true), + range("status", 200L, 200L, true, true), + numericMapper + ); + + // Inclusive/Exclusive boundaries + assertRangeIntersection( + range("status", 200L, 300L, true, false), + range("status", 200L, 300L, false, true), + range("status", 200L, 300L, false, false), + numericMapper + ); + + // Non-overlapping ranges + assertNoIntersection(range("status", 200L, 300L, true, true), range("status", 301L, 400L, true, true), numericMapper); + + // Exactly touching ranges (no overlap) + assertNoIntersection(range("status", 200L, 300L, true, false), range("status", 300L, 400L, true, true), numericMapper); + + // Null bounds (unbounded ranges) + assertRangeIntersection( + range("status", null, 500L, true, true), + range("status", 200L, null, true, true), + range("status", 200L, 500L, true, true), + numericMapper + ); + assertRangeIntersection( + range("status", 200L, null, true, true), + range("status", null, 500L, true, true), + range("status", 200L, 500L, true, true), + numericMapper + ); + + // Single point overlap + assertRangeIntersection( + range("status", 200L, 300L, true, true), + range("status", 300L, 400L, true, true), + range("status", 300L, 300L, true, true), + numericMapper + ); + + // Very large ranges + assertRangeIntersection( + range("status", Long.MIN_VALUE, Long.MAX_VALUE, true, true), + range("status", 200L, 300L, true, true), + range("status", 200L, 300L, true, true), + numericMapper + ); + + // Zero-width ranges + assertNoIntersection(range("status", 200L, 200L, true, true), range("status", 200L, 200L, false, false), numericMapper); + + // incompatible types + assertThrows( + IllegalArgumentException.class, + () -> DimensionFilterMergerUtils.intersect( + range("status", "200", "300", true, true), + range("status", 200, 300, true, true), + numericMapper + ) + ); + } + + public void testExactMatchIntersection() { + // Single value intersection + assertExactMatchIntersection( + exactMatch("status", List.of(200)), + exactMatch("status", List.of(200)), + exactMatch("status", List.of(200)), + numericMapper + ); + + // Multiple values intersection + assertExactMatchIntersection( + exactMatch("status", Arrays.asList(200, 300, 400)), + exactMatch("status", Arrays.asList(300, 400, 500)), + exactMatch("status", Arrays.asList(300, 400)), + numericMapper + ); + + // No intersection + assertNoIntersection(exactMatch("status", List.of(200)), exactMatch("status", List.of(300)), numericMapper); + + // Empty list + assertNoIntersection(exactMatch("status", Collections.emptyList()), exactMatch("status", List.of(200)), numericMapper); + + // Duplicate values + assertExactMatchIntersection( + exactMatch("status", Arrays.asList(200, 200, 300)), + exactMatch("status", Arrays.asList(200, 300, 300)), + exactMatch("status", Arrays.asList(200, 300)), + numericMapper + ); + + // Special characters in string values + assertExactMatchIntersection( + exactMatch("method", Arrays.asList("GET", "GET*")), + exactMatch("method", Arrays.asList("GET", "GET/")), + exactMatch("method", List.of("GET")), + keywordMapper + ); + + // Case sensitivity + assertNoIntersection( + exactMatch("method", Arrays.asList("GET", "Post")), + exactMatch("method", Arrays.asList("get", "POST")), + keywordMapper + ); + } + + public void testRangeExactMatchIntersection() { + // Value in range + assertRangeExactMatchIntersection( + range("status", 200L, 300L, true, true), + exactMatch("status", List.of(250L)), + exactMatch("status", List.of(250L)), + numericMapper + ); + + // Value at range boundaries + assertRangeExactMatchIntersection( + range("status", 200L, 300L, true, true), + exactMatch("status", Arrays.asList(200L, 300L)), + exactMatch("status", Arrays.asList(200L, 300L)), + numericMapper + ); + + // Value at exclusive boundaries + assertRangeExactMatchIntersection( + range("status", 200L, 300L, false, false), + exactMatch("status", Arrays.asList(201L, 299L)), + exactMatch("status", Arrays.asList(201L, 299L)), + numericMapper + ); + + // No values in range + assertNoIntersection(range("status", 200L, 300L, true, true), exactMatch("status", Arrays.asList(199L, 301L)), numericMapper); + + // Multiple values, some in range + assertRangeExactMatchIntersection( + range("status", 200L, 300L, true, true), + exactMatch("status", Arrays.asList(199L, 200L, 250L, 300L, 301L)), + exactMatch("status", Arrays.asList(200L, 250L, 300L)), + numericMapper + ); + } + + public void testDifferentDimensions() { + // Cannot intersect different dimensions + assertThrows( + IllegalArgumentException.class, + () -> DimensionFilterMergerUtils.intersect( + range("status", 200, 300, true, true), + range("port", 80, 443, true, true), + numericMapper + ) + ); + } + + public void testUnsignedLongRangeIntersection() { + // Setup unsigned long mapper + DimensionFilterMapper unsignedLongMapper = DimensionFilterMapper.Factory.fromMappedFieldType( + new NumberFieldMapper.NumberFieldType("unsigned_field", NumberFieldMapper.NumberType.UNSIGNED_LONG) + ); + + // Test case 1: Regular positive values + assertRangeIntersection( + range("unsigned_field", 100L, 500L, true, true), + range("unsigned_field", 200L, 300L, true, true), + range("unsigned_field", 200L, 300L, true, true), + unsignedLongMapper + ); + + // Test case 2: High values (near max unsigned long) + assertRangeIntersection( + range("unsigned_field", -10L, -1L, true, true), // -1L is max unsigned long (2^64 - 1) + range("unsigned_field", -5L, -2L, true, true), + range("unsigned_field", -5L, -2L, true, true), + unsignedLongMapper + ); + + // Test case 3: Crossing the unsigned boundary + assertRangeIntersection( + range("unsigned_field", 2L, -2L, true, true), // -2L is near max unsigned long + range("unsigned_field", 1L, -1L, true, true), // -1L is max unsigned long + range("unsigned_field", 2L, -2L, true, true), + unsignedLongMapper + ); + + // Test case 4: Non-overlapping ranges in unsigned space + assertNoIntersection( + range("unsigned_field", -10L, -5L, true, true), // High unsigned values + range("unsigned_field", 1L, 10L, true, true), // Low unsigned values + unsignedLongMapper + ); + + // Test case 5: Single point intersection at max unsigned value + assertRangeIntersection( + range("unsigned_field", -2L, -1L, true, true), // -1L is max unsigned long + range("unsigned_field", 0L, -1L, true, true), + range("unsigned_field", -2L, -1L, true, true), + unsignedLongMapper + ); + + // Test case 6: Full range + assertRangeIntersection( + range("unsigned_field", 0L, -1L, true, true), // 0 to max unsigned + range("unsigned_field", 100L, 200L, true, true), + range("unsigned_field", 100L, 200L, true, true), + unsignedLongMapper + ); + } + + public void testIntersectValidation() { + // Test null filters + assertNull( + "Should return null for null first filter", + DimensionFilterMergerUtils.intersect(null, exactMatch("status", List.of(200L)), numericMapper) + ); + assertNull( + "Should return null for null second filter", + DimensionFilterMergerUtils.intersect(exactMatch("status", List.of(200L)), null, numericMapper) + ); + assertNull("Should return null for both null filters", DimensionFilterMergerUtils.intersect(null, null, numericMapper)); + + // Test null dimension names + DimensionFilter nullDimFilter1 = new ExactMatchDimFilter(null, List.of(200L)); + DimensionFilter nullDimFilter2 = new ExactMatchDimFilter(null, List.of(300L)); + IllegalArgumentException e1 = assertThrows( + IllegalArgumentException.class, + () -> DimensionFilterMergerUtils.intersect(nullDimFilter1, exactMatch("status", List.of(200L)), numericMapper) + ); + assertEquals("Cannot intersect filters with null dimension name", e1.getMessage()); + + IllegalArgumentException e2 = assertThrows( + IllegalArgumentException.class, + () -> DimensionFilterMergerUtils.intersect(exactMatch("status", List.of(200L)), nullDimFilter2, numericMapper) + ); + assertEquals("Cannot intersect filters with null dimension name", e2.getMessage()); + + IllegalArgumentException e3 = assertThrows( + IllegalArgumentException.class, + () -> DimensionFilterMergerUtils.intersect(nullDimFilter1, nullDimFilter2, numericMapper) + ); + assertEquals("Cannot intersect filters with null dimension name", e3.getMessage()); + + // Test different dimensions + IllegalArgumentException e4 = assertThrows( + IllegalArgumentException.class, + () -> DimensionFilterMergerUtils.intersect( + exactMatch("status", List.of(200L)), + exactMatch("method", List.of("GET")), + numericMapper + ) + ); + assertEquals("Cannot intersect filters for different dimensions: status and method", e4.getMessage()); + } + + public void testUnsupportedFilterCombination() { + // Create a custom filter type for testing + class CustomDimensionFilter implements DimensionFilter { + @Override + public String getDimensionName() { + return "status"; + } + + @Override + public void initialiseForSegment(StarTreeValues starTreeValues, SearchContext searchContext) {} + + @Override + public void matchStarTreeNodes(StarTreeNode parentNode, StarTreeValues starTreeValues, StarTreeNodeCollector collector) {} + + @Override + public boolean matchDimValue(long ordinal, StarTreeValues starTreeValues) { + return false; + } + } + + DimensionFilter customFilter = new CustomDimensionFilter(); + + // Test unsupported combination with ExactMatchDimFilter + IllegalArgumentException e1 = assertThrows( + IllegalArgumentException.class, + () -> DimensionFilterMergerUtils.intersect(customFilter, exactMatch("status", List.of(200L)), numericMapper) + ); + assertEquals("Unsupported filter combination: CustomDimensionFilter and ExactMatchDimFilter", e1.getMessage()); + + // Test unsupported combination with RangeMatchDimFilter + IllegalArgumentException e2 = assertThrows( + IllegalArgumentException.class, + () -> DimensionFilterMergerUtils.intersect(range("status", 200L, 300L, true, true), customFilter, numericMapper) + ); + assertEquals("Unsupported filter combination: RangeMatchDimFilter and CustomDimensionFilter", e2.getMessage()); + } + + // Helper methods + private RangeMatchDimFilter range(String dimension, Object low, Object high, boolean includeLow, boolean includeHigh) { + return new RangeMatchDimFilter(dimension, low, high, includeLow, includeHigh); + } + + private ExactMatchDimFilter exactMatch(String dimension, List values) { + return new ExactMatchDimFilter(dimension, values); + } + + private void assertRangeIntersection( + RangeMatchDimFilter filter1, + RangeMatchDimFilter filter2, + RangeMatchDimFilter expected, + DimensionFilterMapper mapper + ) { + DimensionFilter result = DimensionFilterMergerUtils.intersect(filter1, filter2, mapper); + assertTrue(result instanceof RangeMatchDimFilter); + RangeMatchDimFilter rangeResult = (RangeMatchDimFilter) result; + assertEquals(expected.getLow(), rangeResult.getLow()); + assertEquals(expected.getHigh(), rangeResult.getHigh()); + assertEquals(expected.isIncludeLow(), rangeResult.isIncludeLow()); + assertEquals(expected.isIncludeHigh(), rangeResult.isIncludeHigh()); + } + + private void assertExactMatchIntersection( + ExactMatchDimFilter filter1, + ExactMatchDimFilter filter2, + ExactMatchDimFilter expected, + DimensionFilterMapper mapper + ) { + DimensionFilter result = DimensionFilterMergerUtils.intersect(filter1, filter2, mapper); + assertTrue(result instanceof ExactMatchDimFilter); + ExactMatchDimFilter exactResult = (ExactMatchDimFilter) result; + assertEquals(new HashSet<>(expected.getRawValues()), new HashSet<>(exactResult.getRawValues())); + } + + private void assertRangeExactMatchIntersection( + RangeMatchDimFilter range, + ExactMatchDimFilter exact, + ExactMatchDimFilter expected, + DimensionFilterMapper mapper + ) { + DimensionFilter result = DimensionFilterMergerUtils.intersect(range, exact, mapper); + assertTrue(result instanceof ExactMatchDimFilter); + ExactMatchDimFilter exactResult = (ExactMatchDimFilter) result; + assertEquals(new HashSet<>(expected.getRawValues()), new HashSet<>(exactResult.getRawValues())); + } + + private void assertNoIntersection(DimensionFilter filter1, DimensionFilter filter2, DimensionFilterMapper mapper) { + assertNull(DimensionFilterMergerUtils.intersect(filter1, filter2, mapper)); + } +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java index 4555382700f21..6640b691449c1 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/MetricAggregatorTests.java @@ -52,6 +52,7 @@ import org.opensearch.index.mapper.MappedFieldType; import org.opensearch.index.mapper.MapperService; import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.index.query.BoolQueryBuilder; import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.RangeQueryBuilder; @@ -255,7 +256,9 @@ private void testStarTreeDocValuesInternal(Codec codec, List for (int cases = 0; cases < 15; cases++) { // Get all types of queries (Term/Terms/Range) for all the given dimensions. List allFieldQueries = dimensionFieldData.stream() - .flatMap(x -> Stream.of(x.getTermQueryBuilder(), x.getTermsQueryBuilder(), x.getRangeQueryBuilder())) + .flatMap( + x -> Stream.of(x.getTermQueryBuilder(), x.getTermsQueryBuilder(), x.getRangeQueryBuilder(), x.getBoolQueryBuilder()) + ) .toList(); for (QueryBuilder qb : allFieldQueries) { @@ -557,6 +560,24 @@ public QueryBuilder getRangeQueryBuilder() { .includeUpper(randomBoolean()); } + public QueryBuilder getBoolQueryBuilder() { + // MUST only + BoolQueryBuilder mustOnly = new BoolQueryBuilder().must(getTermQueryBuilder()).must(getRangeQueryBuilder()); + + // MUST with nested SHOULD on same dimension + BoolQueryBuilder mustWithShould = new BoolQueryBuilder().must(getTermQueryBuilder()) + .must( + new BoolQueryBuilder().should(new TermQueryBuilder(fieldName, valueSupplier.get())) + .should(new TermQueryBuilder(fieldName, valueSupplier.get())) + ); + + // SHOULD only on same dimension + BoolQueryBuilder shouldOnly = new BoolQueryBuilder().should(new TermQueryBuilder(fieldName, valueSupplier.get())) + .should(new RangeQueryBuilder(fieldName).from(valueSupplier.get()).to(valueSupplier.get())); + + return randomFrom(mustOnly, mustWithShould, shouldOnly); + } + public String getFieldType() { return fieldType; } diff --git a/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java b/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java index cd2943f23be7a..dbd52aab6a687 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/startree/StarTreeFilterTests.java @@ -49,6 +49,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; @@ -481,4 +482,41 @@ public static XContentBuilder getExpandedMapping( b.endObject(); }); } + + public void testStarTreeFilterWithBoolQueries() throws IOException { + List docs = new ArrayList<>(); + Directory directory = createStarTreeIndex(5, true, docs); + DirectoryReader ir = DirectoryReader.open(directory); + initValuesSourceRegistry(); + LeafReaderContext context = ir.leaves().get(0); + SegmentReader reader = Lucene.segmentReader(context.reader()); + CompositeIndexReader starTreeDocValuesReader = (CompositeIndexReader) reader.getDocValuesReader(); + + MapperService mapperService = Mockito.mock(MapperService.class); + SearchContext searchContext = Mockito.mock(SearchContext.class); + Mockito.when(searchContext.mapperService()).thenReturn(mapperService); + Mockito.when(mapperService.fieldType(SNDV)) + .thenReturn(new NumberFieldMapper.NumberFieldType(SNDV, NumberFieldMapper.NumberType.INTEGER)); + Mockito.when(mapperService.fieldType(DV)) + .thenReturn(new NumberFieldMapper.NumberFieldType(DV, NumberFieldMapper.NumberType.INTEGER)); + + // Test 'MUST' clause + StarTreeFilter mustFilter = new StarTreeFilter( + Map.of(SNDV, List.of(new ExactMatchDimFilter(SNDV, List.of(0L))), DV, List.of(new ExactMatchDimFilter(DV, List.of(0L)))) + ); + long starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, mustFilter, context, searchContext); + long docCount = getDocCount(docs, Map.of(SNDV, 0L, DV, 0L)); + assertEquals(docCount, starTreeDocCount); + + // Test 'SHOULD' clause (same dimension) + StarTreeFilter shouldFilter = new StarTreeFilter( + Map.of(SNDV, Arrays.asList(new ExactMatchDimFilter(SNDV, List.of(0L)), new ExactMatchDimFilter(SNDV, List.of(1L)))) + ); + starTreeDocCount = getDocCountFromStarTree(starTreeDocValuesReader, shouldFilter, context, searchContext); + docCount = getDocCount(docs, Map.of(SNDV, 0L)) + getDocCount(docs, Map.of(SNDV, 1L)); + assertEquals(docCount, starTreeDocCount); + + ir.close(); + directory.close(); + } }