Skip to content
This repository was archived by the owner on Aug 2, 2022. It is now read-only.

Expression pushdown optimization #663

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
5198a38
Test serializer
dai-chen Jul 20, 2020
75005eb
Merge branch 'develop' into expression-pushdown-optimization
dai-chen Jul 30, 2020
705397b
Add JDK serializer impl
dai-chen Jul 31, 2020
defdd08
Add UT
dai-chen Jul 31, 2020
6a51c26
Support boolean literal
dai-chen Jul 31, 2020
24a4423
Add UT for comparison expression
dai-chen Jul 31, 2020
2283de1
Add UT for function expression
dai-chen Jul 31, 2020
bd37727
Add UT for multiple fields
dai-chen Jul 31, 2020
8e4364b
Use expr value factory
dai-chen Aug 3, 2020
5e42d86
Add comments
dai-chen Aug 3, 2020
1614040
Merge branch 'develop' into expression-pushdown-optimization
dai-chen Aug 3, 2020
cca0479
Test coverage
dai-chen Aug 4, 2020
de03a58
Refactor package and class
dai-chen Aug 4, 2020
f77a739
Add UT for script factory and leaf factory
dai-chen Aug 4, 2020
c95c2e1
Add support for date doc value
dai-chen Aug 5, 2020
3193fdc
More UT
dai-chen Aug 5, 2020
f29b402
Refactor edge case UT
dai-chen Aug 5, 2020
8bb9405
Add UT for value factory
dai-chen Aug 5, 2020
0c0b0d5
Add UT for serializer error case
dai-chen Aug 5, 2020
45bf93d
Fix checkstyle
dai-chen Aug 5, 2020
8ccd384
Add expression visitor and UT
dai-chen Aug 5, 2020
4d0c97b
Use expression visitor in script engine
dai-chen Aug 5, 2020
f839f8f
Add UT for visitor
dai-chen Aug 5, 2020
0bac7b5
Push down query
dai-chen Aug 5, 2020
da2ece3
Handle text keyword
dai-chen Aug 6, 2020
81acd11
Cast long/double doc value to int/float
dai-chen Aug 6, 2020
0ddcab4
Don't push down if illegal state exception thrown
dai-chen Aug 6, 2020
6d259fc
Fix broken IT due to field type change
dai-chen Aug 6, 2020
154358a
Prepare PR
dai-chen Aug 6, 2020
f6eafcb
Address PR comments
dai-chen Aug 7, 2020
af8beb3
Address PR comments
dai-chen Aug 8, 2020
5952930
Merge branch 'develop' into expression-pushdown-optimization
dai-chen Aug 11, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,15 @@ public interface Expression extends Serializable {
* The type of the expression.
*/
ExprType type();

/**
* Accept a visitor to visit current expression node.
* @param visitor visitor
* @param context context
* @param <T> result type
* @param <C> context type
* @return result accumulated by visitor when visiting
*/
<T, C> T accept(ExpressionNodeVisitor<T, C> visitor, C context);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*
*/

package com.amazon.opendistroforelasticsearch.sql.expression;

import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator;
import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionImplementation;

/**
* Abstract visitor for expression tree nodes.
* @param <T> type of return value to accumulate when visiting.
* @param <C> type of context.
*/
public abstract class ExpressionNodeVisitor<T, C> {

public T visitNode(Expression node, C context) {
return null;
}

/**
* Visit children nodes in function arguments.
* @param node function node
* @param context context
* @return result
*/
public T visitChildren(FunctionImplementation node, C context) {
T result = defaultResult();

for (Expression child : node.getArguments()) {
T childResult = child.accept(this, context);
result = aggregateResult(result, childResult);
}
return result;
}

private T defaultResult() {
return null;
}

private T aggregateResult(T aggregate, T nextResult) {
return nextResult;
}

public T visitLiteral(LiteralExpression node, C context) {
return visitNode(node, context);
}

public T visitNamed(NamedExpression node, C context) {
return visitNode(node, context);
}

public T visitReference(ReferenceExpression node, C context) {
return visitNode(node, context);
}

public T visitFunction(FunctionExpression node, C context) {
return visitChildren(node, context);
}

public T visitAggregator(Aggregator<?> node, C context) {
return visitChildren(node, context);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,10 @@ public abstract class FunctionExpression implements Expression, FunctionImplemen

@Getter
private final List<Expression> arguments;

@Override
public <T, C> T accept(ExpressionNodeVisitor<T, C> visitor, C context) {
return visitor.visitFunction(this, context);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,11 @@ public ExprType type() {
return exprValue.type();
}

@Override
public <T, C> T accept(ExpressionNodeVisitor<T, C> visitor, C context) {
return visitor.visitLiteral(this, context);
}

@Override
public String toString() {
return exprValue.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,4 +71,9 @@ public String getName() {
return Strings.isNullOrEmpty(alias) ? name : alias;
}

@Override
public <T, C> T accept(ExpressionNodeVisitor<T, C> visitor, C context) {
return visitor.visitNamed(this, context);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ public ExprType type() {
return type;
}

@Override
public <T, C> T accept(ExpressionNodeVisitor<T, C> visitor, C context) {
return visitor.visitReference(this, context);
}

@Override
public String toString() {
return attr;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
import com.amazon.opendistroforelasticsearch.sql.exception.ExpressionEvaluationException;
import com.amazon.opendistroforelasticsearch.sql.expression.Expression;
import com.amazon.opendistroforelasticsearch.sql.expression.ExpressionNodeVisitor;
import com.amazon.opendistroforelasticsearch.sql.expression.env.Environment;
import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionImplementation;
import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName;
Expand Down Expand Up @@ -70,4 +71,10 @@ public ExprValue valueOf(Environment<Expression, ExprValue> valueEnv) {
public ExprType type() {
return returnType;
}

@Override
public <T, C> T accept(ExpressionNodeVisitor<T, C> visitor, C context) {
return visitor.visitAggregator(this, context);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*
*/

package com.amazon.opendistroforelasticsearch.sql.expression;

import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.INTEGER;
import static com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType.STRING;
import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.literal;
import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.named;
import static com.amazon.opendistroforelasticsearch.sql.expression.DSL.ref;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;

import com.amazon.opendistroforelasticsearch.sql.expression.aggregation.Aggregator;
import com.amazon.opendistroforelasticsearch.sql.expression.config.ExpressionConfig;
import java.util.List;
import java.util.stream.Collectors;
import org.junit.jupiter.api.DisplayNameGeneration;
import org.junit.jupiter.api.DisplayNameGenerator;
import org.junit.jupiter.api.Test;

@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class)
class ExpressionNodeVisitorTest {

private final DSL dsl = new ExpressionConfig().dsl(new ExpressionConfig().functionRepository());

@Test
void should_return_null_by_default() {
ExpressionNodeVisitor<Object, Object> visitor = new ExpressionNodeVisitor<Object, Object>(){};
assertNull(literal(10).accept(visitor, null));
assertNull(ref("name", STRING).accept(visitor, null));
assertNull(named("bool", literal(true)).accept(visitor, null));
assertNull(dsl.abs(literal(-10)).accept(visitor, null));
assertNull(dsl.sum(literal(10)).accept(visitor, null));
}

@Test
void can_visit_all_types_of_expression_node() {
Expression expr =
dsl.sum(
dsl.add(
ref("balance", INTEGER),
literal(10)));

Expression actual = expr.accept(new ExpressionNodeVisitor<Expression, Object>() {
@Override
public Expression visitLiteral(LiteralExpression node, Object context) {
return node;
}

@Override
public Expression visitReference(ReferenceExpression node, Object context) {
return node;
}

@Override
public Expression visitFunction(FunctionExpression node, Object context) {
return dsl.add(visitArguments(node.getArguments(), context));
}

@Override
public Expression visitAggregator(Aggregator<?> node, Object context) {
return dsl.sum(visitArguments(node.getArguments(), context));
}

private Expression[] visitArguments(List<Expression> arguments, Object context) {
return arguments.stream()
.map(arg -> arg.accept(this, context))
.toArray(Expression[]::new);
}
}, null);

assertEquals(expr, actual);
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*
*/

package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value;

import static com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.type.ElasticsearchDataType.ES_TEXT_KEYWORD;

import com.amazon.opendistroforelasticsearch.sql.data.model.ExprStringValue;
import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;

/**
* Expression Text Keyword Value, it is an extension of the ExprValue by Elasticsearch.
* This mostly represents a multi-field in Elasticsearch which has a text field and a
* keyword field inside to preserve the original text.
*/
public class ElasticsearchExprTextKeywordValue extends ExprStringValue {

public ElasticsearchExprTextKeywordValue(String value) {
super(value);
}

@Override
public ExprType type() {
return ES_TEXT_KEYWORD;
}

}
Loading