Opendistro Release 1.9.0 (#532)

* prepare odfe 1.9

* Fix all ES 7.8 compile and build errors

* Revert changes as Lombok is working now

* Update CustomExternalTestCluster.java

* Fix license headers check

* Use splitFieldsByMetadata to separate fields when calling SearchHit constructor

* More fixes for ODFE 1.9

* Remove todo statement

* Add ODFE 1.9.0 release notes

Author: joshuali925

build.gradle

@@ -15,7 +15,7 @@
 
 buildscript {
     ext {
-        es_version = "7.7.0"
+        es_version = "7.8.0"
     }
     // This isn't applying from repositories.gradle so repeating it here
     repositories {
@@ -40,7 +40,7 @@ repositories {
 }
 
 ext {
-    opendistroVersion = '1.8.0'
+    opendistroVersion = '1.9.0'
     isSnapshot = "true" == System.getProperty("build.snapshot", "true")
 }
 
@@ -97,6 +97,10 @@ licenseHeaders {
     excludes = ['com/amazon/opendistroforelasticsearch/sql/antlr/parser/**']
 }
 
+tasks.withType(licenseHeaders.class) {
+    additionalLicense 'AL   ', 'Apache', 'Licensed under the Apache License, Version 2.0 (the "License")'
+}
+
 // TODO: need to fix java doc to enable JavaDoc
 javadoc.enabled = false
 esplugin {
@@ -272,14 +276,16 @@ dependencies {
     testCompile group: "org.elasticsearch.client", name: 'transport', version: "${es_version}"
 
     // JDBC drivers for comparison test. Somehow Apache Derby throws security permission exception.
-    testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.3.0.0'
+    testCompile group: 'com.amazon.opendistroforelasticsearch.client', name: 'opendistro-sql-jdbc', version: '1.8.0.0'
     testCompile group: 'com.h2database', name: 'h2', version: '1.4.200'
     testCompile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.28.0'
     //testCompile group: 'org.apache.derby', name: 'derby', version: '10.15.1.3'
 }
 
 apply plugin: 'nebula.ospackage'
 
+validateNebulaPom.enabled = false
+
 // This is afterEvaluate because the bundlePlugin ZIP task is updated afterEvaluate and changes the ZIP name to match the plugin name
 afterEvaluate {
     ospackage {

gradle.properties

@@ -13,4 +13,4 @@
 #   permissions and limitations under the License.
 #
 
-version=1.8.0
+version=1.9.0

gradle/wrapper/gradle-wrapper.properties

@@ -13,7 +13,7 @@
 #   permissions and limitations under the License.
 #
 
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists

release-notes/opendistro-elasticsearch-sql.release-notes-1.9.0.md

@@ -0,0 +1,16 @@
+## 2020-06-23 Version 1.9.0.0
+
+### Features
+#### Elasticsearch Compatibility
+* Feature [#532](https://github.com/opendistro-for-elasticsearch/sql/pull/532): Elasticsearch 7.8.0 compatibility (issue: [#531](https://github.com/opendistro-for-elasticsearch/sql/issues/531))
+
+#### Documentation
+* Feature [#486](https://github.com/opendistro-for-elasticsearch/sql/pull/486): Add Github badges to README
+
+### Enhancements
+#### SQL Features
+* Feature [#473](https://github.com/opendistro-for-elasticsearch/sql/pull/473): Support Integration Tests for Security enabled ODFE cluster
+
+### Bugfixes
+* Bugfix [#489](https://github.com/opendistro-for-elasticsearch/sql/pull/489): Fix ANTLR grammar for negative integer and floating point number (issue: [#488](https://github.com/opendistro-for-elasticsearch/sql/issues/488))
+* Bugfix [#522](https://github.com/opendistro-for-elasticsearch/sql/pull/522): Bug fix, support long type for aggregation (issue: [#521](https://github.com/opendistro-for-elasticsearch/sql/issues/521))

src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/LocalClusterState.java

@@ -109,11 +109,11 @@ public void setClusterService(ClusterService clusterService) {
         this.clusterService = clusterService;
 
         clusterService.addListener(event -> {
-            if (event.metaDataChanged()) {
+            if (event.metadataChanged()) {
                 // State in cluster service is already changed to event.state() before listener fired
                 if (LOG.isDebugEnabled()) {
                     LOG.debug("Metadata in cluster state changed: {}",
-                            new IndexMappings(clusterService.state().metaData()));
+                            new IndexMappings(clusterService.state().metadata()));
                 }
                 cache.invalidateAll();
             }
@@ -169,8 +169,8 @@ public IndexMappings getFieldMappings(String[] indices, String[] types) {
     }
 
     /**
-     * Get field mappings by index expressions, type and field filter. Because IndexMetaData/MappingMetaData
-     * is hard to convert to FieldMappingMetaData, custom mapping domain objects are being used here. In future,
+     * Get field mappings by index expressions, type and field filter. Because IndexMetadata/MappingMetadata
+     * is hard to convert to FieldMappingMetadata, custom mapping domain objects are being used here. In future,
      * it should be moved to domain model layer for all ES specific knowledge.
      * <p>
      * Note that cluster state may be change inside ES so it's possible to read different state in 2 accesses
@@ -222,7 +222,7 @@ private IndexMappings findMappings(ClusterState state, String[] indices, String[
                                        Function<String, Predicate<String>> fieldFilter) throws IOException {
         LOG.debug("Cache didn't help. Load and parse mapping in cluster state");
         return new IndexMappings(
-                state.metaData().findMappings(indices, types, fieldFilter)
+                state.metadata().findMappings(indices, types, fieldFilter)
         );
     }
 

src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMapping.java

@@ -21,7 +21,7 @@
 import java.util.Map;
 
 import static java.util.Collections.emptyMap;
-import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetaData;
+import static org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsResponse.FieldMappingMetadata;
 
 /**
  * Field mapping that parses native ES mapping.
@@ -39,7 +39,7 @@ public class FieldMapping {
     /**
      * Native mapping information returned from ES
      */
-    private final Map<String, FieldMappingMetaData> typeMappings;
+    private final Map<String, FieldMappingMetadata> typeMappings;
 
     /**
      * Maps a field name to Field object that specified in query explicitly
@@ -51,7 +51,7 @@ public FieldMapping(String fieldName) {
     }
 
     public FieldMapping(String fieldName,
-                        Map<String, FieldMappingMetaData> typeMappings,
+                        Map<String, FieldMappingMetadata> typeMappings,
                         Map<String, Field> specifiedFieldByNames) {
 
         this.fieldName = fieldName;
@@ -119,16 +119,16 @@ public String path() {
     }
 
     /**
-     * Used to retrieve the type of fields from metaData map structures for both regular and nested fields
+     * Used to retrieve the type of fields from metadata map structures for both regular and nested fields
      */
     @SuppressWarnings("unchecked")
     public String type() {
-        FieldMappingMetaData metaData = typeMappings.get(fieldName);
-        Map<String, Object> source = metaData.sourceAsMap();
+        FieldMappingMetadata metadata = typeMappings.get(fieldName);
+        Map<String, Object> source = metadata.sourceAsMap();
         String[] fieldPath = fieldName.split("\\.");
 
         /*
-         * When field is not nested the metaData source is fieldName -> type
+         * When field is not nested the metadata source is fieldName -> type
          * When it is nested or contains "." in general (ex. fieldName.nestedName) the source is nestedName -> type
          */
         String root = (fieldPath.length == 1) ? fieldName : fieldPath[1];

src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/FieldMappings.java

@@ -15,7 +15,7 @@
 
 package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;
 
-import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.json.JSONObject;
 
 import java.util.HashMap;
@@ -61,7 +61,7 @@ public class FieldMappings implements Mappings<Map<String, Object>> {
      */
     private final Map<String, Object> fieldMappings;
 
-    public FieldMappings(MappingMetaData mappings) {
+    public FieldMappings(MappingMetadata mappings) {
         fieldMappings = mappings.sourceAsMap();
     }
 

src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/IndexMappings.java

@@ -15,8 +15,8 @@
 
 package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;
 
-import org.elasticsearch.cluster.metadata.MappingMetaData;
-import org.elasticsearch.cluster.metadata.MetaData;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
+import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 
 import java.util.Map;
@@ -55,12 +55,12 @@ public IndexMappings() {
         this.indexMappings = emptyMap();
     }
 
-    public IndexMappings(MetaData metaData) {
-        this.indexMappings = buildMappings(metaData.indices(),
-                indexMetaData -> new TypeMappings(indexMetaData.getMappings()));
+    public IndexMappings(Metadata metadata) {
+        this.indexMappings = buildMappings(metadata.indices(),
+                indexMetadata -> new TypeMappings(indexMetadata.getMappings()));
     }
 
-    public IndexMappings(ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings) {
+    public IndexMappings(ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings) {
         this.indexMappings = buildMappings(mappings, TypeMappings::new);
     }
 

src/main/java/com/amazon/opendistroforelasticsearch/sql/esdomain/mapping/TypeMappings.java

@@ -15,7 +15,7 @@
 
 package com.amazon.opendistroforelasticsearch.sql.esdomain.mapping;
 
-import org.elasticsearch.cluster.metadata.MappingMetaData;
+import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.collect.ImmutableOpenMap;
 
 import java.util.Map;
@@ -36,7 +36,7 @@ public class TypeMappings implements Mappings<FieldMappings> {
      */
     private final Map<String, FieldMappings> typeMappings;
 
-    public TypeMappings(ImmutableOpenMap<String, MappingMetaData> mappings) {
+    public TypeMappings(ImmutableOpenMap<String, MappingMetadata> mappings) {
         typeMappings = buildMappings(mappings, FieldMappings::new);
     }
 

src/main/java/com/amazon/opendistroforelasticsearch/sql/executor/AsyncRestExecutor.java

@@ -143,7 +143,7 @@ private void async(Client client, Map<String, String> params, QueryAction queryA
 
         // Preserve context of calling thread to ensure headers of requests are forwarded when running blocking actions
         threadPool.schedule(
-                threadPool.preserveContext(LogUtils.withCurrentContext(runnable)),
+                LogUtils.withCurrentContext(runnable),
                 new TimeValue(0L),
                 SQL_WORKER_THREAD_POOL_NAME
         );