KNOX-3154: Implement CM discovery support for the Apache Iceberg REST Catalog Service

gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/hive/IcebergRestServiceModelGenerator.java

@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.discovery.cm.model.hive;
+
+import com.cloudera.api.swagger.client.ApiException;
+import com.cloudera.api.swagger.model.ApiConfigList;
+import com.cloudera.api.swagger.model.ApiRole;
+import com.cloudera.api.swagger.model.ApiService;
+import com.cloudera.api.swagger.model.ApiServiceConfig;
+import org.apache.knox.gateway.topology.discovery.cm.ServiceModel;
+import org.apache.knox.gateway.topology.discovery.cm.ServiceModelGeneratorHandleResponse;
+import org.apache.knox.gateway.topology.discovery.cm.model.AbstractServiceModelGenerator;
+
+import java.util.Locale;
+
+public class IcebergRestServiceModelGenerator extends AbstractServiceModelGenerator {
+
+  public static final String SERVICE = "ICEBERG-REST";
+  public static final String SERVICE_TYPE = "HIVE";
+  public static final String ROLE_TYPE = "HIVEMETASTORE";
+
+  static final String HTTP_PORT = "hive_metastore_catalog_servlet_port";
+  static final String HTTP_PATH = "hive_metastore_catalog_servlet_path";
+  static final String REST_CATALOG_ENABLED = "hive_rest_catalog_enabled";
+
+  static final String DEFAULT_HTTP_PATH = "icecli";
+
+  @Override
+  public String getService() {
+    return SERVICE;
+  }
+
+  @Override
+  public String getServiceType() {
+    return SERVICE_TYPE;
+  }
+
+  @Override
+  public String getRoleType() {
+    return ROLE_TYPE;
+  }
+
+  @Override
+  public ServiceModel.Type getModelType() {
+    return ServiceModel.Type.API;
+  }
+
+  @Override
+  public ServiceModelGeneratorHandleResponse handles(ApiService service, ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig) {
+    final ServiceModelGeneratorHandleResponse response = super.handles(service, serviceConfig, role, roleConfig);
+    if (response.handled()) {
+      validateCatalogEnabled(serviceConfig, response);
+    }
+    return response;
+  }
+
+  @Override
+  public ServiceModel generateService(ApiService service,
+                                      ApiServiceConfig serviceConfig,
+                                      ApiRole role,
+                                      ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
+    String hostname = role.getHostRef().getHostname();
+    String scheme = "http";
+    String port = getHttpPort(serviceConfig);
+    String httpPath = getHttpPath(serviceConfig);
+
+    if (httpPath == null) {
+      httpPath = DEFAULT_HTTP_PATH;
+    }
+
+    ServiceModel model =
+        createServiceModel(String.format(Locale.getDefault(), "%s://%s:%s/%s", scheme, hostname, port, httpPath));
+    model.addServiceProperty(HTTP_PORT, getHttpPort(serviceConfig));
+    model.addServiceProperty(HTTP_PATH, getHttpPath(serviceConfig));
+    model.addServiceProperty(REST_CATALOG_ENABLED, getRestCatalogEnabled(serviceConfig));
+
+    return model;
+  }
+
+  protected String getHttpPort(ApiServiceConfig serviceConfig) {
+    return getServiceConfigValue(serviceConfig, HTTP_PORT);
+  }
+
+  protected String getHttpPath(ApiServiceConfig serviceConfig) {
+    return getServiceConfigValue(serviceConfig, HTTP_PATH);
+  }
+
+  protected String getRestCatalogEnabled(ApiServiceConfig serviceConfig) {
+    return getServiceConfigValue(serviceConfig, REST_CATALOG_ENABLED);
+  }
+
+  protected void validateCatalogEnabled(ApiServiceConfig serviceConfig, ServiceModelGeneratorHandleResponse response) {
+    String catalogEnabledValue = getRestCatalogEnabled(serviceConfig);
+    boolean isCatalogEnabled = Boolean.parseBoolean(catalogEnabledValue);
+    if (!isCatalogEnabled) {
+      response.addConfigurationIssue("Invalid configuration: " + REST_CATALOG_ENABLED +
+          ". Expected=true; Found=" + catalogEnabledValue);
+    }
+  }
+
+}

gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator

@@ -27,6 +27,7 @@ org.apache.knox.gateway.topology.discovery.cm.model.hdfs.WebHdfsServiceModelGene
 org.apache.knox.gateway.topology.discovery.cm.model.hive.HiveServiceModelGenerator
 org.apache.knox.gateway.topology.discovery.cm.model.hive.HiveOnTezServiceModelGenerator
 org.apache.knox.gateway.topology.discovery.cm.model.hive.WebHCatServiceModelGenerator
+org.apache.knox.gateway.topology.discovery.cm.model.hive.IcebergRestServiceModelGenerator
 org.apache.knox.gateway.topology.discovery.cm.model.hue.HueServiceModelGenerator
 org.apache.knox.gateway.topology.discovery.cm.model.hue.HueLBServiceModelGenerator
 org.apache.knox.gateway.topology.discovery.cm.model.impala.ImpalaServiceModelGenerator

gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java

@@ -18,6 +18,7 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
 
 import com.cloudera.api.swagger.client.ApiException;
 import com.cloudera.api.swagger.client.ApiResponse;
@@ -41,6 +42,7 @@
 import org.apache.knox.gateway.topology.discovery.cm.model.hbase.HBaseUIServiceModelGenerator;
 import org.apache.knox.gateway.topology.discovery.cm.model.hbase.WebHBaseServiceModelGenerator;
 import org.apache.knox.gateway.topology.discovery.cm.model.hdfs.NameNodeServiceModelGenerator;
+import org.apache.knox.gateway.topology.discovery.cm.model.hive.IcebergRestServiceModelGenerator;
 import org.apache.knox.gateway.topology.discovery.cm.model.hive.HiveOnTezServiceModelGenerator;
 import org.apache.knox.gateway.topology.discovery.cm.model.hive.HiveServiceModelGenerator;
 import org.apache.knox.gateway.topology.discovery.cm.model.hive.WebHCatServiceModelGenerator;
@@ -230,6 +232,42 @@ private void doTestHiveOnTezServiceDiscovery(final String thriftPath, final bool
     assertEquals((expectedScheme + "://" + hostName + ":" + thriftPort + "/" + expectedThriftPath), hiveURLs.get(0));
   }
 
+  @Test
+  public void testHiveMetastoreRestCatalogServiceDiscovery() {
+    final String hostName = "test-host-1";
+    final String port = "10001";
+    final String hostName2 = "test-host-2";
+    final String defaultPort = "8090";
+
+    doTestHMSRestCatalogServiceDiscovery(hostName, port, null, null);
+    doTestHMSRestCatalogServiceDiscovery(hostName, port, "non_default_path", null);
+    doTestHMSRestCatalogServiceDiscovery(hostName, port, "icecli", null);
+
+    doTestHMSRestCatalogServiceDiscovery(hostName2, defaultPort, null, "false");
+    doTestHMSRestCatalogServiceDiscovery(hostName2, defaultPort, "non_default_path", "false");
+    doTestHMSRestCatalogServiceDiscovery(hostName2, defaultPort, "icecli", "false");
+
+    doTestHMSRestCatalogServiceDiscovery(hostName, port, null, "true");
+    doTestHMSRestCatalogServiceDiscovery(hostName, port, "non_default_path", "true");
+    doTestHMSRestCatalogServiceDiscovery(hostName, port, "icecli", "true");
+  }
+
+  private void doTestHMSRestCatalogServiceDiscovery(final String hostName, final String port, final String httpPath, final String enabled) {
+    final String expectedScheme = "http";
+    final String expectedPath = httpPath != null ? httpPath : "icecli";
+
+    ServiceDiscovery.Cluster cluster =
+        doHMSRestCatalogServiceDiscovery(hostName, port, httpPath, enabled);
+    List<String> serviceURLs = cluster.getServiceURLs("ICEBERG-REST");
+
+    if (Boolean.parseBoolean(enabled)) {
+      assertNotNull(serviceURLs);
+      assertEquals(1, serviceURLs.size());
+      assertEquals((expectedScheme + "://" + hostName + ":" + port + "/" + expectedPath), serviceURLs.get(0));
+    } else {
+      assertTrue(serviceURLs.isEmpty());
+    }
+  }
 
   @Test
   public void testWebHDFSServiceDiscovery() {
@@ -1059,6 +1097,28 @@ private ServiceDiscovery.Cluster doTestHiveOnTezServiceDiscovery(final String  h
                            roleProperties);
   }
 
+  private ServiceDiscovery.Cluster doHMSRestCatalogServiceDiscovery(final String  hostName,
+                                                                    final String  port,
+                                                                    final String  path,
+                                                                    final String enabled) {
+
+
+
+    Map<String, String> roleProperties = new HashMap<>();
+
+    final Map<String, String> serviceProperties = new HashMap<>();
+    serviceProperties.put("hive_metastore_catalog_servlet_port", port);
+    serviceProperties.put("hive_metastore_catalog_servlet_path", path);
+    serviceProperties.put("hive_rest_catalog_enabled", enabled);
+
+    return doTestDiscovery(hostName,
+        "HIVE-1",
+        IcebergRestServiceModelGenerator.SERVICE_TYPE,
+        "HIVE-1-HIVEMETASTORE-12345",
+        IcebergRestServiceModelGenerator.ROLE_TYPE,
+        serviceProperties,
+        roleProperties);
+  }
 
   private ServiceDiscovery.Cluster doTestHDFSDiscovery(final String hostName,
                                                        final String nameService,

gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/AbstractServiceModelGeneratorTest.java

@@ -113,7 +113,7 @@ protected void validateServiceModel(ServiceModel        candidate,
     // Validate the role configuration
     Map<String, String> modelRoleProperties = candidate.getRoleProperties().get(getRoleType());
     if (validateCounts) {
-      assertEquals(roleConfig.size(), modelRoleProperties.size());
+      assertEquals(roleConfig.size(), modelRoleProperties == null ? 0 : modelRoleProperties.size());
     }
     for (Map.Entry<String, String> roleProp : roleConfig.entrySet()) {
       assertTrue(modelRoleProperties.containsKey(roleProp.getKey()));

gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/hive/IcebergRestServiceModelGeneratorTest.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.knox.gateway.topology.discovery.cm.model.hive;
+
+import org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator;
+import org.apache.knox.gateway.topology.discovery.cm.model.AbstractServiceModelGeneratorTest;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class IcebergRestServiceModelGeneratorTest extends AbstractServiceModelGeneratorTest {
+
+  @Test
+  @Override
+  public void testHandles() {
+    final Map<String, String> serviceConfig = new HashMap<>();
+    serviceConfig.put(IcebergRestServiceModelGenerator.REST_CATALOG_ENABLED, "true");
+
+    assertTrue(doTestHandles(newGenerator(), getServiceType(), serviceConfig, getRoleType(), Collections.emptyMap()));
+  }
+
+  @Test
+  public void testHandlesWhenEnabledIsFalse() {
+    final Map<String, String> serviceConfig = new HashMap<>();
+    serviceConfig.put(IcebergRestServiceModelGenerator.REST_CATALOG_ENABLED, "false");
+
+    assertFalse(doTestHandles(newGenerator(), getServiceType(), serviceConfig, getRoleType(), Collections.emptyMap()));
+  }
+
+
+  @Test
+  public void testServiceModel() {
+    final Map<String, String> serviceConfig = new HashMap<>();
+    serviceConfig.put("hive_metastore_catalog_servlet_port", "8090");
+    serviceConfig.put("hive_metastore_catalog_servlet_path", "icecli");
+    serviceConfig.put("hive_rest_catalog_enabled", "true");
+
+    final Map<String, String> roleConfig = Collections.emptyMap();
+
+    validateServiceModel(createServiceModel(serviceConfig, roleConfig), serviceConfig, roleConfig);
+  }
+
+  @Override
+  protected String getServiceType() {
+    return IcebergRestServiceModelGenerator.SERVICE_TYPE;
+  }
+
+  @Override
+  protected String getRoleType() {
+    return IcebergRestServiceModelGenerator.ROLE_TYPE;
+  }
+
+  @Override
+  protected ServiceModelGenerator newGenerator() {
+    return new IcebergRestServiceModelGenerator();
+  }
+
+}