diff --git a/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/hive/IcebergRestServiceModelGenerator.java b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/hive/IcebergRestServiceModelGenerator.java new file mode 100644 index 0000000000..f25f914db3 --- /dev/null +++ b/gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/hive/IcebergRestServiceModelGenerator.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.cm.model.hive; + +import com.cloudera.api.swagger.client.ApiException; +import com.cloudera.api.swagger.model.ApiConfigList; +import com.cloudera.api.swagger.model.ApiRole; +import com.cloudera.api.swagger.model.ApiService; +import com.cloudera.api.swagger.model.ApiServiceConfig; +import org.apache.knox.gateway.topology.discovery.cm.ServiceModel; +import org.apache.knox.gateway.topology.discovery.cm.ServiceModelGeneratorHandleResponse; +import org.apache.knox.gateway.topology.discovery.cm.model.AbstractServiceModelGenerator; + +import java.util.Locale; + +public class IcebergRestServiceModelGenerator extends AbstractServiceModelGenerator { + + public static final String SERVICE = "ICEBERG-REST"; + public static final String SERVICE_TYPE = "HIVE"; + public static final String ROLE_TYPE = "HIVEMETASTORE"; + + static final String HTTP_PORT = "hive_metastore_catalog_servlet_port"; + static final String HTTP_PATH = "hive_metastore_catalog_servlet_path"; + static final String REST_CATALOG_ENABLED = "hive_rest_catalog_enabled"; + + static final String DEFAULT_HTTP_PATH = "icecli"; + + @Override + public String getService() { + return SERVICE; + } + + @Override + public String getServiceType() { + return SERVICE_TYPE; + } + + @Override + public String getRoleType() { + return ROLE_TYPE; + } + + @Override + public ServiceModel.Type getModelType() { + return ServiceModel.Type.API; + } + + @Override + public ServiceModelGeneratorHandleResponse handles(ApiService service, ApiServiceConfig serviceConfig, ApiRole role, ApiConfigList roleConfig) { + final ServiceModelGeneratorHandleResponse response = super.handles(service, serviceConfig, role, roleConfig); + if (response.handled()) { + validateCatalogEnabled(serviceConfig, response); + } + return response; + } + + @Override + public ServiceModel generateService(ApiService service, + ApiServiceConfig serviceConfig, + ApiRole role, + ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException { + String hostname = role.getHostRef().getHostname(); + String scheme = "http"; + String port = getHttpPort(serviceConfig); + String httpPath = getHttpPath(serviceConfig); + + if (httpPath == null) { + httpPath = DEFAULT_HTTP_PATH; + } + + ServiceModel model = + createServiceModel(String.format(Locale.getDefault(), "%s://%s:%s/%s", scheme, hostname, port, httpPath)); + model.addServiceProperty(HTTP_PORT, getHttpPort(serviceConfig)); + model.addServiceProperty(HTTP_PATH, getHttpPath(serviceConfig)); + model.addServiceProperty(REST_CATALOG_ENABLED, getRestCatalogEnabled(serviceConfig)); + + return model; + } + + protected String getHttpPort(ApiServiceConfig serviceConfig) { + return getServiceConfigValue(serviceConfig, HTTP_PORT); + } + + protected String getHttpPath(ApiServiceConfig serviceConfig) { + return getServiceConfigValue(serviceConfig, HTTP_PATH); + } + + protected String getRestCatalogEnabled(ApiServiceConfig serviceConfig) { + return getServiceConfigValue(serviceConfig, REST_CATALOG_ENABLED); + } + + protected void validateCatalogEnabled(ApiServiceConfig serviceConfig, ServiceModelGeneratorHandleResponse response) { + String catalogEnabledValue = getRestCatalogEnabled(serviceConfig); + boolean isCatalogEnabled = Boolean.parseBoolean(catalogEnabledValue); + if (!isCatalogEnabled) { + response.addConfigurationIssue("Invalid configuration: " + REST_CATALOG_ENABLED + + ". Expected=true; Found=" + catalogEnabledValue); + } + } + +} diff --git a/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator b/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator index dbea07ab1c..e1b0c6c941 100644 --- a/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator +++ b/gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator @@ -27,6 +27,7 @@ org.apache.knox.gateway.topology.discovery.cm.model.hdfs.WebHdfsServiceModelGene org.apache.knox.gateway.topology.discovery.cm.model.hive.HiveServiceModelGenerator org.apache.knox.gateway.topology.discovery.cm.model.hive.HiveOnTezServiceModelGenerator org.apache.knox.gateway.topology.discovery.cm.model.hive.WebHCatServiceModelGenerator +org.apache.knox.gateway.topology.discovery.cm.model.hive.IcebergRestServiceModelGenerator org.apache.knox.gateway.topology.discovery.cm.model.hue.HueServiceModelGenerator org.apache.knox.gateway.topology.discovery.cm.model.hue.HueLBServiceModelGenerator org.apache.knox.gateway.topology.discovery.cm.model.impala.ImpalaServiceModelGenerator diff --git a/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java b/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java index 9d98643ba7..d64f6242e2 100644 --- a/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java +++ b/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscoveryTest.java @@ -18,6 +18,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import com.cloudera.api.swagger.client.ApiException; import com.cloudera.api.swagger.client.ApiResponse; @@ -41,6 +42,7 @@ import org.apache.knox.gateway.topology.discovery.cm.model.hbase.HBaseUIServiceModelGenerator; import org.apache.knox.gateway.topology.discovery.cm.model.hbase.WebHBaseServiceModelGenerator; import org.apache.knox.gateway.topology.discovery.cm.model.hdfs.NameNodeServiceModelGenerator; +import org.apache.knox.gateway.topology.discovery.cm.model.hive.IcebergRestServiceModelGenerator; import org.apache.knox.gateway.topology.discovery.cm.model.hive.HiveOnTezServiceModelGenerator; import org.apache.knox.gateway.topology.discovery.cm.model.hive.HiveServiceModelGenerator; import org.apache.knox.gateway.topology.discovery.cm.model.hive.WebHCatServiceModelGenerator; @@ -230,6 +232,42 @@ private void doTestHiveOnTezServiceDiscovery(final String thriftPath, final bool assertEquals((expectedScheme + "://" + hostName + ":" + thriftPort + "/" + expectedThriftPath), hiveURLs.get(0)); } + @Test + public void testHiveMetastoreRestCatalogServiceDiscovery() { + final String hostName = "test-host-1"; + final String port = "10001"; + final String hostName2 = "test-host-2"; + final String defaultPort = "8090"; + + doTestHMSRestCatalogServiceDiscovery(hostName, port, null, null); + doTestHMSRestCatalogServiceDiscovery(hostName, port, "non_default_path", null); + doTestHMSRestCatalogServiceDiscovery(hostName, port, "icecli", null); + + doTestHMSRestCatalogServiceDiscovery(hostName2, defaultPort, null, "false"); + doTestHMSRestCatalogServiceDiscovery(hostName2, defaultPort, "non_default_path", "false"); + doTestHMSRestCatalogServiceDiscovery(hostName2, defaultPort, "icecli", "false"); + + doTestHMSRestCatalogServiceDiscovery(hostName, port, null, "true"); + doTestHMSRestCatalogServiceDiscovery(hostName, port, "non_default_path", "true"); + doTestHMSRestCatalogServiceDiscovery(hostName, port, "icecli", "true"); + } + + private void doTestHMSRestCatalogServiceDiscovery(final String hostName, final String port, final String httpPath, final String enabled) { + final String expectedScheme = "http"; + final String expectedPath = httpPath != null ? httpPath : "icecli"; + + ServiceDiscovery.Cluster cluster = + doHMSRestCatalogServiceDiscovery(hostName, port, httpPath, enabled); + List serviceURLs = cluster.getServiceURLs("ICEBERG-REST"); + + if (Boolean.parseBoolean(enabled)) { + assertNotNull(serviceURLs); + assertEquals(1, serviceURLs.size()); + assertEquals((expectedScheme + "://" + hostName + ":" + port + "/" + expectedPath), serviceURLs.get(0)); + } else { + assertTrue(serviceURLs.isEmpty()); + } + } @Test public void testWebHDFSServiceDiscovery() { @@ -1059,6 +1097,28 @@ private ServiceDiscovery.Cluster doTestHiveOnTezServiceDiscovery(final String h roleProperties); } + private ServiceDiscovery.Cluster doHMSRestCatalogServiceDiscovery(final String hostName, + final String port, + final String path, + final String enabled) { + + + + Map roleProperties = new HashMap<>(); + + final Map serviceProperties = new HashMap<>(); + serviceProperties.put("hive_metastore_catalog_servlet_port", port); + serviceProperties.put("hive_metastore_catalog_servlet_path", path); + serviceProperties.put("hive_rest_catalog_enabled", enabled); + + return doTestDiscovery(hostName, + "HIVE-1", + IcebergRestServiceModelGenerator.SERVICE_TYPE, + "HIVE-1-HIVEMETASTORE-12345", + IcebergRestServiceModelGenerator.ROLE_TYPE, + serviceProperties, + roleProperties); + } private ServiceDiscovery.Cluster doTestHDFSDiscovery(final String hostName, final String nameService, diff --git a/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/AbstractServiceModelGeneratorTest.java b/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/AbstractServiceModelGeneratorTest.java index baaf347410..efa3445fe0 100644 --- a/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/AbstractServiceModelGeneratorTest.java +++ b/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/AbstractServiceModelGeneratorTest.java @@ -113,7 +113,7 @@ protected void validateServiceModel(ServiceModel candidate, // Validate the role configuration Map modelRoleProperties = candidate.getRoleProperties().get(getRoleType()); if (validateCounts) { - assertEquals(roleConfig.size(), modelRoleProperties.size()); + assertEquals(roleConfig.size(), modelRoleProperties == null ? 0 : modelRoleProperties.size()); } for (Map.Entry roleProp : roleConfig.entrySet()) { assertTrue(modelRoleProperties.containsKey(roleProp.getKey())); diff --git a/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/hive/IcebergRestServiceModelGeneratorTest.java b/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/hive/IcebergRestServiceModelGeneratorTest.java new file mode 100644 index 0000000000..97179bbeae --- /dev/null +++ b/gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/hive/IcebergRestServiceModelGeneratorTest.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.knox.gateway.topology.discovery.cm.model.hive; + +import org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator; +import org.apache.knox.gateway.topology.discovery.cm.model.AbstractServiceModelGeneratorTest; +import org.junit.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class IcebergRestServiceModelGeneratorTest extends AbstractServiceModelGeneratorTest { + + @Test + @Override + public void testHandles() { + final Map serviceConfig = new HashMap<>(); + serviceConfig.put(IcebergRestServiceModelGenerator.REST_CATALOG_ENABLED, "true"); + + assertTrue(doTestHandles(newGenerator(), getServiceType(), serviceConfig, getRoleType(), Collections.emptyMap())); + } + + @Test + public void testHandlesWhenEnabledIsFalse() { + final Map serviceConfig = new HashMap<>(); + serviceConfig.put(IcebergRestServiceModelGenerator.REST_CATALOG_ENABLED, "false"); + + assertFalse(doTestHandles(newGenerator(), getServiceType(), serviceConfig, getRoleType(), Collections.emptyMap())); + } + + + @Test + public void testServiceModel() { + final Map serviceConfig = new HashMap<>(); + serviceConfig.put("hive_metastore_catalog_servlet_port", "8090"); + serviceConfig.put("hive_metastore_catalog_servlet_path", "icecli"); + serviceConfig.put("hive_rest_catalog_enabled", "true"); + + final Map roleConfig = Collections.emptyMap(); + + validateServiceModel(createServiceModel(serviceConfig, roleConfig), serviceConfig, roleConfig); + } + + @Override + protected String getServiceType() { + return IcebergRestServiceModelGenerator.SERVICE_TYPE; + } + + @Override + protected String getRoleType() { + return IcebergRestServiceModelGenerator.ROLE_TYPE; + } + + @Override + protected ServiceModelGenerator newGenerator() { + return new IcebergRestServiceModelGenerator(); + } + +}