diff --git a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/ConfigReloadAutoConfiguration.java b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/ConfigReloadAutoConfiguration.java index a78e265be9..509609d1a1 100644 --- a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/ConfigReloadAutoConfiguration.java +++ b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/ConfigReloadAutoConfiguration.java @@ -24,7 +24,6 @@ import org.springframework.boot.actuate.autoconfigure.endpoint.EndpointAutoConfiguration; import org.springframework.boot.actuate.autoconfigure.info.InfoEndpointAutoConfiguration; import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.boot.autoconfigure.condition.AnyNestedCondition; import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -36,6 +35,8 @@ import org.springframework.cloud.context.restart.RestartEndpoint; import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; +import org.springframework.cloud.kubernetes.config.reload.condition.EventReloadDetectionMode; +import org.springframework.cloud.kubernetes.config.reload.condition.PollingReloadDetectionMode; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Conditional; @@ -49,6 +50,7 @@ * Definition of beans needed for the automatic reload of configuration. * * @author Nicolla Ferraro + * @author Kris Iyer */ @Configuration(proxyBeanMethods = false) @ConditionalOnProperty(value = "spring.cloud.kubernetes.enabled", matchIfMissing = true) @@ -56,7 +58,6 @@ @AutoConfigureAfter({ InfoEndpointAutoConfiguration.class, RefreshEndpointAutoConfiguration.class, RefreshAutoConfiguration.class }) @EnableConfigurationProperties(ConfigReloadProperties.class) - public class ConfigReloadAutoConfiguration { /** @@ -75,25 +76,71 @@ protected static class ConfigReloadAutoConfigurationBeans { private KubernetesClient kubernetesClient; /** + * Polling configMap ConfigurationChangeDetector. * @param properties config reload properties * @param strategy configuration update strategy + * @param configMapPropertySourceLocator configMap property source locator * @return a bean that listen to configuration changes and fire a reload. */ @Bean - @Conditional(OnConfigEnabledOrSecretsEnabled.class) - public ConfigurationChangeDetector propertyChangeWatcher(ConfigReloadProperties properties, - ConfigurationUpdateStrategy strategy, - @Autowired(required = false) ConfigMapPropertySourceLocator configMapPropertySourceLocator, - @Autowired(required = false) SecretsPropertySourceLocator secretsPropertySourceLocator) { - switch (properties.getMode()) { - case POLLING: - return new PollingConfigurationChangeDetector(this.environment, properties, this.kubernetesClient, - strategy, configMapPropertySourceLocator, secretsPropertySourceLocator); - case EVENT: - return new EventBasedConfigurationChangeDetector(this.environment, properties, this.kubernetesClient, - strategy, configMapPropertySourceLocator, secretsPropertySourceLocator); - } - throw new IllegalStateException("Unsupported configuration reload mode: " + properties.getMode()); + @ConditionalOnBean(ConfigMapPropertySourceLocator.class) + @Conditional(PollingReloadDetectionMode.class) + public ConfigurationChangeDetector configMapPropertyChangePollingWatcher(ConfigReloadProperties properties, + ConfigurationUpdateStrategy strategy, ConfigMapPropertySourceLocator configMapPropertySourceLocator) { + + return new PollingConfigMapChangeDetector(this.environment, properties, this.kubernetesClient, strategy, + configMapPropertySourceLocator); + } + + /** + * Polling secrets ConfigurationChangeDetector. + * @param properties config reload properties + * @param strategy configuration update strategy + * @param secretsPropertySourceLocator secrets property source locator + * @return a bean that listen to configuration changes and fire a reload. + */ + @Bean + @ConditionalOnBean(SecretsPropertySourceLocator.class) + @Conditional(PollingReloadDetectionMode.class) + public ConfigurationChangeDetector secretsPropertyChangePollingWatcher(ConfigReloadProperties properties, + ConfigurationUpdateStrategy strategy, SecretsPropertySourceLocator secretsPropertySourceLocator) { + + return new PollingSecretsChangeDetector(this.environment, properties, this.kubernetesClient, strategy, + secretsPropertySourceLocator); + } + + /** + * Event Based configMap ConfigurationChangeDetector. + * @param properties config reload properties + * @param strategy configuration update strategy + * @param configMapPropertySourceLocator configMap property source locator + * @return a bean that listen to configMap change events and fire a reload. + */ + @Bean + @ConditionalOnBean(ConfigMapPropertySourceLocator.class) + @Conditional(EventReloadDetectionMode.class) + public ConfigurationChangeDetector configMapPropertyChangeEventWatcher(ConfigReloadProperties properties, + ConfigurationUpdateStrategy strategy, ConfigMapPropertySourceLocator configMapPropertySourceLocator) { + + return new EventBasedConfigMapChangeDetector(this.environment, properties, this.kubernetesClient, strategy, + configMapPropertySourceLocator); + } + + /** + * Event Based secrets ConfigurationChangeDetector. + * @param properties config reload properties + * @param strategy configuration update strategy + * @param secretsPropertySourceLocator secrets property source locator + * @return a bean that listen to secrets change events and fire a reload. + */ + @Bean + @ConditionalOnBean(SecretsPropertySourceLocator.class) + @Conditional(EventReloadDetectionMode.class) + public ConfigurationChangeDetector secretsPropertyChangeEventWatcher(ConfigReloadProperties properties, + ConfigurationUpdateStrategy strategy, SecretsPropertySourceLocator secretsPropertySourceLocator) { + + return new EventBasedSecretsChangeDetector(this.environment, properties, this.kubernetesClient, strategy, + secretsPropertySourceLocator); } /** @@ -135,24 +182,6 @@ private static void wait(ConfigReloadProperties properties) { } } - private static class OnConfigEnabledOrSecretsEnabled extends AnyNestedCondition { - - OnConfigEnabledOrSecretsEnabled() { - super(ConfigurationPhase.REGISTER_BEAN); - } - - @ConditionalOnBean(ConfigMapPropertySourceLocator.class) - static class configEnabled { - - } - - @ConditionalOnBean(SecretsPropertySourceLocator.class) - static class secretsEnabled { - - } - - } - } } diff --git a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/ConfigReloadDefaultAutoConfiguration.java b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/ConfigReloadDefaultAutoConfiguration.java index 66f3200e29..621bf03d67 100644 --- a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/ConfigReloadDefaultAutoConfiguration.java +++ b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/ConfigReloadDefaultAutoConfiguration.java @@ -21,13 +21,17 @@ import io.fabric8.kubernetes.client.KubernetesClient; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; +import org.springframework.cloud.kubernetes.config.reload.condition.EventReloadDetectionMode; +import org.springframework.cloud.kubernetes.config.reload.condition.PollingReloadDetectionMode; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.AbstractEnvironment; import org.springframework.scheduling.annotation.EnableAsync; @@ -35,6 +39,7 @@ /** * @author Ryan Baxter + * @author Kris Iyer */ @Configuration(proxyBeanMethods = false) @ConditionalOnProperty(value = "spring.cloud.kubernetes.enabled", matchIfMissing = true) @@ -63,30 +68,76 @@ protected static class ConfigReloadAutoConfigurationBeans { private SecretsPropertySourceLocator secretsPropertySourceLocator; /** + * Polling configMap ConfigurationChangeDetector. * @param properties config reload properties * @param strategy configuration update strategy + * @param configMapPropertySourceLocator configMap property source locator * @return a bean that listen to configuration changes and fire a reload. */ @Bean - @ConditionalOnMissingBean - public ConfigurationChangeDetector propertyChangeWatcher(ConfigReloadProperties properties, - ConfigurationUpdateStrategy strategy) { - switch (properties.getMode()) { - case POLLING: - return new PollingConfigurationChangeDetector(this.environment, properties, this.kubernetesClient, - strategy, this.configMapPropertySourceLocator, this.secretsPropertySourceLocator); - case EVENT: - return new EventBasedConfigurationChangeDetector(this.environment, properties, this.kubernetesClient, - strategy, this.configMapPropertySourceLocator, this.secretsPropertySourceLocator); - } - throw new IllegalStateException("Unsupported configuration reload mode: " + properties.getMode()); + @ConditionalOnBean(ConfigMapPropertySourceLocator.class) + @Conditional(PollingReloadDetectionMode.class) + public ConfigurationChangeDetector configMapPropertyChangePollingWatcher(ConfigReloadProperties properties, + ConfigurationUpdateStrategy strategy, ConfigMapPropertySourceLocator configMapPropertySourceLocator) { + + return new PollingConfigMapChangeDetector(this.environment, properties, this.kubernetesClient, strategy, + configMapPropertySourceLocator); + } + + /** + * Polling secrets ConfigurationChangeDetector. + * @param properties config reload properties + * @param strategy configuration update strategy + * @param secretsPropertySourceLocator secrets property source locator + * @return a bean that listen to configuration changes and fire a reload. + */ + @Bean + @ConditionalOnBean(SecretsPropertySourceLocator.class) + @Conditional(PollingReloadDetectionMode.class) + public ConfigurationChangeDetector secretsPropertyChangePollingWatcher(ConfigReloadProperties properties, + ConfigurationUpdateStrategy strategy, SecretsPropertySourceLocator secretsPropertySourceLocator) { + + return new PollingSecretsChangeDetector(this.environment, properties, this.kubernetesClient, strategy, + secretsPropertySourceLocator); + } + + /** + * Event Based configMap ConfigurationChangeDetector. + * @param properties config reload properties + * @param strategy configuration update strategy + * @param configMapPropertySourceLocator configMap property source locator + * @return a bean that listen to configMap change events and fire a reload. + */ + @Bean + @ConditionalOnBean(ConfigMapPropertySourceLocator.class) + @Conditional(EventReloadDetectionMode.class) + public ConfigurationChangeDetector configMapPropertyChangeEventWatcher(ConfigReloadProperties properties, + ConfigurationUpdateStrategy strategy, ConfigMapPropertySourceLocator configMapPropertySourceLocator) { + + return new EventBasedConfigMapChangeDetector(this.environment, properties, this.kubernetesClient, strategy, + configMapPropertySourceLocator); + } + + /** + * Event Based secrets ConfigurationChangeDetector. + * @param properties config reload properties + * @param strategy configuration update strategy + * @param secretsPropertySourceLocator secrets property source locator + * @return a bean that listen to secrets change events and fire a reload. + */ + @Bean + @ConditionalOnBean(SecretsPropertySourceLocator.class) + @Conditional(EventReloadDetectionMode.class) + public ConfigurationChangeDetector secretsPropertyChangeEventWatcher(ConfigReloadProperties properties, + ConfigurationUpdateStrategy strategy, SecretsPropertySourceLocator secretsPropertySourceLocator) { + + return new EventBasedSecretsChangeDetector(this.environment, properties, this.kubernetesClient, strategy, + secretsPropertySourceLocator); } /** * @param properties config reload properties * @param ctx application context - * @param restarter restart endpoint - * @param refresher context refresher * @return provides the action to execute when the configuration changes. */ @Bean diff --git a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/EventBasedConfigurationChangeDetector.java b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/EventBasedConfigMapChangeDetector.java similarity index 61% rename from spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/EventBasedConfigurationChangeDetector.java rename to spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/EventBasedConfigMapChangeDetector.java index af712dbb7a..c7567d807b 100644 --- a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/EventBasedConfigurationChangeDetector.java +++ b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/EventBasedConfigMapChangeDetector.java @@ -23,7 +23,6 @@ import javax.annotation.PreDestroy; import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.KubernetesClientException; import io.fabric8.kubernetes.client.Watch; @@ -31,33 +30,28 @@ import org.springframework.cloud.kubernetes.config.ConfigMapPropertySource; import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; -import org.springframework.cloud.kubernetes.config.SecretsPropertySource; -import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; import org.springframework.core.env.AbstractEnvironment; /** - * A change detector that subscribes to changes in secrets and configmaps and fire a + * An Event Based change detector that subscribes to changes in configMaps and fire a * reload when something changes. * * @author Nicola Ferraro * @author Haytham Mohamed + * @author Kris Iyer */ -public class EventBasedConfigurationChangeDetector extends ConfigurationChangeDetector { +public class EventBasedConfigMapChangeDetector extends ConfigurationChangeDetector { private final ConfigMapPropertySourceLocator configMapPropertySourceLocator; - private final SecretsPropertySourceLocator secretsPropertySourceLocator; - private final Map watches; - public EventBasedConfigurationChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, + public EventBasedConfigMapChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, KubernetesClient kubernetesClient, ConfigurationUpdateStrategy strategy, - ConfigMapPropertySourceLocator configMapPropertySourceLocator, - SecretsPropertySourceLocator secretsPropertySourceLocator) { + ConfigMapPropertySourceLocator configMapPropertySourceLocator) { super(environment, properties, kubernetesClient, strategy); this.configMapPropertySourceLocator = configMapPropertySourceLocator; - this.secretsPropertySourceLocator = secretsPropertySourceLocator; this.watches = new HashMap<>(); } @@ -65,9 +59,9 @@ public EventBasedConfigurationChangeDetector(AbstractEnvironment environment, Co public void watch() { boolean activated = false; - if (this.properties.isMonitoringConfigMaps() && this.configMapPropertySourceLocator != null) { + if (this.properties.isMonitoringConfigMaps()) { try { - String name = "config-maps-watch"; + String name = "config-maps-watch-event"; this.watches.put(name, this.kubernetesClient.configMaps().watch(new Watcher() { @Override public void eventReceived(Action action, ConfigMap configMap) { @@ -91,34 +85,8 @@ public void onClose(KubernetesClientException e) { } } - if (this.properties.isMonitoringSecrets() && this.secretsPropertySourceLocator != null) { - try { - activated = false; - String name = "secrets-watch"; - this.watches.put(name, this.kubernetesClient.secrets().watch(new Watcher() { - @Override - public void eventReceived(Action action, Secret secret) { - if (log.isDebugEnabled()) { - log.debug(name + " received and event for Secret " + secret.getMetadata().getName()); - } - onEvent(secret); - } - - @Override - public void onClose(KubernetesClientException e) { - } - })); - activated = true; - this.log.info("Added new Kubernetes watch: " + name); - } - catch (Exception e) { - this.log.error("Error while establishing a connection to watch secrets: configuration may remain stale", - e); - } - } - if (activated) { - this.log.info("Kubernetes event-based configuration change detector activated"); + this.log.info("Kubernetes event-based configMap change detector activated"); } } @@ -147,13 +115,4 @@ protected void onEvent(ConfigMap configMap) { } } - protected void onEvent(Secret secret) { - boolean changed = changed(locateMapPropertySources(this.secretsPropertySourceLocator, this.environment), - findPropertySources(SecretsPropertySource.class)); - if (changed) { - this.log.info("Detected change in secrets"); - reloadProperties(); - } - } - } diff --git a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/EventBasedSecretsChangeDetector.java b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/EventBasedSecretsChangeDetector.java new file mode 100644 index 0000000000..f3f24cb145 --- /dev/null +++ b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/EventBasedSecretsChangeDetector.java @@ -0,0 +1,118 @@ +/* + * Copyright 2013-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.config.reload; + +import java.util.HashMap; +import java.util.Map; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.client.KubernetesClient; +import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.Watch; +import io.fabric8.kubernetes.client.Watcher; + +import org.springframework.cloud.kubernetes.config.SecretsPropertySource; +import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; +import org.springframework.core.env.AbstractEnvironment; + +/** + * An event based change detector that subscribes to changes in secrets and fire a reload + * when something changes. + * + * @author Nicola Ferraro + * @author Haytham Mohamed + * @author Kris Iyer + */ +public class EventBasedSecretsChangeDetector extends ConfigurationChangeDetector { + + private SecretsPropertySourceLocator secretsPropertySourceLocator; + + private Map watches; + + public EventBasedSecretsChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, + KubernetesClient kubernetesClient, ConfigurationUpdateStrategy strategy, + SecretsPropertySourceLocator secretsPropertySourceLocator) { + super(environment, properties, kubernetesClient, strategy); + + this.secretsPropertySourceLocator = secretsPropertySourceLocator; + this.watches = new HashMap<>(); + } + + @PostConstruct + public void watch() { + boolean activated = false; + + if (this.properties.isMonitoringSecrets()) { + try { + activated = false; + String name = "secrets-watch-event"; + this.watches.put(name, this.kubernetesClient.secrets().watch(new Watcher() { + @Override + public void eventReceived(Action action, Secret secret) { + if (log.isDebugEnabled()) { + log.debug(name + " received event for Secret " + secret.getMetadata().getName()); + } + onEvent(secret); + } + + @Override + public void onClose(KubernetesClientException e) { + } + })); + activated = true; + this.log.info("Added new Kubernetes watch: " + name); + } + catch (Exception e) { + this.log.error("Error while establishing a connection to watch secrets: configuration may remain stale", + e); + } + } + + if (activated) { + this.log.info("Kubernetes event-based secrets change detector activated"); + } + } + + @PreDestroy + public void unwatch() { + if (this.watches != null) { + for (Map.Entry entry : this.watches.entrySet()) { + try { + this.log.debug("Closing the watch " + entry.getKey()); + entry.getValue().close(); + + } + catch (Exception e) { + this.log.error("Error while closing the watch connection", e); + } + } + } + } + + protected void onEvent(Secret secret) { + boolean changed = changed(locateMapPropertySources(this.secretsPropertySourceLocator, this.environment), + findPropertySources(SecretsPropertySource.class)); + if (changed) { + this.log.info("Detected change in secrets"); + reloadProperties(); + } + } + +} diff --git a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/PollingConfigMapChangeDetector.java b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/PollingConfigMapChangeDetector.java new file mode 100644 index 0000000000..28b859aa07 --- /dev/null +++ b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/PollingConfigMapChangeDetector.java @@ -0,0 +1,85 @@ +/* + * Copyright 2013-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.config.reload; + +import java.util.List; + +import javax.annotation.PostConstruct; + +import io.fabric8.kubernetes.client.KubernetesClient; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.cloud.kubernetes.config.ConfigMapPropertySource; +import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; +import org.springframework.core.env.AbstractEnvironment; +import org.springframework.core.env.MapPropertySource; +import org.springframework.scheduling.annotation.Scheduled; + +/** + * A change detector that periodically retrieves configmaps and fire a reload when + * something changes. + * + * @author Nicola Ferraro + * @author Haytham Mohamed + * @author Kris Iyer + */ +public class PollingConfigMapChangeDetector extends ConfigurationChangeDetector { + + protected Log log = LogFactory.getLog(getClass()); + + private ConfigMapPropertySourceLocator configMapPropertySourceLocator; + + public PollingConfigMapChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, + KubernetesClient kubernetesClient, ConfigurationUpdateStrategy strategy, + ConfigMapPropertySourceLocator configMapPropertySourceLocator) { + super(environment, properties, kubernetesClient, strategy); + + this.configMapPropertySourceLocator = configMapPropertySourceLocator; + } + + @PostConstruct + public void init() { + this.log.info("Kubernetes polling configMap change detector activated"); + } + + @Scheduled(initialDelayString = "${spring.cloud.kubernetes.reload.period:15000}", + fixedDelayString = "${spring.cloud.kubernetes.reload.period:15000}") + public void executeCycle() { + + boolean changedConfigMap = false; + if (this.properties.isMonitoringConfigMaps()) { + if (log.isDebugEnabled()) { + log.debug("Polling for changes in config maps"); + } + List currentConfigMapSources = findPropertySources( + ConfigMapPropertySource.class); + + if (!currentConfigMapSources.isEmpty()) { + changedConfigMap = changed( + locateMapPropertySources(this.configMapPropertySourceLocator, this.environment), + currentConfigMapSources); + } + } + + if (changedConfigMap) { + this.log.info("Detected change in config maps"); + reloadProperties(); + } + } + +} diff --git a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/PollingConfigurationChangeDetector.java b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/PollingSecretsChangeDetector.java similarity index 65% rename from spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/PollingConfigurationChangeDetector.java rename to spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/PollingSecretsChangeDetector.java index 941ad47e81..d1e1ee97ac 100644 --- a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/PollingConfigurationChangeDetector.java +++ b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/PollingSecretsChangeDetector.java @@ -24,8 +24,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.springframework.cloud.kubernetes.config.ConfigMapPropertySource; -import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; import org.springframework.cloud.kubernetes.config.SecretsPropertySource; import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; import org.springframework.core.env.AbstractEnvironment; @@ -33,53 +31,41 @@ import org.springframework.scheduling.annotation.Scheduled; /** - * A change detector that periodically retrieves secrets and configmaps and fire a reload - * when something changes. + * A change detector that periodically retrieves secrets and fire a reload when something + * changes. * * @author Nicola Ferraro * @author Haytham Mohamed + * @author Kris Iyer */ -public class PollingConfigurationChangeDetector extends ConfigurationChangeDetector { +public class PollingSecretsChangeDetector extends ConfigurationChangeDetector { protected Log log = LogFactory.getLog(getClass()); - private final ConfigMapPropertySourceLocator configMapPropertySourceLocator; - private final SecretsPropertySourceLocator secretsPropertySourceLocator; - public PollingConfigurationChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, + public PollingSecretsChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, KubernetesClient kubernetesClient, ConfigurationUpdateStrategy strategy, - ConfigMapPropertySourceLocator configMapPropertySourceLocator, SecretsPropertySourceLocator secretsPropertySourceLocator) { super(environment, properties, kubernetesClient, strategy); - this.configMapPropertySourceLocator = configMapPropertySourceLocator; this.secretsPropertySourceLocator = secretsPropertySourceLocator; } @PostConstruct public void init() { - this.log.info("Kubernetes polling configuration change detector activated"); + this.log.info("Kubernetes polling secrets change detector activated"); } @Scheduled(initialDelayString = "${spring.cloud.kubernetes.reload.period:15000}", fixedDelayString = "${spring.cloud.kubernetes.reload.period:15000}") public void executeCycle() { - boolean changedConfigMap = false; - if (this.properties.isMonitoringConfigMaps() && this.configMapPropertySourceLocator != null) { - List currentConfigMapSources = findPropertySources( - ConfigMapPropertySource.class); - - if (!currentConfigMapSources.isEmpty()) { - changedConfigMap = changed( - locateMapPropertySources(this.configMapPropertySourceLocator, this.environment), - currentConfigMapSources); - } - } - boolean changedSecrets = false; if (this.properties.isMonitoringSecrets()) { + if (log.isDebugEnabled()) { + log.debug("Polling for changes in secrets"); + } List currentSecretSources = locateMapPropertySources(this.secretsPropertySourceLocator, this.environment); if (currentSecretSources != null && !currentSecretSources.isEmpty()) { @@ -88,7 +74,8 @@ public void executeCycle() { } } - if (changedConfigMap || changedSecrets) { + if (changedSecrets) { + this.log.info("Detected change in secrets"); reloadProperties(); } } diff --git a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/condition/EventReloadDetectionMode.java b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/condition/EventReloadDetectionMode.java new file mode 100644 index 0000000000..04702a873c --- /dev/null +++ b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/condition/EventReloadDetectionMode.java @@ -0,0 +1,48 @@ +/* + * Copyright 2013-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.config.reload.condition; + +import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties.ReloadDetectionMode; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * A condition for Event ReloadDetectionMode and auto configuration. + * + * @author Kris Iyer + * + */ +public class EventReloadDetectionMode implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + Environment environment = context.getEnvironment(); + if (!environment.containsProperty("spring.cloud.kubernetes.reload.mode")) { + return true; + } + else { + if (environment.getProperty("spring.cloud.kubernetes.reload.mode") + .equalsIgnoreCase(ReloadDetectionMode.EVENT.name())) { + return true; + } + } + return false; + } + +} diff --git a/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/condition/PollingReloadDetectionMode.java b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/condition/PollingReloadDetectionMode.java new file mode 100644 index 0000000000..39e2091078 --- /dev/null +++ b/spring-cloud-kubernetes-config/src/main/java/org/springframework/cloud/kubernetes/config/reload/condition/PollingReloadDetectionMode.java @@ -0,0 +1,48 @@ +/* + * Copyright 2013-2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.config.reload.condition; + +import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties.ReloadDetectionMode; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; + +/** + * A condition for Polling ReloadDetectionMode and auto configuration. + * + * @author Kris Iyer + * + */ +public class PollingReloadDetectionMode implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + Environment environment = context.getEnvironment(); + if (!environment.containsProperty("spring.cloud.kubernetes.reload.mode")) { + return false; + } + else { + if (environment.getProperty("spring.cloud.kubernetes.reload.mode") + .equalsIgnoreCase(ReloadDetectionMode.POLLING.name())) { + return true; + } + } + return false; + } + +} diff --git a/spring-cloud-kubernetes-config/src/test/java/org/springframework/cloud/kubernetes/config/KubernetesConfigConfigurationTest.java b/spring-cloud-kubernetes-config/src/test/java/org/springframework/cloud/kubernetes/config/KubernetesConfigConfigurationTest.java index 3491a629fc..a58f6851e0 100644 --- a/spring-cloud-kubernetes-config/src/test/java/org/springframework/cloud/kubernetes/config/KubernetesConfigConfigurationTest.java +++ b/spring-cloud-kubernetes-config/src/test/java/org/springframework/cloud/kubernetes/config/KubernetesConfigConfigurationTest.java @@ -33,6 +33,7 @@ /** * @author Ryan Dawson + * @author Kris Iyer - Add tests for #643 */ public class KubernetesConfigConfigurationTest { @@ -85,7 +86,20 @@ public void kubernetesReloadEnabled() throws Exception { setup("spring.cloud.kubernetes.enabled=true", "spring.cloud.kubernetes.reload.enabled=true"); assertThat(this.context.containsBean("configMapPropertySourceLocator")).isTrue(); assertThat(this.context.containsBean("secretsPropertySourceLocator")).isTrue(); - assertThat(this.context.containsBean("propertyChangeWatcher")).isTrue(); + assertThat(this.context.containsBean("configMapPropertyChangeEventWatcher")).isTrue(); + assertThat(this.context.containsBean("secretsPropertyChangeEventWatcher")).isTrue(); + } + + @Test + public void kubernetesReloadEnabledWithPolling() throws Exception { + setup("spring.cloud.kubernetes.enabled=true", "spring.cloud.kubernetes.reload.enabled=true", + "spring.cloud.kubernetes.reload.mode=polling"); + assertThat(this.context.containsBean("configMapPropertySourceLocator")).isTrue(); + assertThat(this.context.containsBean("secretsPropertySourceLocator")).isTrue(); + assertThat(this.context.containsBean("configMapPropertyChangePollingWatcher")).isTrue(); + assertThat(this.context.containsBean("secretsPropertyChangePollingWatcher")).isTrue(); + assertThat(this.context.containsBean("configMapPropertyChangeEventWatcher")).isFalse(); + assertThat(this.context.containsBean("secretsPropertyChangeEventWatcher")).isFalse(); } @Test @@ -94,7 +108,8 @@ public void kubernetesReloadEnabledButSecretDisabled() throws Exception { "spring.cloud.kubernetes.secrets.enabled=false", "spring.cloud.kubernetes.reload.enabled=true"); assertThat(this.context.containsBean("configMapPropertySourceLocator")).isTrue(); assertThat(this.context.containsBean("secretsPropertySourceLocator")).isFalse(); - assertThat(this.context.containsBean("propertyChangeWatcher")).isTrue(); + assertThat(this.context.containsBean("configMapPropertyChangeEventWatcher")).isTrue(); + assertThat(this.context.containsBean("secretsPropertyChangeEventWatcher")).isFalse(); } @Test @@ -103,7 +118,8 @@ public void kubernetesReloadEnabledButSecretAndConfigDisabled() throws Exception "spring.cloud.kubernetes.secrets.enabled=false", "spring.cloud.kubernetes.reload.enabled=true"); assertThat(this.context.containsBean("configMapPropertySourceLocator")).isFalse(); assertThat(this.context.containsBean("secretsPropertySourceLocator")).isFalse(); - assertThat(this.context.containsBean("propertyChangeWatcher")).isFalse(); + assertThat(this.context.containsBean("configMapPropertyChangeEventWatcher")).isFalse(); + assertThat(this.context.containsBean("secretsPropertyChangeEventWatcher")).isFalse(); } private void setup(String... env) { diff --git a/spring-cloud-kubernetes-config/src/test/java/org/springframework/cloud/kubernetes/config/reload/EventBasedConfigurationChangeDetectorTests.java b/spring-cloud-kubernetes-config/src/test/java/org/springframework/cloud/kubernetes/config/reload/EventBasedConfigurationChangeDetectorTests.java index 49f56c7681..2e96c23a5c 100644 --- a/spring-cloud-kubernetes-config/src/test/java/org/springframework/cloud/kubernetes/config/reload/EventBasedConfigurationChangeDetectorTests.java +++ b/spring-cloud-kubernetes-config/src/test/java/org/springframework/cloud/kubernetes/config/reload/EventBasedConfigurationChangeDetectorTests.java @@ -31,7 +31,6 @@ import org.springframework.cloud.bootstrap.config.BootstrapPropertySource; import org.springframework.cloud.kubernetes.config.ConfigMapPropertySource; import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; -import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; import org.springframework.mock.env.MockEnvironment; import static org.assertj.core.api.Assertions.assertThat; @@ -65,9 +64,8 @@ public void verifyConfigChangesAccountsForBootstrapPropertySources() { ConfigurationUpdateStrategy configurationUpdateStrategy = mock(ConfigurationUpdateStrategy.class); ConfigMapPropertySourceLocator configMapLocator = mock(ConfigMapPropertySourceLocator.class); - SecretsPropertySourceLocator secretsLocator = mock(SecretsPropertySourceLocator.class); - EventBasedConfigurationChangeDetector detector = new EventBasedConfigurationChangeDetector(env, - configReloadProperties, k8sClient, configurationUpdateStrategy, configMapLocator, secretsLocator); + EventBasedConfigMapChangeDetector detector = new EventBasedConfigMapChangeDetector(env, configReloadProperties, + k8sClient, configurationUpdateStrategy, configMapLocator); List sources = detector.findPropertySources(ConfigMapPropertySource.class); assertThat(sources.size()).isEqualTo(1); assertThat(sources.get(0).getProperty("foo")).isEqualTo("bar"); diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigurationWatcherChangeDetector.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigMapWatcherChangeDetector.java similarity index 77% rename from spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigurationWatcherChangeDetector.java rename to spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigMapWatcherChangeDetector.java index afeb17b607..956a7d9a65 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigurationWatcherChangeDetector.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigMapWatcherChangeDetector.java @@ -17,14 +17,12 @@ package org.springframework.cloud.kubernetes.configuration.watcher; import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.KubernetesClient; import reactor.core.publisher.Mono; import org.springframework.cloud.bus.BusProperties; import org.springframework.cloud.bus.event.RefreshRemoteApplicationEvent; import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; -import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties; import org.springframework.cloud.kubernetes.config.reload.ConfigurationUpdateStrategy; import org.springframework.context.ApplicationEventPublisher; @@ -34,33 +32,26 @@ /** * @author Ryan Baxter + * @author Kris Iyer */ -public class BusEventBasedConfigurationWatcherChangeDetector extends ConfigurationWatcherChangeDetector +public class BusEventBasedConfigMapWatcherChangeDetector extends ConfigMapWatcherChangeDetector implements ApplicationEventPublisherAware { private ApplicationEventPublisher applicationEventPublisher; private BusProperties busProperties; - public BusEventBasedConfigurationWatcherChangeDetector(AbstractEnvironment environment, + public BusEventBasedConfigMapWatcherChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, KubernetesClient kubernetesClient, ConfigurationUpdateStrategy strategy, - ConfigMapPropertySourceLocator configMapPropertySourceLocator, - SecretsPropertySourceLocator secretsPropertySourceLocator, BusProperties busProperties, + ConfigMapPropertySourceLocator configMapPropertySourceLocator, BusProperties busProperties, ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, ThreadPoolTaskExecutor threadPoolTaskExecutor) { super(environment, properties, kubernetesClient, strategy, configMapPropertySourceLocator, - secretsPropertySourceLocator, k8SConfigurationProperties, threadPoolTaskExecutor); + k8SConfigurationProperties, threadPoolTaskExecutor); this.busProperties = busProperties; } - @Override - protected Mono triggerRefresh(Secret secret) { - this.applicationEventPublisher.publishEvent( - new RefreshRemoteApplicationEvent(secret, busProperties.getId(), secret.getMetadata().getName())); - return Mono.empty(); - } - @Override protected Mono triggerRefresh(ConfigMap configMap) { this.applicationEventPublisher.publishEvent( diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedSecretsWatcherChangeDetector.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedSecretsWatcherChangeDetector.java new file mode 100644 index 0000000000..74bce19d23 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedSecretsWatcherChangeDetector.java @@ -0,0 +1,67 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher; + +import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.client.KubernetesClient; +import reactor.core.publisher.Mono; + +import org.springframework.cloud.bus.BusProperties; +import org.springframework.cloud.bus.event.RefreshRemoteApplicationEvent; +import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; +import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties; +import org.springframework.cloud.kubernetes.config.reload.ConfigurationUpdateStrategy; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.ApplicationEventPublisherAware; +import org.springframework.core.env.AbstractEnvironment; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +/** + * @author Ryan Baxter + * @author Kris Iyer + */ +public class BusEventBasedSecretsWatcherChangeDetector extends SecretsWatcherChangeDetector + implements ApplicationEventPublisherAware { + + private ApplicationEventPublisher applicationEventPublisher; + + private BusProperties busProperties; + + public BusEventBasedSecretsWatcherChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, + KubernetesClient kubernetesClient, ConfigurationUpdateStrategy strategy, + SecretsPropertySourceLocator secretsPropertySourceLocator, BusProperties busProperties, + ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, + ThreadPoolTaskExecutor threadPoolTaskExecutor) { + super(environment, properties, kubernetesClient, strategy, secretsPropertySourceLocator, + k8SConfigurationProperties, threadPoolTaskExecutor); + + this.busProperties = busProperties; + } + + @Override + protected Mono triggerRefresh(Secret secret) { + this.applicationEventPublisher.publishEvent( + new RefreshRemoteApplicationEvent(secret, busProperties.getId(), secret.getMetadata().getName())); + return Mono.empty(); + } + + @Override + public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) { + this.applicationEventPublisher = applicationEventPublisher; + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherChangeDetector.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigMapWatcherChangeDetector.java similarity index 67% rename from spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherChangeDetector.java rename to spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigMapWatcherChangeDetector.java index 5b8ed7358b..8266b603e6 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherChangeDetector.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigMapWatcherChangeDetector.java @@ -21,35 +21,36 @@ import java.util.concurrent.TimeUnit; import io.fabric8.kubernetes.api.model.ConfigMap; -import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.KubernetesClient; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import reactor.core.publisher.Mono; import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; -import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties; import org.springframework.cloud.kubernetes.config.reload.ConfigurationUpdateStrategy; -import org.springframework.cloud.kubernetes.config.reload.EventBasedConfigurationChangeDetector; +import org.springframework.cloud.kubernetes.config.reload.EventBasedConfigMapChangeDetector; import org.springframework.core.env.AbstractEnvironment; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; /** * @author Ryan Baxter + * @author Kris Iyer */ -public abstract class ConfigurationWatcherChangeDetector extends EventBasedConfigurationChangeDetector { +public abstract class ConfigMapWatcherChangeDetector extends EventBasedConfigMapChangeDetector { + + protected Log log = LogFactory.getLog(getClass()); private ScheduledExecutorService executorService; protected ConfigurationWatcherConfigurationProperties k8SConfigurationProperties; - public ConfigurationWatcherChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, + public ConfigMapWatcherChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, KubernetesClient kubernetesClient, ConfigurationUpdateStrategy strategy, ConfigMapPropertySourceLocator configMapPropertySourceLocator, - SecretsPropertySourceLocator secretsPropertySourceLocator, ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, ThreadPoolTaskExecutor threadPoolTaskExecutor) { - super(environment, properties, kubernetesClient, strategy, configMapPropertySourceLocator, - secretsPropertySourceLocator); + super(environment, properties, kubernetesClient, strategy, configMapPropertySourceLocator); this.executorService = Executors.newScheduledThreadPool(k8SConfigurationProperties.getThreadPoolSize(), threadPoolTaskExecutor); this.k8SConfigurationProperties = k8SConfigurationProperties; @@ -82,35 +83,6 @@ protected boolean isSpringCloudKubernetesConfig(ConfigMap configMap) { configMap.getMetadata().getLabels().getOrDefault(k8SConfigurationProperties.getConfigLabel(), "false")); } - protected boolean isSpringCloudKubernetesSecret(Secret secret) { - if (secret.getMetadata() == null || secret.getMetadata().getLabels() == null) { - return false; - } - return Boolean.parseBoolean( - secret.getMetadata().getLabels().getOrDefault(k8SConfigurationProperties.getSecretLabel(), "false")); - } - - protected abstract Mono triggerRefresh(Secret secret); - protected abstract Mono triggerRefresh(ConfigMap configMap); - @Override - protected void onEvent(Secret secret) { - if (isSpringCloudKubernetesSecret(secret)) { - if (log.isDebugEnabled()) { - log.debug("Scheduling remote refresh event to be published for Secret " + secret.getMetadata().getName() - + " to be published in " + k8SConfigurationProperties.getRefreshDelay().toMillis() - + " milliseconds"); - } - executorService.schedule(() -> triggerRefresh(secret).subscribe(), - k8SConfigurationProperties.getRefreshDelay().toMillis(), TimeUnit.MILLISECONDS); - } - else { - if (log.isDebugEnabled()) { - log.debug("Not publishing event. Secret " + secret.getMetadata().getName() - + " does not contain the label " + k8SConfigurationProperties.getSecretLabel()); - } - } - } - } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java index 0c8084d747..058c52d8d8 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/ConfigurationWatcherAutoConfiguration.java @@ -17,8 +17,6 @@ package org.springframework.cloud.kubernetes.configuration.watcher; import io.fabric8.kubernetes.client.KubernetesClient; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.springframework.boot.actuate.autoconfigure.amqp.RabbitHealthContributorAutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; @@ -40,13 +38,12 @@ /** * @author Ryan Baxter + * @author Kris Iyer */ @Configuration(proxyBeanMethods = false) @EnableConfigurationProperties({ ConfigurationWatcherConfigurationProperties.class }) public class ConfigurationWatcherAutoConfiguration { - protected Log log = LogFactory.getLog(getClass()); - @Bean @ConditionalOnMissingBean public WebClient webClient(WebClient.Builder webClientBuilder) { @@ -54,17 +51,30 @@ public WebClient webClient(WebClient.Builder webClientBuilder) { } @Bean - @ConditionalOnMissingBean(ConfigurationWatcherChangeDetector.class) - public ConfigurationWatcherChangeDetector httpBasedConfigurationWatchChangeDetector(AbstractEnvironment environment, + @ConditionalOnMissingBean(ConfigMapWatcherChangeDetector.class) + public ConfigMapWatcherChangeDetector httpBasedConfigMapWatchChangeDetector(AbstractEnvironment environment, KubernetesClient kubernetesClient, ConfigMapPropertySourceLocator configMapPropertySourceLocator, SecretsPropertySourceLocator secretsPropertySourceLocator, ConfigReloadProperties properties, ConfigurationUpdateStrategy strategy, ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, ThreadPoolTaskExecutor threadFactory, WebClient webClient, KubernetesReactiveDiscoveryClient kubernetesReactiveDiscoveryClient) { - return new HttpBasedConfigurationWatchChangeDetector(environment, properties, kubernetesClient, strategy, - configMapPropertySourceLocator, secretsPropertySourceLocator, k8SConfigurationProperties, threadFactory, - webClient, kubernetesReactiveDiscoveryClient); + return new HttpBasedConfigMapWatchChangeDetector(environment, properties, kubernetesClient, strategy, + configMapPropertySourceLocator, k8SConfigurationProperties, threadFactory, webClient, + kubernetesReactiveDiscoveryClient); + } + + @Bean + @ConditionalOnMissingBean(SecretsWatcherChangeDetector.class) + public SecretsWatcherChangeDetector httpBasedSecretsWatchChangeDetector(AbstractEnvironment environment, + KubernetesClient kubernetesClient, SecretsPropertySourceLocator secretsPropertySourceLocator, + ConfigReloadProperties properties, ConfigurationUpdateStrategy strategy, + ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, + ThreadPoolTaskExecutor threadFactory, WebClient webClient, + KubernetesReactiveDiscoveryClient kubernetesReactiveDiscoveryClient) { + return new HttpBasedSecretsWatchChangeDetector(environment, properties, kubernetesClient, strategy, + secretsPropertySourceLocator, k8SConfigurationProperties, threadFactory, webClient, + kubernetesReactiveDiscoveryClient); } @Configuration @@ -73,17 +83,27 @@ public ConfigurationWatcherChangeDetector httpBasedConfigurationWatchChangeDetec static class BusConfiguration { @Bean - @ConditionalOnMissingBean(ConfigurationWatcherChangeDetector.class) - public ConfigurationWatcherChangeDetector busPropertyChangeWatcher(BusProperties busProperties, + @ConditionalOnMissingBean(ConfigMapWatcherChangeDetector.class) + public ConfigMapWatcherChangeDetector busConfigMapChangeWatcher(BusProperties busProperties, + AbstractEnvironment environment, KubernetesClient kubernetesClient, + ConfigMapPropertySourceLocator configMapPropertySourceLocator, ConfigReloadProperties properties, + ConfigurationUpdateStrategy strategy, + ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, + ThreadPoolTaskExecutor threadFactory) { + return new BusEventBasedConfigMapWatcherChangeDetector(environment, properties, kubernetesClient, strategy, + configMapPropertySourceLocator, busProperties, k8SConfigurationProperties, threadFactory); + } + + @Bean + @ConditionalOnMissingBean(SecretsWatcherChangeDetector.class) + public SecretsWatcherChangeDetector busSecretsChangeWatcher(BusProperties busProperties, AbstractEnvironment environment, KubernetesClient kubernetesClient, - ConfigMapPropertySourceLocator configMapPropertySourceLocator, SecretsPropertySourceLocator secretsPropertySourceLocator, ConfigReloadProperties properties, ConfigurationUpdateStrategy strategy, ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, ThreadPoolTaskExecutor threadFactory) { - return new BusEventBasedConfigurationWatcherChangeDetector(environment, properties, kubernetesClient, - strategy, configMapPropertySourceLocator, secretsPropertySourceLocator, busProperties, - k8SConfigurationProperties, threadFactory); + return new BusEventBasedSecretsWatcherChangeDetector(environment, properties, kubernetesClient, strategy, + secretsPropertySourceLocator, busProperties, k8SConfigurationProperties, threadFactory); } } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigurationWatchChangeDetector.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigMapWatchChangeDetector.java similarity index 89% rename from spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigurationWatchChangeDetector.java rename to spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigMapWatchChangeDetector.java index 75c9242845..7d910db76e 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigurationWatchChangeDetector.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigMapWatchChangeDetector.java @@ -20,14 +20,14 @@ import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.ObjectMeta; -import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.KubernetesClient; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import org.springframework.cloud.client.ServiceInstance; import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; -import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties; import org.springframework.cloud.kubernetes.config.reload.ConfigurationUpdateStrategy; import org.springframework.cloud.kubernetes.discovery.reactive.KubernetesReactiveDiscoveryClient; @@ -40,8 +40,11 @@ /** * @author Ryan Baxter + * @author Kris Iyer */ -public class HttpBasedConfigurationWatchChangeDetector extends ConfigurationWatcherChangeDetector { +public class HttpBasedConfigMapWatchChangeDetector extends ConfigMapWatcherChangeDetector { + + private Log log = LogFactory.getLog(getClass()); /** * Annotation key for actuator port and path. @@ -52,24 +55,18 @@ public class HttpBasedConfigurationWatchChangeDetector extends ConfigurationWatc private KubernetesReactiveDiscoveryClient kubernetesReactiveDiscoveryClient; - public HttpBasedConfigurationWatchChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, + public HttpBasedConfigMapWatchChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, KubernetesClient kubernetesClient, ConfigurationUpdateStrategy strategy, ConfigMapPropertySourceLocator configMapPropertySourceLocator, - SecretsPropertySourceLocator secretsPropertySourceLocator, ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, ThreadPoolTaskExecutor threadPoolTaskExecutor, WebClient webClient, KubernetesReactiveDiscoveryClient k8sReactiveDiscoveryClient) { super(environment, properties, kubernetesClient, strategy, configMapPropertySourceLocator, - secretsPropertySourceLocator, k8SConfigurationProperties, threadPoolTaskExecutor); + k8SConfigurationProperties, threadPoolTaskExecutor); this.webClient = webClient; this.kubernetesReactiveDiscoveryClient = k8sReactiveDiscoveryClient; } - @Override - protected Mono triggerRefresh(Secret secret) { - return refresh(secret.getMetadata()).then(); - } - private void setActuatorUriFromAnnotation(UriComponentsBuilder actuatorUriBuilder, String metadataUri) { URI annotationUri = URI.create(metadataUri); actuatorUriBuilder.path(annotationUri.getPath() + "/refresh"); diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedSecretsWatchChangeDetector.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedSecretsWatchChangeDetector.java new file mode 100644 index 0000000000..c9192508c4 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedSecretsWatchChangeDetector.java @@ -0,0 +1,134 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher; + +import java.net.URI; + +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.client.KubernetesClient; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; +import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties; +import org.springframework.cloud.kubernetes.config.reload.ConfigurationUpdateStrategy; +import org.springframework.cloud.kubernetes.discovery.reactive.KubernetesReactiveDiscoveryClient; +import org.springframework.core.env.AbstractEnvironment; +import org.springframework.http.ResponseEntity; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.util.StringUtils; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.util.UriComponentsBuilder; + +/** + * @author Ryan Baxter + * @author Kris Iyer + */ +public class HttpBasedSecretsWatchChangeDetector extends SecretsWatcherChangeDetector { + + /** + * Annotation key for actuator port and path. + */ + public static String ANNOTATION_KEY = "boot.spring.io/actuator"; + + private WebClient webClient; + + private KubernetesReactiveDiscoveryClient kubernetesReactiveDiscoveryClient; + + public HttpBasedSecretsWatchChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, + KubernetesClient kubernetesClient, ConfigurationUpdateStrategy strategy, + SecretsPropertySourceLocator secretsPropertySourceLocator, + ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, + ThreadPoolTaskExecutor threadPoolTaskExecutor, WebClient webClient, + KubernetesReactiveDiscoveryClient k8sReactiveDiscoveryClient) { + super(environment, properties, kubernetesClient, strategy, secretsPropertySourceLocator, + k8SConfigurationProperties, threadPoolTaskExecutor); + this.webClient = webClient; + this.kubernetesReactiveDiscoveryClient = k8sReactiveDiscoveryClient; + } + + @Override + protected Mono triggerRefresh(Secret secret) { + return refresh(secret.getMetadata()).then(); + } + + private void setActuatorUriFromAnnotation(UriComponentsBuilder actuatorUriBuilder, String metadataUri) { + URI annotationUri = URI.create(metadataUri); + actuatorUriBuilder.path(annotationUri.getPath() + "/refresh"); + + // The URI may not contain a host so if that is the case the port in the URI will + // be -1 + // The authority of the URI will be : for example :9090, we just need the + // 9090 in this case + if (annotationUri.getPort() < 0) { + if (annotationUri.getAuthority() != null) { + actuatorUriBuilder.port(annotationUri.getAuthority().replaceFirst(":", "")); + } + } + else { + actuatorUriBuilder.port(annotationUri.getPort()); + } + } + + private URI getActuatorUri(ServiceInstance si) { + String metadataUri = si.getMetadata().getOrDefault(ANNOTATION_KEY, ""); + if (log.isDebugEnabled()) { + log.debug("Metadata actuator uri is: " + metadataUri); + } + + UriComponentsBuilder actuatorUriBuilder = UriComponentsBuilder.newInstance().scheme(si.getScheme()) + .host(si.getHost()); + + if (!StringUtils.isEmpty(metadataUri)) { + if (log.isDebugEnabled()) { + log.debug("Found actuator URI in service instance metadata"); + } + setActuatorUriFromAnnotation(actuatorUriBuilder, metadataUri); + } + else { + Integer port = k8SConfigurationProperties.getActuatorPort() < 0 ? si.getPort() + : k8SConfigurationProperties.getActuatorPort(); + actuatorUriBuilder = actuatorUriBuilder.path(k8SConfigurationProperties.getActuatorPath() + "/refresh") + .port(port); + } + + return actuatorUriBuilder.build().toUri(); + } + + protected Flux> refresh(ObjectMeta objectMeta) { + + return kubernetesReactiveDiscoveryClient.getInstances(objectMeta.getName()).flatMap(si -> { + URI actuatorUri = getActuatorUri(si); + if (log.isDebugEnabled()) { + log.debug("Sending refresh request for " + objectMeta.getName() + " to URI " + actuatorUri.toString()); + } + Mono> response = webClient.post().uri(actuatorUri).retrieve().toBodilessEntity() + .doOnSuccess(re -> { + if (log.isDebugEnabled()) { + log.debug("Refresh sent to " + objectMeta.getName() + " at URI address " + actuatorUri + + " returned a " + re.getStatusCode().toString()); + } + }).doOnError(t -> { + log.warn("Refresh sent to " + objectMeta.getName() + " failed", t); + }); + return response; + }); + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretsWatcherChangeDetector.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretsWatcherChangeDetector.java new file mode 100644 index 0000000000..01b714d004 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/main/java/org/springframework/cloud/kubernetes/configuration/watcher/SecretsWatcherChangeDetector.java @@ -0,0 +1,88 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.client.KubernetesClient; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import reactor.core.publisher.Mono; + +import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; +import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties; +import org.springframework.cloud.kubernetes.config.reload.ConfigurationUpdateStrategy; +import org.springframework.cloud.kubernetes.config.reload.EventBasedSecretsChangeDetector; +import org.springframework.core.env.AbstractEnvironment; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +/** + * @author Ryan Baxter + * @author Kris Iyer + */ +public abstract class SecretsWatcherChangeDetector extends EventBasedSecretsChangeDetector { + + protected Log log = LogFactory.getLog(getClass()); + + private ScheduledExecutorService executorService; + + protected ConfigurationWatcherConfigurationProperties k8SConfigurationProperties; + + public SecretsWatcherChangeDetector(AbstractEnvironment environment, ConfigReloadProperties properties, + KubernetesClient kubernetesClient, ConfigurationUpdateStrategy strategy, + SecretsPropertySourceLocator secretsPropertySourceLocator, + ConfigurationWatcherConfigurationProperties k8SConfigurationProperties, + ThreadPoolTaskExecutor threadPoolTaskExecutor) { + super(environment, properties, kubernetesClient, strategy, secretsPropertySourceLocator); + this.executorService = Executors.newScheduledThreadPool(k8SConfigurationProperties.getThreadPoolSize(), + threadPoolTaskExecutor); + this.k8SConfigurationProperties = k8SConfigurationProperties; + } + + protected boolean isSpringCloudKubernetesSecret(Secret secret) { + if (secret.getMetadata() == null || secret.getMetadata().getLabels() == null) { + return false; + } + return Boolean.parseBoolean( + secret.getMetadata().getLabels().getOrDefault(k8SConfigurationProperties.getSecretLabel(), "false")); + } + + protected abstract Mono triggerRefresh(Secret secret); + + @Override + protected void onEvent(Secret secret) { + if (isSpringCloudKubernetesSecret(secret)) { + if (log.isDebugEnabled()) { + log.debug("Scheduling remote refresh event to be published for Secret " + secret.getMetadata().getName() + + " to be published in " + k8SConfigurationProperties.getRefreshDelay().toMillis() + + " milliseconds"); + } + executorService.schedule(() -> triggerRefresh(secret).subscribe(), + k8SConfigurationProperties.getRefreshDelay().toMillis(), TimeUnit.MILLISECONDS); + } + else { + if (log.isDebugEnabled()) { + log.debug("Not publishing event. Secret " + secret.getMetadata().getName() + + " does not contain the label " + k8SConfigurationProperties.getSecretLabel()); + } + } + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigurationWatcherChangeDetectorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigMapWatcherChangeDetectorTests.java similarity index 73% rename from spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigurationWatcherChangeDetectorTests.java rename to spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigMapWatcherChangeDetectorTests.java index 5a38e2b069..6c8f62d234 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigurationWatcherChangeDetectorTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedConfigMapWatcherChangeDetectorTests.java @@ -18,7 +18,6 @@ import io.fabric8.kubernetes.api.model.ConfigMap; import io.fabric8.kubernetes.api.model.ObjectMeta; -import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.KubernetesClient; import org.junit.Before; import org.junit.Test; @@ -30,7 +29,6 @@ import org.springframework.cloud.bus.BusProperties; import org.springframework.cloud.bus.event.RefreshRemoteApplicationEvent; import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; -import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties; import org.springframework.cloud.kubernetes.config.reload.ConfigurationUpdateStrategy; import org.springframework.context.ApplicationEventPublisher; @@ -42,9 +40,10 @@ /** * @author Ryan Baxter + * @author Kris Iyer */ @RunWith(MockitoJUnitRunner.class) -public class BusEventBasedConfigurationWatcherChangeDetectorTests { +public class BusEventBasedConfigMapWatcherChangeDetectorTests { @Mock private KubernetesClient client; @@ -55,16 +54,13 @@ public class BusEventBasedConfigurationWatcherChangeDetectorTests { @Mock private ConfigMapPropertySourceLocator configMapPropertySourceLocator; - @Mock - private SecretsPropertySourceLocator secretsPropertySourceLocator; - @Mock private ThreadPoolTaskExecutor threadPoolTaskExecutor; @Mock private ApplicationEventPublisher applicationEventPublisher; - private BusEventBasedConfigurationWatcherChangeDetector changeDetector; + private BusEventBasedConfigMapWatcherChangeDetector changeDetector; private ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties; @@ -76,8 +72,8 @@ public void setup() { ConfigReloadProperties configReloadProperties = new ConfigReloadProperties(); configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); busProperties = new BusProperties(); - changeDetector = new BusEventBasedConfigurationWatcherChangeDetector(mockEnvironment, configReloadProperties, - client, updateStrategy, configMapPropertySourceLocator, secretsPropertySourceLocator, busProperties, + changeDetector = new BusEventBasedConfigMapWatcherChangeDetector(mockEnvironment, configReloadProperties, + client, updateStrategy, configMapPropertySourceLocator, busProperties, configurationWatcherConfigurationProperties, threadPoolTaskExecutor); changeDetector.setApplicationEventPublisher(applicationEventPublisher); } @@ -97,19 +93,4 @@ public void triggerRefreshWithConfigMap() { assertThat(argumentCaptor.getValue().getDestinationService()).isEqualTo("foo:**"); } - @Test - public void triggerRefreshWithSecret() { - ObjectMeta objectMeta = new ObjectMeta(); - objectMeta.setName("foo"); - Secret secret = new Secret(); - secret.setMetadata(objectMeta); - changeDetector.triggerRefresh(secret); - ArgumentCaptor argumentCaptor = ArgumentCaptor - .forClass(RefreshRemoteApplicationEvent.class); - verify(applicationEventPublisher).publishEvent(argumentCaptor.capture()); - assertThat(argumentCaptor.getValue().getSource()).isEqualTo(secret); - assertThat(argumentCaptor.getValue().getOriginService()).isEqualTo(busProperties.getId()); - assertThat(argumentCaptor.getValue().getDestinationService()).isEqualTo("foo:**"); - } - } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedSecretsWatcherChangeDetectorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedSecretsWatcherChangeDetectorTests.java new file mode 100644 index 0000000000..0266f06915 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/BusEventBasedSecretsWatcherChangeDetectorTests.java @@ -0,0 +1,96 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher; + +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.client.KubernetesClient; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import org.springframework.cloud.bus.BusProperties; +import org.springframework.cloud.bus.event.RefreshRemoteApplicationEvent; +import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; +import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties; +import org.springframework.cloud.kubernetes.config.reload.ConfigurationUpdateStrategy; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.mock.env.MockEnvironment; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.verify; + +/** + * @author Ryan Baxter + * @author Kris Iyer + */ +@RunWith(MockitoJUnitRunner.class) +public class BusEventBasedSecretsWatcherChangeDetectorTests { + + @Mock + private KubernetesClient client; + + @Mock + private ConfigurationUpdateStrategy updateStrategy; + + @Mock + private SecretsPropertySourceLocator secretsPropertySourceLocator; + + @Mock + private ThreadPoolTaskExecutor threadPoolTaskExecutor; + + @Mock + private ApplicationEventPublisher applicationEventPublisher; + + private BusEventBasedSecretsWatcherChangeDetector changeDetector; + + private ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties; + + private BusProperties busProperties; + + @Before + public void setup() { + MockEnvironment mockEnvironment = new MockEnvironment(); + ConfigReloadProperties configReloadProperties = new ConfigReloadProperties(); + configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); + busProperties = new BusProperties(); + changeDetector = new BusEventBasedSecretsWatcherChangeDetector(mockEnvironment, configReloadProperties, client, + updateStrategy, secretsPropertySourceLocator, busProperties, + configurationWatcherConfigurationProperties, threadPoolTaskExecutor); + changeDetector.setApplicationEventPublisher(applicationEventPublisher); + } + + @Test + public void triggerRefreshWithSecret() { + ObjectMeta objectMeta = new ObjectMeta(); + objectMeta.setName("foo"); + Secret secret = new Secret(); + secret.setMetadata(objectMeta); + changeDetector.triggerRefresh(secret); + ArgumentCaptor argumentCaptor = ArgumentCaptor + .forClass(RefreshRemoteApplicationEvent.class); + verify(applicationEventPublisher).publishEvent(argumentCaptor.capture()); + assertThat(argumentCaptor.getValue().getSource()).isEqualTo(secret); + assertThat(argumentCaptor.getValue().getOriginService()).isEqualTo(busProperties.getId()); + assertThat(argumentCaptor.getValue().getDestinationService()).isEqualTo("foo:**"); + } + +} diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigurationWatchChangeDetectorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigMapWatchChangeDetectorTests.java similarity index 68% rename from spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigurationWatchChangeDetectorTests.java rename to spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigMapWatchChangeDetectorTests.java index 7985738713..e77fc279d9 100644 --- a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigurationWatchChangeDetectorTests.java +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedConfigMapWatchChangeDetectorTests.java @@ -27,7 +27,6 @@ import io.fabric8.kubernetes.api.model.EndpointAddress; import io.fabric8.kubernetes.api.model.EndpointPort; import io.fabric8.kubernetes.api.model.ObjectMeta; -import io.fabric8.kubernetes.api.model.Secret; import io.fabric8.kubernetes.client.KubernetesClient; import org.junit.Before; import org.junit.Rule; @@ -40,7 +39,6 @@ import org.springframework.cloud.client.ServiceInstance; import org.springframework.cloud.kubernetes.config.ConfigMapPropertySourceLocator; -import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties; import org.springframework.cloud.kubernetes.config.reload.ConfigurationUpdateStrategy; import org.springframework.cloud.kubernetes.discovery.KubernetesServiceInstance; @@ -57,13 +55,14 @@ import static com.github.tomakehurst.wiremock.client.WireMock.verify; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; -import static org.springframework.cloud.kubernetes.configuration.watcher.HttpBasedConfigurationWatchChangeDetector.ANNOTATION_KEY; +import static org.springframework.cloud.kubernetes.configuration.watcher.HttpBasedConfigMapWatchChangeDetector.ANNOTATION_KEY; /** * @author Ryan Baxter + * @author Kris Iyer */ @RunWith(MockitoJUnitRunner.class) -public class HttpBasedConfigurationWatchChangeDetectorTests { +public class HttpBasedConfigMapWatchChangeDetectorTests { @Rule public WireMockRule wireMockRule = new WireMockRule(0); @@ -77,16 +76,13 @@ public class HttpBasedConfigurationWatchChangeDetectorTests { @Mock private ConfigMapPropertySourceLocator configMapPropertySourceLocator; - @Mock - private SecretsPropertySourceLocator secretsPropertySourceLocator; - @Mock private ThreadPoolTaskExecutor threadPoolTaskExecutor; @Mock private KubernetesReactiveDiscoveryClient reactiveDiscoveryClient; - private HttpBasedConfigurationWatchChangeDetector changeDetector; + private HttpBasedConfigMapWatchChangeDetector changeDetector; private ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties; @@ -106,10 +102,9 @@ public void setup() { ConfigReloadProperties configReloadProperties = new ConfigReloadProperties(); configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); WebClient webClient = WebClient.builder().build(); - changeDetector = new HttpBasedConfigurationWatchChangeDetector(mockEnvironment, configReloadProperties, client, - updateStrategy, configMapPropertySourceLocator, secretsPropertySourceLocator, - configurationWatcherConfigurationProperties, threadPoolTaskExecutor, webClient, - reactiveDiscoveryClient); + changeDetector = new HttpBasedConfigMapWatchChangeDetector(mockEnvironment, configReloadProperties, client, + updateStrategy, configMapPropertySourceLocator, configurationWatcherConfigurationProperties, + threadPoolTaskExecutor, webClient, reactiveDiscoveryClient); } @Test @@ -124,18 +119,6 @@ public void triggerConfigMapRefresh() { verify(postRequestedFor(urlEqualTo("/actuator/refresh"))); } - @Test - public void triggerSecretRefresh() throws InterruptedException { - Secret secret = new Secret(); - ObjectMeta objectMeta = new ObjectMeta(); - objectMeta.setName("foo"); - secret.setMetadata(objectMeta); - WireMock.configureFor("localhost", wireMockRule.port()); - stubFor(post(WireMock.urlEqualTo("/actuator/refresh")).willReturn(aResponse().withStatus(200))); - StepVerifier.create(changeDetector.triggerRefresh(secret)).verifyComplete(); - verify(postRequestedFor(urlEqualTo("/actuator/refresh"))); - } - @Test public void triggerConfigMapRefreshWithPropertiesBasedActuatorPath() throws InterruptedException { configurationWatcherConfigurationProperties.setActuatorPath("/my/custom/actuator"); @@ -149,19 +132,6 @@ public void triggerConfigMapRefreshWithPropertiesBasedActuatorPath() throws Inte verify(postRequestedFor(urlEqualTo("/my/custom/actuator/refresh"))); } - @Test - public void triggerSecretRefreshWithPropertiesBasedActuatorPath() throws InterruptedException { - configurationWatcherConfigurationProperties.setActuatorPath("/my/custom/actuator"); - Secret secret = new Secret(); - ObjectMeta objectMeta = new ObjectMeta(); - objectMeta.setName("foo"); - secret.setMetadata(objectMeta); - WireMock.configureFor("localhost", wireMockRule.port()); - stubFor(post(WireMock.urlEqualTo("/my/custom/actuator/refresh")).willReturn(aResponse().withStatus(200))); - StepVerifier.create(changeDetector.triggerRefresh(secret)).verifyComplete(); - verify(postRequestedFor(urlEqualTo("/my/custom/actuator/refresh"))); - } - @Test public void triggerConfigMapRefreshWithAnnotationActuatorPath() { Map metadata = new HashMap<>(); @@ -185,27 +155,4 @@ public void triggerConfigMapRefreshWithAnnotationActuatorPath() { verify(postRequestedFor(urlEqualTo("/my/custom/actuator/refresh"))); } - @Test - public void triggerSecretRefreshWithAnnotationActuatorPath() { - Map metadata = new HashMap<>(); - metadata.put(ANNOTATION_KEY, "http://:" + wireMockRule.port() + "/my/custom/actuator"); - EndpointAddress fooEndpointAddress = new EndpointAddress(); - fooEndpointAddress.setIp("127.0.0.1"); - fooEndpointAddress.setHostname("localhost"); - EndpointPort fooEndpointPort = new EndpointPort(); - fooEndpointPort.setPort(wireMockRule.port()); - List instances = new ArrayList<>(); - KubernetesServiceInstance fooServiceInstance = new KubernetesServiceInstance("foo", "foo", - fooEndpointAddress.getIp(), fooEndpointPort.getPort(), metadata, false); - instances.add(fooServiceInstance); - when(reactiveDiscoveryClient.getInstances(eq("foo"))).thenReturn(Flux.fromIterable(instances)); - Secret secret = new Secret(); - ObjectMeta objectMeta = new ObjectMeta(); - objectMeta.setName("foo"); - secret.setMetadata(objectMeta); - stubFor(post(WireMock.urlEqualTo("/my/custom/actuator/refresh")).willReturn(aResponse().withStatus(200))); - StepVerifier.create(changeDetector.triggerRefresh(secret)).verifyComplete(); - verify(postRequestedFor(urlEqualTo("/my/custom/actuator/refresh"))); - } - } diff --git a/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedSecretsWatchChangeDetectorTests.java b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedSecretsWatchChangeDetectorTests.java new file mode 100644 index 0000000000..e9ff158892 --- /dev/null +++ b/spring-cloud-kubernetes-controllers/spring-cloud-kubernetes-configuration-watcher/src/test/java/org/springframework/cloud/kubernetes/configuration/watcher/HttpBasedSecretsWatchChangeDetectorTests.java @@ -0,0 +1,158 @@ +/* + * Copyright 2013-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.kubernetes.configuration.watcher; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.junit.WireMockRule; +import io.fabric8.kubernetes.api.model.EndpointAddress; +import io.fabric8.kubernetes.api.model.EndpointPort; +import io.fabric8.kubernetes.api.model.ObjectMeta; +import io.fabric8.kubernetes.api.model.Secret; +import io.fabric8.kubernetes.client.KubernetesClient; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; +import reactor.core.publisher.Flux; +import reactor.test.StepVerifier; + +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.cloud.kubernetes.config.SecretsPropertySourceLocator; +import org.springframework.cloud.kubernetes.config.reload.ConfigReloadProperties; +import org.springframework.cloud.kubernetes.config.reload.ConfigurationUpdateStrategy; +import org.springframework.cloud.kubernetes.discovery.KubernetesServiceInstance; +import org.springframework.cloud.kubernetes.discovery.reactive.KubernetesReactiveDiscoveryClient; +import org.springframework.mock.env.MockEnvironment; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.web.reactive.function.client.WebClient; + +import static com.github.tomakehurst.wiremock.client.WireMock.aResponse; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.postRequestedFor; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo; +import static com.github.tomakehurst.wiremock.client.WireMock.verify; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.when; +import static org.springframework.cloud.kubernetes.configuration.watcher.HttpBasedConfigMapWatchChangeDetector.ANNOTATION_KEY; + +/** + * @author Ryan Baxter + * @author Kris Iyer + */ +@RunWith(MockitoJUnitRunner.class) +public class HttpBasedSecretsWatchChangeDetectorTests { + + @Rule + public WireMockRule wireMockRule = new WireMockRule(0); + + @Mock + private KubernetesClient client; + + @Mock + private ConfigurationUpdateStrategy updateStrategy; + + @Mock + private SecretsPropertySourceLocator secretsPropertySourceLocator; + + @Mock + private ThreadPoolTaskExecutor threadPoolTaskExecutor; + + @Mock + private KubernetesReactiveDiscoveryClient reactiveDiscoveryClient; + + private HttpBasedSecretsWatchChangeDetector changeDetector; + + private ConfigurationWatcherConfigurationProperties configurationWatcherConfigurationProperties; + + @Before + public void setup() { + EndpointAddress fooEndpointAddress = new EndpointAddress(); + fooEndpointAddress.setIp("127.0.0.1"); + fooEndpointAddress.setHostname("localhost"); + EndpointPort fooEndpointPort = new EndpointPort(); + fooEndpointPort.setPort(wireMockRule.port()); + List instances = new ArrayList<>(); + KubernetesServiceInstance fooServiceInstance = new KubernetesServiceInstance("foo", "foo", + fooEndpointAddress.getIp(), fooEndpointPort.getPort(), new HashMap<>(), false); + instances.add(fooServiceInstance); + when(reactiveDiscoveryClient.getInstances(eq("foo"))).thenReturn(Flux.fromIterable(instances)); + MockEnvironment mockEnvironment = new MockEnvironment(); + ConfigReloadProperties configReloadProperties = new ConfigReloadProperties(); + configurationWatcherConfigurationProperties = new ConfigurationWatcherConfigurationProperties(); + WebClient webClient = WebClient.builder().build(); + changeDetector = new HttpBasedSecretsWatchChangeDetector(mockEnvironment, configReloadProperties, client, + updateStrategy, secretsPropertySourceLocator, configurationWatcherConfigurationProperties, + threadPoolTaskExecutor, webClient, reactiveDiscoveryClient); + } + + @Test + public void triggerSecretRefresh() throws InterruptedException { + Secret secret = new Secret(); + ObjectMeta objectMeta = new ObjectMeta(); + objectMeta.setName("foo"); + secret.setMetadata(objectMeta); + WireMock.configureFor("localhost", wireMockRule.port()); + stubFor(post(WireMock.urlEqualTo("/actuator/refresh")).willReturn(aResponse().withStatus(200))); + StepVerifier.create(changeDetector.triggerRefresh(secret)).verifyComplete(); + verify(postRequestedFor(urlEqualTo("/actuator/refresh"))); + } + + @Test + public void triggerSecretRefreshWithPropertiesBasedActuatorPath() throws InterruptedException { + configurationWatcherConfigurationProperties.setActuatorPath("/my/custom/actuator"); + Secret secret = new Secret(); + ObjectMeta objectMeta = new ObjectMeta(); + objectMeta.setName("foo"); + secret.setMetadata(objectMeta); + WireMock.configureFor("localhost", wireMockRule.port()); + stubFor(post(WireMock.urlEqualTo("/my/custom/actuator/refresh")).willReturn(aResponse().withStatus(200))); + StepVerifier.create(changeDetector.triggerRefresh(secret)).verifyComplete(); + verify(postRequestedFor(urlEqualTo("/my/custom/actuator/refresh"))); + } + + @Test + public void triggerSecretRefreshWithAnnotationActuatorPath() { + Map metadata = new HashMap<>(); + metadata.put(ANNOTATION_KEY, "http://:" + wireMockRule.port() + "/my/custom/actuator"); + EndpointAddress fooEndpointAddress = new EndpointAddress(); + fooEndpointAddress.setIp("127.0.0.1"); + fooEndpointAddress.setHostname("localhost"); + EndpointPort fooEndpointPort = new EndpointPort(); + fooEndpointPort.setPort(wireMockRule.port()); + List instances = new ArrayList<>(); + KubernetesServiceInstance fooServiceInstance = new KubernetesServiceInstance("foo", "foo", + fooEndpointAddress.getIp(), fooEndpointPort.getPort(), metadata, false); + instances.add(fooServiceInstance); + when(reactiveDiscoveryClient.getInstances(eq("foo"))).thenReturn(Flux.fromIterable(instances)); + Secret secret = new Secret(); + ObjectMeta objectMeta = new ObjectMeta(); + objectMeta.setName("foo"); + secret.setMetadata(objectMeta); + stubFor(post(WireMock.urlEqualTo("/my/custom/actuator/refresh")).willReturn(aResponse().withStatus(200))); + StepVerifier.create(changeDetector.triggerRefresh(secret)).verifyComplete(); + verify(postRequestedFor(urlEqualTo("/my/custom/actuator/refresh"))); + } + +}