From 55d08d90e2593bd8dfa0767d463d1641bac5f871 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Fri, 20 Feb 2026 11:31:30 +0000 Subject: [PATCH 01/21] feat: add kafak metrics to apoc cdc handler --- .../kafka/configuration/Neo4jConfiguration.kt | 10 ++++ .../connectors/kafka/metrics/JmxMetrics.kt | 32 ++++++++++ .../connectors/kafka/metrics/KafkaMetrics.kt | 41 +++++++++++++ .../kafka/metrics/MetricsFactory.kt | 59 +++++++++++++++++++ packaging/pom.xml | 5 ++ .../src/test/kotlin/ConfigPropertiesTest.kt | 42 ++++++++----- pom.xml | 2 +- .../connectors/kafka/sink/Neo4jSinkTask.kt | 13 +++- .../kafka/sink/SinkConfiguration.kt | 7 +-- .../connectors/kafka/sink/SinkStrategy.kt | 26 ++++++-- .../sink/strategy/cdc/apoc/ApocCdcHandler.kt | 29 +++++++++ .../strategy/cdc/apoc/ApocCdcSchemaHandler.kt | 4 +- .../cdc/apoc/ApocCdcSourceIdHandler.kt | 4 +- .../kafka/sink/SinkConfigurationTest.kt | 59 +++++++++++-------- .../cdc/apoc/ApocCdcSchemaHandlerTaskIT.kt | 6 +- .../cdc/apoc/ApocCdcSchemaHandlerTest.kt | 25 ++++---- .../cdc/apoc/ApocCdcSourceIdHandlerTaskIT.kt | 6 +- .../cdc/apoc/ApocCdcSourceIdHandlerTest.kt | 5 ++ .../batch/BatchedCdcSchemaHandlerTaskIT.kt | 6 +- .../batch/BatchedCdcSourceIdHandlerTaskIT.kt | 6 +- 20 files changed, 317 insertions(+), 70 deletions(-) create mode 100644 common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt create mode 100644 common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt create mode 100644 common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt index 258f0c53..0c71e1ad 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt @@ -239,6 +239,12 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty } } + val connectorName + get(): String = getString(CONNECTOR_NAME) + + val taskId + get(): Int = getInt(TASK_ID) + companion object { val DEFAULT_MAX_RETRY_DURATION = 30.seconds @@ -271,6 +277,10 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty const val SECURITY_TRUST_STRATEGY = "neo4j.security.trust-strategy" const val SECURITY_CERT_FILES = "neo4j.security.cert-files" + // internal properties + const val CONNECTOR_NAME = "name" + const val TASK_ID = "neo4j.task.id" + /** Perform validation on dependent configuration items */ fun validate(config: org.apache.kafka.common.config.Config) { // authentication configuration diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt new file mode 100644 index 00000000..3b4dfa6a --- /dev/null +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt @@ -0,0 +1,32 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.neo4j.connectors.kafka.metrics + +import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration + +class JmxMetrics(private val config: Neo4jConfiguration) : Metrics { + + private val connectorName: String = config.connectorName + private val taskId: Int = config.taskId + + override fun addGauge( + name: String, + description: String, + tags: LinkedHashMap, + valueProvider: () -> T?, + ) {} +} diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt new file mode 100644 index 00000000..7936c075 --- /dev/null +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt @@ -0,0 +1,41 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.neo4j.connectors.kafka.metrics + +import org.apache.kafka.common.metrics.Gauge +import org.apache.kafka.common.metrics.MetricConfig +import org.apache.kafka.common.metrics.PluginMetrics + +class KafkaMetrics(private val pluginMetrics: PluginMetrics) : Metrics { + + override fun addGauge( + name: String, + description: String, + tags: LinkedHashMap, + valueProvider: () -> T?, + ) { + val metricName = pluginMetrics.metricName(name, description, tags) + pluginMetrics.addMetric( + metricName, + object : Gauge { + override fun value(config: MetricConfig?, now: Long): T? { + return valueProvider() + } + }, + ) + } +} diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt new file mode 100644 index 00000000..3ead127f --- /dev/null +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt @@ -0,0 +1,59 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.neo4j.connectors.kafka.metrics + +import org.apache.kafka.connect.sink.SinkTaskContext +import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class MetricsFactory { + + fun createMetrics(context: SinkTaskContext, config: Neo4jConfiguration): Metrics { + return createKafkaMetrics(context) ?: createJmxMetrics(config) + } + + private fun createKafkaMetrics(context: SinkTaskContext): KafkaMetrics? { + return try { + val metrics = KafkaMetrics(context.pluginMetrics()) + log.info("Plugin metrics support detected") + metrics + } catch (_: NoSuchMethodError) { + null + } catch (_: NoClassDefFoundError) { + null + } + } + + private fun createJmxMetrics(config: Neo4jConfiguration): JmxMetrics { + log.error("TTT No plugin metrics support detected. Using JMX only metrics") + return JmxMetrics(config) + } + + companion object { + private val log: Logger = LoggerFactory.getLogger(MetricsFactory::class.java) + } +} + +interface Metrics { + fun addGauge( + name: String, + description: String, + tags: LinkedHashMap, + valueProvider: () -> T?, + ) +} diff --git a/packaging/pom.xml b/packaging/pom.xml index 37467665..cecf8697 100644 --- a/packaging/pom.xml +++ b/packaging/pom.xml @@ -52,6 +52,11 @@ junit-jupiter test + + org.mockito.kotlin + mockito-kotlin + test + diff --git a/packaging/src/test/kotlin/ConfigPropertiesTest.kt b/packaging/src/test/kotlin/ConfigPropertiesTest.kt index 7b995e4d..3220397d 100644 --- a/packaging/src/test/kotlin/ConfigPropertiesTest.kt +++ b/packaging/src/test/kotlin/ConfigPropertiesTest.kt @@ -26,12 +26,14 @@ import org.junit.jupiter.api.Test import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.MethodSource +import org.mockito.Mockito.mock import org.neo4j.caniuse.Neo4j import org.neo4j.caniuse.Neo4jDeploymentType import org.neo4j.caniuse.Neo4jEdition import org.neo4j.caniuse.Neo4jVersion import org.neo4j.cdc.client.selector.NodeSelector import org.neo4j.cdc.client.selector.RelationshipSelector +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.SinkConfiguration import org.neo4j.connectors.kafka.sink.SinkStrategyHandler import org.neo4j.connectors.kafka.sink.strategy.CdcSchemaHandler @@ -48,6 +50,8 @@ import org.neo4j.cypherdsl.core.renderer.Renderer class ConfigPropertiesTest { + private val metricsMock: Metrics = mock() + // This test checks that the number of config files is fixed. // If a new file is added, the test will fail, reminding the developer to update this test and add // unit tests for the new file. @@ -72,10 +76,11 @@ class ConfigPropertiesTest { SinkConfiguration(properties, Renderer.getDefaultRenderer(), neo4j, apocDoITAvailable) } - config.topicHandlers.keys shouldBe setOf("creates", "updates", "deletes") - config.topicHandlers["creates"] shouldBe instanceOf(expectedHandlerType) - config.topicHandlers["updates"] shouldBe instanceOf(expectedHandlerType) - config.topicHandlers["deletes"] shouldBe instanceOf(expectedHandlerType) + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers.keys shouldBe setOf("creates", "updates", "deletes") + topicHandlers["creates"] shouldBe instanceOf(expectedHandlerType) + topicHandlers["updates"] shouldBe instanceOf(expectedHandlerType) + topicHandlers["deletes"] shouldBe instanceOf(expectedHandlerType) } @ParameterizedTest @@ -93,10 +98,11 @@ class ConfigPropertiesTest { SinkConfiguration(properties, Renderer.getDefaultRenderer(), neo4j, apocDoITAvailable) } - config.topicHandlers.keys shouldBe setOf("creates", "updates", "deletes") - config.topicHandlers["creates"] shouldBe instanceOf(expectedHandlerType) - config.topicHandlers["updates"] shouldBe instanceOf(expectedHandlerType) - config.topicHandlers["deletes"] shouldBe instanceOf(expectedHandlerType) + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers.keys shouldBe setOf("creates", "updates", "deletes") + topicHandlers["creates"] shouldBe instanceOf(expectedHandlerType) + topicHandlers["updates"] shouldBe instanceOf(expectedHandlerType) + topicHandlers["deletes"] shouldBe instanceOf(expectedHandlerType) } @Test @@ -107,8 +113,9 @@ class ConfigPropertiesTest { val config = shouldNotThrowAny { SinkConfiguration(properties, Renderer.getDefaultRenderer()) } - config.topicHandlers.keys shouldBe setOf("people") - config.topicHandlers["people"].shouldBeInstanceOf() + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers.keys shouldBe setOf("people") + topicHandlers["people"].shouldBeInstanceOf() } @Test @@ -119,8 +126,9 @@ class ConfigPropertiesTest { val config = shouldNotThrowAny { SinkConfiguration(properties, Renderer.getDefaultRenderer()) } - config.topicHandlers.keys shouldBe setOf("people") - config.topicHandlers["people"].shouldBeInstanceOf() + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers.keys shouldBe setOf("people") + topicHandlers["people"].shouldBeInstanceOf() } @Test @@ -131,8 +139,9 @@ class ConfigPropertiesTest { val config = shouldNotThrowAny { SinkConfiguration(properties, Renderer.getDefaultRenderer()) } - config.topicHandlers.keys shouldBe setOf("people") - config.topicHandlers["people"].shouldBeInstanceOf() + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers.keys shouldBe setOf("people") + topicHandlers["people"].shouldBeInstanceOf() } @Test @@ -143,8 +152,9 @@ class ConfigPropertiesTest { val config = shouldNotThrowAny { SinkConfiguration(properties, Renderer.getDefaultRenderer()) } - config.topicHandlers.keys shouldBe setOf("knows") - config.topicHandlers["knows"].shouldBeInstanceOf() + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers.keys shouldBe setOf("knows") + topicHandlers["knows"].shouldBeInstanceOf() } @Test diff --git a/pom.xml b/pom.xml index afcfb84e..e70b1e98 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ - 3.8.1 + 4.1.1 6.1.3 1.10.2 2.3.10 diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt index d385b50b..99336626 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt @@ -22,14 +22,18 @@ import org.apache.kafka.connect.sink.SinkRecord import org.apache.kafka.connect.sink.SinkTask import org.jetbrains.annotations.VisibleForTesting import org.neo4j.connectors.kafka.configuration.helpers.VersionUtil +import org.neo4j.connectors.kafka.metrics.Metrics +import org.neo4j.connectors.kafka.metrics.MetricsFactory import org.slf4j.Logger import org.slf4j.LoggerFactory -class Neo4jSinkTask : SinkTask() { +class Neo4jSinkTask(private val metricsFactory: MetricsFactory = MetricsFactory()) : SinkTask() { private val log: Logger = LoggerFactory.getLogger(Neo4jSinkTask::class.java) private lateinit var settings: Map @VisibleForTesting lateinit var config: SinkConfiguration + private lateinit var metrics: Metrics + private lateinit var topicHandlers: Map override fun version(): String = VersionUtil.version(Neo4jSinkTask::class.java) @@ -38,6 +42,10 @@ class Neo4jSinkTask : SinkTask() { settings = props!! config = SinkConfiguration(settings) + + metrics = metricsFactory.createMetrics(context, config) + topicHandlers = SinkStrategyHandler.createFrom(config, metrics) + log.error("TTT handlers: {}", topicHandlers.mapValues { it.value.javaClass.name }) } override fun stop() { @@ -53,7 +61,7 @@ class Neo4jSinkTask : SinkTask() { records ?.map { SinkMessage(it) } ?.groupBy { it.topic } - ?.mapKeys { config.topicHandlers.getValue(it.key) } + ?.mapKeys { topicHandlers.getValue(it.key) } ?.forEach { (handler, messages) -> processMessages(handler, messages) } } log.info("processed {} records in {} ms", records?.size ?: 0, duration.inWholeMilliseconds) @@ -83,6 +91,7 @@ class Neo4jSinkTask : SinkTask() { ) log.trace("after write transaction for group {}", index) } + handler.postProcessLastMessageBatch(group) handled.addAll(group.flatMap { it.messages }) } diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt index da58cfe9..b21c95b3 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt @@ -51,7 +51,6 @@ class SinkConfiguration : Neo4jConfiguration { fixedRenderer = renderer _neo4j = neo4j this.apocCypherDoItAvailable = apocCypherDoItAvailable - validateAllTopics() } val batchSize @@ -125,14 +124,10 @@ class SinkConfiguration : Neo4jConfiguration { originalsStrings()[SinkTask.TOPICS_CONFIG]?.split(',')?.map { it.trim() }?.toList() ?: emptyList() - val topicHandlers: Map by lazy { - SinkStrategyHandler.createFrom(this) - } - override fun userAgentComment(): String = SinkStrategyHandler.configuredStrategies(this).sorted().joinToString("; ") - private fun validateAllTopics() { + fun validateAllTopics(topicHandlers: Map) { // todo what is this for val sourceTopics = topicNames.toSet() val configuredTopics = topicHandlers.keys diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkStrategy.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkStrategy.kt index 171068be..fd1c51f5 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkStrategy.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkStrategy.kt @@ -28,6 +28,7 @@ import org.neo4j.connectors.kafka.data.cdcTxId import org.neo4j.connectors.kafka.data.cdcTxSeq import org.neo4j.connectors.kafka.data.fetchConstraintData import org.neo4j.connectors.kafka.data.isCdcMessage +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.strategy.CdcSchemaHandler import org.neo4j.connectors.kafka.sink.strategy.CdcSourceIdHandler import org.neo4j.connectors.kafka.sink.strategy.CudHandler @@ -41,6 +42,7 @@ import org.neo4j.connectors.kafka.sink.strategy.pattern.Pattern import org.neo4j.connectors.kafka.sink.strategy.pattern.RelationshipPattern import org.neo4j.connectors.kafka.utils.JSONUtils import org.neo4j.driver.Query +import org.slf4j.LoggerFactory data class SinkMessage(val record: SinkRecord) { val topic @@ -134,13 +136,21 @@ interface SinkStrategyHandler { */ fun handle(messages: Iterable): Iterable> + fun postProcessLastMessageBatch(batch: Iterable) {} + companion object { - fun createFrom(config: SinkConfiguration): Map { - return config.topicNames.associateWith { topic -> createForTopic(topic, config) } + private val logger = LoggerFactory.getLogger(SinkStrategyHandler::class.java) + + fun createFrom(config: SinkConfiguration, metrics: Metrics): Map { + return config.topicNames.associateWith { topic -> createForTopic(topic, config, metrics) } } - private fun createForTopic(topic: String, config: SinkConfiguration): SinkStrategyHandler { + private fun createForTopic( + topic: String, + config: SinkConfiguration, + metrics: Metrics, + ): SinkStrategyHandler { var handler: SinkStrategyHandler? = null val originals = config.originalsStrings() @@ -226,6 +236,7 @@ interface SinkStrategyHandler { config.neo4j(), config.batchSize, config.eosOffsetLabel, + metrics, labelName, propertyName, ) @@ -244,9 +255,16 @@ interface SinkStrategyHandler { canIUse(Cypher.setDynamicLabels()).withNeo4j(config.neo4j()) && canIUse(Cypher.removeDynamicLabels()).withNeo4j(config.neo4j()) ) - ApocCdcSchemaHandler(topic, config.neo4j(), config.batchSize, config.eosOffsetLabel) + ApocCdcSchemaHandler( + topic, + config.neo4j(), + config.batchSize, + config.eosOffsetLabel, + metrics, + ) else CdcSchemaHandler(topic, config.renderer) } + logger val cudTopics = config.getList(SinkConfiguration.CUD_TOPICS) if (cudTopics.contains(topic)) { diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcHandler.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcHandler.kt index a3c42a28..18d94b11 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcHandler.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcHandler.kt @@ -28,6 +28,7 @@ import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.cdc.client.model.EntityOperation import org.neo4j.cdc.client.model.NodeEvent import org.neo4j.cdc.client.model.RelationshipEvent +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.ChangeQuery import org.neo4j.connectors.kafka.sink.SinkMessage import org.neo4j.connectors.kafka.sink.SinkStrategyHandler @@ -40,15 +41,36 @@ abstract class ApocCdcHandler( private val neo4j: Neo4j, private val batchSize: Int, private val eosOffsetLabel: String, + private val metrics: Metrics, ) : SinkStrategyHandler { private val logger: Logger = LoggerFactory.getLogger(javaClass) + private var lastTxCommitTs: Long? = null + private var lastTxStartTs: Long? = null + data class MessageToEvent( val message: SinkMessage, val changeEvent: ChangeEvent, val cdcData: CdcData, ) + init { + metrics.addGauge( + "last_tx_commit_timestamp", + "The transaction commit timestamp of the last written CDC message", + linkedMapOf(), + ) { + lastTxCommitTs + } + metrics.addGauge( + "last_tx_start_timestamp", + "The transaction start timestamp of the last written CDC message", + linkedMapOf(), + ) { + lastTxStartTs + } + } + override fun handle(messages: Iterable): Iterable> { val (topic, partition) = messages.firstOrNull()?.let { it.record.topic() to it.record.kafkaPartition() } @@ -164,4 +186,11 @@ abstract class ApocCdcHandler( protected abstract fun transformUpdate(event: RelationshipEvent): CdcData protected abstract fun transformDelete(event: RelationshipEvent): CdcData + + override fun postProcessLastMessageBatch(batch: Iterable) { + batch.lastOrNull()?.messages?.lastOrNull()?.toChangeEvent()?.metadata?.let { + lastTxCommitTs = it.txCommitTime.toEpochSecond() + lastTxStartTs = it.txStartTime.toEpochSecond() + } + } } diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandler.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandler.kt index b615c977..ce4b7978 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandler.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandler.kt @@ -21,6 +21,7 @@ import org.neo4j.cdc.client.model.EntityOperation import org.neo4j.cdc.client.model.NodeEvent import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.connectors.kafka.exceptions.InvalidDataException +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.SinkStrategy import org.neo4j.connectors.kafka.sink.strategy.addedLabels import org.neo4j.connectors.kafka.sink.strategy.mutatedProperties @@ -33,7 +34,8 @@ class ApocCdcSchemaHandler( neo4j: Neo4j, batchSize: Int, eosOffsetLabel: String = "", -) : ApocCdcHandler(neo4j, batchSize, eosOffsetLabel) { + metrics: Metrics, +) : ApocCdcHandler(neo4j, batchSize, eosOffsetLabel, metrics) { private val logger: Logger = LoggerFactory.getLogger(javaClass) init { diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandler.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandler.kt index 991c5509..aeb106ff 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandler.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandler.kt @@ -21,6 +21,7 @@ import org.neo4j.cdc.client.model.EntityOperation import org.neo4j.cdc.client.model.NodeEvent import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.connectors.kafka.exceptions.InvalidDataException +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.SinkConfiguration import org.neo4j.connectors.kafka.sink.SinkStrategy import org.neo4j.connectors.kafka.sink.strategy.addedLabels @@ -34,9 +35,10 @@ class ApocCdcSourceIdHandler( neo4j: Neo4j, batchSize: Int, eosOffsetLabel: String = "", + metrics: Metrics, val labelName: String = SinkConfiguration.DEFAULT_SOURCE_ID_LABEL_NAME, val propertyName: String = SinkConfiguration.DEFAULT_SOURCE_ID_PROPERTY_NAME, -) : ApocCdcHandler(neo4j, batchSize, eosOffsetLabel) { +) : ApocCdcHandler(neo4j, batchSize, eosOffsetLabel, metrics) { private val logger: Logger = LoggerFactory.getLogger(javaClass) init { diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt index 94593500..5477a13e 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt @@ -29,11 +29,13 @@ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.EnumSource import org.junit.jupiter.params.provider.MethodSource +import org.mockito.kotlin.mock import org.neo4j.caniuse.Neo4j import org.neo4j.caniuse.Neo4jDeploymentType import org.neo4j.caniuse.Neo4jEdition import org.neo4j.caniuse.Neo4jVersion import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.strategy.CdcHandler import org.neo4j.connectors.kafka.sink.strategy.CdcSourceIdHandler import org.neo4j.connectors.kafka.sink.strategy.CudHandler @@ -47,6 +49,8 @@ import org.neo4j.driver.TransactionConfig class SinkConfigurationTest { + private val metricsMock: Metrics = mock() + @Test fun `should throw a ConfigException because of mismatch`() { shouldThrow { @@ -96,9 +100,10 @@ class SinkConfigurationTest { val config = SinkConfiguration(originals, Renderer.getDefaultRenderer()) config.batchSize shouldBe 10 - config.topicHandlers shouldHaveKey "foo" - config.topicHandlers["foo"] shouldBe instanceOf() - (config.topicHandlers["foo"] as CypherHandler).query shouldBe + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers shouldHaveKey "foo" + topicHandlers["foo"] shouldBe instanceOf() + (topicHandlers["foo"] as CypherHandler).query shouldBe "CREATE (p:Person{name: event.firstName})" } @@ -116,9 +121,10 @@ class SinkConfigurationTest { val config = SinkConfiguration(originals, Renderer.getDefaultRenderer()) config.batchSize shouldBe 10 - config.topicHandlers shouldHaveKey "foo" - config.topicHandlers["foo"] shouldBe instanceOf() - (config.topicHandlers["foo"] as NodePatternHandler).pattern shouldBe + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers shouldHaveKey "foo" + topicHandlers["foo"] shouldBe instanceOf() + (topicHandlers["foo"] as NodePatternHandler).pattern shouldBe NodePattern( setOf("Foo"), false, @@ -127,9 +133,9 @@ class SinkConfigurationTest { emptySet(), ) - config.topicHandlers shouldHaveKey "bar" - config.topicHandlers["bar"] shouldBe instanceOf() - (config.topicHandlers["bar"] as NodePatternHandler).pattern shouldBe + topicHandlers shouldHaveKey "bar" + topicHandlers["bar"] shouldBe instanceOf() + (topicHandlers["bar"] as NodePatternHandler).pattern shouldBe NodePattern( setOf("Bar"), false, @@ -186,15 +192,16 @@ class SinkConfigurationTest { val config = SinkConfiguration(originals, Renderer.getDefaultRenderer(), apocCypherDoItAvailable = false) - config.topicHandlers shouldHaveKey "foo" - config.topicHandlers["foo"] shouldBe instanceOf() - (config.topicHandlers["foo"] as CdcSourceIdHandler).labelName shouldBe "TestCdcLabel" - (config.topicHandlers["foo"] as CdcSourceIdHandler).propertyName shouldBe "test_id" + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers shouldHaveKey "foo" + topicHandlers["foo"] shouldBe instanceOf() + (topicHandlers["foo"] as CdcSourceIdHandler).labelName shouldBe "TestCdcLabel" + (topicHandlers["foo"] as CdcSourceIdHandler).propertyName shouldBe "test_id" - config.topicHandlers shouldHaveKey "bar" - config.topicHandlers["bar"] shouldBe instanceOf() - (config.topicHandlers["bar"] as CdcSourceIdHandler).labelName shouldBe "TestCdcLabel" - (config.topicHandlers["bar"] as CdcSourceIdHandler).propertyName shouldBe "test_id" + topicHandlers shouldHaveKey "bar" + topicHandlers["bar"] shouldBe instanceOf() + (topicHandlers["bar"] as CdcSourceIdHandler).labelName shouldBe "TestCdcLabel" + (topicHandlers["bar"] as CdcSourceIdHandler).propertyName shouldBe "test_id" } @ParameterizedTest @@ -219,11 +226,12 @@ class SinkConfigurationTest { apocCypherDoItAvailable = apocDoItAvailable, ) - config.topicHandlers shouldHaveKey "foo" - config.topicHandlers["foo"] shouldBe instanceOf(clazz) + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers shouldHaveKey "foo" + topicHandlers["foo"] shouldBe instanceOf(clazz) - config.topicHandlers shouldHaveKey "bar" - config.topicHandlers["bar"] shouldBe instanceOf(clazz) + topicHandlers shouldHaveKey "bar" + topicHandlers["bar"] shouldBe instanceOf(clazz) } @Test @@ -237,11 +245,12 @@ class SinkConfigurationTest { ) val config = SinkConfiguration(originals, Renderer.getDefaultRenderer()) - config.topicHandlers shouldHaveKey "foo" - config.topicHandlers["foo"] shouldBe instanceOf() + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers shouldHaveKey "foo" + topicHandlers["foo"] shouldBe instanceOf() - config.topicHandlers shouldHaveKey "bar" - config.topicHandlers["bar"] shouldBe instanceOf() + topicHandlers shouldHaveKey "bar" + topicHandlers["bar"] shouldBe instanceOf() } @ParameterizedTest diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandlerTaskIT.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandlerTaskIT.kt index 39401b36..61c03303 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandlerTaskIT.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandlerTaskIT.kt @@ -40,8 +40,10 @@ import org.neo4j.cdc.client.model.NodeEvent import org.neo4j.cdc.client.model.NodeState import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.cdc.client.model.RelationshipState +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.Neo4jSinkTask import org.neo4j.connectors.kafka.sink.SinkStrategy.CDC_SCHEMA +import org.neo4j.connectors.kafka.sink.SinkStrategyHandler import org.neo4j.connectors.kafka.sink.strategy.TestUtils.newChangeEventMessage import org.neo4j.connectors.kafka.sink.strategy.TestUtils.verifyEosOffsetIfEnabled import org.neo4j.connectors.kafka.testing.DatabaseSupport.createDatabase @@ -129,7 +131,9 @@ abstract class ApocCdcSchemaHandlerTaskIT(val eosOffsetLabel: String) { } ) - task.config.topicHandlers["my-topic"] shouldBe instanceOf(ApocCdcSchemaHandler::class) + val metricsMock: Metrics = mock() + SinkStrategyHandler.createFrom(task.config, metricsMock)["my-topic"] shouldBe + instanceOf(ApocCdcSchemaHandler::class) } @Test diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandlerTest.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandlerTest.kt index 0633aace..0c48f2c8 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandlerTest.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSchemaHandlerTest.kt @@ -25,6 +25,7 @@ import java.time.LocalDate import kotlin.collections.emptyList import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.mockito.Mockito.mock import org.neo4j.caniuse.Neo4j import org.neo4j.caniuse.Neo4jDeploymentType import org.neo4j.caniuse.Neo4jEdition @@ -38,6 +39,7 @@ import org.neo4j.cdc.client.model.RelationshipState import org.neo4j.connectors.kafka.data.StreamsTransactionEventExtensions.toChangeEvent import org.neo4j.connectors.kafka.events.StreamsTransactionEvent import org.neo4j.connectors.kafka.exceptions.InvalidDataException +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.ChangeQuery import org.neo4j.connectors.kafka.sink.SinkMessage import org.neo4j.connectors.kafka.sink.strategy.TestUtils.newChangeEventMessage @@ -80,6 +82,8 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected private val neo4j = Neo4j(Neo4jVersion(2025, 12, 1), Neo4jEdition.ENTERPRISE, Neo4jDeploymentType.SELF_MANAGED) + private val metricsMock: Metrics = mock() + @Test fun `should fail on empty keys`() { listOf( @@ -152,7 +156,8 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected ) .forEach { shouldThrow { - val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel) + val handler = + ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel, metricsMock) handler.handle(listOf(it)) } @@ -1052,7 +1057,7 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected @Test fun `should split changes over batch size`() { - val handler = ApocCdcSchemaHandler("my-topic", neo4j, 2, eosOffsetLabel) + val handler = ApocCdcSchemaHandler("my-topic", neo4j, 2, eosOffsetLabel, metricsMock) val result = handler.handle( @@ -1092,7 +1097,7 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected @Test fun `should fail on null 'after' field with node create operation`() { - val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel) + val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel, metricsMock) val nodeChangeEventMessage = newChangeEventMessage( @@ -1116,7 +1121,7 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected @Test fun `should fail on null 'after' field with relationship create operation`() { - val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel) + val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel, metricsMock) val relationshipChangeEventMessage = newChangeEventMessage( @@ -1150,7 +1155,7 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected @Test fun `should fail on null 'before' field with node update operation`() { - val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel) + val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel, metricsMock) val nodeChangeEventMessage = newChangeEventMessage( @@ -1174,7 +1179,7 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected @Test fun `should fail on null 'before' field with relationship update operation`() { - val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel) + val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel, metricsMock) val relationshipChangeEventMessage = newChangeEventMessage( @@ -1208,7 +1213,7 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected @Test fun `should fail on null 'after' field with node update operation`() { - val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel) + val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel, metricsMock) val nodeChangeEventMessage = newChangeEventMessage( @@ -1232,7 +1237,7 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected @Test fun `should fail on null 'after' field with relationship update operation`() { - val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel) + val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel, metricsMock) val relationshipChangeEventMessage = newChangeEventMessage( @@ -2433,7 +2438,7 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected private fun assertInvalidDataException(sinkMessage: SinkMessage) { shouldThrow { - val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel) + val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel, metricsMock) handler.handle(listOf(sinkMessage)) } @@ -2446,7 +2451,7 @@ abstract class ApocCdcSchemaHandlerTest(val eosOffsetLabel: String, val expected } private fun verify(messages: Iterable, expected: Iterable>) { - val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel) + val handler = ApocCdcSchemaHandler("my-topic", neo4j, 1000, eosOffsetLabel, metricsMock) val result = handler.handle(messages) diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandlerTaskIT.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandlerTaskIT.kt index b8b12e43..dd5fd963 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandlerTaskIT.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandlerTaskIT.kt @@ -40,8 +40,10 @@ import org.neo4j.cdc.client.model.NodeEvent import org.neo4j.cdc.client.model.NodeState import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.cdc.client.model.RelationshipState +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.Neo4jSinkTask import org.neo4j.connectors.kafka.sink.SinkStrategy.CDC_SOURCE_ID +import org.neo4j.connectors.kafka.sink.SinkStrategyHandler import org.neo4j.connectors.kafka.sink.strategy.TestUtils.newChangeEventMessage import org.neo4j.connectors.kafka.sink.strategy.TestUtils.verifyEosOffsetIfEnabled import org.neo4j.connectors.kafka.testing.DatabaseSupport.createDatabase @@ -131,7 +133,9 @@ abstract class ApocCdcSourceIdHandlerTaskIT(val eosOffsetLabel: String) { } ) - task.config.topicHandlers["my-topic"] shouldBe instanceOf(ApocCdcSourceIdHandler::class) + val metricsMock: Metrics = mock() + SinkStrategyHandler.createFrom(task.config, metricsMock)["my-topic"] shouldBe + instanceOf(ApocCdcSourceIdHandler::class) } @Test diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandlerTest.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandlerTest.kt index b8678532..bdddb40b 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandlerTest.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/apoc/ApocCdcSourceIdHandlerTest.kt @@ -23,6 +23,7 @@ import io.kotest.matchers.throwable.shouldHaveMessage import java.time.LocalDate import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertThrows +import org.mockito.kotlin.mock import org.neo4j.caniuse.Neo4j import org.neo4j.caniuse.Neo4jDeploymentType import org.neo4j.caniuse.Neo4jEdition @@ -34,6 +35,7 @@ import org.neo4j.cdc.client.model.NodeState import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.cdc.client.model.RelationshipState import org.neo4j.connectors.kafka.exceptions.InvalidDataException +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.ChangeQuery import org.neo4j.connectors.kafka.sink.SinkMessage import org.neo4j.connectors.kafka.sink.strategy.TestUtils.newChangeEventMessage @@ -75,6 +77,8 @@ abstract class ApocCdcSourceIdHandlerTest(val eosOffsetLabel: String, val expect private val neo4j = Neo4j(Neo4jVersion(2025, 12, 1), Neo4jEdition.ENTERPRISE, Neo4jDeploymentType.SELF_MANAGED) + private val metricsMock: Metrics = mock() + @Test fun `should generate correct statement for node creation events`() { val sinkMessage = @@ -1048,6 +1052,7 @@ abstract class ApocCdcSourceIdHandlerTest(val eosOffsetLabel: String, val expect neo4j, batchSize, eosOffsetLabel, + metricsMock, "SourceEvent", "sourceElementId", ) diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/batch/BatchedCdcSchemaHandlerTaskIT.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/batch/BatchedCdcSchemaHandlerTaskIT.kt index c8b346cc..3eb7ef8c 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/batch/BatchedCdcSchemaHandlerTaskIT.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/batch/BatchedCdcSchemaHandlerTaskIT.kt @@ -38,7 +38,9 @@ import org.neo4j.cdc.client.model.NodeEvent import org.neo4j.cdc.client.model.NodeState import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.cdc.client.model.RelationshipState +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.Neo4jSinkTask +import org.neo4j.connectors.kafka.sink.SinkStrategyHandler import org.neo4j.connectors.kafka.sink.strategy.TestUtils.newChangeEventMessage import org.neo4j.connectors.kafka.testing.DatabaseSupport.createDatabase import org.neo4j.connectors.kafka.testing.DatabaseSupport.dropDatabase @@ -111,7 +113,9 @@ class BatchedCdcSchemaHandlerTaskIT { ) ) - task.config.topicHandlers["my-topic"] shouldBe instanceOf(BatchedCdcSchemaHandler::class) + val metricsMock: Metrics = mock() + SinkStrategyHandler.createFrom(task.config, metricsMock)["my-topic"] shouldBe + instanceOf(BatchedCdcSchemaHandler::class) } @Test diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/batch/BatchedCdcSourceIdHandlerTaskIT.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/batch/BatchedCdcSourceIdHandlerTaskIT.kt index 0aa0dcd0..3e3778b7 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/batch/BatchedCdcSourceIdHandlerTaskIT.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/batch/BatchedCdcSourceIdHandlerTaskIT.kt @@ -38,7 +38,9 @@ import org.neo4j.cdc.client.model.NodeEvent import org.neo4j.cdc.client.model.NodeState import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.cdc.client.model.RelationshipState +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.Neo4jSinkTask +import org.neo4j.connectors.kafka.sink.SinkStrategyHandler import org.neo4j.connectors.kafka.sink.strategy.TestUtils.newChangeEventMessage import org.neo4j.connectors.kafka.testing.DatabaseSupport.createDatabase import org.neo4j.connectors.kafka.testing.DatabaseSupport.dropDatabase @@ -114,7 +116,9 @@ class BatchedCdcSourceIdHandlerTaskIT { ) ) - task.config.topicHandlers["my-topic"] shouldBe instanceOf(BatchedCdcSourceIdHandler::class) + val metricsMock: Metrics = mock() + SinkStrategyHandler.createFrom(task.config, metricsMock)["my-topic"] shouldBe + instanceOf(BatchedCdcSourceIdHandler::class) } @Test From d2d7b7ab3da3daa917834071fab0688bed899116 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Mon, 23 Feb 2026 14:09:01 +0000 Subject: [PATCH 02/21] feat: add metrics data class --- .../kafka/sink/strategy/cdc/CdcMetricsData.kt | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt new file mode 100644 index 00000000..45cd7e6d --- /dev/null +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt @@ -0,0 +1,36 @@ +package org.neo4j.connectors.kafka.sink.strategy.cdc + +import org.neo4j.connectors.kafka.metrics.Metrics +import org.neo4j.cdc.client.model.Metadata as CdcMetadata + +class CdcMetricsData( + metrics: Metrics, + tags: LinkedHashMap = linkedMapOf(), +) { + + private var lastTxCommitTs: Long? = null + private var lastTxStartTs: Long? = null + + init { + metrics.addGauge( + "last_cdc_tx_commit_timestamp", + "The transaction commit timestamp of the last written CDC message", + tags, + ) { + lastTxCommitTs + } + metrics.addGauge( + "last_cdc_tx_start_timestamp", + "The transaction start timestamp of the last written CDC message", + tags, + ) { + lastTxStartTs + } + } + + fun applyMetadata(metadata: CdcMetadata) { + lastTxCommitTs = metadata.txCommitTime.toEpochSecond() + lastTxStartTs = metadata.txStartTime.toEpochSecond() + } + +} From 9b98c9cefad82ed8ebb8ddc95dfa5e1425ef53f0 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Mon, 23 Feb 2026 14:25:08 +0000 Subject: [PATCH 03/21] feat: update cdc sink handler --- .../connectors/kafka/sink/SinkStrategy.kt | 21 ++++++-- .../kafka/sink/strategy/cdc/CdcHandler.kt | 8 +++ .../kafka/sink/strategy/cdc/CdcMetricsData.kt | 37 +++++++++---- .../kafka/sink/SinkConfigurationTest.kt | 53 +++++++++++-------- .../sink/strategy/cdc/CdcSchemaHandlerIT.kt | 5 +- .../sink/strategy/cdc/CdcSourceIdHandlerIT.kt | 5 +- 6 files changed, 93 insertions(+), 36 deletions(-) diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkStrategy.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkStrategy.kt index 57f70bad..0b61a095 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkStrategy.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkStrategy.kt @@ -26,6 +26,7 @@ import org.neo4j.connectors.kafka.data.cdcTxId import org.neo4j.connectors.kafka.data.cdcTxSeq import org.neo4j.connectors.kafka.data.fetchConstraintData import org.neo4j.connectors.kafka.data.isCdcMessage +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.strategy.CudHandler import org.neo4j.connectors.kafka.sink.strategy.CypherHandler import org.neo4j.connectors.kafka.sink.strategy.NodePatternHandler @@ -133,13 +134,19 @@ interface SinkStrategyHandler { */ fun handle(messages: Iterable): Iterable> + fun postProcessLastMessageBatch(group: Iterable) {} + companion object { - fun createFrom(config: SinkConfiguration): Map { - return config.topicNames.associateWith { topic -> createForTopic(topic, config) } + fun createFrom(config: SinkConfiguration, metrics: Metrics): Map { + return config.topicNames.associateWith { topic -> createForTopic(topic, config, metrics) } } - private fun createForTopic(topic: String, config: SinkConfiguration): SinkStrategyHandler { + private fun createForTopic( + topic: String, + config: SinkConfiguration, + metrics: Metrics, + ): SinkStrategyHandler { var handler: SinkStrategyHandler? = null val originals = config.originalsStrings() @@ -237,6 +244,7 @@ interface SinkStrategyHandler { SinkStrategy.CDC_SOURCE_ID, batchStrategy, CdcSourceIdEventTransformer(topic, labelName, propertyName), + metrics, ) } @@ -265,7 +273,12 @@ interface SinkStrategyHandler { } handler = - CdcHandler(SinkStrategy.CDC_SCHEMA, batchStrategy, CdcSchemaEventTransformer(topic)) + CdcHandler( + SinkStrategy.CDC_SCHEMA, + batchStrategy, + CdcSchemaEventTransformer(topic), + metrics, + ) } val cudTopics = config.getList(SinkConfiguration.CUD_TOPICS) diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt index d8d5301b..57da96b7 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt @@ -20,6 +20,7 @@ import org.apache.kafka.connect.data.Struct import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.connectors.kafka.data.StreamsTransactionEventExtensions.toChangeEvent import org.neo4j.connectors.kafka.data.toChangeEvent +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.ChangeQuery import org.neo4j.connectors.kafka.sink.SinkMessage import org.neo4j.connectors.kafka.sink.SinkStrategy @@ -31,13 +32,20 @@ class CdcHandler( private val strategy: SinkStrategy, internal val batchStrategy: CdcBatchStrategy, internal val eventTransformer: CdcEventTransformer, + metrics: Metrics, ) : SinkStrategyHandler { + private val metricsData = CdcMetricsData(metrics) + override fun strategy(): SinkStrategy = strategy override fun handle(messages: Iterable): Iterable> { return batchStrategy.handle(messages) { eventTransformer.transform(it) } } + + override fun postProcessLastMessageBatch(group: Iterable) { + group.lastOrNull()?.messages?.lastOrNull()?.toChangeEvent() + } } internal fun SinkMessage.toChangeEvent(): ChangeEvent = diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt index 45cd7e6d..41d76aa8 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt @@ -1,15 +1,29 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.neo4j.connectors.kafka.sink.strategy.cdc +import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.connectors.kafka.metrics.Metrics -import org.neo4j.cdc.client.model.Metadata as CdcMetadata -class CdcMetricsData( - metrics: Metrics, - tags: LinkedHashMap = linkedMapOf(), -) { +class CdcMetricsData(metrics: Metrics, tags: LinkedHashMap = linkedMapOf()) { private var lastTxCommitTs: Long? = null private var lastTxStartTs: Long? = null + private var lastTxId: Long? = null init { metrics.addGauge( @@ -26,11 +40,16 @@ class CdcMetricsData( ) { lastTxStartTs } + metrics.addGauge("last_cdc_tx_id", "The transaction id of the last written CDC message", tags) { + lastTxId + } } - fun applyMetadata(metadata: CdcMetadata) { - lastTxCommitTs = metadata.txCommitTime.toEpochSecond() - lastTxStartTs = metadata.txStartTime.toEpochSecond() + fun update(event: ChangeEvent) { + event.metadata?.let { + lastTxCommitTs = it.txCommitTime.toEpochSecond() + lastTxStartTs = it.txStartTime.toEpochSecond() + } + lastTxId = event.txId } - } diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt index 3446cdc3..150f0910 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt @@ -29,11 +29,13 @@ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.EnumSource import org.junit.jupiter.params.provider.MethodSource +import org.mockito.kotlin.mock import org.neo4j.caniuse.Neo4j import org.neo4j.caniuse.Neo4jDeploymentType import org.neo4j.caniuse.Neo4jEdition import org.neo4j.caniuse.Neo4jVersion import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.strategy.CudHandler import org.neo4j.connectors.kafka.sink.strategy.CypherHandler import org.neo4j.connectors.kafka.sink.strategy.NodePatternHandler @@ -45,6 +47,8 @@ import org.neo4j.driver.TransactionConfig class SinkConfigurationTest { + val metricsMock: Metrics = mock() + @Test fun `should throw a ConfigException because of mismatch`() { shouldThrow { @@ -94,9 +98,11 @@ class SinkConfigurationTest { val config = SinkConfiguration(originals, Renderer.getDefaultRenderer()) config.batchSize shouldBe 10 - config.topicHandlers shouldHaveKey "foo" - config.topicHandlers["foo"] shouldBe instanceOf() - (config.topicHandlers["foo"] as CypherHandler).query shouldBe + + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers shouldHaveKey "foo" + topicHandlers["foo"] shouldBe instanceOf() + (topicHandlers["foo"] as CypherHandler).query shouldBe "CREATE (p:Person{name: event.firstName})" } @@ -114,9 +120,11 @@ class SinkConfigurationTest { val config = SinkConfiguration(originals, Renderer.getDefaultRenderer()) config.batchSize shouldBe 10 - config.topicHandlers shouldHaveKey "foo" - config.topicHandlers["foo"] shouldBe instanceOf() - (config.topicHandlers["foo"] as NodePatternHandler).pattern shouldBe + + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers shouldHaveKey "foo" + topicHandlers["foo"] shouldBe instanceOf() + (topicHandlers["foo"] as NodePatternHandler).pattern shouldBe NodePattern( setOf("Foo"), false, @@ -125,9 +133,9 @@ class SinkConfigurationTest { emptySet(), ) - config.topicHandlers shouldHaveKey "bar" - config.topicHandlers["bar"] shouldBe instanceOf() - (config.topicHandlers["bar"] as NodePatternHandler).pattern shouldBe + topicHandlers shouldHaveKey "bar" + topicHandlers["bar"] shouldBe instanceOf() + (topicHandlers["bar"] as NodePatternHandler).pattern shouldBe NodePattern( setOf("Bar"), false, @@ -199,11 +207,12 @@ class SinkConfigurationTest { neo4j = neo4j5_26, ) - config.topicHandlers shouldHaveKey "foo" - config.topicHandlers["foo"] shouldBe instanceOf() + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers shouldHaveKey "foo" + topicHandlers["foo"] shouldBe instanceOf() - config.topicHandlers shouldHaveKey "bar" - config.topicHandlers["bar"] shouldBe instanceOf() + topicHandlers shouldHaveKey "bar" + topicHandlers["bar"] shouldBe instanceOf() } @ParameterizedTest @@ -228,11 +237,12 @@ class SinkConfigurationTest { apocCypherDoItAvailable = apocDoItAvailable, ) - config.topicHandlers shouldHaveKey "foo" - config.topicHandlers["foo"] shouldBe instanceOf(clazz) + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers shouldHaveKey "foo" + topicHandlers["foo"] shouldBe instanceOf(clazz) - config.topicHandlers shouldHaveKey "bar" - config.topicHandlers["bar"] shouldBe instanceOf(clazz) + topicHandlers shouldHaveKey "bar" + topicHandlers["bar"] shouldBe instanceOf(clazz) } @Test @@ -246,11 +256,12 @@ class SinkConfigurationTest { ) val config = SinkConfiguration(originals, Renderer.getDefaultRenderer()) - config.topicHandlers shouldHaveKey "foo" - config.topicHandlers["foo"] shouldBe instanceOf() + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + topicHandlers shouldHaveKey "foo" + topicHandlers["foo"] shouldBe instanceOf() - config.topicHandlers shouldHaveKey "bar" - config.topicHandlers["bar"] shouldBe instanceOf() + topicHandlers shouldHaveKey "bar" + topicHandlers["bar"] shouldBe instanceOf() } @ParameterizedTest diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcSchemaHandlerIT.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcSchemaHandlerIT.kt index c0011a21..81efe6bc 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcSchemaHandlerIT.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcSchemaHandlerIT.kt @@ -36,8 +36,10 @@ import org.neo4j.cdc.client.model.NodeEvent import org.neo4j.cdc.client.model.NodeState import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.cdc.client.model.RelationshipState +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.Neo4jSinkTask import org.neo4j.connectors.kafka.sink.SinkStrategy.CDC_SCHEMA +import org.neo4j.connectors.kafka.sink.SinkStrategyHandler import org.neo4j.connectors.kafka.sink.strategy.TestUtils.newChangeEventMessage import org.neo4j.connectors.kafka.sink.strategy.TestUtils.verifyEosOffsetIfEnabled import org.neo4j.connectors.kafka.testing.DatabaseSupport.createDatabase @@ -95,7 +97,8 @@ abstract class CdcSchemaHandlerIT( } ) - val handler = task.config.topicHandlers["my-topic"] + val metricsMock: Metrics = mock() + val handler = SinkStrategyHandler.createFrom(task.config, metricsMock)["my-topic"] handler shouldBe instanceOf() val cdcHandler = handler as CdcHandler diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcSourceIdHandlerIT.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcSourceIdHandlerIT.kt index 06196544..49c8e0a3 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcSourceIdHandlerIT.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcSourceIdHandlerIT.kt @@ -36,8 +36,10 @@ import org.neo4j.cdc.client.model.NodeEvent import org.neo4j.cdc.client.model.NodeState import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.cdc.client.model.RelationshipState +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.Neo4jSinkTask import org.neo4j.connectors.kafka.sink.SinkStrategy.CDC_SOURCE_ID +import org.neo4j.connectors.kafka.sink.SinkStrategyHandler import org.neo4j.connectors.kafka.sink.strategy.TestUtils.newChangeEventMessage import org.neo4j.connectors.kafka.sink.strategy.TestUtils.verifyEosOffsetIfEnabled import org.neo4j.connectors.kafka.testing.DatabaseSupport.createDatabase @@ -110,7 +112,8 @@ abstract class CdcSourceIdHandlerIT( } ) - val handler = task.config.topicHandlers["my-topic"] + val metricsMock: Metrics = mock() + val handler = SinkStrategyHandler.createFrom(task.config, metricsMock)["my-topic"] handler shouldBe instanceOf() val cdcHandler = handler as CdcHandler From 94a8d080ec65cba4a49ff4e2d7975e78ecdac145 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Mon, 23 Feb 2026 19:27:30 +0000 Subject: [PATCH 04/21] feat: add source metrics --- .../kafka/metrics}/CdcMetricsData.kt | 13 ++-- .../kafka/metrics/DbTransactionMetricsData.kt | 78 +++++++++++++++++++ .../kafka/metrics/MetricsFactory.kt | 19 ++++- .../kafka/sink/strategy/cdc/CdcHandler.kt | 3 +- .../connectors/kafka/source/Neo4jCdcTask.kt | 36 ++++++++- 5 files changed, 139 insertions(+), 10 deletions(-) rename {sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc => common/src/main/kotlin/org/neo4j/connectors/kafka/metrics}/CdcMetricsData.kt (79%) create mode 100644 common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt similarity index 79% rename from sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt rename to common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt index 41d76aa8..a87ecc97 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcMetricsData.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt @@ -14,10 +14,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.neo4j.connectors.kafka.sink.strategy.cdc +package org.neo4j.connectors.kafka.metrics import org.neo4j.cdc.client.model.ChangeEvent -import org.neo4j.connectors.kafka.metrics.Metrics class CdcMetricsData(metrics: Metrics, tags: LinkedHashMap = linkedMapOf()) { @@ -28,19 +27,23 @@ class CdcMetricsData(metrics: Metrics, tags: LinkedHashMap = lin init { metrics.addGauge( "last_cdc_tx_commit_timestamp", - "The transaction commit timestamp of the last written CDC message", + "The transaction commit timestamp of the last processed CDC message", tags, ) { lastTxCommitTs } metrics.addGauge( "last_cdc_tx_start_timestamp", - "The transaction start timestamp of the last written CDC message", + "The transaction start timestamp of the last processed CDC message", tags, ) { lastTxStartTs } - metrics.addGauge("last_cdc_tx_id", "The transaction id of the last written CDC message", tags) { + metrics.addGauge( + "last_cdc_tx_id", + "The transaction id of the last processed CDC message", + tags, + ) { lastTxId } } diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt new file mode 100644 index 00000000..763ac0be --- /dev/null +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt @@ -0,0 +1,78 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.neo4j.connectors.kafka.metrics + +import java.util.concurrent.atomic.AtomicLong +import kotlin.time.Duration +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import org.neo4j.driver.Driver +import org.neo4j.driver.SessionConfig +import org.neo4j.driver.TransactionConfig + +class DbTransactionMetricsData( + metrics: Metrics, + tags: LinkedHashMap = linkedMapOf(), + refreshTimeout: Duration, + neo4jDriver: Driver, + sessionConfig: SessionConfig, + transactionConfig: TransactionConfig, +) { + + private val lastTransactionId = AtomicLong(0) + + private val scope = CoroutineScope(Dispatchers.Default + Job()) + + init { + metrics.addGauge( + "last_db_tx_id", + "The transaction commit timestamp of the last processed CDC message", + tags, + ) { + lastTransactionId.get() + } + + scope.launch { + val databaseName = sessionConfig.database().orElse("neo4j") + while (isActive) { + val txId = + neo4jDriver.session(sessionConfig).use { session -> + session + .run( + "SHOW DATABASE $databaseName YIELD lastCommittedTxn RETURN lastCommittedTxn as txId", + transactionConfig, + ) + .single() + .get("txId") + .asLong() + } + lastTransactionId.set(txId) + + delay(refreshTimeout) + } + } + } + + fun stop() { + scope.cancel() + } +} diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt index 3ead127f..564eb7a5 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt @@ -17,6 +17,7 @@ package org.neo4j.connectors.kafka.metrics import org.apache.kafka.connect.sink.SinkTaskContext +import org.apache.kafka.connect.source.SourceTaskContext import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -27,6 +28,22 @@ class MetricsFactory { return createKafkaMetrics(context) ?: createJmxMetrics(config) } + fun createMetrics(context: SourceTaskContext, config: Neo4jConfiguration): Metrics { + return createKafkaMetrics(context) ?: createJmxMetrics(config) + } + + private fun createKafkaMetrics(context: SourceTaskContext): KafkaMetrics? { + return try { + val metrics = KafkaMetrics(context.pluginMetrics()) + log.info("Plugin metrics support detected") + metrics + } catch (_: NoSuchMethodError) { + null + } catch (_: NoClassDefFoundError) { + null + } + } + private fun createKafkaMetrics(context: SinkTaskContext): KafkaMetrics? { return try { val metrics = KafkaMetrics(context.pluginMetrics()) @@ -40,7 +57,7 @@ class MetricsFactory { } private fun createJmxMetrics(config: Neo4jConfiguration): JmxMetrics { - log.error("TTT No plugin metrics support detected. Using JMX only metrics") + log.error("No plugin metrics support detected. Using JMX only metrics") return JmxMetrics(config) } diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt index 57da96b7..599edf43 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt @@ -20,6 +20,7 @@ import org.apache.kafka.connect.data.Struct import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.connectors.kafka.data.StreamsTransactionEventExtensions.toChangeEvent import org.neo4j.connectors.kafka.data.toChangeEvent +import org.neo4j.connectors.kafka.metrics.CdcMetricsData import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.sink.ChangeQuery import org.neo4j.connectors.kafka.sink.SinkMessage @@ -44,7 +45,7 @@ class CdcHandler( } override fun postProcessLastMessageBatch(group: Iterable) { - group.lastOrNull()?.messages?.lastOrNull()?.toChangeEvent() + group.lastOrNull()?.messages?.lastOrNull()?.toChangeEvent()?.let { metricsData.update(it) } } } diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index 90b6325a..06acfa8c 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -17,6 +17,8 @@ package org.neo4j.connectors.kafka.source import java.util.concurrent.atomic.AtomicReference +import jdk.internal.platform.Container.metrics +import kotlin.time.Duration.Companion.seconds import kotlin.time.TimeSource import kotlin.time.toJavaDuration import kotlinx.coroutines.ExperimentalCoroutinesApi @@ -24,6 +26,7 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.asFlow import kotlinx.coroutines.flow.flatMapConcat +import kotlinx.coroutines.flow.onEach import kotlinx.coroutines.flow.toList import kotlinx.coroutines.reactive.asFlow import kotlinx.coroutines.runBlocking @@ -37,12 +40,15 @@ import org.neo4j.connectors.kafka.configuration.helpers.VersionUtil import org.neo4j.connectors.kafka.data.ChangeEventConverter import org.neo4j.connectors.kafka.data.Headers import org.neo4j.connectors.kafka.data.ValueConverter +import org.neo4j.connectors.kafka.metrics.CdcMetricsData +import org.neo4j.connectors.kafka.metrics.DbTransactionMetricsData +import org.neo4j.connectors.kafka.metrics.MetricsFactory import org.neo4j.driver.SessionConfig import org.neo4j.driver.TransactionConfig import org.slf4j.Logger import org.slf4j.LoggerFactory -class Neo4jCdcTask : SourceTask() { +class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory()) : SourceTask() { private val log: Logger = LoggerFactory.getLogger(Neo4jCdcTask::class.java) private lateinit var settings: Map @@ -56,6 +62,9 @@ class Neo4jCdcTask : SourceTask() { internal fun latestOffset(): String = offset.get() + private lateinit var metricsData: CdcMetricsData + private lateinit var dbTransactionMetricsData: DbTransactionMetricsData + override fun version(): String = VersionUtil.version(this.javaClass as Class<*>) override fun start(props: Map?) { @@ -70,6 +79,8 @@ class Neo4jCdcTask : SourceTask() { sessionConfig = configBuilder.build() transactionConfig = config.txConfig() + val metrics = metricsFactory.createMetrics(context, config) + cdc = CDCClient( config.driver, @@ -84,17 +95,32 @@ class Neo4jCdcTask : SourceTask() { log.info("resuming from offset: ${offset.get()}") changeEventConverter = ChangeEventConverter(config.payloadMode) + + metricsData = CdcMetricsData(metrics) + // todo enable in config + dbTransactionMetricsData = + DbTransactionMetricsData( + metrics = metrics, + neo4jDriver = config.driver, + sessionConfig = sessionConfig, + transactionConfig = transactionConfig, + refreshTimeout = 30.seconds, // todo configure this + ) } override fun stop() { log.info("stopping") config.close() + if (this::dbTransactionMetricsData.isInitialized) { + dbTransactionMetricsData.stop() + } } @OptIn(ExperimentalCoroutinesApi::class) override fun poll(): MutableList { log.info("polling from offset: ${offset.get()}") val list = mutableListOf() + var lastChangeEvent: ChangeEvent? = null runBlocking { val timeSource = TimeSource.Monotonic @@ -105,6 +131,10 @@ class Neo4jCdcTask : SourceTask() { cdc.query(ChangeIdentifier(offset.get()), { lastKnownId -> offset.set(lastKnownId.id) }) .take(config.batchSize.toLong(), true) .asFlow() + .onEach { + lastChangeEvent = it + offset.set(it.id.id) + } .flatMapConcat { build(it) } .toList(list) if (list.isNotEmpty()) { @@ -114,8 +144,8 @@ class Neo4jCdcTask : SourceTask() { delay(config.cdcPollingInterval) } - if (list.isNotEmpty()) { - offset.set(list.last().sourceOffset()["value"] as String) + if (lastChangeEvent != null) { + metricsData.update(lastChangeEvent) } } From d20ca2017a710bd2cfb02d76b0622e735942cf92 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Tue, 24 Feb 2026 11:34:20 +0000 Subject: [PATCH 05/21] feat: implement jmx reporter --- .../kafka/configuration/Neo4jConfiguration.kt | 4 +- .../connectors/kafka/metrics/JmxMetrics.kt | 90 ++++++++++++++++++- .../connectors/kafka/metrics/KafkaMetrics.kt | 31 +++++-- .../kafka/metrics/MetricsFactory.kt | 5 +- .../configuration/Neo4jConfigurationTest.kt | 17 ++++ .../connectors/kafka/sink/Neo4jConnector.kt | 13 ++- .../connectors/kafka/sink/Neo4jSinkTask.kt | 4 +- .../connectors/kafka/source/Neo4jConnector.kt | 9 +- .../connectors/kafka/source/Neo4jCdcTask.kt | 8 +- 9 files changed, 158 insertions(+), 23 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt index 0c71e1ad..1c3d927d 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt @@ -240,10 +240,10 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty } val connectorName - get(): String = getString(CONNECTOR_NAME) + get(): String = originals()[CONNECTOR_NAME].toString() val taskId - get(): Int = getInt(TASK_ID) + get(): String = originals()[TASK_ID].toString() companion object { val DEFAULT_MAX_RETRY_DURATION = 30.seconds diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt index 3b4dfa6a..0b0fdff8 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt @@ -16,17 +16,101 @@ */ package org.neo4j.connectors.kafka.metrics +import java.lang.management.ManagementFactory +import java.util.Hashtable +import java.util.concurrent.ConcurrentHashMap +import javax.management.Attribute +import javax.management.AttributeList +import javax.management.DynamicMBean +import javax.management.MBeanAttributeInfo +import javax.management.MBeanInfo +import javax.management.ObjectName import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration -class JmxMetrics(private val config: Neo4jConfiguration) : Metrics { +class JmxMetrics(config: Neo4jConfiguration) : Metrics, DynamicMBean { private val connectorName: String = config.connectorName - private val taskId: Int = config.taskId + private val taskId: String = config.taskId + private val objectName: ObjectName = + ObjectName( + "kafka.connect", + Hashtable().apply { + put("type", "plugins") + put("connector", connectorName) + put("task", taskId) + }, + ) + + private val gauges = ConcurrentHashMap>() + private val mbs = ManagementFactory.getPlatformMBeanServer() + + init { + if (mbs.isRegistered(objectName)) { + mbs.unregisterMBean(objectName) + } + mbs.registerMBean(this, objectName) + } override fun addGauge( name: String, description: String, tags: LinkedHashMap, valueProvider: () -> T?, - ) {} + ) { + gauges[name] = Gauge(name, description, valueProvider) + } + + override fun getAttribute(attribute: String?): Any? { + return gauges[attribute]?.valueProvider?.invoke() + } + + override fun setAttribute(attribute: Attribute?) { + throw UnsupportedOperationException("Attributes are read-only") + } + + override fun getAttributes(attributes: Array?): AttributeList { + val list = AttributeList() + attributes?.forEach { name -> + getAttribute(name)?.let { value -> list.add(Attribute(name, value)) } + } + return list + } + + override fun setAttributes(attributes: AttributeList?): AttributeList { + throw UnsupportedOperationException("Attributes are read-only") + } + + override fun invoke( + actionName: String?, + params: Array?, + signature: Array?, + ): Any { + throw UnsupportedOperationException("Operations are not supported") + } + + override fun getMBeanInfo(): MBeanInfo { + val attrs = + gauges.values.map { gauge -> + MBeanAttributeInfo(gauge.name, "java.lang.Number", gauge.description, true, false, false) + } + + return MBeanInfo( + this.javaClass.name, + "Neo4j Kafka Connector JMX Metrics", + attrs.toTypedArray(), + null, + null, + null, + ) + } + + private class Gauge( + val name: String, + val description: String, + val valueProvider: () -> T?, + ) + + override fun close() { + mbs.unregisterMBean(objectName) + } } diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt index 7936c075..39e5e908 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt @@ -16,26 +16,39 @@ */ package org.neo4j.connectors.kafka.metrics +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.withLock +import org.apache.kafka.common.MetricName import org.apache.kafka.common.metrics.Gauge import org.apache.kafka.common.metrics.MetricConfig import org.apache.kafka.common.metrics.PluginMetrics class KafkaMetrics(private val pluginMetrics: PluginMetrics) : Metrics { + private val lock = ReentrantLock() + private val registeredMetrics: MutableSet = mutableSetOf() + override fun addGauge( name: String, description: String, tags: LinkedHashMap, valueProvider: () -> T?, ) { - val metricName = pluginMetrics.metricName(name, description, tags) - pluginMetrics.addMetric( - metricName, - object : Gauge { - override fun value(config: MetricConfig?, now: Long): T? { - return valueProvider() - } - }, - ) + lock.withLock { + val metricName = pluginMetrics.metricName(name, description, tags) + registeredMetrics.add(metricName) + pluginMetrics.addMetric( + metricName, + object : Gauge { + override fun value(config: MetricConfig?, now: Long): T? { + return valueProvider() + } + }, + ) + } + } + + override fun close() { + lock.withLock { registeredMetrics.forEach(pluginMetrics::removeMetric) } } } diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt index 564eb7a5..34c58962 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt @@ -16,6 +16,7 @@ */ package org.neo4j.connectors.kafka.metrics +import java.io.Closeable import org.apache.kafka.connect.sink.SinkTaskContext import org.apache.kafka.connect.source.SourceTaskContext import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration @@ -57,7 +58,7 @@ class MetricsFactory { } private fun createJmxMetrics(config: Neo4jConfiguration): JmxMetrics { - log.error("No plugin metrics support detected. Using JMX only metrics") + log.info("No plugin metrics support detected. Using JMX only metrics") return JmxMetrics(config) } @@ -66,7 +67,7 @@ class MetricsFactory { } } -interface Metrics { +interface Metrics : Closeable { fun addGauge( name: String, description: String, diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt index f1cdc8ed..7b5b2cfa 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt @@ -361,6 +361,23 @@ class Neo4jConfigurationTest { } } + @Test + fun `internal variables`() { + Neo4jConfiguration( + Neo4jConfiguration.config(), + mapOf( + Neo4jConfiguration.URI to "bolt://localhost", + Neo4jConfiguration.TASK_ID to "1", + Neo4jConfiguration.CONNECTOR_NAME to "neo4j-connector", + ), + ConnectorType.SINK, + ) + .run { + assertEquals("1", this.taskId) + assertEquals("neo4j-connector", this.connectorName) + } + } + companion object { fun newTempFile(prefix: String = "test", suffix: String = ".tmp"): File { val f = File.createTempFile(prefix, suffix) diff --git a/sink-connector/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jConnector.kt b/sink-connector/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jConnector.kt index d85e499c..8378f2e9 100644 --- a/sink-connector/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jConnector.kt +++ b/sink-connector/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jConnector.kt @@ -20,6 +20,7 @@ import org.apache.kafka.common.config.Config import org.apache.kafka.common.config.ConfigDef import org.apache.kafka.connect.connector.Task import org.apache.kafka.connect.sink.SinkConnector +import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration.Companion.TASK_ID import org.neo4j.connectors.kafka.utils.PropertiesUtil class Neo4jConnector : SinkConnector() { @@ -27,13 +28,19 @@ class Neo4jConnector : SinkConnector() { override fun version(): String = PropertiesUtil.getVersion() - override fun start(props: MutableMap?) { - this.props = props!!.toMap() + override fun start(props: MutableMap) { + this.props = props.toMap() } override fun taskClass(): Class = Neo4jSinkTask::class.java - override fun taskConfigs(maxTasks: Int): List> = List(maxTasks) { props } + override fun taskConfigs(maxTasks: Int): List> = + (0 until maxTasks).toList().map { + buildMap { + putAll(props) + put(TASK_ID, it.toString()) + } + } override fun stop() {} diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt index 99336626..b5433323 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt @@ -45,7 +45,6 @@ class Neo4jSinkTask(private val metricsFactory: MetricsFactory = MetricsFactory( metrics = metricsFactory.createMetrics(context, config) topicHandlers = SinkStrategyHandler.createFrom(config, metrics) - log.error("TTT handlers: {}", topicHandlers.mapValues { it.value.javaClass.name }) } override fun stop() { @@ -53,6 +52,9 @@ class Neo4jSinkTask(private val metricsFactory: MetricsFactory = MetricsFactory( if (this::config.isInitialized) { config.close() } + if (this::metrics.isInitialized) { + metrics.close() + } } override fun put(records: Collection?) { diff --git a/source-connector/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jConnector.kt b/source-connector/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jConnector.kt index 4ef671f6..a048c2f9 100644 --- a/source-connector/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jConnector.kt +++ b/source-connector/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jConnector.kt @@ -21,6 +21,7 @@ import org.apache.kafka.common.config.ConfigDef import org.apache.kafka.connect.connector.Task import org.apache.kafka.connect.source.ExactlyOnceSupport import org.apache.kafka.connect.source.SourceConnector +import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration.Companion.TASK_ID import org.neo4j.connectors.kafka.configuration.helpers.VersionUtil import org.neo4j.connectors.kafka.source.SourceConfiguration.Companion.STRATEGY @@ -44,7 +45,13 @@ class Neo4jConnector : SourceConnector() { SourceType.QUERY -> Neo4jQueryTask::class.java } - override fun taskConfigs(maxTasks: Int): List> = listOf(props) + override fun taskConfigs(maxTasks: Int): List> = + (0 until maxTasks).toList().map { + buildMap { + putAll(props) + put(TASK_ID, it.toString()) + } + } override fun stop() {} diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index 06acfa8c..7d430cf5 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -17,7 +17,6 @@ package org.neo4j.connectors.kafka.source import java.util.concurrent.atomic.AtomicReference -import jdk.internal.platform.Container.metrics import kotlin.time.Duration.Companion.seconds import kotlin.time.TimeSource import kotlin.time.toJavaDuration @@ -42,6 +41,7 @@ import org.neo4j.connectors.kafka.data.Headers import org.neo4j.connectors.kafka.data.ValueConverter import org.neo4j.connectors.kafka.metrics.CdcMetricsData import org.neo4j.connectors.kafka.metrics.DbTransactionMetricsData +import org.neo4j.connectors.kafka.metrics.Metrics import org.neo4j.connectors.kafka.metrics.MetricsFactory import org.neo4j.driver.SessionConfig import org.neo4j.driver.TransactionConfig @@ -62,6 +62,7 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() internal fun latestOffset(): String = offset.get() + private lateinit var metrics: Metrics private lateinit var metricsData: CdcMetricsData private lateinit var dbTransactionMetricsData: DbTransactionMetricsData @@ -79,7 +80,7 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() sessionConfig = configBuilder.build() transactionConfig = config.txConfig() - val metrics = metricsFactory.createMetrics(context, config) + metrics = metricsFactory.createMetrics(context, config) cdc = CDCClient( @@ -114,6 +115,9 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() if (this::dbTransactionMetricsData.isInitialized) { dbTransactionMetricsData.stop() } + if (this::metrics.isInitialized) { + metrics.close() + } } @OptIn(ExperimentalCoroutinesApi::class) From cc01c153c3df9610aad7743ec5dde3649f265747 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Tue, 24 Feb 2026 11:44:11 +0000 Subject: [PATCH 06/21] refactor: clean up --- .../kafka/metrics/CdcMetricsData.kt | 19 ++++++++++--------- .../kafka/sink/SinkConfiguration.kt | 2 +- .../connectors/kafka/source/Neo4jCdcTask.kt | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt index a87ecc97..775506f1 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt @@ -17,12 +17,13 @@ package org.neo4j.connectors.kafka.metrics import org.neo4j.cdc.client.model.ChangeEvent +import java.util.concurrent.atomic.AtomicLong class CdcMetricsData(metrics: Metrics, tags: LinkedHashMap = linkedMapOf()) { - private var lastTxCommitTs: Long? = null - private var lastTxStartTs: Long? = null - private var lastTxId: Long? = null + private var lastTxCommitTs: AtomicLong = AtomicLong(0L) + private var lastTxStartTs: AtomicLong = AtomicLong(0L) + private var lastTxId: AtomicLong = AtomicLong(0L) init { metrics.addGauge( @@ -30,29 +31,29 @@ class CdcMetricsData(metrics: Metrics, tags: LinkedHashMap = lin "The transaction commit timestamp of the last processed CDC message", tags, ) { - lastTxCommitTs + lastTxCommitTs.get() } metrics.addGauge( "last_cdc_tx_start_timestamp", "The transaction start timestamp of the last processed CDC message", tags, ) { - lastTxStartTs + lastTxStartTs.get() } metrics.addGauge( "last_cdc_tx_id", "The transaction id of the last processed CDC message", tags, ) { - lastTxId + lastTxId.get() } } fun update(event: ChangeEvent) { event.metadata?.let { - lastTxCommitTs = it.txCommitTime.toEpochSecond() - lastTxStartTs = it.txStartTime.toEpochSecond() + lastTxCommitTs.set(it.txCommitTime.toEpochSecond()) + lastTxStartTs.set(it.txStartTime.toEpochSecond()) } - lastTxId = event.txId + lastTxId.set(event.txId) } } diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt index b21c95b3..f76a335c 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt @@ -127,7 +127,7 @@ class SinkConfiguration : Neo4jConfiguration { override fun userAgentComment(): String = SinkStrategyHandler.configuredStrategies(this).sorted().joinToString("; ") - fun validateAllTopics(topicHandlers: Map) { // todo what is this for + private fun validateAllTopics(topicHandlers: Map) { // todo is it used in production val sourceTopics = topicNames.toSet() val configuredTopics = topicHandlers.keys diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index 7d430cf5..e2d78a63 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -137,7 +137,6 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() .asFlow() .onEach { lastChangeEvent = it - offset.set(it.id.id) } .flatMapConcat { build(it) } .toList(list) @@ -149,6 +148,7 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() } if (lastChangeEvent != null) { + offset.set(lastChangeEvent.id.id) metricsData.update(lastChangeEvent) } } From f5fed7c7becfd48a820e19b232a64839f5b56a33 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Tue, 24 Feb 2026 12:26:30 +0000 Subject: [PATCH 07/21] chore: spotless --- .../org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt | 2 +- .../org/neo4j/connectors/kafka/sink/SinkConfiguration.kt | 4 +++- .../kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt | 4 +--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt index 775506f1..dddc2c46 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt @@ -16,8 +16,8 @@ */ package org.neo4j.connectors.kafka.metrics -import org.neo4j.cdc.client.model.ChangeEvent import java.util.concurrent.atomic.AtomicLong +import org.neo4j.cdc.client.model.ChangeEvent class CdcMetricsData(metrics: Metrics, tags: LinkedHashMap = linkedMapOf()) { diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt index f76a335c..7b15d475 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt @@ -127,7 +127,9 @@ class SinkConfiguration : Neo4jConfiguration { override fun userAgentComment(): String = SinkStrategyHandler.configuredStrategies(this).sorted().joinToString("; ") - private fun validateAllTopics(topicHandlers: Map) { // todo is it used in production + private fun validateAllTopics( + topicHandlers: Map + ) { // todo is it used in production val sourceTopics = topicNames.toSet() val configuredTopics = topicHandlers.keys diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index e2d78a63..479065e2 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -135,9 +135,7 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() cdc.query(ChangeIdentifier(offset.get()), { lastKnownId -> offset.set(lastKnownId.id) }) .take(config.batchSize.toLong(), true) .asFlow() - .onEach { - lastChangeEvent = it - } + .onEach { lastChangeEvent = it } .flatMapConcat { build(it) } .toList(list) if (list.isNotEmpty()) { From 8cd6cfd631a8ce18d286953e9a3747708477fb17 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Tue, 24 Feb 2026 12:52:10 +0000 Subject: [PATCH 08/21] refactor: use write access mode to retrieve last db tx id from a leader --- .../kafka/metrics/DbTransactionMetricsData.kt | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt index 763ac0be..ad8bc3af 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt @@ -25,6 +25,7 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch +import org.neo4j.driver.AccessMode import org.neo4j.driver.Driver import org.neo4j.driver.SessionConfig import org.neo4j.driver.TransactionConfig @@ -38,6 +39,18 @@ class DbTransactionMetricsData( transactionConfig: TransactionConfig, ) { + private val writeAccessModeSessionConfig: SessionConfig by lazy { + val builder = SessionConfig.builder() + + sessionConfig.database().ifPresent { builder.withDatabase(it) } + sessionConfig.fetchSize().ifPresent { builder.withFetchSize(it) } + sessionConfig.impersonatedUser().ifPresent { builder.withImpersonatedUser(it) } + sessionConfig.bookmarks()?.let { builder.withBookmarks(it) } + + builder.withDefaultAccessMode(AccessMode.WRITE) + builder.build() + } + private val lastTransactionId = AtomicLong(0) private val scope = CoroutineScope(Dispatchers.Default + Job()) @@ -52,10 +65,10 @@ class DbTransactionMetricsData( } scope.launch { - val databaseName = sessionConfig.database().orElse("neo4j") + val databaseName = writeAccessModeSessionConfig.database().orElse("neo4j") while (isActive) { val txId = - neo4jDriver.session(sessionConfig).use { session -> + neo4jDriver.session(writeAccessModeSessionConfig).use { session -> session .run( "SHOW DATABASE $databaseName YIELD lastCommittedTxn RETURN lastCommittedTxn as txId", From 053fe21e1c99e97badb191126a06c7d67d750f86 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Tue, 24 Feb 2026 14:07:55 +0000 Subject: [PATCH 09/21] refactor: parameterize last db metric --- .../kafka/configuration/Neo4jConfiguration.kt | 33 ++++++++++++------- .../Neo4jConfigurationDeclarations.kt | 28 ++++++++++++++++ .../resources/neo4j-configuration.properties | 4 ++- .../configuration/Neo4jConfigurationTest.kt | 29 ++++++++++++++++ .../connectors/kafka/source/Neo4jCdcTask.kt | 20 +++++------ 5 files changed, 92 insertions(+), 22 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt index 1c3d927d..6d0b2326 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt @@ -45,6 +45,7 @@ import org.neo4j.driver.TransactionConfig import org.neo4j.driver.net.ServerAddress import org.slf4j.Logger import org.slf4j.LoggerFactory +import kotlin.time.Duration enum class ConnectorType(val description: String) { SINK("sink"), @@ -70,33 +71,33 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty get(): List = getList(URI).map { URI(it) } internal val connectionTimeout - get(): kotlin.time.Duration = - kotlin.time.Duration.parseSimpleString(getString(CONNECTION_TIMEOUT)) + get(): Duration = + Duration.parseSimpleString(getString(CONNECTION_TIMEOUT)) internal val maxRetryTime - get(): kotlin.time.Duration = - kotlin.time.Duration.parseSimpleString(getString(MAX_TRANSACTION_RETRY_TIMEOUT)) + get(): Duration = + Duration.parseSimpleString(getString(MAX_TRANSACTION_RETRY_TIMEOUT)) internal val maxConnectionPoolSize get(): Int = getInt(POOL_MAX_CONNECTION_POOL_SIZE) internal val connectionAcquisitionTimeout - get(): kotlin.time.Duration = - kotlin.time.Duration.parseSimpleString(getString(POOL_CONNECTION_ACQUISITION_TIMEOUT)) + get(): Duration = + Duration.parseSimpleString(getString(POOL_CONNECTION_ACQUISITION_TIMEOUT)) internal val idleTimeBeforeTest - get(): kotlin.time.Duration = + get(): Duration = getString(POOL_IDLE_TIME_BEFORE_TEST).orEmpty().run { if (this.isEmpty()) { (-1).milliseconds } else { - kotlin.time.Duration.parseSimpleString(this) + Duration.parseSimpleString(this) } } internal val maxConnectionLifetime - get(): kotlin.time.Duration = - kotlin.time.Duration.parseSimpleString(getString(POOL_MAX_CONNECTION_LIFETIME)) + get(): Duration = + Duration.parseSimpleString(getString(POOL_MAX_CONNECTION_LIFETIME)) internal val encrypted get(): Boolean = getString(SECURITY_ENCRYPTED).toBoolean() @@ -146,6 +147,12 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty } } + val lastDbTxIdEnabled + get(): Boolean = getString(METRIC_LAST_TX_ID_ENABLED).toBoolean() + + val lastDbTxIdRefreshInterval + get(): Duration = Duration.parseSimpleString(getString(METRIC_LAST_TX_ID_REFRESH_INTERVAL)) + val driver: Driver by lazy { val config = Config.builder() @@ -277,6 +284,9 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty const val SECURITY_TRUST_STRATEGY = "neo4j.security.trust-strategy" const val SECURITY_CERT_FILES = "neo4j.security.cert-files" + const val METRIC_LAST_TX_ID_ENABLED = "neo4j.metric.last-tx-id.enabled" + const val METRIC_LAST_TX_ID_REFRESH_INTERVAL = "neo4j.metric.last-tx-id.refresh-interval" + // internal properties const val CONNECTOR_NAME = "name" const val TASK_ID = "neo4j.task.id" @@ -305,5 +315,6 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty .defineEncryptionSettings() .definePoolSettings() .defineRetrySettings() - } + .defineMetricSettings() + } } diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationDeclarations.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationDeclarations.kt index 30f2545c..88f2d6a1 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationDeclarations.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationDeclarations.kt @@ -19,6 +19,7 @@ package org.neo4j.connectors.kafka.configuration import java.net.URI import java.util.function.Predicate import kotlin.time.Duration.Companion.milliseconds +import kotlin.time.Duration.Companion.seconds import org.apache.kafka.common.config.ConfigDef import org.apache.kafka.common.config.ConfigDef.Importance import org.apache.kafka.common.config.ConfigDef.Range @@ -386,3 +387,30 @@ fun ConfigDef.defineRetrySettings(): ConfigDef = validator = Validators.pattern(SIMPLE_DURATION_PATTERN) } ) + +fun ConfigDef.defineMetricSettings(): ConfigDef = + this.define( + ConfigKeyBuilder.of( + Neo4jConfiguration.METRIC_LAST_TX_ID_ENABLED, + ConfigDef.Type.STRING, + ) { + importance = Importance.LOW + defaultValue = "false" + group = Groups.CONNECTOR_ADVANCED.title + validator = Validators.bool() + recommender = Recommenders.bool() + documentation = "Whether the last transaction ID metric is enabled." + } + ) + .define( + ConfigKeyBuilder.of( + Neo4jConfiguration.METRIC_LAST_TX_ID_REFRESH_INTERVAL, + ConfigDef.Type.STRING, + ) { + importance = Importance.LOW + defaultValue = 30.seconds.toSimpleString() + group = Groups.CONNECTOR_ADVANCED.title + validator = Validators.pattern(SIMPLE_DURATION_PATTERN) + documentation = "The refresh interval for the last transaction ID metric." + } + ) diff --git a/common/src/main/resources/neo4j-configuration.properties b/common/src/main/resources/neo4j-configuration.properties index 8d6ec005..0e7b85db 100644 --- a/common/src/main/resources/neo4j-configuration.properties +++ b/common/src/main/resources/neo4j-configuration.properties @@ -32,4 +32,6 @@ neo4j.pool.max-connection-pool-size=Type: Integer;\nDescription: Maximum number neo4j.pool.connection-acquisition-timeout=Type: Duration;\nDescription: Maximum duration to wait for acquiring a connection from the connection pool (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.pool.idle-time-before-connection-test=Type: Duration;\nDescription: Duration after which idle connections are tested for liveness (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.pool.max-connection-lifetime=Type: Duration;\nDescription: Duration after which a connection is dropped from the connection pool (valid units are: ``ms`, `s`, `m`, `h` and `d`; default unit is `s`). -neo4j.max-retry-time=Type: Duration;\nDescription: Maximum duration to retry a transaction. \ No newline at end of file +neo4j.max-retry-time=Type: Duration;\nDescription: Maximum duration to retry a transaction. +neo4j.metrics.last-db-tx-id-enabled=Type: Boolean;\nDescription: Whether to enable the `last_db_tx_id` metric. +neo4j.metrics.last-db-tx-id-refresh-interval=Type: Duration;\nDescription: Interval between metric updates (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). \ No newline at end of file diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt index 7b5b2cfa..bbeeb571 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt @@ -378,6 +378,35 @@ class Neo4jConfigurationTest { } } + @Test + fun `metric settings`() { + Neo4jConfiguration( + Neo4jConfiguration.config(), + mapOf( + Neo4jConfiguration.URI to "bolt://localhost", + ), + ConnectorType.SINK, + ) + .run { + assertFalse(this.lastDbTxIdEnabled) + assertEquals(30.seconds, this.lastDbTxIdRefreshInterval) + } + + Neo4jConfiguration( + Neo4jConfiguration.config(), + mapOf( + Neo4jConfiguration.URI to "bolt://localhost", + Neo4jConfiguration.METRIC_LAST_TX_ID_ENABLED to "true", + Neo4jConfiguration.METRIC_LAST_TX_ID_REFRESH_INTERVAL to "1m", + ), + ConnectorType.SINK, + ) + .run { + assertTrue(this.lastDbTxIdEnabled) + assertEquals(1.minutes, this.lastDbTxIdRefreshInterval) + } + } + companion object { fun newTempFile(prefix: String = "test", suffix: String = ".tmp"): File { val f = File.createTempFile(prefix, suffix) diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index 479065e2..42a7d630 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -17,7 +17,6 @@ package org.neo4j.connectors.kafka.source import java.util.concurrent.atomic.AtomicReference -import kotlin.time.Duration.Companion.seconds import kotlin.time.TimeSource import kotlin.time.toJavaDuration import kotlinx.coroutines.ExperimentalCoroutinesApi @@ -98,15 +97,16 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() changeEventConverter = ChangeEventConverter(config.payloadMode) metricsData = CdcMetricsData(metrics) - // todo enable in config - dbTransactionMetricsData = - DbTransactionMetricsData( - metrics = metrics, - neo4jDriver = config.driver, - sessionConfig = sessionConfig, - transactionConfig = transactionConfig, - refreshTimeout = 30.seconds, // todo configure this - ) + if (config.lastDbTxIdEnabled) { + dbTransactionMetricsData = + DbTransactionMetricsData( + metrics = metrics, + neo4jDriver = config.driver, + sessionConfig = sessionConfig, + transactionConfig = transactionConfig, + refreshTimeout = config.lastDbTxIdRefreshInterval, + ) + } } override fun stop() { From b51e87e03682b2e80a18194e207e18fe867adb47 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Tue, 24 Feb 2026 16:37:33 +0000 Subject: [PATCH 10/21] fix: update unit tests --- .../kafka/configuration/Neo4jConfiguration.kt | 16 ++++++---------- .../connectors/kafka/metrics/MetricsFactory.kt | 4 ++-- .../configuration/Neo4jConfigurationTest.kt | 4 +--- .../data/converter/CompactValueConverterTest.kt | 2 -- .../neo4j/connectors/kafka/sink/Neo4jSinkTask.kt | 1 + .../connectors/kafka/sink/SinkConfiguration.kt | 4 +--- .../kafka/sink/SinkConfigurationTest.kt | 8 ++++++-- 7 files changed, 17 insertions(+), 22 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt index 6d0b2326..8fb541a2 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt @@ -20,6 +20,7 @@ import java.io.Closeable import java.io.File import java.net.URI import java.util.concurrent.TimeUnit +import kotlin.time.Duration import kotlin.time.Duration.Companion.milliseconds import kotlin.time.Duration.Companion.seconds import org.apache.kafka.common.config.AbstractConfig @@ -45,7 +46,6 @@ import org.neo4j.driver.TransactionConfig import org.neo4j.driver.net.ServerAddress import org.slf4j.Logger import org.slf4j.LoggerFactory -import kotlin.time.Duration enum class ConnectorType(val description: String) { SINK("sink"), @@ -71,19 +71,16 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty get(): List = getList(URI).map { URI(it) } internal val connectionTimeout - get(): Duration = - Duration.parseSimpleString(getString(CONNECTION_TIMEOUT)) + get(): Duration = Duration.parseSimpleString(getString(CONNECTION_TIMEOUT)) internal val maxRetryTime - get(): Duration = - Duration.parseSimpleString(getString(MAX_TRANSACTION_RETRY_TIMEOUT)) + get(): Duration = Duration.parseSimpleString(getString(MAX_TRANSACTION_RETRY_TIMEOUT)) internal val maxConnectionPoolSize get(): Int = getInt(POOL_MAX_CONNECTION_POOL_SIZE) internal val connectionAcquisitionTimeout - get(): Duration = - Duration.parseSimpleString(getString(POOL_CONNECTION_ACQUISITION_TIMEOUT)) + get(): Duration = Duration.parseSimpleString(getString(POOL_CONNECTION_ACQUISITION_TIMEOUT)) internal val idleTimeBeforeTest get(): Duration = @@ -96,8 +93,7 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty } internal val maxConnectionLifetime - get(): Duration = - Duration.parseSimpleString(getString(POOL_MAX_CONNECTION_LIFETIME)) + get(): Duration = Duration.parseSimpleString(getString(POOL_MAX_CONNECTION_LIFETIME)) internal val encrypted get(): Boolean = getString(SECURITY_ENCRYPTED).toBoolean() @@ -316,5 +312,5 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty .definePoolSettings() .defineRetrySettings() .defineMetricSettings() - } + } } diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt index 34c58962..8615d87c 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt @@ -35,7 +35,7 @@ class MetricsFactory { private fun createKafkaMetrics(context: SourceTaskContext): KafkaMetrics? { return try { - val metrics = KafkaMetrics(context.pluginMetrics()) + val metrics = KafkaMetrics(context.pluginMetrics() ?: return null) log.info("Plugin metrics support detected") metrics } catch (_: NoSuchMethodError) { @@ -47,7 +47,7 @@ class MetricsFactory { private fun createKafkaMetrics(context: SinkTaskContext): KafkaMetrics? { return try { - val metrics = KafkaMetrics(context.pluginMetrics()) + val metrics = KafkaMetrics(context.pluginMetrics() ?: return null) log.info("Plugin metrics support detected") metrics } catch (_: NoSuchMethodError) { diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt index bbeeb571..906e8888 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt @@ -382,9 +382,7 @@ class Neo4jConfigurationTest { fun `metric settings`() { Neo4jConfiguration( Neo4jConfiguration.config(), - mapOf( - Neo4jConfiguration.URI to "bolt://localhost", - ), + mapOf(Neo4jConfiguration.URI to "bolt://localhost"), ConnectorType.SINK, ) .run { diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/converter/CompactValueConverterTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/converter/CompactValueConverterTest.kt index dd5742fd..45181bcf 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/converter/CompactValueConverterTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/converter/CompactValueConverterTest.kt @@ -608,8 +608,6 @@ class DynamicTypesCompactTest { val schema = converter.schema(map, false) val converted = converter.value(schema, map) - println(schema) - println(converted) converted shouldBe Struct(schema) .put("name", "john") diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt index b5433323..b18c7b24 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt @@ -45,6 +45,7 @@ class Neo4jSinkTask(private val metricsFactory: MetricsFactory = MetricsFactory( metrics = metricsFactory.createMetrics(context, config) topicHandlers = SinkStrategyHandler.createFrom(config, metrics) + config.validateAllTopics(topicHandlers) } override fun stop() { diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt index 7b15d475..6dbc2793 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/SinkConfiguration.kt @@ -127,9 +127,7 @@ class SinkConfiguration : Neo4jConfiguration { override fun userAgentComment(): String = SinkStrategyHandler.configuredStrategies(this).sorted().joinToString("; ") - private fun validateAllTopics( - topicHandlers: Map - ) { // todo is it used in production + fun validateAllTopics(topicHandlers: Map) { val sourceTopics = topicNames.toSet() val configuredTopics = topicHandlers.keys diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt index 150f0910..549d6bec 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/SinkConfigurationTest.kt @@ -60,7 +60,9 @@ class SinkConfigurationTest { "${SinkConfiguration.CYPHER_TOPIC_PREFIX}foo" to "CREATE (p:Person{name: event.firstName})", ) - SinkConfiguration(originals, Renderer.getDefaultRenderer()) + val config = SinkConfiguration(originals, Renderer.getDefaultRenderer()) + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + config.validateAllTopics(topicHandlers) } shouldHaveMessage "Topic 'bar' is not assigned a sink strategy" } @@ -79,7 +81,9 @@ class SinkConfigurationTest { SinkConfiguration.CDC_SOURCE_ID_TOPICS to "foo", ) - SinkConfiguration(originals, Renderer.getDefaultRenderer()) + val config = SinkConfiguration(originals, Renderer.getDefaultRenderer()) + val topicHandlers = SinkStrategyHandler.createFrom(config, metricsMock) + config.validateAllTopics(topicHandlers) } shouldHaveMessage "Topic 'foo' has multiple strategies defined" } From a666fc10f884a65ab7e88e35880d72d996487517 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Wed, 25 Feb 2026 14:25:52 +0000 Subject: [PATCH 11/21] refactor: downgrade to kafka 3.8 --- .../connectors/kafka/metrics/KafkaMetrics.kt | 54 ------------------- .../kafka/metrics/MetricsFactory.kt | 43 +-------------- pom.xml | 2 +- .../connectors/kafka/sink/Neo4jSinkTask.kt | 2 +- .../connectors/kafka/source/Neo4jCdcTask.kt | 2 +- 5 files changed, 4 insertions(+), 99 deletions(-) delete mode 100644 common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt deleted file mode 100644 index 39e5e908..00000000 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/KafkaMetrics.kt +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.neo4j.connectors.kafka.metrics - -import java.util.concurrent.locks.ReentrantLock -import kotlin.concurrent.withLock -import org.apache.kafka.common.MetricName -import org.apache.kafka.common.metrics.Gauge -import org.apache.kafka.common.metrics.MetricConfig -import org.apache.kafka.common.metrics.PluginMetrics - -class KafkaMetrics(private val pluginMetrics: PluginMetrics) : Metrics { - - private val lock = ReentrantLock() - private val registeredMetrics: MutableSet = mutableSetOf() - - override fun addGauge( - name: String, - description: String, - tags: LinkedHashMap, - valueProvider: () -> T?, - ) { - lock.withLock { - val metricName = pluginMetrics.metricName(name, description, tags) - registeredMetrics.add(metricName) - pluginMetrics.addMetric( - metricName, - object : Gauge { - override fun value(config: MetricConfig?, now: Long): T? { - return valueProvider() - } - }, - ) - } - } - - override fun close() { - lock.withLock { registeredMetrics.forEach(pluginMetrics::removeMetric) } - } -} diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt index 8615d87c..3278a2cb 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/MetricsFactory.kt @@ -17,54 +17,13 @@ package org.neo4j.connectors.kafka.metrics import java.io.Closeable -import org.apache.kafka.connect.sink.SinkTaskContext -import org.apache.kafka.connect.source.SourceTaskContext import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration -import org.slf4j.Logger -import org.slf4j.LoggerFactory class MetricsFactory { - fun createMetrics(context: SinkTaskContext, config: Neo4jConfiguration): Metrics { - return createKafkaMetrics(context) ?: createJmxMetrics(config) - } - - fun createMetrics(context: SourceTaskContext, config: Neo4jConfiguration): Metrics { - return createKafkaMetrics(context) ?: createJmxMetrics(config) - } - - private fun createKafkaMetrics(context: SourceTaskContext): KafkaMetrics? { - return try { - val metrics = KafkaMetrics(context.pluginMetrics() ?: return null) - log.info("Plugin metrics support detected") - metrics - } catch (_: NoSuchMethodError) { - null - } catch (_: NoClassDefFoundError) { - null - } - } - - private fun createKafkaMetrics(context: SinkTaskContext): KafkaMetrics? { - return try { - val metrics = KafkaMetrics(context.pluginMetrics() ?: return null) - log.info("Plugin metrics support detected") - metrics - } catch (_: NoSuchMethodError) { - null - } catch (_: NoClassDefFoundError) { - null - } - } - - private fun createJmxMetrics(config: Neo4jConfiguration): JmxMetrics { - log.info("No plugin metrics support detected. Using JMX only metrics") + fun createMetrics(config: Neo4jConfiguration): Metrics { return JmxMetrics(config) } - - companion object { - private val log: Logger = LoggerFactory.getLogger(MetricsFactory::class.java) - } } interface Metrics : Closeable { diff --git a/pom.xml b/pom.xml index a668725b..394c74e9 100644 --- a/pom.xml +++ b/pom.xml @@ -62,7 +62,7 @@ - 4.1.1 + 3.8.1 6.1.3 1.10.2 2.3.10 diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt index b18c7b24..56402f19 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/Neo4jSinkTask.kt @@ -43,7 +43,7 @@ class Neo4jSinkTask(private val metricsFactory: MetricsFactory = MetricsFactory( settings = props!! config = SinkConfiguration(settings) - metrics = metricsFactory.createMetrics(context, config) + metrics = metricsFactory.createMetrics(config) topicHandlers = SinkStrategyHandler.createFrom(config, metrics) config.validateAllTopics(topicHandlers) } diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index 42a7d630..98ff7166 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -79,7 +79,7 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() sessionConfig = configBuilder.build() transactionConfig = config.txConfig() - metrics = metricsFactory.createMetrics(context, config) + metrics = metricsFactory.createMetrics(config) cdc = CDCClient( From 39d659d2ce272295b25ac382afb1f6094ddf7a56 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Wed, 25 Feb 2026 19:55:00 +0000 Subject: [PATCH 12/21] test: add metric tests --- .../kafka/metrics/DbTransactionMetricsData.kt | 24 +-- .../connectors/kafka/metrics/JmxMetrics.kt | 4 +- .../metrics/DbTransactionMetricsDataTest.kt | 163 ++++++++++++++++++ .../kafka/metrics/JmxMetricsTest.kt | 131 ++++++++++++++ .../connectors/kafka/source/Neo4jCdcTask.kt | 4 +- 5 files changed, 312 insertions(+), 14 deletions(-) create mode 100644 common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt create mode 100644 common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetricsTest.kt diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt index ad8bc3af..fb475774 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt @@ -18,6 +18,7 @@ package org.neo4j.connectors.kafka.metrics import java.util.concurrent.atomic.AtomicLong import kotlin.time.Duration +import kotlinx.coroutines.CoroutineDispatcher import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.Job @@ -29,15 +30,17 @@ import org.neo4j.driver.AccessMode import org.neo4j.driver.Driver import org.neo4j.driver.SessionConfig import org.neo4j.driver.TransactionConfig +import java.io.Closeable class DbTransactionMetricsData( - metrics: Metrics, - tags: LinkedHashMap = linkedMapOf(), - refreshTimeout: Duration, - neo4jDriver: Driver, - sessionConfig: SessionConfig, - transactionConfig: TransactionConfig, -) { + metrics: Metrics, + tags: LinkedHashMap = linkedMapOf(), + refreshInterval: Duration, + neo4jDriver: Driver, + sessionConfig: SessionConfig, + transactionConfig: TransactionConfig, + dispatcher: CoroutineDispatcher = Dispatchers.Default, +): Closeable { private val writeAccessModeSessionConfig: SessionConfig by lazy { val builder = SessionConfig.builder() @@ -52,8 +55,7 @@ class DbTransactionMetricsData( } private val lastTransactionId = AtomicLong(0) - - private val scope = CoroutineScope(Dispatchers.Default + Job()) + private val scope = CoroutineScope(dispatcher + Job()) init { metrics.addGauge( @@ -80,12 +82,12 @@ class DbTransactionMetricsData( } lastTransactionId.set(txId) - delay(refreshTimeout) + delay(refreshInterval) } } } - fun stop() { + override fun close() { scope.cancel() } } diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt index 0b0fdff8..73e811fb 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetrics.kt @@ -111,6 +111,8 @@ class JmxMetrics(config: Neo4jConfiguration) : Metrics, DynamicMBean { ) override fun close() { - mbs.unregisterMBean(objectName) + if (mbs.isRegistered(objectName)) { + mbs.unregisterMBean(objectName) + } } } diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt new file mode 100644 index 00000000..755a3818 --- /dev/null +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt @@ -0,0 +1,163 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.neo4j.connectors.kafka.metrics + +import kotlin.time.Duration.Companion.milliseconds +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.test.advanceTimeBy +import kotlinx.coroutines.test.runCurrent +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.assertNotNull +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.verify +import org.neo4j.connectors.kafka.data.TypesTest.Companion.neo4jImage +import org.neo4j.driver.AuthTokens +import org.neo4j.driver.Driver +import org.neo4j.driver.GraphDatabase +import org.neo4j.driver.SessionConfig +import org.neo4j.driver.TransactionConfig +import org.testcontainers.containers.Neo4jContainer +import org.testcontainers.junit.jupiter.Container +import org.testcontainers.junit.jupiter.Testcontainers +import java.util.UUID +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +@OptIn(ExperimentalCoroutinesApi::class) +@Testcontainers +class DbTransactionMetricsDataTest { + + @Test + fun `should register gauge and poll for transaction id`() = runTest { + val metrics = mock() + + val refreshInterval = 100.milliseconds + val sessionConfig = SessionConfig.builder().build() + val transactionConfig = TransactionConfig.builder().build() + + val driver = GraphDatabase.driver(neo4j.boltUrl, AuthTokens.none()) + val dispatcher = this.coroutineContext[kotlinx.coroutines.CoroutineDispatcher]!! + + DbTransactionMetricsData( + metrics = metrics, + refreshInterval = refreshInterval, + neo4jDriver = driver, + sessionConfig = sessionConfig, + transactionConfig = transactionConfig, + dispatcher = dispatcher + ).use { + val gaugeCaptor = argumentCaptor<() -> Long?>() + verify(metrics) + .addGauge( + eq("last_db_tx_id"), + eq("The transaction commit timestamp of the last processed CDC message"), + any(), + gaugeCaptor.capture(), + ) + + val initialValue = gaugeCaptor.firstValue() + assertNotNull(initialValue, "captured transaction id should not be null") + + commitTransaction(driver) + runCurrent() + + val firstIncrement = gaugeCaptor.firstValue() + assertNotNull(firstIncrement, "the first increment of transaction id should not be null") + assertTrue("transaction id should increment") { initialValue < firstIncrement } + + commitTransaction(driver) + advanceTimeBy(refreshInterval) + runCurrent() + + val secondIncrement = gaugeCaptor.firstValue() + assertNotNull(secondIncrement, "the first increment of transaction id should not be null") + assertEquals(firstIncrement + 1, secondIncrement, "transaction id should increment") + } + } + + @Test + fun `should stop polling when stop is called`() = runTest { + val metrics = mock() + + val refreshInterval = 100.milliseconds + val sessionConfig = SessionConfig.builder().build() + val transactionConfig = TransactionConfig.builder().build() + + val driver = GraphDatabase.driver(neo4j.boltUrl, AuthTokens.none()) + val dispatcher = this.coroutineContext[kotlinx.coroutines.CoroutineDispatcher]!! + + DbTransactionMetricsData( + metrics = metrics, + refreshInterval = refreshInterval, + neo4jDriver = driver, + sessionConfig = sessionConfig, + transactionConfig = transactionConfig, + dispatcher = dispatcher + ).use { data -> + val gaugeCaptor = argumentCaptor<() -> Long?>() + verify(metrics) + .addGauge( + eq("last_db_tx_id"), + eq("The transaction commit timestamp of the last processed CDC message"), + any(), + gaugeCaptor.capture(), + ) + + val initialValue = gaugeCaptor.firstValue() + assertNotNull(initialValue, "captured transaction id should not be null") + + commitTransaction(driver) + runCurrent() + + val firstIncrement = gaugeCaptor.firstValue() + assertNotNull(firstIncrement, "the first increment of transaction id should not be null") + assertTrue("transaction id should increment") { initialValue < firstIncrement } + + data.close() + commitTransaction(driver) + advanceTimeBy(refreshInterval) + runCurrent() + + val secondIncrement = gaugeCaptor.firstValue() + assertNotNull(secondIncrement, "the first increment of transaction id should not be null") + assertEquals(firstIncrement, secondIncrement, "transaction id should increment") + } + } + + companion object { + + @Container + val neo4j: Neo4jContainer<*> = + Neo4jContainer(neo4jImage()) + .withEnv("NEO4J_ACCEPT_LICENSE_AGREEMENT", "yes") + .withExposedPorts(7687) + .withoutAuthentication() + + + private fun commitTransaction(driver: Driver) { + driver.session().use { session -> + session.run($$"CREATE (n: TestNode {id: $id})", mapOf("id" to UUID.randomUUID().toString())) + .consume() + } + } + } + +} diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetricsTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetricsTest.kt new file mode 100644 index 00000000..72eddc60 --- /dev/null +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetricsTest.kt @@ -0,0 +1,131 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.neo4j.connectors.kafka.metrics + +import java.lang.management.ManagementFactory +import javax.management.ObjectName +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertTrue +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.doReturn +import org.mockito.kotlin.mock +import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration +import javax.management.MBeanServer + +class JmxMetricsTest { + + private val config = + mock { + on { connectorName } doReturn "my-connector" + on { taskId } doReturn "0" + } + + private lateinit var metrics: JmxMetrics + + @AfterEach + fun tearDown() { + metrics.close() + } + + @Test + fun `should register MBean on initialization`() { + // given MBean is not registered + assertFalse(mbs.isRegistered(objectName)) + + // when + metrics = JmxMetrics(config) + + // then + assertTrue(mbs.isRegistered(objectName)) + } + + @Test + fun `should unregister MBean on close`() { + // given + metrics = JmxMetrics(config) + assertTrue(mbs.isRegistered(objectName)) + + // when + metrics.close() + + // then + assertFalse(mbs.isRegistered(objectName)) + } + + @Test + fun `should provide gauge value via JMX`() { + // given + metrics = JmxMetrics(config) + + // when + metrics.addGauge("test_gauge", "A test gauge", linkedMapOf()) { 42 } + + // then + val value = mbs.getAttribute(objectName, "test_gauge") + assertEquals(42, value) + } + + @Test + fun `should expose MBean info with attributes`() { + // given + metrics = JmxMetrics(config) + + // when + metrics.addGauge("gauge_1", "Description 1", linkedMapOf()) { 1 } + metrics.addGauge("gauge_2", "Description 2", linkedMapOf()) { 2 } + + // then + val info = metrics.getMBeanInfo() + assertNotNull(info) + assertEquals(2, info.attributes.size) + + val attr1 = info.attributes.find { it.name == "gauge_1" } + assertNotNull(attr1) + assertEquals("Description 1", attr1.description) + assertEquals("java.lang.Number", attr1.type) + + val attr2 = info.attributes.find { it.name == "gauge_2" } + assertNotNull(attr2) + assertEquals("Description 2", attr2.description) + assertEquals("java.lang.Number", attr2.type) + } + + @Test + fun `should handle re-registration`() { + // given + metrics = JmxMetrics(config) + assertTrue(mbs.isRegistered(objectName)) + + // create another instance with the same name, should unregister old and register new + val metrics2 = JmxMetrics(config) + assertTrue(mbs.isRegistered(objectName)) + + metrics2.close() + assertFalse(mbs.isRegistered(objectName)) + + // original metrics close should not fail even if already unregistered + metrics.close() + } + + companion object { + val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() + val objectName: ObjectName = ObjectName("kafka.connect:type=plugins,connector=my-connector,task=0") + } +} diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index 98ff7166..330c645e 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -104,7 +104,7 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() neo4jDriver = config.driver, sessionConfig = sessionConfig, transactionConfig = transactionConfig, - refreshTimeout = config.lastDbTxIdRefreshInterval, + refreshInterval = config.lastDbTxIdRefreshInterval, ) } } @@ -113,7 +113,7 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() log.info("stopping") config.close() if (this::dbTransactionMetricsData.isInitialized) { - dbTransactionMetricsData.stop() + dbTransactionMetricsData.close() } if (this::metrics.isInitialized) { metrics.close() From bf7b24bc2177406d715d5e2e577ce96266b46cc3 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Wed, 25 Feb 2026 19:57:51 +0000 Subject: [PATCH 13/21] chore: udate test description --- .../kafka/metrics/DbTransactionMetricsDataTest.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt index 755a3818..9ef1545b 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt @@ -88,7 +88,7 @@ class DbTransactionMetricsDataTest { runCurrent() val secondIncrement = gaugeCaptor.firstValue() - assertNotNull(secondIncrement, "the first increment of transaction id should not be null") + assertNotNull(secondIncrement, "the second increment of transaction id should not be null") assertEquals(firstIncrement + 1, secondIncrement, "transaction id should increment") } } @@ -137,8 +137,8 @@ class DbTransactionMetricsDataTest { runCurrent() val secondIncrement = gaugeCaptor.firstValue() - assertNotNull(secondIncrement, "the first increment of transaction id should not be null") - assertEquals(firstIncrement, secondIncrement, "transaction id should increment") + assertNotNull(secondIncrement, "the second increment of transaction id should not be null") + assertEquals(firstIncrement, secondIncrement, "transaction id should not increment") } } From ba7bdf056b8e9580807569113734323ccabc9fed Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Thu, 26 Feb 2026 09:47:53 +0000 Subject: [PATCH 14/21] chore: spotless apply --- .../kafka/metrics/DbTransactionMetricsData.kt | 18 +-- .../metrics/DbTransactionMetricsDataTest.kt | 153 +++++++++--------- .../kafka/metrics/JmxMetricsTest.kt | 7 +- 3 files changed, 93 insertions(+), 85 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt index fb475774..1c8fc178 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt @@ -16,6 +16,7 @@ */ package org.neo4j.connectors.kafka.metrics +import java.io.Closeable import java.util.concurrent.atomic.AtomicLong import kotlin.time.Duration import kotlinx.coroutines.CoroutineDispatcher @@ -30,17 +31,16 @@ import org.neo4j.driver.AccessMode import org.neo4j.driver.Driver import org.neo4j.driver.SessionConfig import org.neo4j.driver.TransactionConfig -import java.io.Closeable class DbTransactionMetricsData( - metrics: Metrics, - tags: LinkedHashMap = linkedMapOf(), - refreshInterval: Duration, - neo4jDriver: Driver, - sessionConfig: SessionConfig, - transactionConfig: TransactionConfig, - dispatcher: CoroutineDispatcher = Dispatchers.Default, -): Closeable { + metrics: Metrics, + tags: LinkedHashMap = linkedMapOf(), + refreshInterval: Duration, + neo4jDriver: Driver, + sessionConfig: SessionConfig, + transactionConfig: TransactionConfig, + dispatcher: CoroutineDispatcher = Dispatchers.Default, +) : Closeable { private val writeAccessModeSessionConfig: SessionConfig by lazy { val builder = SessionConfig.builder() diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt index 9ef1545b..e9706621 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt @@ -16,6 +16,9 @@ */ package org.neo4j.connectors.kafka.metrics +import java.util.UUID +import kotlin.test.assertEquals +import kotlin.test.assertTrue import kotlin.time.Duration.Companion.milliseconds import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.advanceTimeBy @@ -37,9 +40,6 @@ import org.neo4j.driver.TransactionConfig import org.testcontainers.containers.Neo4jContainer import org.testcontainers.junit.jupiter.Container import org.testcontainers.junit.jupiter.Testcontainers -import java.util.UUID -import kotlin.test.assertEquals -import kotlin.test.assertTrue @OptIn(ExperimentalCoroutinesApi::class) @Testcontainers @@ -57,40 +57,44 @@ class DbTransactionMetricsDataTest { val dispatcher = this.coroutineContext[kotlinx.coroutines.CoroutineDispatcher]!! DbTransactionMetricsData( - metrics = metrics, - refreshInterval = refreshInterval, - neo4jDriver = driver, - sessionConfig = sessionConfig, - transactionConfig = transactionConfig, - dispatcher = dispatcher - ).use { - val gaugeCaptor = argumentCaptor<() -> Long?>() - verify(metrics) - .addGauge( - eq("last_db_tx_id"), - eq("The transaction commit timestamp of the last processed CDC message"), - any(), - gaugeCaptor.capture(), + metrics = metrics, + refreshInterval = refreshInterval, + neo4jDriver = driver, + sessionConfig = sessionConfig, + transactionConfig = transactionConfig, + dispatcher = dispatcher, + ) + .use { + val gaugeCaptor = argumentCaptor<() -> Long?>() + verify(metrics) + .addGauge( + eq("last_db_tx_id"), + eq("The transaction commit timestamp of the last processed CDC message"), + any(), + gaugeCaptor.capture(), + ) + + val initialValue = gaugeCaptor.firstValue() + assertNotNull(initialValue, "captured transaction id should not be null") + + commitTransaction(driver) + runCurrent() + + val firstIncrement = gaugeCaptor.firstValue() + assertNotNull(firstIncrement, "the first increment of transaction id should not be null") + assertTrue("transaction id should increment") { initialValue < firstIncrement } + + commitTransaction(driver) + advanceTimeBy(refreshInterval) + runCurrent() + + val secondIncrement = gaugeCaptor.firstValue() + assertNotNull( + secondIncrement, + "the second increment of transaction id should not be null", ) - - val initialValue = gaugeCaptor.firstValue() - assertNotNull(initialValue, "captured transaction id should not be null") - - commitTransaction(driver) - runCurrent() - - val firstIncrement = gaugeCaptor.firstValue() - assertNotNull(firstIncrement, "the first increment of transaction id should not be null") - assertTrue("transaction id should increment") { initialValue < firstIncrement } - - commitTransaction(driver) - advanceTimeBy(refreshInterval) - runCurrent() - - val secondIncrement = gaugeCaptor.firstValue() - assertNotNull(secondIncrement, "the second increment of transaction id should not be null") - assertEquals(firstIncrement + 1, secondIncrement, "transaction id should increment") - } + assertEquals(firstIncrement + 1, secondIncrement, "transaction id should increment") + } } @Test @@ -105,41 +109,45 @@ class DbTransactionMetricsDataTest { val dispatcher = this.coroutineContext[kotlinx.coroutines.CoroutineDispatcher]!! DbTransactionMetricsData( - metrics = metrics, - refreshInterval = refreshInterval, - neo4jDriver = driver, - sessionConfig = sessionConfig, - transactionConfig = transactionConfig, - dispatcher = dispatcher - ).use { data -> - val gaugeCaptor = argumentCaptor<() -> Long?>() - verify(metrics) - .addGauge( - eq("last_db_tx_id"), - eq("The transaction commit timestamp of the last processed CDC message"), - any(), - gaugeCaptor.capture(), + metrics = metrics, + refreshInterval = refreshInterval, + neo4jDriver = driver, + sessionConfig = sessionConfig, + transactionConfig = transactionConfig, + dispatcher = dispatcher, + ) + .use { data -> + val gaugeCaptor = argumentCaptor<() -> Long?>() + verify(metrics) + .addGauge( + eq("last_db_tx_id"), + eq("The transaction commit timestamp of the last processed CDC message"), + any(), + gaugeCaptor.capture(), + ) + + val initialValue = gaugeCaptor.firstValue() + assertNotNull(initialValue, "captured transaction id should not be null") + + commitTransaction(driver) + runCurrent() + + val firstIncrement = gaugeCaptor.firstValue() + assertNotNull(firstIncrement, "the first increment of transaction id should not be null") + assertTrue("transaction id should increment") { initialValue < firstIncrement } + + data.close() + commitTransaction(driver) + advanceTimeBy(refreshInterval) + runCurrent() + + val secondIncrement = gaugeCaptor.firstValue() + assertNotNull( + secondIncrement, + "the second increment of transaction id should not be null", ) - - val initialValue = gaugeCaptor.firstValue() - assertNotNull(initialValue, "captured transaction id should not be null") - - commitTransaction(driver) - runCurrent() - - val firstIncrement = gaugeCaptor.firstValue() - assertNotNull(firstIncrement, "the first increment of transaction id should not be null") - assertTrue("transaction id should increment") { initialValue < firstIncrement } - - data.close() - commitTransaction(driver) - advanceTimeBy(refreshInterval) - runCurrent() - - val secondIncrement = gaugeCaptor.firstValue() - assertNotNull(secondIncrement, "the second increment of transaction id should not be null") - assertEquals(firstIncrement, secondIncrement, "transaction id should not increment") - } + assertEquals(firstIncrement, secondIncrement, "transaction id should not increment") + } } companion object { @@ -151,13 +159,12 @@ class DbTransactionMetricsDataTest { .withExposedPorts(7687) .withoutAuthentication() - private fun commitTransaction(driver: Driver) { driver.session().use { session -> - session.run($$"CREATE (n: TestNode {id: $id})", mapOf("id" to UUID.randomUUID().toString())) + session + .run("CREATE (n: TestNode {id: ${'$'}id})", mapOf("id" to UUID.randomUUID().toString())) .consume() } } } - } diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetricsTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetricsTest.kt index 72eddc60..68143c66 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetricsTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/JmxMetricsTest.kt @@ -17,6 +17,7 @@ package org.neo4j.connectors.kafka.metrics import java.lang.management.ManagementFactory +import javax.management.MBeanServer import javax.management.ObjectName import kotlin.test.assertEquals import kotlin.test.assertFalse @@ -27,7 +28,6 @@ import org.junit.jupiter.api.Test import org.mockito.kotlin.doReturn import org.mockito.kotlin.mock import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration -import javax.management.MBeanServer class JmxMetricsTest { @@ -119,13 +119,14 @@ class JmxMetricsTest { metrics2.close() assertFalse(mbs.isRegistered(objectName)) - + // original metrics close should not fail even if already unregistered metrics.close() } companion object { val mbs: MBeanServer = ManagementFactory.getPlatformMBeanServer() - val objectName: ObjectName = ObjectName("kafka.connect:type=plugins,connector=my-connector,task=0") + val objectName: ObjectName = + ObjectName("kafka.connect:type=plugins,connector=my-connector,task=0") } } From 5d464ad78bb27dff126ce0d4ee9300d40e45372e Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Thu, 26 Feb 2026 13:56:38 +0000 Subject: [PATCH 15/21] refactor: move tx id properties to source config --- .../kafka/configuration/Neo4jConfiguration.kt | 10 ------- .../Neo4jConfigurationDeclarations.kt | 28 ----------------- .../resources/neo4j-configuration.properties | 4 +-- .../neo4j-source-configuration.properties | 3 ++ .../configuration/Neo4jConfigurationTest.kt | 27 ----------------- .../connectors/kafka/source/Neo4jCdcTask.kt | 2 -- .../kafka/source/SourceConfiguration.kt | 30 +++++++++++++++++++ .../kafka/source/SourceConfigurationTest.kt | 24 +++++++++++++++ 8 files changed, 58 insertions(+), 70 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt index f87a8478..62ec1ecc 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfiguration.kt @@ -143,12 +143,6 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty } } - val lastDbTxIdEnabled - get(): Boolean = getString(METRIC_LAST_TX_ID_ENABLED).toBoolean() - - val lastDbTxIdRefreshInterval - get(): Duration = Duration.parseSimpleString(getString(METRIC_LAST_TX_ID_REFRESH_INTERVAL)) - val driver: Driver by lazy { val config = Config.builder() @@ -280,9 +274,6 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty const val SECURITY_TRUST_STRATEGY = "neo4j.security.trust-strategy" const val SECURITY_CERT_FILES = "neo4j.security.cert-files" - const val METRIC_LAST_TX_ID_ENABLED = "neo4j.metric.last-tx-id.enabled" - const val METRIC_LAST_TX_ID_REFRESH_INTERVAL = "neo4j.metric.last-tx-id.refresh-interval" - // internal properties const val CONNECTOR_NAME = "name" const val TASK_ID = "neo4j.task.id" @@ -311,6 +302,5 @@ open class Neo4jConfiguration(configDef: ConfigDef, originals: Map<*, *>, val ty .defineEncryptionSettings() .definePoolSettings() .defineRetrySettings() - .defineMetricSettings() } } diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationDeclarations.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationDeclarations.kt index 88f2d6a1..30f2545c 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationDeclarations.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationDeclarations.kt @@ -19,7 +19,6 @@ package org.neo4j.connectors.kafka.configuration import java.net.URI import java.util.function.Predicate import kotlin.time.Duration.Companion.milliseconds -import kotlin.time.Duration.Companion.seconds import org.apache.kafka.common.config.ConfigDef import org.apache.kafka.common.config.ConfigDef.Importance import org.apache.kafka.common.config.ConfigDef.Range @@ -387,30 +386,3 @@ fun ConfigDef.defineRetrySettings(): ConfigDef = validator = Validators.pattern(SIMPLE_DURATION_PATTERN) } ) - -fun ConfigDef.defineMetricSettings(): ConfigDef = - this.define( - ConfigKeyBuilder.of( - Neo4jConfiguration.METRIC_LAST_TX_ID_ENABLED, - ConfigDef.Type.STRING, - ) { - importance = Importance.LOW - defaultValue = "false" - group = Groups.CONNECTOR_ADVANCED.title - validator = Validators.bool() - recommender = Recommenders.bool() - documentation = "Whether the last transaction ID metric is enabled." - } - ) - .define( - ConfigKeyBuilder.of( - Neo4jConfiguration.METRIC_LAST_TX_ID_REFRESH_INTERVAL, - ConfigDef.Type.STRING, - ) { - importance = Importance.LOW - defaultValue = 30.seconds.toSimpleString() - group = Groups.CONNECTOR_ADVANCED.title - validator = Validators.pattern(SIMPLE_DURATION_PATTERN) - documentation = "The refresh interval for the last transaction ID metric." - } - ) diff --git a/common/src/main/resources/neo4j-configuration.properties b/common/src/main/resources/neo4j-configuration.properties index 0e7b85db..8d6ec005 100644 --- a/common/src/main/resources/neo4j-configuration.properties +++ b/common/src/main/resources/neo4j-configuration.properties @@ -32,6 +32,4 @@ neo4j.pool.max-connection-pool-size=Type: Integer;\nDescription: Maximum number neo4j.pool.connection-acquisition-timeout=Type: Duration;\nDescription: Maximum duration to wait for acquiring a connection from the connection pool (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.pool.idle-time-before-connection-test=Type: Duration;\nDescription: Duration after which idle connections are tested for liveness (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.pool.max-connection-lifetime=Type: Duration;\nDescription: Duration after which a connection is dropped from the connection pool (valid units are: ``ms`, `s`, `m`, `h` and `d`; default unit is `s`). -neo4j.max-retry-time=Type: Duration;\nDescription: Maximum duration to retry a transaction. -neo4j.metrics.last-db-tx-id-enabled=Type: Boolean;\nDescription: Whether to enable the `last_db_tx_id` metric. -neo4j.metrics.last-db-tx-id-refresh-interval=Type: Duration;\nDescription: Interval between metric updates (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). \ No newline at end of file +neo4j.max-retry-time=Type: Duration;\nDescription: Maximum duration to retry a transaction. \ No newline at end of file diff --git a/common/src/main/resources/neo4j-source-configuration.properties b/common/src/main/resources/neo4j-source-configuration.properties index ab6bc857..47c507bf 100644 --- a/common/src/main/resources/neo4j-source-configuration.properties +++ b/common/src/main/resources/neo4j-source-configuration.properties @@ -31,3 +31,6 @@ neo4j.cdc.use-leader=Type: Boolean;\nDescription: Whether to use leader for chan neo4j.cdc.poll-duration=Type: Duration;\nDescription: Maximum amount of time Kafka Connect poll request will wait for a change to be received from the database (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.cdc.poll-interval=Type: Duration;\nDescription: The interval in which the database will be polled for changes (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.payload-mode=Type: Enum;\nDescription: Defines the structure of change messages generated. `COMPACT` produces messages which are simpler but with potential schema compatibility and type safety issues, while `EXTENDED` produces messages with extra type information which removes the limitations of `COMPACT` mode. For example, a property type change will lead to schema compatibility failures in `COMPACT` mode. `RAW_JSON_STRING` mode generates messages which are serialized as raw JSON strings. Default is `EXTENDED`. +neo4j.cdc.metric.last-tx-id.enabled=Type: Boolean;\nDescription: Whether to enable the `last_db_tx_id` metric. +neo4j.cdc.metric.last-tx-id.refresh-interval=Type: Duration;\nDescription: Interval between metric updates (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). + diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt index 906e8888..7b5b2cfa 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/configuration/Neo4jConfigurationTest.kt @@ -378,33 +378,6 @@ class Neo4jConfigurationTest { } } - @Test - fun `metric settings`() { - Neo4jConfiguration( - Neo4jConfiguration.config(), - mapOf(Neo4jConfiguration.URI to "bolt://localhost"), - ConnectorType.SINK, - ) - .run { - assertFalse(this.lastDbTxIdEnabled) - assertEquals(30.seconds, this.lastDbTxIdRefreshInterval) - } - - Neo4jConfiguration( - Neo4jConfiguration.config(), - mapOf( - Neo4jConfiguration.URI to "bolt://localhost", - Neo4jConfiguration.METRIC_LAST_TX_ID_ENABLED to "true", - Neo4jConfiguration.METRIC_LAST_TX_ID_REFRESH_INTERVAL to "1m", - ), - ConnectorType.SINK, - ) - .run { - assertTrue(this.lastDbTxIdEnabled) - assertEquals(1.minutes, this.lastDbTxIdRefreshInterval) - } - } - companion object { fun newTempFile(prefix: String = "test", suffix: String = ".tmp"): File { val f = File.createTempFile(prefix, suffix) diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index 330c645e..3379eac0 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -37,7 +37,6 @@ import org.neo4j.cdc.client.model.ChangeIdentifier import org.neo4j.connectors.kafka.configuration.helpers.VersionUtil import org.neo4j.connectors.kafka.data.ChangeEventConverter import org.neo4j.connectors.kafka.data.Headers -import org.neo4j.connectors.kafka.data.ValueConverter import org.neo4j.connectors.kafka.metrics.CdcMetricsData import org.neo4j.connectors.kafka.metrics.DbTransactionMetricsData import org.neo4j.connectors.kafka.metrics.Metrics @@ -56,7 +55,6 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() private lateinit var transactionConfig: TransactionConfig private lateinit var cdc: CDCService private lateinit var offset: AtomicReference - private lateinit var converter: ValueConverter private lateinit var changeEventConverter: ChangeEventConverter internal fun latestOffset(): String = offset.get() diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt index ac487579..f2b85a1f 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt @@ -22,6 +22,7 @@ import kotlin.time.Duration.Companion.seconds import kotlin.time.toJavaDuration import org.apache.kafka.common.config.Config import org.apache.kafka.common.config.ConfigDef +import org.apache.kafka.common.config.ConfigDef.Importance import org.apache.kafka.common.config.ConfigDef.Range import org.apache.kafka.common.config.ConfigException import org.neo4j.cdc.client.model.EntityOperation @@ -119,6 +120,12 @@ class SourceConfiguration(originals: Map<*, *>) : val cdcPollingDuration get(): Duration = Duration.parseSimpleString(getString(CDC_POLL_DURATION)) + val lastDbTxIdEnabled + get(): Boolean = getString(CDC_METRIC_LAST_TX_ID_ENABLED).toBoolean() + + val lastDbTxIdRefreshInterval + get(): Duration = Duration.parseSimpleString(getString(CDC_METRIC_LAST_TX_ID_REFRESH_INTERVAL)) + val cdcSelectorsToTopics: Map> by lazy { when (strategy) { SourceType.CDC -> { @@ -515,6 +522,10 @@ class SourceConfiguration(originals: Map<*, *>) : "^neo4j\\.cdc\\.topic\\.(?<$GROUP_NAME_TOPIC>[a-zA-Z0-9._-]+)(\\.patterns)\\.(?<$GROUP_NAME_INDEX>[0-9]+)(\\.metadata)\\.(?<$GROUP_NAME_METADATA>[a-zA-Z0-9._-]+)$" ) + const val CDC_METRIC_LAST_TX_ID_ENABLED = "neo4j.cdc.metric.last-tx-id.enabled" + const val CDC_METRIC_LAST_TX_ID_REFRESH_INTERVAL = + "neo4j.cdc.metric.last-tx-id.refresh-interval" + private val DEFAULT_QUERY_POLL_INTERVAL = 1.seconds private val DEFAULT_QUERY_POLL_DURATION = 5.seconds private const val DEFAULT_BATCH_SIZE = 1000 @@ -753,5 +764,24 @@ class SourceConfiguration(originals: Map<*, *>) : recommender = Recommenders.enum(PayloadMode::class.java) } ) + .define( + ConfigKeyBuilder.of(CDC_METRIC_LAST_TX_ID_ENABLED, ConfigDef.Type.STRING) { + importance = Importance.LOW + defaultValue = "false" + group = Groups.CONNECTOR_ADVANCED.title + validator = Validators.bool() + recommender = Recommenders.bool() + documentation = "Whether the last transaction ID metric is enabled." + } + ) + .define( + ConfigKeyBuilder.of(CDC_METRIC_LAST_TX_ID_REFRESH_INTERVAL, ConfigDef.Type.STRING) { + importance = Importance.LOW + defaultValue = 30.seconds.toSimpleString() + group = Groups.CONNECTOR_ADVANCED.title + validator = Validators.pattern(SIMPLE_DURATION_PATTERN) + documentation = "The refresh interval for the last transaction ID metric." + } + ) } } diff --git a/source/src/test/kotlin/org/neo4j/connectors/kafka/source/SourceConfigurationTest.kt b/source/src/test/kotlin/org/neo4j/connectors/kafka/source/SourceConfigurationTest.kt index 4073cd9d..d19d35d8 100644 --- a/source/src/test/kotlin/org/neo4j/connectors/kafka/source/SourceConfigurationTest.kt +++ b/source/src/test/kotlin/org/neo4j/connectors/kafka/source/SourceConfigurationTest.kt @@ -23,6 +23,8 @@ import io.kotest.matchers.shouldBe import io.kotest.matchers.throwable.shouldHaveMessage import kotlin.test.assertEquals import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertTrue import kotlin.time.Duration.Companion.minutes import kotlin.time.Duration.Companion.seconds import org.apache.kafka.common.config.ConfigException @@ -35,6 +37,8 @@ import org.neo4j.cdc.client.selector.RelationshipSelector import org.neo4j.connectors.kafka.configuration.AuthenticationType import org.neo4j.connectors.kafka.configuration.Neo4jConfiguration import org.neo4j.connectors.kafka.configuration.PayloadMode +import org.neo4j.connectors.kafka.source.SourceConfiguration.Companion.CDC_METRIC_LAST_TX_ID_ENABLED +import org.neo4j.connectors.kafka.source.SourceConfiguration.Companion.CDC_METRIC_LAST_TX_ID_REFRESH_INTERVAL import org.neo4j.driver.AccessMode import org.neo4j.driver.TransactionConfig @@ -796,4 +800,24 @@ class SourceConfigurationTest { config.txConfig() shouldBe TransactionConfig.builder().withMetadata(mapOf("app" to "kafka-source")).build() } + + @Test + fun `metric settings`() { + SourceConfiguration(mapOf(Neo4jConfiguration.URI to "bolt://localhost")).run { + assertFalse(this.lastDbTxIdEnabled) + assertEquals(30.seconds, this.lastDbTxIdRefreshInterval) + } + + SourceConfiguration( + mapOf( + Neo4jConfiguration.URI to "bolt://localhost", + CDC_METRIC_LAST_TX_ID_ENABLED to "true", + CDC_METRIC_LAST_TX_ID_REFRESH_INTERVAL to "1m", + ) + ) + .run { + assertTrue(this.lastDbTxIdEnabled) + assertEquals(1.minutes, this.lastDbTxIdRefreshInterval) + } + } } From 167f58a3d4c9ff94810cca2d5b54cbde5cfc80e6 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Thu, 26 Feb 2026 15:06:58 +0000 Subject: [PATCH 16/21] chore: update default value --- common/src/main/resources/neo4j-source-configuration.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/resources/neo4j-source-configuration.properties b/common/src/main/resources/neo4j-source-configuration.properties index 47c507bf..d89be1d5 100644 --- a/common/src/main/resources/neo4j-source-configuration.properties +++ b/common/src/main/resources/neo4j-source-configuration.properties @@ -32,5 +32,5 @@ neo4j.cdc.poll-duration=Type: Duration;\nDescription: Maximum amount of time Kaf neo4j.cdc.poll-interval=Type: Duration;\nDescription: The interval in which the database will be polled for changes (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.payload-mode=Type: Enum;\nDescription: Defines the structure of change messages generated. `COMPACT` produces messages which are simpler but with potential schema compatibility and type safety issues, while `EXTENDED` produces messages with extra type information which removes the limitations of `COMPACT` mode. For example, a property type change will lead to schema compatibility failures in `COMPACT` mode. `RAW_JSON_STRING` mode generates messages which are serialized as raw JSON strings. Default is `EXTENDED`. neo4j.cdc.metric.last-tx-id.enabled=Type: Boolean;\nDescription: Whether to enable the `last_db_tx_id` metric. -neo4j.cdc.metric.last-tx-id.refresh-interval=Type: Duration;\nDescription: Interval between metric updates (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). +neo4j.cdc.metric.last-tx-id.refresh-interval=Type: Duration;\nDescription: Interval between metric updates (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). Default value is 30 seconds. From a0e463571f453151f402b876ddf5786d839dcb67 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Fri, 27 Feb 2026 11:51:48 +0000 Subject: [PATCH 17/21] refactor: review fixes * update descriptions * update property keys * exception handling for last db tx is metric --- .../kafka/metrics/DbTransactionMetricsData.kt | 58 +++++++++---------- .../neo4j-source-configuration.properties | 4 +- .../kafka/source/SourceConfiguration.kt | 10 ++-- 3 files changed, 34 insertions(+), 38 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt index 1c8fc178..98501a66 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt @@ -27,10 +27,11 @@ import kotlinx.coroutines.cancel import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch -import org.neo4j.driver.AccessMode import org.neo4j.driver.Driver import org.neo4j.driver.SessionConfig import org.neo4j.driver.TransactionConfig +import org.slf4j.Logger +import org.slf4j.LoggerFactory class DbTransactionMetricsData( metrics: Metrics, @@ -42,45 +43,36 @@ class DbTransactionMetricsData( dispatcher: CoroutineDispatcher = Dispatchers.Default, ) : Closeable { - private val writeAccessModeSessionConfig: SessionConfig by lazy { - val builder = SessionConfig.builder() - - sessionConfig.database().ifPresent { builder.withDatabase(it) } - sessionConfig.fetchSize().ifPresent { builder.withFetchSize(it) } - sessionConfig.impersonatedUser().ifPresent { builder.withImpersonatedUser(it) } - sessionConfig.bookmarks()?.let { builder.withBookmarks(it) } - - builder.withDefaultAccessMode(AccessMode.WRITE) - builder.build() - } - private val lastTransactionId = AtomicLong(0) private val scope = CoroutineScope(dispatcher + Job()) init { - metrics.addGauge( - "last_db_tx_id", - "The transaction commit timestamp of the last processed CDC message", - tags, - ) { + metrics.addGauge("last_db_tx_id", "The last committed transaction id in the database", tags) { lastTransactionId.get() } scope.launch { - val databaseName = writeAccessModeSessionConfig.database().orElse("neo4j") + val databaseName = sessionConfig.database().orElse("neo4j") while (isActive) { - val txId = - neo4jDriver.session(writeAccessModeSessionConfig).use { session -> - session - .run( - "SHOW DATABASE $databaseName YIELD lastCommittedTxn RETURN lastCommittedTxn as txId", - transactionConfig, - ) - .single() - .get("txId") - .asLong() - } - lastTransactionId.set(txId) + try { + val txId: Long = + neo4jDriver.session(sessionConfig).use { session -> + session.writeTransaction( + { tx -> + tx.run( + "SHOW DATABASE $databaseName YIELD lastCommittedTxn RETURN lastCommittedTxn as txId" + ) + .single() + .get("txId") + .asLong() + }, + transactionConfig, + ) + } + lastTransactionId.set(txId) + } catch (e: Throwable) { + log.warn("Unexpected error occurred while fetching last committed transaction id", e) + } delay(refreshInterval) } @@ -90,4 +82,8 @@ class DbTransactionMetricsData( override fun close() { scope.cancel() } + + companion object { + private val log: Logger = LoggerFactory.getLogger(DbTransactionMetricsData::class.java) + } } diff --git a/common/src/main/resources/neo4j-source-configuration.properties b/common/src/main/resources/neo4j-source-configuration.properties index d89be1d5..be492048 100644 --- a/common/src/main/resources/neo4j-source-configuration.properties +++ b/common/src/main/resources/neo4j-source-configuration.properties @@ -31,6 +31,6 @@ neo4j.cdc.use-leader=Type: Boolean;\nDescription: Whether to use leader for chan neo4j.cdc.poll-duration=Type: Duration;\nDescription: Maximum amount of time Kafka Connect poll request will wait for a change to be received from the database (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.cdc.poll-interval=Type: Duration;\nDescription: The interval in which the database will be polled for changes (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.payload-mode=Type: Enum;\nDescription: Defines the structure of change messages generated. `COMPACT` produces messages which are simpler but with potential schema compatibility and type safety issues, while `EXTENDED` produces messages with extra type information which removes the limitations of `COMPACT` mode. For example, a property type change will lead to schema compatibility failures in `COMPACT` mode. `RAW_JSON_STRING` mode generates messages which are serialized as raw JSON strings. Default is `EXTENDED`. -neo4j.cdc.metric.last-tx-id.enabled=Type: Boolean;\nDescription: Whether to enable the `last_db_tx_id` metric. -neo4j.cdc.metric.last-tx-id.refresh-interval=Type: Duration;\nDescription: Interval between metric updates (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). Default value is 30 seconds. +neo4j.cdc.metric.last-db-tx-id.enabled=Type: Boolean;\nDescription: Whether to enable the `last_db_tx_id` metric. +neo4j.cdc.metric.last-db-tx-id.refresh-interval=Type: Duration;\nDescription: The refresh interval of the `last_db_tx_id` metric. (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). The default value is 30 seconds. diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt index f2b85a1f..0b796970 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt @@ -522,9 +522,9 @@ class SourceConfiguration(originals: Map<*, *>) : "^neo4j\\.cdc\\.topic\\.(?<$GROUP_NAME_TOPIC>[a-zA-Z0-9._-]+)(\\.patterns)\\.(?<$GROUP_NAME_INDEX>[0-9]+)(\\.metadata)\\.(?<$GROUP_NAME_METADATA>[a-zA-Z0-9._-]+)$" ) - const val CDC_METRIC_LAST_TX_ID_ENABLED = "neo4j.cdc.metric.last-tx-id.enabled" + const val CDC_METRIC_LAST_TX_ID_ENABLED = "neo4j.cdc.metric.last-db-tx-id.enabled" const val CDC_METRIC_LAST_TX_ID_REFRESH_INTERVAL = - "neo4j.cdc.metric.last-tx-id.refresh-interval" + "neo4j.cdc.metric.last-db-tx-id.refresh-interval" private val DEFAULT_QUERY_POLL_INTERVAL = 1.seconds private val DEFAULT_QUERY_POLL_DURATION = 5.seconds @@ -532,7 +532,7 @@ class SourceConfiguration(originals: Map<*, *>) : private val DEFAULT_QUERY_TIMEOUT = 0.seconds private const val DEFAULT_QUERY_FORCE_MAPS_AS_STRUCT = true - private val DEFAULT_CDC_USE_LEADER = false + private const val DEFAULT_CDC_USE_LEADER = false private val DEFAULT_CDC_POLL_INTERVAL = 1.seconds private val DEFAULT_CDC_POLL_DURATION = 5.seconds private const val DEFAULT_STREAMING_PROPERTY = "timestamp" @@ -771,7 +771,7 @@ class SourceConfiguration(originals: Map<*, *>) : group = Groups.CONNECTOR_ADVANCED.title validator = Validators.bool() recommender = Recommenders.bool() - documentation = "Whether the last transaction ID metric is enabled." + documentation = "Whether to enable the `last_db_tx_id` metric." } ) .define( @@ -780,7 +780,7 @@ class SourceConfiguration(originals: Map<*, *>) : defaultValue = 30.seconds.toSimpleString() group = Groups.CONNECTOR_ADVANCED.title validator = Validators.pattern(SIMPLE_DURATION_PATTERN) - documentation = "The refresh interval for the last transaction ID metric." + documentation = "The refresh interval of the `last_db_tx_id` metric." } ) } From bfa81f691616d30a0815be5a67c12ece719d1d87 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Fri, 27 Feb 2026 12:08:15 +0000 Subject: [PATCH 18/21] refactor: pass databaseName to db tx metric data --- .../connectors/kafka/metrics/DbTransactionMetricsData.kt | 9 ++++----- .../kafka/metrics/DbTransactionMetricsDataTest.kt | 7 ++----- .../org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt | 2 +- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt index 98501a66..0d80e887 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt @@ -28,7 +28,6 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.isActive import kotlinx.coroutines.launch import org.neo4j.driver.Driver -import org.neo4j.driver.SessionConfig import org.neo4j.driver.TransactionConfig import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -38,8 +37,8 @@ class DbTransactionMetricsData( tags: LinkedHashMap = linkedMapOf(), refreshInterval: Duration, neo4jDriver: Driver, - sessionConfig: SessionConfig, transactionConfig: TransactionConfig, + databaseName: String, dispatcher: CoroutineDispatcher = Dispatchers.Default, ) : Closeable { @@ -52,15 +51,15 @@ class DbTransactionMetricsData( } scope.launch { - val databaseName = sessionConfig.database().orElse("neo4j") while (isActive) { try { + val explicitDatabaseName = databaseName.ifBlank { "neo4j" } val txId: Long = - neo4jDriver.session(sessionConfig).use { session -> + neo4jDriver.session().use { session -> session.writeTransaction( { tx -> tx.run( - "SHOW DATABASE $databaseName YIELD lastCommittedTxn RETURN lastCommittedTxn as txId" + "SHOW DATABASE $explicitDatabaseName YIELD lastCommittedTxn RETURN lastCommittedTxn as txId" ) .single() .get("txId") diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt index e9706621..d478c78e 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt @@ -35,7 +35,6 @@ import org.neo4j.connectors.kafka.data.TypesTest.Companion.neo4jImage import org.neo4j.driver.AuthTokens import org.neo4j.driver.Driver import org.neo4j.driver.GraphDatabase -import org.neo4j.driver.SessionConfig import org.neo4j.driver.TransactionConfig import org.testcontainers.containers.Neo4jContainer import org.testcontainers.junit.jupiter.Container @@ -50,7 +49,6 @@ class DbTransactionMetricsDataTest { val metrics = mock() val refreshInterval = 100.milliseconds - val sessionConfig = SessionConfig.builder().build() val transactionConfig = TransactionConfig.builder().build() val driver = GraphDatabase.driver(neo4j.boltUrl, AuthTokens.none()) @@ -60,7 +58,7 @@ class DbTransactionMetricsDataTest { metrics = metrics, refreshInterval = refreshInterval, neo4jDriver = driver, - sessionConfig = sessionConfig, + databaseName = "", transactionConfig = transactionConfig, dispatcher = dispatcher, ) @@ -102,7 +100,6 @@ class DbTransactionMetricsDataTest { val metrics = mock() val refreshInterval = 100.milliseconds - val sessionConfig = SessionConfig.builder().build() val transactionConfig = TransactionConfig.builder().build() val driver = GraphDatabase.driver(neo4j.boltUrl, AuthTokens.none()) @@ -112,7 +109,7 @@ class DbTransactionMetricsDataTest { metrics = metrics, refreshInterval = refreshInterval, neo4jDriver = driver, - sessionConfig = sessionConfig, + databaseName = "", transactionConfig = transactionConfig, dispatcher = dispatcher, ) diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index 3379eac0..f87b6969 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -100,7 +100,7 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() DbTransactionMetricsData( metrics = metrics, neo4jDriver = config.driver, - sessionConfig = sessionConfig, + databaseName = config.database, transactionConfig = transactionConfig, refreshInterval = config.lastDbTxIdRefreshInterval, ) From d3356cbaadb35eb8520f5396ddd889461af4311c Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Fri, 27 Feb 2026 13:40:16 +0000 Subject: [PATCH 19/21] fix: update unit test --- .../connectors/kafka/metrics/DbTransactionMetricsDataTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt index d478c78e..7b6492e4 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsDataTest.kt @@ -67,7 +67,7 @@ class DbTransactionMetricsDataTest { verify(metrics) .addGauge( eq("last_db_tx_id"), - eq("The transaction commit timestamp of the last processed CDC message"), + eq("The last committed transaction id in the database"), any(), gaugeCaptor.capture(), ) @@ -118,7 +118,7 @@ class DbTransactionMetricsDataTest { verify(metrics) .addGauge( eq("last_db_tx_id"), - eq("The transaction commit timestamp of the last processed CDC message"), + eq("The last committed transaction id in the database"), any(), gaugeCaptor.capture(), ) From c6d906b295af0364577bc45e5ea6f8f60d871cf7 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Fri, 27 Feb 2026 14:06:02 +0000 Subject: [PATCH 20/21] fix: typo --- common/src/main/resources/neo4j-source-configuration.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/resources/neo4j-source-configuration.properties b/common/src/main/resources/neo4j-source-configuration.properties index be492048..83d5fa39 100644 --- a/common/src/main/resources/neo4j-source-configuration.properties +++ b/common/src/main/resources/neo4j-source-configuration.properties @@ -32,5 +32,5 @@ neo4j.cdc.poll-duration=Type: Duration;\nDescription: Maximum amount of time Kaf neo4j.cdc.poll-interval=Type: Duration;\nDescription: The interval in which the database will be polled for changes (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.payload-mode=Type: Enum;\nDescription: Defines the structure of change messages generated. `COMPACT` produces messages which are simpler but with potential schema compatibility and type safety issues, while `EXTENDED` produces messages with extra type information which removes the limitations of `COMPACT` mode. For example, a property type change will lead to schema compatibility failures in `COMPACT` mode. `RAW_JSON_STRING` mode generates messages which are serialized as raw JSON strings. Default is `EXTENDED`. neo4j.cdc.metric.last-db-tx-id.enabled=Type: Boolean;\nDescription: Whether to enable the `last_db_tx_id` metric. -neo4j.cdc.metric.last-db-tx-id.refresh-interval=Type: Duration;\nDescription: The refresh interval of the `last_db_tx_id` metric. (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). The default value is 30 seconds. +neo4j.cdc.metric.last-db-tx-id.refresh-interval=Type: Duration;\nDescription: The refresh interval of the `last_db_tx_id` metric (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). The default value is 30 seconds. From 1f3e3feb8d8f8e73ecdcbd5dd731f025d5c3c8c6 Mon Sep 17 00:00:00 2001 From: Eugene Rubanov Date: Fri, 27 Feb 2026 16:18:00 +0000 Subject: [PATCH 21/21] refactor: update metrics description --- .../kafka/metrics/CdcMetricsData.kt | 21 +++++++++++++++---- .../kafka/metrics/DbTransactionMetricsData.kt | 3 ++- .../kafka/sink/strategy/cdc/CdcHandler.kt | 3 ++- .../connectors/kafka/source/Neo4jCdcTask.kt | 3 ++- 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt index dddc2c46..78e9707b 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/CdcMetricsData.kt @@ -18,8 +18,13 @@ package org.neo4j.connectors.kafka.metrics import java.util.concurrent.atomic.AtomicLong import org.neo4j.cdc.client.model.ChangeEvent +import org.neo4j.connectors.kafka.configuration.ConnectorType -class CdcMetricsData(metrics: Metrics, tags: LinkedHashMap = linkedMapOf()) { +class CdcMetricsData( + metrics: Metrics, + connectorType: ConnectorType, + tags: LinkedHashMap = linkedMapOf(), +) { private var lastTxCommitTs: AtomicLong = AtomicLong(0L) private var lastTxStartTs: AtomicLong = AtomicLong(0L) @@ -28,21 +33,21 @@ class CdcMetricsData(metrics: Metrics, tags: LinkedHashMap = lin init { metrics.addGauge( "last_cdc_tx_commit_timestamp", - "The transaction commit timestamp of the last processed CDC message", + "The transaction commit timestamp of the last ${connectorType.descriptionActionVerb()} CDC message", tags, ) { lastTxCommitTs.get() } metrics.addGauge( "last_cdc_tx_start_timestamp", - "The transaction start timestamp of the last processed CDC message", + "The transaction start timestamp of the last ${connectorType.descriptionActionVerb()} CDC message", tags, ) { lastTxStartTs.get() } metrics.addGauge( "last_cdc_tx_id", - "The transaction id of the last processed CDC message", + "The transaction id of the last ${connectorType.descriptionActionVerb()} CDC message", tags, ) { lastTxId.get() @@ -56,4 +61,12 @@ class CdcMetricsData(metrics: Metrics, tags: LinkedHashMap = lin } lastTxId.set(event.txId) } + + companion object { + private fun ConnectorType.descriptionActionVerb(): String = + when (this) { + ConnectorType.SOURCE -> "polled" + ConnectorType.SINK -> "pushed" + } + } } diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt index 0d80e887..9ebefa12 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/metrics/DbTransactionMetricsData.kt @@ -59,7 +59,8 @@ class DbTransactionMetricsData( session.writeTransaction( { tx -> tx.run( - "SHOW DATABASE $explicitDatabaseName YIELD lastCommittedTxn RETURN lastCommittedTxn as txId" + "SHOW DATABASE ${"$"}dbName YIELD lastCommittedTxn RETURN lastCommittedTxn as txId", + mapOf("dbName" to explicitDatabaseName), ) .single() .get("txId") diff --git a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt index 599edf43..3ae3ef13 100644 --- a/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt +++ b/sink/src/main/kotlin/org/neo4j/connectors/kafka/sink/strategy/cdc/CdcHandler.kt @@ -18,6 +18,7 @@ package org.neo4j.connectors.kafka.sink.strategy.cdc import org.apache.kafka.connect.data.Struct import org.neo4j.cdc.client.model.ChangeEvent +import org.neo4j.connectors.kafka.configuration.ConnectorType.SINK import org.neo4j.connectors.kafka.data.StreamsTransactionEventExtensions.toChangeEvent import org.neo4j.connectors.kafka.data.toChangeEvent import org.neo4j.connectors.kafka.metrics.CdcMetricsData @@ -36,7 +37,7 @@ class CdcHandler( metrics: Metrics, ) : SinkStrategyHandler { - private val metricsData = CdcMetricsData(metrics) + private val metricsData = CdcMetricsData(metrics, SINK) override fun strategy(): SinkStrategy = strategy diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index f87b6969..f8c1c047 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -34,6 +34,7 @@ import org.neo4j.cdc.client.CDCClient import org.neo4j.cdc.client.CDCService import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.cdc.client.model.ChangeIdentifier +import org.neo4j.connectors.kafka.configuration.ConnectorType.SOURCE import org.neo4j.connectors.kafka.configuration.helpers.VersionUtil import org.neo4j.connectors.kafka.data.ChangeEventConverter import org.neo4j.connectors.kafka.data.Headers @@ -94,7 +95,7 @@ class Neo4jCdcTask(private val metricsFactory: MetricsFactory = MetricsFactory() changeEventConverter = ChangeEventConverter(config.payloadMode) - metricsData = CdcMetricsData(metrics) + metricsData = CdcMetricsData(metrics, SOURCE) if (config.lastDbTxIdEnabled) { dbTransactionMetricsData = DbTransactionMetricsData(