Merge branch 'main' into connect_utils_removal

ali-ince · web-flow · commit f66171f8c408 · 2023-10-15T12:10:47.000+01:00
diff --git a/.teamcity/builds/Build.kt b/.teamcity/builds/Build.kt
@@ -26,8 +26,8 @@ class Build(name: String, branchFilter: String, forPullRequests: Boolean) :
                 artifacts(packaging) {
                   artifactRules =
                       """
-                        +:packaging/*.jar => docker/plugins
-                        -:packaging/*-kc-oss.jar
+                        +:packages/*.jar => docker/plugins
+                        -:packages/*-kc-oss.jar
                       """
                           .trimIndent()
                 }
diff --git a/.teamcity/builds/Common.kt b/.teamcity/builds/Common.kt
@@ -13,6 +13,8 @@ import jetbrains.buildServer.configs.kotlin.buildFeatures.pullRequests
 
 const val GITHUB_OWNER = "neo4j"
 const val GITHUB_REPOSITORY = "neo4j-kafka-connector"
+const val MAVEN_DEFAULT_ARGS =
+    "--no-transfer-progress --batch-mode -Dmaven.repo.local=%teamcity.build.checkoutDir%/.m2"
 
 enum class LinuxSize(val value: String) {
   SMALL("small"),
diff --git a/.teamcity/builds/IntegrationTests.kt b/.teamcity/builds/IntegrationTests.kt
@@ -1,8 +1,8 @@
 package builds
 
 import jetbrains.buildServer.configs.kotlin.BuildStep
-import jetbrains.buildServer.configs.kotlin.BuildStepConditions
 import jetbrains.buildServer.configs.kotlin.BuildType
+import jetbrains.buildServer.configs.kotlin.buildFeatures.buildCache
 import jetbrains.buildServer.configs.kotlin.buildFeatures.dockerSupport
 import jetbrains.buildServer.configs.kotlin.buildSteps.MavenBuildStep
 import jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep
@@ -34,23 +34,36 @@ class IntegrationTests(id: String, name: String, init: BuildType.() -> Unit) :
       </settings>
        */
 
-      artifactRules = "packaging/neo4j-kafka-connect-neo4j-*-SNAPSHOT.jar => docker/plugins"
       params {
         text("env.PACKAGES_USERNAME", "%github-packages-user%")
         password("env.PACKAGES_PASSWORD", "%github-packages-token%")
+
+        text("env.BROKER_EXTERNAL_HOST", "broker:29092")
+        text("env.SCHEMA_CONTROL_REGISTRY_URI", "http://schema-registry:8081")
+        text("env.SCHEMA_CONTROL_REGISTRY_EXTERNAL_URI", "http://schema-registry:8081")
+        text("env.KAFKA_CONNECT_EXTERNAL_URI", "http://connect:8083")
+        text("env.NEO4J_URI", "neo4j://neo4j")
+        text("env.NEO4J_EXTERNAL_URI", "neo4j://neo4j")
+        text("env.NEO4J_USER", "neo4j")
+        text("env.NEO4J_PASSWORD", "password")
       }
 
       steps {
         script {
-          scriptContent = """
+          scriptContent =
+              """
                 #!/bin/bash -eu
                 # TODO: publish custom image instead
                 apt-get update
                 apt-get install --yes ruby-full
                 gem install dip
                 curl -fsSL https://get.docker.com | sh
                 dip compose up -d neo4j zookeeper broker schema-registry control-center
-            """.trimIndent()
+                until [ "`docker inspect -f {{.State.Health.Status}} control-center`"=="healthy" ]; do
+                    sleep 0.1;
+                done;
+            """
+                  .trimIndent()
           formatStderrAsError = true
 
           dockerImagePlatform = ScriptBuildStep.ImagePlatform.Linux
@@ -59,25 +72,30 @@ class IntegrationTests(id: String, name: String, init: BuildType.() -> Unit) :
         }
         maven {
           this.goals = "verify"
-          this.runnerArgs = "-DskipUnitTests"
+          this.runnerArgs = "$MAVEN_DEFAULT_ARGS -DskipUnitTests"
 
           // this is the settings name we uploaded to Connectors project
           userSettingsSelection = "github"
+          localRepoScope = MavenBuildStep.RepositoryScope.MAVEN_DEFAULT
 
           dockerImagePlatform = MavenBuildStep.ImagePlatform.Linux
           dockerImage = "eclipse-temurin:11-jdk"
-          dockerRunParameters = "--volume /var/run/docker.sock:/var/run/docker.sock"
+          dockerRunParameters =
+              "--volume /var/run/docker.sock:/var/run/docker.sock --network neo4j-kafka-connector_default"
         }
         script {
-          scriptContent = """
+          scriptContent =
+              """
                 #!/bin/bash -eu
                 # TODO: publish custom image instead
                 apt-get update
                 apt-get install --yes ruby-full
                 gem install dip
                 curl -fsSL https://get.docker.com | sh
+                dip compose logs --no-color
                 dip compose down --rmi local
-            """.trimIndent()
+            """
+                  .trimIndent()
           formatStderrAsError = true
 
           executionMode = BuildStep.ExecutionMode.ALWAYS
@@ -87,8 +105,17 @@ class IntegrationTests(id: String, name: String, init: BuildType.() -> Unit) :
         }
       }
 
-      features { dockerSupport {} }
+      features {
+        dockerSupport {}
 
-      requirements { runOnLinux(LinuxSize.LARGE) }
+        buildCache {
+          this.name = "neo4j-kafka-connector"
+          publish = true
+          use = true
+          publishOnlyChanged = true
+          rules = ".m2/repository"
+        }
+      }
 
+      requirements { runOnLinux(LinuxSize.LARGE) }
     })
diff --git a/.teamcity/builds/Maven.kt b/.teamcity/builds/Maven.kt
@@ -1,6 +1,7 @@
 package builds
 
 import jetbrains.buildServer.configs.kotlin.BuildType
+import jetbrains.buildServer.configs.kotlin.buildFeatures.buildCache
 import jetbrains.buildServer.configs.kotlin.buildFeatures.dockerSupport
 import jetbrains.buildServer.configs.kotlin.buildSteps.MavenBuildStep
 import jetbrains.buildServer.configs.kotlin.buildSteps.maven
@@ -37,18 +38,29 @@ class Maven(id: String, name: String, goals: String, args: String? = null) :
       steps {
         maven {
           this.goals = goals
-          this.runnerArgs = args
+          this.runnerArgs = "$MAVEN_DEFAULT_ARGS ${args ?: ""}"
 
           // this is the settings name we uploaded to Connectors project
           userSettingsSelection = "github"
+          localRepoScope = MavenBuildStep.RepositoryScope.MAVEN_DEFAULT
 
           dockerImagePlatform = MavenBuildStep.ImagePlatform.Linux
           dockerImage = "eclipse-temurin:11-jdk"
           dockerRunParameters = "--volume /var/run/docker.sock:/var/run/docker.sock"
         }
       }
 
-      features { dockerSupport {} }
+      features {
+        dockerSupport {}
+
+        buildCache {
+          this.name = "neo4j-kafka-connector"
+          publish = true
+          use = true
+          publishOnlyChanged = true
+          rules = ".m2/repository"
+        }
+      }
 
       requirements { runOnLinux(LinuxSize.SMALL) }
     })
diff --git a/dip.yml b/dip.yml
@@ -23,7 +23,7 @@ interaction:
     description: verify connector packages
     runner: local
     command: >
-      sh -c "./mvnw verify"
+      sh -c "./mvnw verify -DskipUnitTests"
     environment:
       BROKER_EXTERNAL_HOST: "localhost:9092"
       SCHEMA_CONTROL_REGISTRY_URI: "http://schema-registry:8081"
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -12,6 +12,13 @@ services:
       NEO4J_ACCEPT_LICENSE_AGREEMENT: "yes"
       NEO4J_server_memory_heap_max__size: "4G"
       NEO4J_internal_dbms_change__data__capture: "true"
+    healthcheck:
+      test: [ "CMD", "cypher-shell", "-u", "neo4j", "-p", "password", "RETURN 1" ]
+      start_period: 2m
+      start_interval: 10s
+      interval: 30s
+      timeout: 10s
+      retries: 5
 
   zookeeper:
     image: confluentinc/cp-zookeeper:7.5.0
@@ -22,6 +29,13 @@ services:
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "localhost", "2181" ]
+      start_period: 5m
+      start_interval: 10s
+      interval: 1m
+      timeout: 10s
+      retries: 5
 
   broker:
     image: confluentinc/cp-server:7.5.0
@@ -51,6 +65,13 @@ services:
       CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
       CONFLUENT_METRICS_ENABLE: 'true'
       CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "localhost", "9092" ]
+      start_period: 5m
+      start_interval: 10s
+      interval: 1m
+      timeout: 10s
+      retries: 5
 
   schema-registry:
     image: confluentinc/cp-schema-registry:7.5.0
@@ -64,6 +85,13 @@ services:
       SCHEMA_REGISTRY_HOST_NAME: schema-registry
       SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
       SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "localhost", "8081" ]
+      start_period: 5m
+      start_interval: 10s
+      interval: 1m
+      timeout: 10s
+      retries: 5
 
   connect:
     image: confluentinc/cp-server-connect:7.5.0
@@ -101,15 +129,27 @@ services:
       - -c
       - |
         /etc/confluent/docker/run
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "localhost", "8083" ]
+      start_period: 5m
+      start_interval: 10s
+      interval: 1m
+      timeout: 10s
+      retries: 5
 
   control-center:
     image: confluentinc/cp-enterprise-control-center:7.5.0
     hostname: control-center
     container_name: control-center
     depends_on:
-      - broker
-      - schema-registry
-      - connect
+      broker:
+        condition: service_healthy
+      schema-registry:
+        condition: service_healthy
+      connect:
+        condition: service_healthy
+      neo4j: # not required but makes it easier to make sure everything is started
+        condition: service_healthy
     ports:
       - "9021:9021"
     environment:
@@ -121,3 +161,10 @@ services:
       CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
       CONFLUENT_METRICS_TOPIC_REPLICATION: 1
       PORT: 9021
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://localhost:9021" ]
+      start_period: 5m
+      start_interval: 10s
+      interval: 1m
+      timeout: 10s
+      retries: 5
diff --git a/source/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceIT.kt b/source/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceIT.kt
@@ -37,7 +37,7 @@ class Neo4jSourceIT {
       streamingFrom = StreamingFrom.ALL,
       streamingQuery =
           "MATCH (ts:TestSource) WHERE ts.timestamp > \$lastCheck RETURN ts.name AS name, ts.surname AS surname, ts.timestamp AS timestamp, ts.execId AS execId",
-      consumerOffset = "latest",
+      consumerOffset = "earliest",
   )
   @Test
   fun `reads latest changes from Neo4j source`(

Original file line number	Diff line number	Diff line change
`@@ -37,7 +37,7 @@ class Neo4jSourceIT {`
`37`	`37`	`streamingFrom = StreamingFrom.ALL,`
`38`	`38`	`streamingQuery =`
`39`	`39`	`"MATCH (ts:TestSource) WHERE ts.timestamp > \$lastCheck RETURN ts.name AS name, ts.surname AS surname, ts.timestamp AS timestamp, ts.execId AS execId",`
`40`		`- consumerOffset = "latest",`
	`40`	`+ consumerOffset = "earliest",`
`41`	`41`	`)`
`42`	`42`	`@Test`
`43`	`43`	fun `reads latest changes from Neo4j source`(