From 247cd68b7070749dc87aa984907134eb04f53752 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Fri, 26 Jul 2024 22:47:40 +0100 Subject: [PATCH 1/9] refactor: rework schema derivation to be more extensible --- .../kafka/data/ChangeEventExtensions.kt | 166 ++--- .../org/neo4j/connectors/kafka/data/Types.kt | 659 ++++++++++-------- .../kafka/data/ChangeEventExtensionsTest.kt | 375 ++++------ .../connectors/kafka/data/DynamicTypesTest.kt | 525 +++++++------- .../neo4j/connectors/kafka/data/TypesTest.kt | 273 ++++---- .../neo4j/connectors/kafka/sink/Neo4jCudIT.kt | 29 +- .../connectors/kafka/sink/Neo4jCypherIT.kt | 78 +-- .../kafka/sink/Neo4jNodePatternIT.kt | 11 +- .../kafka/sink/Neo4jRelationshipPatternIT.kt | 12 +- .../sink/strategy/NodePatternHandlerTest.kt | 9 +- .../kafka/source/Neo4jCdcSourceIT.kt | 26 +- .../kafka/source/Neo4jSourceQueryIT.kt | 26 +- 12 files changed, 1032 insertions(+), 1157 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt index 79dea2b33..9d8b77775 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt @@ -44,9 +44,9 @@ class ChangeEventConverter( private fun toConnectSchema(changeEvent: ChangeEvent): Schema = SchemaBuilder.struct() - .field("id", SimpleTypes.STRING.schema()) - .field("txId", SimpleTypes.LONG.schema()) - .field("seq", SimpleTypes.LONG.schema()) + .field("id", Schema.STRING_SCHEMA) + .field("txId", Schema.INT64_SCHEMA) + .field("seq", Schema.INT64_SCHEMA) .field("metadata", metadataToConnectSchema(changeEvent.metadata)) .field("event", eventToConnectSchema(changeEvent.event)) .build() @@ -64,15 +64,15 @@ class ChangeEventConverter( internal fun metadataToConnectSchema(metadata: Metadata): Schema = SchemaBuilder.struct() - .field("authenticatedUser", SimpleTypes.STRING.schema()) - .field("executingUser", SimpleTypes.STRING.schema()) - .field("connectionType", SimpleTypes.STRING.schema(true)) - .field("connectionClient", SimpleTypes.STRING.schema(true)) - .field("connectionServer", SimpleTypes.STRING.schema(true)) - .field("serverId", SimpleTypes.STRING.schema()) - .field("captureMode", SimpleTypes.STRING.schema()) - .field("txStartTime", SimpleTypes.ZONEDDATETIME_STRUCT.schema()) - .field("txCommitTime", SimpleTypes.ZONEDDATETIME_STRUCT.schema()) + .field("authenticatedUser", Schema.STRING_SCHEMA) + .field("executingUser", Schema.STRING_SCHEMA) + .field("connectionType", Schema.OPTIONAL_STRING_SCHEMA) + .field("connectionClient", Schema.OPTIONAL_STRING_SCHEMA) + .field("connectionServer", Schema.OPTIONAL_STRING_SCHEMA) + .field("serverId", Schema.STRING_SCHEMA) + .field("captureMode", Schema.STRING_SCHEMA) + .field("txStartTime", propertyType) + .field("txCommitTime", propertyType) .field( "txMetadata", toConnectSchema( @@ -102,14 +102,8 @@ class ChangeEventConverter( it.put("connectionServer", metadata.connectionServer) it.put("serverId", metadata.serverId) it.put("captureMode", metadata.captureMode.name) - it.put( - "txStartTime", - DynamicTypes.toConnectValue( - SimpleTypes.ZONEDDATETIME_STRUCT.schema(), metadata.txStartTime)) - it.put( - "txCommitTime", - DynamicTypes.toConnectValue( - SimpleTypes.ZONEDDATETIME_STRUCT.schema(), metadata.txCommitTime)) + it.put("txStartTime", DynamicTypes.toConnectValue(propertyType, metadata.txStartTime)) + it.put("txCommitTime", DynamicTypes.toConnectValue(propertyType, metadata.txCommitTime)) it.put( "txMetadata", DynamicTypes.toConnectValue(schema.field("txMetadata").schema(), metadata.txMetadata)) @@ -138,10 +132,10 @@ class ChangeEventConverter( internal fun nodeEventToConnectSchema(nodeEvent: NodeEvent): Schema = SchemaBuilder.struct() - .field("elementId", SimpleTypes.STRING.schema()) - .field("eventType", SimpleTypes.STRING.schema()) - .field("operation", SimpleTypes.STRING.schema()) - .field("labels", SchemaBuilder.array(SimpleTypes.STRING.schema()).build()) + .field("elementId", Schema.STRING_SCHEMA) + .field("eventType", Schema.STRING_SCHEMA) + .field("operation", Schema.STRING_SCHEMA) + .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field("keys", schemaForKeysByLabel(nodeEvent.keys)) .field("state", nodeStateSchema(nodeEvent.before, nodeEvent.after)) .build() @@ -161,13 +155,13 @@ class ChangeEventConverter( internal fun relationshipEventToConnectSchema(relationshipEvent: RelationshipEvent): Schema = SchemaBuilder.struct() - .field("elementId", SimpleTypes.STRING.schema()) - .field("eventType", SimpleTypes.STRING.schema()) - .field("operation", SimpleTypes.STRING.schema()) - .field("type", SimpleTypes.STRING.schema()) + .field("elementId", Schema.STRING_SCHEMA) + .field("eventType", Schema.STRING_SCHEMA) + .field("operation", Schema.STRING_SCHEMA) + .field("type", Schema.STRING_SCHEMA) .field("start", nodeToConnectSchema(relationshipEvent.start)) .field("end", nodeToConnectSchema(relationshipEvent.end)) - .field("keys", schemaForKeys(relationshipEvent.keys)) + .field("keys", schemaForKeys()) .field( "state", relationshipStateSchema(relationshipEvent.before, relationshipEvent.after)) .build() @@ -194,8 +188,8 @@ class ChangeEventConverter( internal fun nodeToConnectSchema(node: Node): Schema { return SchemaBuilder.struct() - .field("elementId", SimpleTypes.STRING.schema()) - .field("labels", SchemaBuilder.array(SimpleTypes.STRING.schema()).build()) + .field("elementId", Schema.STRING_SCHEMA) + .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field("keys", schemaForKeysByLabel(node.keys)) .build() } @@ -209,31 +203,13 @@ class ChangeEventConverter( private fun schemaForKeysByLabel(keys: Map>>?): Schema { return SchemaBuilder.struct() - .apply { keys?.forEach { field(it.key, schemaForKeys(it.value)) } } + .apply { keys?.forEach { field(it.key, schemaForKeys()) } } .optional() .build() } - private fun schemaForKeys(keys: List>?): Schema { - return SchemaBuilder.array( - // We need to define a uniform structure of key array elements. Because all elements - // must have identical structure, we list all available keys as optional fields. - SchemaBuilder.struct() - .apply { - keys?.forEach { key -> - key.forEach { - field( - it.key, - toConnectSchema( - it.value, - optional = true, - forceMapsAsStruct = true, - temporalDataSchemaType = temporalDataSchemaType)) - } - } - } - .optional() - .build()) + private fun schemaForKeys(): Schema { + return SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build() } @@ -242,27 +218,9 @@ class ChangeEventConverter( val stateSchema = SchemaBuilder.struct() .apply { - this.field("labels", SchemaBuilder.array(SimpleTypes.STRING.schema()).build()) + this.field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) this.field( - "properties", - SchemaBuilder.struct() - .also { - // TODO: should we check for incompatible types for the existing value, - // and what happens in that case? - val combinedProperties = - (before?.properties ?: mapOf()) + (after?.properties ?: mapOf()) - combinedProperties.toSortedMap().forEach { entry -> - if (it.field(entry.key) == null) { - it.field( - entry.key, - toConnectSchema( - entry.value, - optional = true, - temporalDataSchemaType = temporalDataSchemaType)) - } - } - } - .build()) + "properties", SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) } .optional() .build() @@ -279,8 +237,9 @@ class ChangeEventConverter( it.put("labels", before.labels) it.put( "properties", - DynamicTypes.toConnectValue( - it.schema().field("properties").schema(), before.properties)) + before.properties.mapValues { e -> + DynamicTypes.toConnectValue(propertyType, e.value) + }) }) } @@ -291,8 +250,9 @@ class ChangeEventConverter( it.put("labels", after.labels) it.put( "properties", - DynamicTypes.toConnectValue( - it.schema().field("properties").schema(), after.properties)) + after.properties.mapValues { e -> + DynamicTypes.toConnectValue(propertyType, e.value) + }) }) } } @@ -305,25 +265,7 @@ class ChangeEventConverter( SchemaBuilder.struct() .apply { this.field( - "properties", - SchemaBuilder.struct() - .also { - // TODO: should we check for incompatible types for the existing value, - // and what happens in that case? - val combinedProperties = - (before?.properties ?: mapOf()) + (after?.properties ?: mapOf()) - combinedProperties.toSortedMap().forEach { entry -> - if (it.field(entry.key) == null) { - it.field( - entry.key, - toConnectSchema( - entry.value, - optional = true, - temporalDataSchemaType = temporalDataSchemaType)) - } - } - } - .build()) + "properties", SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) } .optional() .build() @@ -343,8 +285,9 @@ class ChangeEventConverter( Struct(this.schema().field("before").schema()).also { it.put( "properties", - DynamicTypes.toConnectValue( - it.schema().field("properties").schema(), before.properties)) + before.properties.mapValues { e -> + DynamicTypes.toConnectValue(propertyType, e.value) + }) }) } @@ -354,8 +297,9 @@ class ChangeEventConverter( Struct(this.schema().field("after").schema()).also { it.put( "properties", - DynamicTypes.toConnectValue( - it.schema().field("properties").schema(), after.properties)) + after.properties.mapValues { e -> + DynamicTypes.toConnectValue(propertyType, e.value) + }) }) } } @@ -424,20 +368,20 @@ internal fun Struct.toNodeState(): Pair = Pair( getStruct("before")?.let { val labels = it.getArray("labels") - val properties = it.getStruct("properties") + val properties = it.getMap("properties") NodeState( labels, - DynamicTypes.fromConnectValue(properties.schema(), properties, true) - as Map, + DynamicTypes.fromConnectValue( + it.schema().field("properties").schema(), properties, true) as Map, ) }, getStruct("after")?.let { val labels = it.getArray("labels") - val properties = it.getStruct("properties") + val properties = it.getMap("properties") NodeState( labels, - DynamicTypes.fromConnectValue(properties.schema(), properties, true) - as Map, + DynamicTypes.fromConnectValue( + it.schema().field("properties").schema(), properties, true) as Map, ) }, ) @@ -446,17 +390,17 @@ internal fun Struct.toNodeState(): Pair = internal fun Struct.toRelationshipState(): Pair = Pair( getStruct("before")?.let { - val properties = it.getStruct("properties") + val properties = it.getMap("properties") RelationshipState( - DynamicTypes.fromConnectValue(properties.schema(), properties, true) - as Map, + DynamicTypes.fromConnectValue( + it.schema().field("properties").schema(), properties, true) as Map, ) }, getStruct("after")?.let { - val properties = it.getStruct("properties") + val properties = it.getMap("properties") RelationshipState( - DynamicTypes.fromConnectValue(properties.schema(), properties, true) - as Map, + DynamicTypes.fromConnectValue( + it.schema().field("properties").schema(), properties, true) as Map, ) }, ) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt index c45a96f01..73139e6de 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt @@ -17,13 +17,11 @@ package org.neo4j.connectors.kafka.data import java.nio.ByteBuffer -import java.time.Instant import java.time.LocalDate import java.time.LocalDateTime import java.time.LocalTime import java.time.OffsetDateTime import java.time.OffsetTime -import java.time.ZoneId import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter @@ -61,111 +59,249 @@ const val DIMENSION = "dimension" const val TWO_D: Byte = 2 const val THREE_D: Byte = 3 -@Suppress("SpellCheckingInspection") -enum class SimpleTypes(builder: () -> SchemaBuilder) { - NULL({ SchemaBuilder.struct().optional().namespaced("NULL") }), - BOOLEAN({ SchemaBuilder.bool() }), - LONG({ SchemaBuilder.int64() }), - FLOAT({ SchemaBuilder.float64() }), - STRING({ SchemaBuilder.string() }), - BYTES({ SchemaBuilder.bytes() }), - LOCALDATE({ SchemaBuilder.string().namespaced("LocalDate") }), - LOCALDATETIME({ SchemaBuilder.string().namespaced("LocalDateTime") }), - LOCALTIME({ SchemaBuilder.string().namespaced("LocalTime") }), - ZONEDDATETIME({ SchemaBuilder.string().namespaced("ZonedDateTime") }), - OFFSETTIME({ SchemaBuilder.string().namespaced("OffsetTime") }), - // PROTOBUF converter does not persist our provided version value hence looses it during reads - // That's why we suffix the type names with an explicit version specifier - LOCALDATE_STRUCT({ - SchemaBuilder.struct().namespaced("LocalDateStruct").field(EPOCH_DAYS, Schema.INT64_SCHEMA) - }), - LOCALDATETIME_STRUCT({ - SchemaBuilder.struct() - .namespaced("LocalDateTimeStruct") - .field(EPOCH_DAYS, Schema.INT64_SCHEMA) - .field(NANOS_OF_DAY, Schema.INT64_SCHEMA) - }), - LOCALTIME_STRUCT({ - SchemaBuilder.struct().namespaced("LocalTimeStruct").field(NANOS_OF_DAY, Schema.INT64_SCHEMA) - }), - ZONEDDATETIME_STRUCT({ - SchemaBuilder.struct() - .namespaced("ZonedDateTimeStruct") - .field(EPOCH_SECONDS, Schema.INT64_SCHEMA) - .field(NANOS_OF_SECOND, Schema.INT32_SCHEMA) - .field(ZONE_ID, Schema.STRING_SCHEMA) - }), - OFFSETTIME_STRUCT({ - SchemaBuilder.struct() - .namespaced("OffsetTimeStruct") - .field(NANOS_OF_DAY, Schema.INT64_SCHEMA) - .field(ZONE_ID, Schema.STRING_SCHEMA) - }), - DURATION({ +val durationSchema: Schema = SchemaBuilder(Schema.Type.STRUCT) - .namespaced("Duration") .field(MONTHS, Schema.INT64_SCHEMA) .field(DAYS, Schema.INT64_SCHEMA) .field(SECONDS, Schema.INT64_SCHEMA) .field(NANOS, Schema.INT32_SCHEMA) - }), - POINT({ + .optional() + .build() + +val pointSchema: Schema = SchemaBuilder(Schema.Type.STRUCT) - .namespaced("Point") .field(DIMENSION, Schema.INT8_SCHEMA) .field(SR_ID, Schema.INT32_SCHEMA) .field(X, Schema.FLOAT64_SCHEMA) .field(Y, Schema.FLOAT64_SCHEMA) .field(Z, Schema.OPTIONAL_FLOAT64_SCHEMA) - }); - - // fully namespaced schema name - val id: String - // just schema name, excluding namespace - private val shortId: String - val schema: Schema = builder().build() - private val optionalSchema: Schema = builder().optional().build() - - init { - if (schema.type() == Schema.Type.STRUCT && schema.name().isNullOrBlank()) { - throw IllegalArgumentException("schema name is required for STRUCT types") - } - - id = schema.id() - shortId = schema.shortId() - } - - fun schema(optional: Boolean = false) = if (optional) optionalSchema else schema - - fun matches(other: Schema): Boolean { - return this.id == other.id() || this.shortId == other.shortId() - } + .optional() + .build() + +const val BOOLEAN = "B" +const val BOOLEAN_LIST = "LB" +const val LONG = "I64" +const val LONG_LIST = "LI64" +const val FLOAT = "F64" +const val FLOAT_LIST = "LF64" +const val STRING = "S" +const val STRING_LIST = "LS" +const val BYTES = "BA" +const val LOCAL_DATE = "TLD" +const val LOCAL_DATE_LIST = "LTLD" +const val LOCAL_DATE_TIME = "TLDT" +const val LOCAL_DATE_TIME_LIST = "LTLDT" +const val LOCAL_TIME = "TLT" +const val LOCAL_TIME_LIST = "LTLT" +const val ZONED_DATE_TIME = "TZDT" +const val ZONED_DATE_TIME_LIST = "LZDT" +const val OFFSET_TIME = "TOT" +const val OFFSET_TIME_LIST = "LTOT" +const val DURATION = "TD" +const val DURATION_LIST = "LTD" +const val POINT = "SP" +const val POINT_LIST = "LSP" + +val propertyType: Schema = + SchemaBuilder.struct() + .namespaced("Neo4jSimpleType") + .field(BOOLEAN, Schema.OPTIONAL_BOOLEAN_SCHEMA) + .field(LONG, Schema.OPTIONAL_INT64_SCHEMA) + .field(FLOAT, Schema.OPTIONAL_FLOAT64_SCHEMA) + .field(STRING, Schema.OPTIONAL_STRING_SCHEMA) + .field(BYTES, Schema.OPTIONAL_BYTES_SCHEMA) + .field(LOCAL_DATE, Schema.OPTIONAL_STRING_SCHEMA) + .field(LOCAL_DATE_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(LOCAL_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(ZONED_DATE_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(OFFSET_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(DURATION, durationSchema) + .field(POINT, pointSchema) + .field(BOOLEAN_LIST, SchemaBuilder.array(Schema.BOOLEAN_SCHEMA).optional().build()) + .field(LONG_LIST, SchemaBuilder.array(Schema.INT64_SCHEMA).optional().build()) + .field(FLOAT_LIST, SchemaBuilder.array(Schema.FLOAT64_SCHEMA).optional().build()) + .field(STRING_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(LOCAL_DATE_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(LOCAL_DATE_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(LOCAL_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(ZONED_DATE_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(OFFSET_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(DURATION_LIST, SchemaBuilder.array(durationSchema).optional().build()) + .field(POINT_LIST, SchemaBuilder.array(pointSchema).optional().build()) + .optional() + .build() + +fun Schema.matches(other: Schema): Boolean { + return this.id() == other.id() || this.shortId() == other.shortId() } object DynamicTypes { + @Suppress("UNCHECKED_CAST") fun toConnectValue(schema: Schema, value: Any?): Any? { if (value == null) { return null } + if (schema == propertyType) { + return when (value) { + is Boolean -> Struct(propertyType).put(BOOLEAN, value) + is Float -> Struct(propertyType).put(FLOAT, value.toDouble()) + is Double -> Struct(propertyType).put(FLOAT, value) + is Number -> Struct(propertyType).put(LONG, value.toLong()) + is String -> Struct(propertyType).put(STRING, value) + is Char -> Struct(propertyType).put(STRING, value.toString()) + is CharArray -> Struct(propertyType).put(STRING, String(value)) + is ByteArray -> Struct(propertyType).put(BYTES, value) + is ByteBuffer -> Struct(propertyType).put(BYTES, value.array()) + is LocalDate -> + Struct(propertyType).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(value)) + is LocalDateTime -> + Struct(propertyType).put(LOCAL_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) + is LocalTime -> + Struct(propertyType).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(value)) + is OffsetDateTime -> + Struct(propertyType).put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) + is ZonedDateTime -> + Struct(propertyType).put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) + is OffsetTime -> + Struct(propertyType).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(value)) + is IsoDuration -> + Struct(propertyType) + .put( + DURATION, + Struct(durationSchema) + .put(MONTHS, value.months()) + .put(DAYS, value.days()) + .put(SECONDS, value.seconds()) + .put(NANOS, value.nanoseconds())) + is Point -> + Struct(propertyType) + .put( + POINT, + Struct(pointSchema) + .put(SR_ID, value.srid()) + .put(X, value.x()) + .put(Y, value.y()) + .also { + it.put(DIMENSION, if (value.z().isNaN()) TWO_D else THREE_D) + if (!value.z().isNaN()) { + it.put(Z, value.z()) + } + }) + is ShortArray -> Struct(propertyType).put(LONG_LIST, value.map { s -> s.toLong() }.toList()) + is IntArray -> Struct(propertyType).put(LONG_LIST, value.map { s -> s.toLong() }.toList()) + is LongArray -> Struct(propertyType).put(LONG_LIST, value.toList()) + is FloatArray -> + Struct(propertyType).put(FLOAT_LIST, value.map { s -> s.toDouble() }.toList()) + is DoubleArray -> Struct(propertyType).put(FLOAT_LIST, value.toList()) + is BooleanArray -> Struct(propertyType).put(BOOLEAN_LIST, value.toList()) + is Array<*> -> + when (val componentType = value::class.java.componentType.kotlin) { + Boolean::class -> Struct(propertyType).put(BOOLEAN_LIST, value.toList()) + Byte::class -> Struct(propertyType).put(BYTES, (value as Array).toByteArray()) + Short::class -> + Struct(propertyType) + .put(LONG_LIST, (value as Array).map { s -> s.toLong() }.toList()) + Int::class -> + Struct(propertyType) + .put(LONG_LIST, (value as Array).map { s -> s.toLong() }.toList()) + Long::class -> Struct(propertyType).put(LONG_LIST, (value as Array).toList()) + Float::class -> + Struct(propertyType) + .put(FLOAT_LIST, (value as Array).map { s -> s.toDouble() }.toList()) + Double::class -> + Struct(propertyType).put(FLOAT_LIST, (value as Array).toList()) + String::class -> Struct(propertyType).put(STRING_LIST, value.toList()) + LocalDate::class -> + Struct(propertyType) + .put( + LOCAL_DATE_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_DATE.format(s) } + .toList()) + LocalDateTime::class -> + Struct(propertyType) + .put( + LOCAL_DATE_TIME_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } + .toList()) + LocalTime::class -> + Struct(propertyType) + .put( + LOCAL_TIME_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_TIME.format(s) } + .toList()) + OffsetDateTime::class -> + Struct(propertyType) + .put( + ZONED_DATE_TIME_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } + .toList()) + ZonedDateTime::class -> + Struct(propertyType) + .put( + ZONED_DATE_TIME_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } + .toList()) + OffsetTime::class -> + Struct(propertyType) + .put( + OFFSET_TIME_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_TIME.format(s) } + .toList()) + else -> + if (IsoDuration::class.java.isAssignableFrom(componentType.java)) { + Struct(propertyType) + .put( + DURATION_LIST, + value + .map { s -> s as IsoDuration } + .map { + Struct(durationSchema) + .put(MONTHS, it.months()) + .put(DAYS, it.days()) + .put(SECONDS, it.seconds()) + .put(NANOS, it.nanoseconds()) + } + .toList()) + } else if (Point::class.java.isAssignableFrom(componentType.java)) { + Struct(propertyType) + .put( + POINT_LIST, + value + .map { s -> s as Point } + .map { s -> + Struct(pointSchema) + .put(SR_ID, s.srid()) + .put(X, s.x()) + .put(Y, s.y()) + .also { + it.put(DIMENSION, if (s.z().isNaN()) TWO_D else THREE_D) + if (!s.z().isNaN()) { + it.put(Z, s.z()) + } + } + } + .toList()) + } else { + throw IllegalArgumentException( + "unsupported array type: array of ${value.javaClass.componentType.name}") + } + } + else -> throw IllegalArgumentException("unsupported property type: ${value.javaClass.name}") + } + } + return when (schema.type()) { - Schema.Type.BYTES -> - when (value) { - is ByteArray -> value - is ByteBuffer -> value.array() - else -> throw IllegalArgumentException("unsupported bytes type ${value.javaClass.name}") - } Schema.Type.ARRAY -> when (value) { is Collection<*> -> value.map { toConnectValue(schema.valueSchema(), it) } - is Array<*> -> value.map { toConnectValue(schema.valueSchema(), it) }.toList() - is ShortArray -> value.map { s -> s.toLong() }.toList() - is IntArray -> value.map { i -> i.toLong() }.toList() - is FloatArray -> value.map { f -> f.toDouble() }.toList() - is BooleanArray -> value.toList() - is LongArray -> value.toList() - is DoubleArray -> value.toList() else -> throw IllegalArgumentException("unsupported array type ${value.javaClass.name}") } Schema.Type.MAP -> @@ -175,43 +311,12 @@ object DynamicTypes { } Schema.Type.STRUCT -> when (value) { - is LocalDate -> Struct(schema).put(EPOCH_DAYS, value.toEpochDay()) - is LocalTime -> Struct(schema).put(NANOS_OF_DAY, value.toNanoOfDay()) - is LocalDateTime -> - Struct(schema) - .put(EPOCH_DAYS, value.toLocalDate().toEpochDay()) - .put(NANOS_OF_DAY, value.toLocalTime().toNanoOfDay()) - is OffsetTime -> - Struct(schema) - .put(NANOS_OF_DAY, value.toLocalTime().toNanoOfDay()) - .put(ZONE_ID, value.offset.id) - is ZonedDateTime -> - Struct(schema) - .put(EPOCH_SECONDS, value.toEpochSecond()) - .put(NANOS_OF_SECOND, value.nano) - .put(ZONE_ID, value.zone.id) - is OffsetDateTime -> - Struct(schema) - .put(EPOCH_SECONDS, value.toEpochSecond()) - .put(NANOS_OF_SECOND, value.nano) - .put(ZONE_ID, value.offset.id) - is IsoDuration -> - Struct(schema) - .put(MONTHS, value.months()) - .put(DAYS, value.days()) - .put(SECONDS, value.seconds()) - .put(NANOS, value.nanoseconds()) - is Point -> - Struct(schema).put(SR_ID, value.srid()).put(X, value.x()).put(Y, value.y()).also { - it.put(DIMENSION, if (value.z().isNaN()) TWO_D else THREE_D) - if (!value.z().isNaN()) { - it.put(Z, value.z()) - } - } is Node -> Struct(schema).apply { - put("", value.id()) - put("", value.labels()) + put("", toConnectValue(propertyType, value.id())) + put( + "", + toConnectValue(propertyType, value.labels().toList().toTypedArray())) value .asMap { it.asObject() } @@ -221,10 +326,10 @@ object DynamicTypes { } is Relationship -> Struct(schema).apply { - put("", value.id()) - put("", value.type()) - put("", value.startNodeId()) - put("", value.endNodeId()) + put("", toConnectValue(propertyType, value.id())) + put("", toConnectValue(propertyType, value.type())) + put("", toConnectValue(propertyType, value.startNodeId())) + put("", toConnectValue(propertyType, value.endNodeId())) value .asMap { it.asObject() } @@ -250,29 +355,6 @@ object DynamicTypes { else -> throw IllegalArgumentException("unsupported struct type ${value.javaClass.name}") } - Schema.Type.STRING -> - when (value) { - is LocalDate -> DateTimeFormatter.ISO_DATE.format(value) - is LocalDateTime -> DateTimeFormatter.ISO_DATE_TIME.format(value) - is LocalTime -> DateTimeFormatter.ISO_TIME.format(value) - is OffsetDateTime -> DateTimeFormatter.ISO_DATE_TIME.format(value) - is ZonedDateTime -> DateTimeFormatter.ISO_DATE_TIME.format(value) - is OffsetTime -> DateTimeFormatter.ISO_TIME.format(value) - is String -> value - is Char -> value.toString() - is CharArray -> String(value) - else -> - throw IllegalArgumentException("unsupported string type ${value.javaClass.name}") - } - Schema.Type.INT64, - Schema.Type.FLOAT64 -> - when (value) { - is Float -> value.toDouble() - is Double -> value - is Number -> value.toLong() - else -> - throw IllegalArgumentException("unsupported numeric type ${value.javaClass.name}") - } else -> value } } @@ -297,95 +379,106 @@ object DynamicTypes { else -> throw IllegalArgumentException("unsupported bytes type ${value.javaClass.name}") } Schema.Type.STRING -> - when { - SimpleTypes.LOCALDATE.matches(schema) -> - (value as String?)?.let { - DateTimeFormatter.ISO_DATE.parse(it) { parsed -> LocalDate.from(parsed) } - } - SimpleTypes.LOCALTIME.matches(schema) -> - (value as String?)?.let { - DateTimeFormatter.ISO_TIME.parse(it) { parsed -> LocalTime.from(parsed) } - } - SimpleTypes.LOCALDATETIME.matches(schema) -> - (value as String?)?.let { - DateTimeFormatter.ISO_DATE_TIME.parse(it) { parsed -> LocalDateTime.from(parsed) } - } - SimpleTypes.ZONEDDATETIME.matches(schema) -> - (value as String?)?.let { - DateTimeFormatter.ISO_DATE_TIME.parse(it) { parsed -> - val zoneId = parsed.query(TemporalQueries.zone()) - - if (zoneId is ZoneOffset) { - OffsetDateTime.from(parsed) - } else { - ZonedDateTime.from(parsed) - } - } - } - SimpleTypes.OFFSETTIME.matches(schema) -> - (value as String?)?.let { - DateTimeFormatter.ISO_TIME.parse(it) { parsed -> OffsetTime.from(parsed) } - } - else -> value as String? + when (value) { + is Char -> value.toString() + is CharArray -> value.toString() + is CharSequence -> value.toString() + else -> + throw IllegalArgumentException("unsupported string type ${value.javaClass.name}") } Schema.Type.STRUCT -> when { - SimpleTypes.LOCALDATE_STRUCT.matches(schema) -> - (value as Struct?)?.let { LocalDate.ofEpochDay(it.getInt64(EPOCH_DAYS)) } - SimpleTypes.LOCALTIME_STRUCT.matches(schema) -> - (value as Struct?)?.let { LocalTime.ofNanoOfDay(it.getInt64(NANOS_OF_DAY)) } - SimpleTypes.LOCALDATETIME_STRUCT.matches(schema) -> - (value as Struct?)?.let { - LocalDateTime.of( - LocalDate.ofEpochDay(it.getInt64(EPOCH_DAYS)), - LocalTime.ofNanoOfDay(it.getInt64(NANOS_OF_DAY))) - } - SimpleTypes.ZONEDDATETIME_STRUCT.matches(schema) -> - (value as Struct?)?.let { - val instant = - Instant.ofEpochSecond( - it.getInt64(EPOCH_SECONDS), it.getInt32(NANOS_OF_SECOND).toLong()) - val zoneId = ZoneId.of(it.getString(ZONE_ID)) - if (zoneId is ZoneOffset) { - OffsetDateTime.ofInstant(instant, zoneId) - } else { - ZonedDateTime.ofInstant(instant, zoneId) - } - } - SimpleTypes.OFFSETTIME_STRUCT.matches(schema) -> + propertyType.matches(schema) -> (value as Struct?)?.let { - OffsetTime.of( - LocalTime.ofNanoOfDay(it.getInt64(NANOS_OF_DAY)), - ZoneOffset.of(it.getString(ZONE_ID))) - } - SimpleTypes.DURATION.matches(schema) -> - (value as Struct?) - ?.let { - Values.isoDuration( - it.getInt64(MONTHS), - it.getInt64(DAYS), - it.getInt64(SECONDS), - it.getInt32(NANOS)) + for (f in it.schema().fields()) { + if (it.getWithoutDefault(f.name()) == null) { + continue } - ?.asIsoDuration() - SimpleTypes.POINT.matches(schema) -> - (value as Struct?) - ?.let { - when (val dimension = it.getInt8(DIMENSION)) { - TWO_D -> - Values.point(it.getInt32(SR_ID), it.getFloat64(X), it.getFloat64(Y)) - THREE_D -> - Values.point( - it.getInt32(SR_ID), - it.getFloat64(X), - it.getFloat64(Y), - it.getFloat64(Z)) - else -> - throw IllegalArgumentException( - "unsupported dimension value ${dimension}") - } + + return when (f.name()) { + BOOLEAN -> it.get(f) as Boolean? + BOOLEAN_LIST -> it.get(f) as List<*>? + LONG -> it.get(f) as Long? + LONG_LIST -> it.get(f) as List<*>? + FLOAT -> it.get(f) as Double? + FLOAT_LIST -> it.get(f) as List<*>? + STRING -> it.get(f) as String? + STRING_LIST -> it.get(f) as List<*>? + BYTES -> it.get(f) as ByteArray? + LOCAL_DATE -> + (it.get(f) as String?)?.let { s -> + DateTimeFormatter.ISO_DATE.parse(s) { parsed -> LocalDate.from(parsed) } + } + LOCAL_DATE_LIST -> it.get(f) as List<*>? + LOCAL_TIME -> + (it.get(f) as String?)?.let { s -> + DateTimeFormatter.ISO_TIME.parse(s) { parsed -> LocalTime.from(parsed) } + } + LOCAL_TIME_LIST -> it.get(f) as List<*>? + LOCAL_DATE_TIME -> + (it.get(f) as String?)?.let { s -> + DateTimeFormatter.ISO_DATE_TIME.parse(s) { parsed -> + LocalDateTime.from(parsed) + } + } + LOCAL_DATE_TIME_LIST -> it.get(f) as List<*>? + ZONED_DATE_TIME -> + (it.get(f) as String?)?.let { s -> + DateTimeFormatter.ISO_DATE_TIME.parse(s) { parsed -> + val zoneId = parsed.query(TemporalQueries.zone()) + + if (zoneId is ZoneOffset) { + OffsetDateTime.from(parsed) + } else { + ZonedDateTime.from(parsed) + } + } + } + ZONED_DATE_TIME_LIST -> it.get(f) as List<*>? + OFFSET_TIME -> + (it.get(f) as String?)?.let { s -> + DateTimeFormatter.ISO_TIME.parse(s) { parsed -> + OffsetTime.from(parsed) + } + } + OFFSET_TIME_LIST -> it.get(f) as List<*>? + DURATION -> + (it.get(f) as Struct?) + ?.let { s -> + Values.isoDuration( + s.getInt64(MONTHS), + s.getInt64(DAYS), + s.getInt64(SECONDS), + s.getInt32(NANOS)) + } + ?.asIsoDuration() + DURATION_LIST -> it.get(f) as List<*>? + POINT -> + (it.get(f) as Struct?) + ?.let { s -> + when (val dimension = s.getInt8(DIMENSION)) { + TWO_D -> + Values.point( + s.getInt32(SR_ID), s.getFloat64(X), s.getFloat64(Y)) + THREE_D -> + Values.point( + s.getInt32(SR_ID), + s.getFloat64(X), + s.getFloat64(Y), + s.getFloat64(Z)) + else -> + throw IllegalArgumentException( + "unsupported dimension value ${dimension}") + } + } + ?.asPoint() + POINT_LIST -> it.get(f) as List<*>? + else -> throw IllegalArgumentException("unsupported neo4j type: ${f.name()}") } - ?.asPoint() + } + + return null + } else -> { val result = mutableMapOf() val struct = value as Struct @@ -462,67 +555,64 @@ object DynamicTypes { temporalDataSchemaType: TemporalDataSchemaType = TemporalDataSchemaType.STRUCT, ): Schema = when (value) { - null -> SimpleTypes.NULL.schema(true) - is Boolean -> SimpleTypes.BOOLEAN.schema(optional) + null -> propertyType + is Boolean -> propertyType is Float, - is Double -> SimpleTypes.FLOAT.schema(optional) - is Number -> SimpleTypes.LONG.schema(optional) + is Double -> propertyType + is Number -> propertyType is Char, is CharArray, - is CharSequence -> SimpleTypes.STRING.schema(optional) + is CharSequence -> propertyType is ByteBuffer, - is ByteArray -> SimpleTypes.BYTES.schema(optional) + is ByteArray -> propertyType is ShortArray, is IntArray, - is LongArray -> - SchemaBuilder.array(Schema.INT64_SCHEMA).apply { if (optional) optional() }.build() + is LongArray -> propertyType is FloatArray, - is DoubleArray -> - SchemaBuilder.array(Schema.FLOAT64_SCHEMA).apply { if (optional) optional() }.build() - is BooleanArray -> - SchemaBuilder.array(Schema.BOOLEAN_SCHEMA).apply { if (optional) optional() }.build() + is DoubleArray -> propertyType + is BooleanArray -> propertyType is Array<*> -> { - val first = value.firstOrNull { it.notNullOrEmpty() } - val schema = toConnectSchema(first, optional, forceMapsAsStruct, temporalDataSchemaType) - SchemaBuilder.array(schema).apply { if (optional) optional() }.build() + when (val componentType = value::class.java.componentType.kotlin) { + Boolean::class, + Byte::class, + Short::class, + Int::class, + Long::class, + Float::class, + Double::class, + String::class, + LocalDate::class, + LocalDateTime::class, + LocalTime::class, + OffsetDateTime::class, + ZonedDateTime::class, + OffsetTime::class -> propertyType + else -> + if (IsoDuration::class.java.isAssignableFrom(componentType.java)) { + propertyType + } else if (Point::class.java.isAssignableFrom(componentType.java)) { + propertyType + } else { + val first = value.firstOrNull { it.notNullOrEmpty() } + val schema = + toConnectSchema(first, optional, forceMapsAsStruct, temporalDataSchemaType) + SchemaBuilder.array(schema).apply { if (optional) optional() }.build() + } + } } - is LocalDate -> - when (temporalDataSchemaType) { - TemporalDataSchemaType.STRUCT -> SimpleTypes.LOCALDATE_STRUCT.schema(optional) - TemporalDataSchemaType.STRING -> SimpleTypes.LOCALDATE.schema(optional) - } - is LocalDateTime -> - when (temporalDataSchemaType) { - TemporalDataSchemaType.STRUCT -> SimpleTypes.LOCALDATETIME_STRUCT.schema(optional) - TemporalDataSchemaType.STRING -> SimpleTypes.LOCALDATETIME.schema(optional) - } - is LocalTime -> - when (temporalDataSchemaType) { - TemporalDataSchemaType.STRUCT -> SimpleTypes.LOCALTIME_STRUCT.schema(optional) - TemporalDataSchemaType.STRING -> SimpleTypes.LOCALTIME.schema(optional) - } - is OffsetDateTime -> - when (temporalDataSchemaType) { - TemporalDataSchemaType.STRUCT -> SimpleTypes.ZONEDDATETIME_STRUCT.schema(optional) - TemporalDataSchemaType.STRING -> SimpleTypes.ZONEDDATETIME.schema(optional) - } - is ZonedDateTime -> - when (temporalDataSchemaType) { - TemporalDataSchemaType.STRUCT -> SimpleTypes.ZONEDDATETIME_STRUCT.schema(optional) - TemporalDataSchemaType.STRING -> SimpleTypes.ZONEDDATETIME.schema(optional) - } - is OffsetTime -> - when (temporalDataSchemaType) { - TemporalDataSchemaType.STRUCT -> SimpleTypes.OFFSETTIME_STRUCT.schema(optional) - TemporalDataSchemaType.STRING -> SimpleTypes.OFFSETTIME.schema(optional) - } - is IsoDuration -> SimpleTypes.DURATION.schema(optional) - is Point -> SimpleTypes.POINT.schema(optional) + is LocalDate -> propertyType + is LocalDateTime -> propertyType + is LocalTime -> propertyType + is OffsetDateTime -> propertyType + is ZonedDateTime -> propertyType + is OffsetTime -> propertyType + is IsoDuration -> propertyType + is Point -> propertyType is Node -> SchemaBuilder.struct() .apply { - field("", SimpleTypes.LONG.schema()) - field("", SchemaBuilder.array(SimpleTypes.STRING.schema()).build()) + field("", propertyType) + field("", propertyType) value.keys().forEach { field( @@ -540,10 +630,10 @@ object DynamicTypes { is Relationship -> SchemaBuilder.struct() .apply { - field("", SimpleTypes.LONG.schema()) - field("", SimpleTypes.STRING.schema()) - field("", SimpleTypes.LONG.schema()) - field("", SimpleTypes.LONG.schema()) + field("", propertyType) + field("", propertyType) + field("", propertyType) + field("", propertyType) value.keys().forEach { field( @@ -565,10 +655,7 @@ object DynamicTypes { .map { toConnectSchema(it, optional, forceMapsAsStruct, temporalDataSchemaType) } when (nonEmptyElementTypes.toSet().size) { - 0 -> - SchemaBuilder.array(SimpleTypes.NULL.schema(true)) - .apply { if (optional) optional() } - .build() + 0 -> SchemaBuilder.array(propertyType).apply { if (optional) optional() }.build() 1 -> SchemaBuilder.array(nonEmptyElementTypes.first()) .apply { if (optional) optional() } @@ -637,7 +724,7 @@ object DynamicTypes { else -> throw IllegalArgumentException("unsupported type ${value.javaClass.name}") } - fun Any?.notNullOrEmpty(): Boolean = + private fun Any?.notNullOrEmpty(): Boolean = when (val value = this) { null -> false is Collection<*> -> value.isNotEmpty() && value.any { it.notNullOrEmpty() } diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt index 4906c5c28..3bc33d206 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt @@ -19,6 +19,7 @@ package org.neo4j.connectors.kafka.data import io.kotest.matchers.shouldBe import java.time.LocalDate import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct @@ -65,13 +66,13 @@ class ChangeEventExtensionsTest { .field("connectionServer", Schema.OPTIONAL_STRING_SCHEMA) .field("serverId", Schema.STRING_SCHEMA) .field("captureMode", Schema.STRING_SCHEMA) - .field("txStartTime", SimpleTypes.ZONEDDATETIME_STRUCT.schema()) - .field("txCommitTime", SimpleTypes.ZONEDDATETIME_STRUCT.schema()) + .field("txStartTime", propertyType) + .field("txCommitTime", propertyType) .field( "txMetadata", SchemaBuilder.struct() - .field("user", Schema.OPTIONAL_STRING_SCHEMA) - .field("app", Schema.OPTIONAL_STRING_SCHEMA) + .field("user", propertyType) + .field("app", propertyType) .optional() .build()) .build() @@ -91,24 +92,20 @@ class ChangeEventExtensionsTest { .put( "txStartTime", change.metadata.txStartTime.let { - Struct(SimpleTypes.ZONEDDATETIME_STRUCT.schema) - .put(EPOCH_SECONDS, it.toEpochSecond()) - .put(NANOS_OF_SECOND, it.nano) - .put(ZONE_ID, it.zone.id) + Struct(propertyType) + .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it)) }) .put( "txCommitTime", change.metadata.txCommitTime.let { - Struct(SimpleTypes.ZONEDDATETIME_STRUCT.schema) - .put(EPOCH_SECONDS, it.toEpochSecond()) - .put(NANOS_OF_SECOND, it.nano) - .put(ZONE_ID, it.zone.id) + Struct(propertyType) + .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it)) }) .put( "txMetadata", Struct(schema.nestedSchema("metadata.txMetadata")) - .put("user", "app_user") - .put("app", "hr")) + .put("user", Struct(propertyType).put(STRING, "app_user")) + .put("app", Struct(propertyType).put(STRING, "hr"))) } @Test @@ -139,20 +136,13 @@ class ChangeEventExtensionsTest { .field( "Label1", SchemaBuilder.array( - SchemaBuilder.struct() - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .field("surname", Schema.OPTIONAL_STRING_SCHEMA) - .optional() - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( "Label2", SchemaBuilder.array( - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .optional() - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .optional() @@ -166,11 +156,7 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .field("surname", Schema.OPTIONAL_STRING_SCHEMA) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( @@ -179,11 +165,7 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .field("surname", Schema.OPTIONAL_STRING_SCHEMA) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .build()) @@ -201,13 +183,10 @@ class ChangeEventExtensionsTest { .put( "Label1", listOf( - Struct(schema.nestedValueSchema("event.keys.Label1")) - .put("name", "john") - .put("surname", "doe"))) - .put( - "Label2", - listOf( - Struct(schema.nestedValueSchema("event.keys.Label2")).put("id", 5L)))) + mapOf( + "name" to Struct(propertyType).put(STRING, "john"), + "surname" to Struct(propertyType).put(STRING, "doe")))) + .put("Label2", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L))))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -217,10 +196,10 @@ class ChangeEventExtensionsTest { .put("labels", listOf("Label1", "Label2")) .put( "properties", - Struct(schema.nestedSchema("event.state.after.properties")) - .put("id", 5L) - .put("name", "john") - .put("surname", "doe")))) + mapOf( + "id" to Struct(propertyType).put(LONG, 5L), + "name" to Struct(propertyType).put(STRING, "john"), + "surname" to Struct(propertyType).put(STRING, "doe"))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -256,20 +235,13 @@ class ChangeEventExtensionsTest { .field( "Label1", SchemaBuilder.array( - SchemaBuilder.struct() - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .field("surname", Schema.OPTIONAL_STRING_SCHEMA) - .optional() - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( "Label2", SchemaBuilder.array( - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .optional() - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .optional() @@ -283,12 +255,7 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.struct() - .field("age", Schema.OPTIONAL_INT64_SCHEMA) - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .field("surname", Schema.OPTIONAL_STRING_SCHEMA) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( @@ -297,12 +264,7 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.struct() - .field("age", Schema.OPTIONAL_INT64_SCHEMA) - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .field("surname", Schema.OPTIONAL_STRING_SCHEMA) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .build()) @@ -320,13 +282,10 @@ class ChangeEventExtensionsTest { .put( "Label1", listOf( - Struct(schema.nestedValueSchema("event.keys.Label1")) - .put("name", "john") - .put("surname", "doe"))) - .put( - "Label2", - listOf( - Struct(schema.nestedValueSchema("event.keys.Label2")).put("id", 5L)))) + mapOf( + "name" to Struct(propertyType).put(STRING, "john"), + "surname" to Struct(propertyType).put(STRING, "doe")))) + .put("Label2", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L))))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -336,21 +295,21 @@ class ChangeEventExtensionsTest { .put("labels", listOf("Label1", "Label2")) .put( "properties", - Struct(schema.nestedSchema("event.state.before.properties")) - .put("id", 5L) - .put("name", "john") - .put("surname", "doe"))) + mapOf( + "id" to Struct(propertyType).put(LONG, 5L), + "name" to Struct(propertyType).put(STRING, "john"), + "surname" to Struct(propertyType).put(STRING, "doe")))) .put( "after", Struct(schema.nestedSchema("event.state.after")) .put("labels", listOf("Label1", "Label2", "Label3")) .put( "properties", - Struct(schema.nestedSchema("event.state.after.properties")) - .put("id", 5L) - .put("name", "john") - .put("surname", "doe") - .put("age", 25L)))) + mapOf( + "id" to Struct(propertyType).put(LONG, 5L), + "name" to Struct(propertyType).put(STRING, "john"), + "surname" to Struct(propertyType).put(STRING, "doe"), + "age" to Struct(propertyType).put(LONG, 25L))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -384,20 +343,13 @@ class ChangeEventExtensionsTest { .field( "Label1", SchemaBuilder.array( - SchemaBuilder.struct() - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .field("surname", Schema.OPTIONAL_STRING_SCHEMA) - .optional() - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( "Label2", SchemaBuilder.array( - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .optional() - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .optional() @@ -411,12 +363,7 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.struct() - .field("age", Schema.OPTIONAL_INT64_SCHEMA) - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .field("surname", Schema.OPTIONAL_STRING_SCHEMA) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( @@ -425,12 +372,7 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.struct() - .field("age", Schema.OPTIONAL_INT64_SCHEMA) - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .field("surname", Schema.OPTIONAL_STRING_SCHEMA) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .build()) @@ -448,13 +390,10 @@ class ChangeEventExtensionsTest { .put( "Label1", listOf( - Struct(schema.nestedValueSchema("event.keys.Label1")) - .put("name", "john") - .put("surname", "doe"))) - .put( - "Label2", - listOf( - Struct(schema.nestedValueSchema("event.keys.Label2")).put("id", 5L)))) + mapOf( + "name" to Struct(propertyType).put(STRING, "john"), + "surname" to Struct(propertyType).put(STRING, "doe")))) + .put("Label2", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L))))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -464,11 +403,11 @@ class ChangeEventExtensionsTest { .put("labels", listOf("Label1", "Label2", "Label3")) .put( "properties", - Struct(schema.nestedSchema("event.state.before.properties")) - .put("id", 5L) - .put("name", "john") - .put("surname", "doe") - .put("age", 25L)))) + mapOf( + "id" to Struct(propertyType).put(LONG, 5L), + "name" to Struct(propertyType).put(STRING, "john"), + "surname" to Struct(propertyType).put(STRING, "doe"), + "age" to Struct(propertyType).put(LONG, 25L))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -509,10 +448,8 @@ class ChangeEventExtensionsTest { .field( "Person", SchemaBuilder.array( - SchemaBuilder.struct() - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .optional() - .schema()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + .build()) .optional() .schema()) .optional() @@ -529,10 +466,8 @@ class ChangeEventExtensionsTest { .field( "Company", SchemaBuilder.array( - SchemaBuilder.struct() - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .optional() - .schema()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + .build()) .optional() .schema()) .optional() @@ -540,11 +475,7 @@ class ChangeEventExtensionsTest { .build()) .field( "keys", - SchemaBuilder.array( - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .optional() - .build()) + SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( @@ -555,10 +486,7 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("since", SimpleTypes.LOCALDATE_STRUCT.schema(true)) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( @@ -566,10 +494,7 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("since", SimpleTypes.LOCALDATE_STRUCT.schema(true)) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .build()) @@ -591,9 +516,7 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.start.keys")) .put( "Person", - listOf( - Struct(schema.nestedValueSchema("event.start.keys.Person")) - .put("name", "john"))))) + listOf(mapOf("name" to Struct(propertyType).put(STRING, "john")))))) .put( "end", Struct(schema.nestedSchema("event.end")) @@ -605,9 +528,9 @@ class ChangeEventExtensionsTest { .put( "Company", listOf( - Struct(schema.nestedValueSchema("event.end.keys.Company")) - .put("name", "acme corp"))))) - .put("keys", listOf(Struct(schema.nestedValueSchema("event.keys")).put("id", 5L))) + mapOf( + "name" to Struct(propertyType).put(STRING, "acme corp")))))) + .put("keys", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L)))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -616,14 +539,14 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.state.after")) .put( "properties", - Struct(schema.nestedSchema("event.state.after.properties")) - .put("id", 5L) - .put( - "since", - Struct(SimpleTypes.LOCALDATE_STRUCT.schema(true)) + mapOf( + "id" to Struct(propertyType).put(LONG, 5L), + "since" to + Struct(propertyType) .put( - EPOCH_DAYS, - LocalDate.of(1999, 12, 31).toEpochDay()))))) + LOCAL_DATE, + DateTimeFormatter.ISO_DATE.format( + LocalDate.of(1999, 12, 31))))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -664,10 +587,8 @@ class ChangeEventExtensionsTest { .field( "Person", SchemaBuilder.array( - SchemaBuilder.struct() - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .optional() - .schema()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + .build()) .optional() .schema()) .optional() @@ -684,10 +605,8 @@ class ChangeEventExtensionsTest { .field( "Company", SchemaBuilder.array( - SchemaBuilder.struct() - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .optional() - .schema()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + .build()) .optional() .schema()) .optional() @@ -695,11 +614,7 @@ class ChangeEventExtensionsTest { .build()) .field( "keys", - SchemaBuilder.array( - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .optional() - .build()) + SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( @@ -710,10 +625,7 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("since", SimpleTypes.LOCALDATE_STRUCT.schema(true)) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( @@ -721,10 +633,7 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("since", SimpleTypes.LOCALDATE_STRUCT.schema(true)) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .build()) @@ -746,9 +655,7 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.start.keys")) .put( "Person", - listOf( - Struct(schema.nestedValueSchema("event.start.keys.Person")) - .put("name", "john"))))) + listOf(mapOf("name" to Struct(propertyType).put(STRING, "john")))))) .put( "end", Struct(schema.nestedSchema("event.end")) @@ -760,9 +667,9 @@ class ChangeEventExtensionsTest { .put( "Company", listOf( - Struct(schema.nestedValueSchema("event.end.keys.Company")) - .put("name", "acme corp"))))) - .put("keys", listOf(Struct(schema.nestedValueSchema("event.keys")).put("id", 5L))) + mapOf( + "name" to Struct(propertyType).put(STRING, "acme corp")))))) + .put("keys", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L)))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -771,27 +678,27 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.state.before")) .put( "properties", - Struct(schema.nestedSchema("event.state.before.properties")) - .put("id", 5L) - .put( - "since", - Struct(SimpleTypes.LOCALDATE_STRUCT.schema(true)) + mapOf( + "id" to Struct(propertyType).put(LONG, 5L), + "since" to + Struct(propertyType) .put( - EPOCH_DAYS, - LocalDate.of(1999, 12, 31).toEpochDay())))) + LOCAL_DATE, + DateTimeFormatter.ISO_DATE.format( + LocalDate.of(1999, 12, 31)))))) .put( "after", Struct(schema.nestedSchema("event.state.after")) .put( "properties", - Struct(schema.nestedSchema("event.state.after.properties")) - .put("id", 5L) - .put( - "since", - Struct(SimpleTypes.LOCALDATE_STRUCT.schema(true)) + mapOf( + "id" to Struct(propertyType).put(LONG, 5L), + "since" to + Struct(propertyType) .put( - EPOCH_DAYS, - LocalDate.of(2000, 1, 1).toEpochDay()))))) + LOCAL_DATE, + DateTimeFormatter.ISO_DATE.format( + LocalDate.of(2000, 1, 1))))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -832,9 +739,7 @@ class ChangeEventExtensionsTest { .field( "Person", SchemaBuilder.array( - SchemaBuilder.struct() - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .optional() + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) .build()) .optional() .build()) @@ -852,9 +757,7 @@ class ChangeEventExtensionsTest { .field( "Company", SchemaBuilder.array( - SchemaBuilder.struct() - .field("name", Schema.OPTIONAL_STRING_SCHEMA) - .optional() + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) .build()) .optional() .build()) @@ -863,11 +766,7 @@ class ChangeEventExtensionsTest { .build()) .field( "keys", - SchemaBuilder.array( - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .optional() - .build()) + SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .schema()) .field( @@ -878,10 +777,7 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("since", SimpleTypes.LOCALDATE_STRUCT.schema(true)) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .field( @@ -889,10 +785,7 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_INT64_SCHEMA) - .field("since", SimpleTypes.LOCALDATE_STRUCT.schema(true)) - .build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) .optional() .build()) .build()) @@ -914,9 +807,7 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.start.keys")) .put( "Person", - listOf( - Struct(schema.nestedValueSchema("event.start.keys.Person")) - .put("name", "john"))))) + listOf(mapOf("name" to Struct(propertyType).put(STRING, "john")))))) .put( "end", Struct(schema.nestedSchema("event.end")) @@ -928,9 +819,9 @@ class ChangeEventExtensionsTest { .put( "Company", listOf( - Struct(schema.nestedValueSchema("event.end.keys.Company")) - .put("name", "acme corp"))))) - .put("keys", listOf(Struct(schema.nestedValueSchema("event.keys")).put("id", 5L))) + mapOf( + "name" to Struct(propertyType).put(STRING, "acme corp")))))) + .put("keys", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L)))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -939,14 +830,14 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.state.before")) .put( "properties", - Struct(schema.nestedSchema("event.state.before.properties")) - .put("id", 5L) - .put( - "since", - Struct(SimpleTypes.LOCALDATE_STRUCT.schema(true)) + mapOf( + "id" to Struct(propertyType).put(LONG, 5L), + "since" to + Struct(propertyType) .put( - EPOCH_DAYS, - LocalDate.of(2000, 1, 1).toEpochDay()))))) + LOCAL_DATE, + DateTimeFormatter.ISO_DATE.format( + LocalDate.of(2000, 1, 1))))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -984,7 +875,9 @@ class ChangeEventExtensionsTest { null)) val expectedKeySchema = - SchemaBuilder.array(SchemaBuilder.struct().optional().build()).optional().build() + SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + .optional() + .build() schema.nestedSchema("event.keys") shouldBe expectedKeySchema value.nestedValue("event.keys") shouldBe emptyList() } @@ -1023,29 +916,27 @@ class ChangeEventExtensionsTest { .put( "txStartTime", startTime.let { - Struct(SimpleTypes.ZONEDDATETIME_STRUCT.schema) - .put(EPOCH_SECONDS, it.toEpochSecond()) - .put(NANOS_OF_SECOND, it.nano) - .put(ZONE_ID, it.zone.id) + Struct(propertyType) + .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it)) }) .put( "txCommitTime", commitTime.let { - Struct(SimpleTypes.ZONEDDATETIME_STRUCT.schema) - .put(EPOCH_SECONDS, it.toEpochSecond()) - .put(NANOS_OF_SECOND, it.nano) - .put(ZONE_ID, it.zone.id) + Struct(propertyType) + .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it)) }) .put( "txMetadata", Struct(schema.nestedSchema("txMetadata").schema()) - .put("user", "app_user") - .put("app", "hr") + .put("user", Struct(propertyType).put(STRING, "app_user")) + .put("app", Struct(propertyType).put(STRING, "hr")) .put( "xyz", - Struct(schema.nestedSchema("txMetadata.xyz")).put("a", 1L).put("b", 2L))) - .put("new_field", "abc") - .put("another_field", 1L) + Struct(schema.nestedSchema("txMetadata.xyz")) + .put("a", Struct(propertyType).put(LONG, 1L)) + .put("b", Struct(propertyType).put(LONG, 2L)))) + .put("new_field", Struct(propertyType).put(STRING, "abc")) + .put("another_field", Struct(propertyType).put(LONG, 1L)) val reverted = converted.toMetadata() reverted shouldBe metadata @@ -1170,16 +1061,16 @@ class ChangeEventExtensionsTest { .put( "Person", listOf( - Struct(schema.nestedSchema("keys.Person").valueSchema()).put("id", 1L), - Struct(schema.nestedSchema("keys.Person").valueSchema()) - .put("name", "john") - .put("surname", "doe"))) + mapOf("id" to Struct(propertyType).put(LONG, 1L)), + mapOf( + "name" to Struct(propertyType).put(STRING, "john"), + "surname" to Struct(propertyType).put(STRING, "doe")))) .put( "Employee", listOf( - Struct(schema.nestedSchema("keys.Employee").valueSchema()) - .put("id", 5L) - .put("company_id", 7L)))) + mapOf( + "id" to Struct(propertyType).put(LONG, 5L), + "company_id" to Struct(propertyType).put(LONG, 7L))))) val reverted = converted.toNode() reverted shouldBe node diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt index c8790da92..418afafd9 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt @@ -29,6 +29,7 @@ import java.time.OffsetTime import java.time.ZoneId import java.time.ZoneOffset import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter import java.util.function.Function import java.util.stream.Stream import org.apache.kafka.connect.data.Schema @@ -50,22 +51,22 @@ class DynamicTypesTest { @Test fun `should derive schema for simple types correctly`() { // NULL - DynamicTypes.toConnectSchema(null, false) shouldBe SimpleTypes.NULL.schema() - DynamicTypes.toConnectSchema(null, true) shouldBe SimpleTypes.NULL.schema() + DynamicTypes.toConnectSchema(null, false) shouldBe propertyType + DynamicTypes.toConnectSchema(null, true) shouldBe propertyType // Integer, Long, etc. listOf(8.toByte(), 8.toShort(), 8.toInt(), 8.toLong()).forEach { number -> withClue(number) { - DynamicTypes.toConnectSchema(number, false) shouldBe SchemaBuilder.INT64_SCHEMA - DynamicTypes.toConnectSchema(number, true) shouldBe SchemaBuilder.OPTIONAL_INT64_SCHEMA + DynamicTypes.toConnectSchema(number, false) shouldBe propertyType + DynamicTypes.toConnectSchema(number, true) shouldBe propertyType } } // Float, Double listOf(8.toFloat(), 8.toDouble()).forEach { number -> withClue(number) { - DynamicTypes.toConnectSchema(number, false) shouldBe SchemaBuilder.FLOAT64_SCHEMA - DynamicTypes.toConnectSchema(number, true) shouldBe SchemaBuilder.OPTIONAL_FLOAT64_SCHEMA + DynamicTypes.toConnectSchema(number, false) shouldBe propertyType + DynamicTypes.toConnectSchema(number, true) shouldBe propertyType } } @@ -87,46 +88,40 @@ class DynamicTypesTest { }) .forEach { string -> withClue(string) { - DynamicTypes.toConnectSchema(string, false) shouldBe SchemaBuilder.STRING_SCHEMA - DynamicTypes.toConnectSchema(string, true) shouldBe SchemaBuilder.OPTIONAL_STRING_SCHEMA + DynamicTypes.toConnectSchema(string, false) shouldBe propertyType + DynamicTypes.toConnectSchema(string, true) shouldBe propertyType } } // Byte Array listOf(ByteArray(0), ByteBuffer.allocate(0)).forEach { bytes -> withClue(bytes) { - DynamicTypes.toConnectSchema(bytes, false) shouldBe SchemaBuilder.BYTES_SCHEMA - DynamicTypes.toConnectSchema(bytes, true) shouldBe SchemaBuilder.OPTIONAL_BYTES_SCHEMA + DynamicTypes.toConnectSchema(bytes, false) shouldBe propertyType + DynamicTypes.toConnectSchema(bytes, true) shouldBe propertyType } } // Boolean Array (boolean[]) listOf(BooleanArray(0), BooleanArray(1) { true }).forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe - SchemaBuilder.array(Schema.BOOLEAN_SCHEMA).build() - DynamicTypes.toConnectSchema(array, true) shouldBe - SchemaBuilder.array(Schema.BOOLEAN_SCHEMA).optional().build() + DynamicTypes.toConnectSchema(array, false) shouldBe propertyType + DynamicTypes.toConnectSchema(array, true) shouldBe propertyType } } // Array of Boolean (Boolean[]) listOf(Array(1) { true }).forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe - SchemaBuilder.array(Schema.BOOLEAN_SCHEMA).build() - DynamicTypes.toConnectSchema(array, true) shouldBe - SchemaBuilder.array(Schema.OPTIONAL_BOOLEAN_SCHEMA).optional().build() + DynamicTypes.toConnectSchema(array, false) shouldBe propertyType + DynamicTypes.toConnectSchema(array, true) shouldBe propertyType } } // Int Arrays (short[], int[], long[]) listOf(ShortArray(1), IntArray(1), LongArray(1)).forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe - SchemaBuilder.array(Schema.INT64_SCHEMA).build() - DynamicTypes.toConnectSchema(array, true) shouldBe - SchemaBuilder.array(Schema.INT64_SCHEMA).optional().build() + DynamicTypes.toConnectSchema(array, false) shouldBe propertyType + DynamicTypes.toConnectSchema(array, true) shouldBe propertyType } } @@ -134,162 +129,131 @@ class DynamicTypesTest { listOf(Array(1) { i -> i }, Array(1) { i -> i.toShort() }, Array(1) { i -> i.toLong() }) .forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe - SchemaBuilder.array(Schema.INT64_SCHEMA).build() - DynamicTypes.toConnectSchema(array, true) shouldBe - SchemaBuilder.array(Schema.OPTIONAL_INT64_SCHEMA).optional().build() + DynamicTypes.toConnectSchema(array, false) shouldBe propertyType + DynamicTypes.toConnectSchema(array, true) shouldBe propertyType } } // Float Arrays (float[], double[]) listOf(FloatArray(1), DoubleArray(1)).forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe - SchemaBuilder.array(Schema.FLOAT64_SCHEMA).build() - DynamicTypes.toConnectSchema(array, true) shouldBe - SchemaBuilder.array(Schema.FLOAT64_SCHEMA).optional().build() + DynamicTypes.toConnectSchema(array, false) shouldBe propertyType + DynamicTypes.toConnectSchema(array, true) shouldBe propertyType } } // Float Arrays (Float[], Double[]) listOf(Array(1) { i -> i.toFloat() }, Array(1) { i -> i.toDouble() }).forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe - SchemaBuilder.array(Schema.FLOAT64_SCHEMA).build() - DynamicTypes.toConnectSchema(array, true) shouldBe - SchemaBuilder.array(Schema.OPTIONAL_FLOAT64_SCHEMA).optional().build() + DynamicTypes.toConnectSchema(array, false) shouldBe propertyType + DynamicTypes.toConnectSchema(array, true) shouldBe propertyType } } // String Array - DynamicTypes.toConnectSchema(Array(1) { "a" }, false) shouldBe - SchemaBuilder.array(Schema.STRING_SCHEMA).build() - DynamicTypes.toConnectSchema(Array(1) { "a" }, true) shouldBe - SchemaBuilder.array(Schema.OPTIONAL_STRING_SCHEMA).optional().build() + DynamicTypes.toConnectSchema(Array(1) { "a" }, false) shouldBe propertyType + DynamicTypes.toConnectSchema(Array(1) { "a" }, true) shouldBe propertyType // Temporal Types - DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), false) shouldBe - SimpleTypes.LOCALDATE_STRUCT.schema() - DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), true) shouldBe - SimpleTypes.LOCALDATE_STRUCT.schema(true) + DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), false) shouldBe propertyType + DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), true) shouldBe propertyType DynamicTypes.toConnectSchema( LocalDate.of(1999, 12, 31), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.LOCALDATE.schema() + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema( LocalDate.of(1999, 12, 31), optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.LOCALDATE.schema(true) + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType - DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), false) shouldBe - SimpleTypes.LOCALTIME_STRUCT.schema() - DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), true) shouldBe - SimpleTypes.LOCALTIME_STRUCT.schema(true) + DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), false) shouldBe propertyType + DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), true) shouldBe propertyType DynamicTypes.toConnectSchema( LocalTime.of(23, 59, 59), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.LOCALTIME.schema() + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema( LocalTime.of(23, 59, 59), optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.LOCALTIME.schema(true) + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema(LocalDateTime.of(1999, 12, 31, 23, 59, 59), false) shouldBe - SimpleTypes.LOCALDATETIME_STRUCT.schema() + propertyType DynamicTypes.toConnectSchema(LocalDateTime.of(1999, 12, 31, 23, 59, 59), true) shouldBe - SimpleTypes.LOCALDATETIME_STRUCT.schema(true) + propertyType DynamicTypes.toConnectSchema( LocalDateTime.of(1999, 12, 31, 23, 59, 59), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.LOCALDATETIME.schema() + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema( LocalDateTime.of(1999, 12, 31, 23, 59, 59), optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.LOCALDATETIME.schema(true) + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema(OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), false) shouldBe - SimpleTypes.OFFSETTIME_STRUCT.schema() + propertyType DynamicTypes.toConnectSchema(OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), true) shouldBe - SimpleTypes.OFFSETTIME_STRUCT.schema(true) + propertyType DynamicTypes.toConnectSchema( OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.OFFSETTIME.schema() + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema( OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.OFFSETTIME.schema(true) + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema( - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), false) shouldBe - SimpleTypes.ZONEDDATETIME_STRUCT.schema() + OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), false) shouldBe propertyType DynamicTypes.toConnectSchema( - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), true) shouldBe - SimpleTypes.ZONEDDATETIME_STRUCT.schema(true) + OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), true) shouldBe propertyType DynamicTypes.toConnectSchema( OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.ZONEDDATETIME.schema() + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema( OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.ZONEDDATETIME.schema(true) + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), false) shouldBe - SimpleTypes.ZONEDDATETIME_STRUCT.schema() + propertyType DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), true) shouldBe - SimpleTypes.ZONEDDATETIME_STRUCT.schema(true) + propertyType DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.ZONEDDATETIME.schema() + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe - SimpleTypes.ZONEDDATETIME.schema(true) + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType DynamicTypes.toConnectSchema( - Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), false) shouldBe - SimpleTypes.DURATION.schema() + Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), false) shouldBe propertyType DynamicTypes.toConnectSchema( - Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), true) shouldBe - SimpleTypes.DURATION.schema(true) + Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), true) shouldBe propertyType // Point listOf(Values.point(4326, 1.0, 2.0).asPoint(), Values.point(4326, 1.0, 2.0, 3.0).asPoint()) .forEach { point -> withClue(point) { - DynamicTypes.toConnectSchema(point, false) shouldBe SimpleTypes.POINT.schema() - DynamicTypes.toConnectSchema(point, true) shouldBe SimpleTypes.POINT.schema(true) + DynamicTypes.toConnectSchema(point, false) shouldBe propertyType + DynamicTypes.toConnectSchema(point, true) shouldBe propertyType } } // Node DynamicTypes.toConnectSchema(TestNode(0, emptyList(), emptyMap()), false) shouldBe - SchemaBuilder.struct() - .field("", Schema.INT64_SCHEMA) - .field("", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) - .build() + SchemaBuilder.struct().field("", propertyType).field("", propertyType).build() DynamicTypes.toConnectSchema( TestNode( @@ -298,19 +262,19 @@ class DynamicTypesTest { mapOf("name" to Values.value("john"), "surname" to Values.value("doe"))), false) shouldBe SchemaBuilder.struct() - .field("", Schema.INT64_SCHEMA) - .field("", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) - .field("name", Schema.STRING_SCHEMA) - .field("surname", Schema.STRING_SCHEMA) + .field("", propertyType) + .field("", propertyType) + .field("name", propertyType) + .field("surname", propertyType) .build() // Relationship DynamicTypes.toConnectSchema(TestRelationship(0, 1, 2, "KNOWS", emptyMap()), false) shouldBe SchemaBuilder.struct() - .field("", Schema.INT64_SCHEMA) - .field("", SchemaBuilder.STRING_SCHEMA) - .field("", Schema.INT64_SCHEMA) - .field("", Schema.INT64_SCHEMA) + .field("", propertyType) + .field("", propertyType) + .field("", propertyType) + .field("", propertyType) .build() DynamicTypes.toConnectSchema( TestRelationship( @@ -321,12 +285,12 @@ class DynamicTypesTest { mapOf("name" to Values.value("john"), "surname" to Values.value("doe"))), false) shouldBe SchemaBuilder.struct() - .field("", Schema.INT64_SCHEMA) - .field("", SchemaBuilder.STRING_SCHEMA) - .field("", Schema.INT64_SCHEMA) - .field("", Schema.INT64_SCHEMA) - .field("name", Schema.STRING_SCHEMA) - .field("surname", Schema.STRING_SCHEMA) + .field("", propertyType) + .field("", propertyType) + .field("", propertyType) + .field("", propertyType) + .field("name", propertyType) + .field("surname", propertyType) .build() } @@ -335,48 +299,32 @@ class DynamicTypesTest { listOf(listOf(), setOf(), arrayOf()).forEach { collection -> withClue(collection) { DynamicTypes.toConnectSchema(collection, false) shouldBe - SchemaBuilder.array(SimpleTypes.NULL.schema()).build() + SchemaBuilder.array(propertyType).build() DynamicTypes.toConnectSchema(collection, true) shouldBe - SchemaBuilder.array(SimpleTypes.NULL.schema()).optional().build() + SchemaBuilder.array(propertyType).optional().build() } } } @Test fun `collections with elements of single type should map to an array schema`() { - listOf>( - Triple(listOf(1, 2, 3), Schema.INT64_SCHEMA, Schema.OPTIONAL_INT64_SCHEMA), - Triple(listOf("a", "b", "c"), Schema.STRING_SCHEMA, Schema.OPTIONAL_STRING_SCHEMA), - Triple(setOf(true), Schema.BOOLEAN_SCHEMA, Schema.OPTIONAL_BOOLEAN_SCHEMA), - ) - .forEach { (collection, elementSchema, elementOptionalSchema) -> - withClue(collection) { - DynamicTypes.toConnectSchema(collection, false) shouldBe - SchemaBuilder.array(elementSchema).build() - DynamicTypes.toConnectSchema(collection, true) shouldBe - SchemaBuilder.array(elementOptionalSchema).optional().build() - } - } + listOf(listOf(1, 2, 3), listOf("a", "b", "c"), setOf(true)).forEach { collection -> + withClue(collection) { + DynamicTypes.toConnectSchema(collection, false) shouldBe + SchemaBuilder.array(propertyType).build() + DynamicTypes.toConnectSchema(collection, true) shouldBe + SchemaBuilder.array(propertyType).optional().build() + } + } } @Test fun `collections with elements of different types should map to a struct schema`() { DynamicTypes.toConnectSchema(listOf(1, true, "a", 5.toFloat()), false) shouldBe - SchemaBuilder.struct() - .field("e0", Schema.INT64_SCHEMA) - .field("e1", Schema.BOOLEAN_SCHEMA) - .field("e2", Schema.STRING_SCHEMA) - .field("e3", Schema.FLOAT64_SCHEMA) - .build() + SchemaBuilder.array(propertyType).build() DynamicTypes.toConnectSchema(listOf(1, true, "a", 5.toFloat()), true) shouldBe - SchemaBuilder.struct() - .field("e0", Schema.OPTIONAL_INT64_SCHEMA) - .field("e1", Schema.OPTIONAL_BOOLEAN_SCHEMA) - .field("e2", Schema.OPTIONAL_STRING_SCHEMA) - .field("e3", Schema.OPTIONAL_FLOAT64_SCHEMA) - .optional() - .build() + SchemaBuilder.array(propertyType).optional().build() } @Test @@ -395,11 +343,11 @@ class DynamicTypesTest { } @Test - fun `maps with single typed values should map to a map schema`() { + fun `maps with simple typed values should map to a map schema`() { listOf( - mapOf("a" to 1, "b" to 2, "c" to 3) to Schema.INT64_SCHEMA, - mapOf("a" to "a", "b" to "b", "c" to "c") to Schema.STRING_SCHEMA, - mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong()) to Schema.INT64_SCHEMA) + mapOf("a" to 1, "b" to 2, "c" to 3) to propertyType, + mapOf("a" to "a", "b" to "b", "c" to "c") to propertyType, + mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong()) to propertyType) .forEach { (map, valueSchema) -> withClue("not optional: $map") { DynamicTypes.toConnectSchema(map, false) shouldBe @@ -408,9 +356,9 @@ class DynamicTypesTest { } listOf( - mapOf("a" to 1, "b" to 2, "c" to 3) to Schema.OPTIONAL_INT64_SCHEMA, - mapOf("a" to "a", "b" to "b", "c" to "c") to Schema.OPTIONAL_STRING_SCHEMA, - mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong()) to Schema.OPTIONAL_INT64_SCHEMA) + mapOf("a" to 1, "b" to 2, "c" to 3) to propertyType, + mapOf("a" to "a", "b" to "b", "c" to "c") to propertyType, + mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong()) to propertyType) .forEach { (map, valueSchema) -> withClue("optional: $map") { DynamicTypes.toConnectSchema(map, true) shouldBe @@ -420,25 +368,14 @@ class DynamicTypesTest { } @Test - fun `maps with values of different types should map to a struct schema`() { + fun `maps with values of different types should map to a map of struct schema`() { DynamicTypes.toConnectSchema( mapOf("a" to 1, "b" to true, "c" to "string", "d" to 5.toFloat()), false) shouldBe - SchemaBuilder.struct() - .field("a", Schema.INT64_SCHEMA) - .field("b", Schema.BOOLEAN_SCHEMA) - .field("c", Schema.STRING_SCHEMA) - .field("d", Schema.FLOAT64_SCHEMA) - .build() + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build() DynamicTypes.toConnectSchema( mapOf("a" to 1, "b" to true, "c" to "string", "d" to 5.toFloat()), true) shouldBe - SchemaBuilder.struct() - .field("a", Schema.OPTIONAL_INT64_SCHEMA) - .field("b", Schema.OPTIONAL_BOOLEAN_SCHEMA) - .field("c", Schema.OPTIONAL_STRING_SCHEMA) - .field("d", Schema.OPTIONAL_FLOAT64_SCHEMA) - .optional() - .build() + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).optional().build() } @Test @@ -454,25 +391,28 @@ class DynamicTypesTest { @Test fun `simple types should be converted to themselves and should be converted back`() { listOf( - true to true, - false to false, - 1.toShort() to 1.toLong(), - 2 to 2.toLong(), - 3.toLong() to 3.toLong(), - 4.toFloat() to 4.toDouble(), - 5.toDouble() to 5.toDouble(), - 'c' to "c", - "string" to "string", - "string".toCharArray() to "string", - "string".toByteArray() to "string".toByteArray()) - .forEach { (value, expected) -> + Triple(true, Struct(propertyType).put(BOOLEAN, true), true), + Triple(false, Struct(propertyType).put(BOOLEAN, false), false), + Triple(1.toShort(), Struct(propertyType).put(LONG, 1.toLong()), 1L), + Triple(2, Struct(propertyType).put(LONG, 2.toLong()), 2L), + Triple(3.toLong(), Struct(propertyType).put(LONG, 3.toLong()), 3L), + Triple(4.toFloat(), Struct(propertyType).put(FLOAT, 4.toDouble()), 4.toDouble()), + Triple(5.toDouble(), Struct(propertyType).put(FLOAT, 5.toDouble()), 5.toDouble()), + Triple('c', Struct(propertyType).put(STRING, "c"), "c"), + Triple("string", Struct(propertyType).put(STRING, "string"), "string"), + Triple("string".toCharArray(), Struct(propertyType).put(STRING, "string"), "string"), + Triple( + "string".toByteArray(), + Struct(propertyType).put(BYTES, "string".toByteArray()), + "string".toByteArray())) + .forEach { (value, expected, expectedValue) -> withClue(value) { val schema = DynamicTypes.toConnectSchema(value, false) val converted = DynamicTypes.toConnectValue(schema, value) val reverted = DynamicTypes.fromConnectValue(schema, converted) converted shouldBe expected - reverted shouldBe expected + reverted shouldBe expectedValue } } } @@ -497,41 +437,33 @@ class DynamicTypesTest { return Stream.of( LocalDate.of(1999, 12, 31).let { Arguments.of( - it, Struct(SimpleTypes.LOCALDATE_STRUCT.schema).put(EPOCH_DAYS, it.toEpochDay())) + it, Struct(propertyType).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(it))) }, LocalTime.of(23, 59, 59, 9999).let { Arguments.of( - it, Struct(SimpleTypes.LOCALTIME_STRUCT.schema).put(NANOS_OF_DAY, it.toNanoOfDay())) + it, Struct(propertyType).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(it))) }, LocalDateTime.of(1999, 12, 31, 23, 59, 59, 9999).let { Arguments.of( it, - Struct(SimpleTypes.LOCALDATETIME_STRUCT.schema) - .put(EPOCH_DAYS, it.toLocalDate().toEpochDay()) - .put(NANOS_OF_DAY, it.toLocalTime().toNanoOfDay())) + Struct(propertyType) + .put(LOCAL_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }, OffsetTime.of(23, 59, 59, 9999, ZoneOffset.UTC).let { Arguments.of( - it, - Struct(SimpleTypes.OFFSETTIME_STRUCT.schema) - .put(NANOS_OF_DAY, it.toLocalTime().toNanoOfDay()) - .put(ZONE_ID, it.offset.id)) + it, Struct(propertyType).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(it))) }, OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 9999, ZoneOffset.ofHours(1)).let { Arguments.of( it, - Struct(SimpleTypes.ZONEDDATETIME_STRUCT.schema) - .put(EPOCH_SECONDS, it.toEpochSecond()) - .put(NANOS_OF_SECOND, it.nano) - .put(ZONE_ID, it.offset.id)) + Struct(propertyType) + .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }, ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 9999, ZoneId.of("Europe/Istanbul")).let { Arguments.of( it, - Struct(SimpleTypes.ZONEDDATETIME_STRUCT.schema) - .put(EPOCH_SECONDS, it.toEpochSecond()) - .put(NANOS_OF_SECOND, it.nano) - .put(ZONE_ID, it.zone.id)) + Struct(propertyType) + .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }) } } @@ -540,11 +472,14 @@ class DynamicTypesTest { fun `duration types should be returned as structs and should be converted back`() { listOf( Values.isoDuration(5, 2, 0, 9999).asIsoDuration() to - Struct(SimpleTypes.DURATION.schema()) - .put("months", 5L) - .put("days", 2L) - .put("seconds", 0L) - .put("nanoseconds", 9999)) + Struct(propertyType) + .put( + DURATION, + Struct(durationSchema) + .put("months", 5L) + .put("days", 2L) + .put("seconds", 0L) + .put("nanoseconds", 9999))) .forEach { (value, expected) -> withClue(value) { val schema = DynamicTypes.toConnectSchema(value, false) @@ -559,7 +494,7 @@ class DynamicTypesTest { } @Test - fun `arrays and collections should be returned as arrays and should be converted back`() { + fun `arrays should be returned as list of simple types and should be converted back`() { fun primitiveToArray(value: Any): Any = when (value) { is BooleanArray -> value.toList() @@ -574,18 +509,22 @@ class DynamicTypesTest { } listOf( - ShortArray(1) { 1 } to LongArray(1) { 1.toLong() }, - IntArray(1) { 1 } to LongArray(1) { 1.toLong() }, - LongArray(1) { 1 } to LongArray(1) { 1 }, - FloatArray(1) { 1F } to DoubleArray(1) { 1.0 }, - DoubleArray(1) { 1.0 } to DoubleArray(1) { 1.0 }, - BooleanArray(1) { true } to BooleanArray(1) { true }, - Array(1) { 1 } to Array(1) { 1L }, - Array(1) { 1.toShort() } to Array(1) { 1L }, - Array(1) { "string" } to Array(1) { "string" }, - listOf(1, 2, 3) to arrayOf(1L, 2L, 3L), - listOf("a", "b", "c") to arrayOf("a", "b", "c"), - setOf(true, false) to arrayOf(true, false)) + ShortArray(1) { 1 } to + Struct(propertyType).put(LONG_LIST, LongArray(1) { 1.toLong() }.toList()), + IntArray(1) { 1 } to + Struct(propertyType).put(LONG_LIST, LongArray(1) { 1.toLong() }.toList()), + LongArray(1) { 1 } to Struct(propertyType).put(LONG_LIST, LongArray(1) { 1 }.toList()), + FloatArray(1) { 1F } to + Struct(propertyType).put(FLOAT_LIST, DoubleArray(1) { 1.0 }.toList()), + DoubleArray(1) { 1.0 } to + Struct(propertyType).put(FLOAT_LIST, DoubleArray(1) { 1.0 }.toList()), + BooleanArray(1) { true } to + Struct(propertyType).put(BOOLEAN_LIST, BooleanArray(1) { true }.toList()), + Array(1) { 1 } to Struct(propertyType).put(LONG_LIST, Array(1) { 1L }.toList()), + Array(1) { 1.toShort() } to + Struct(propertyType).put(LONG_LIST, Array(1) { 1L }.toList()), + Array(1) { "string" } to + Struct(propertyType).put(STRING_LIST, Array(1) { "string" }.toList())) .forEach { (value, expected) -> withClue(value) { val schema = DynamicTypes.toConnectSchema(value, false) @@ -600,10 +539,61 @@ class DynamicTypesTest { } @Test - fun `maps should be returned as maps and should be converted back`() { + fun `collections should be returned as arrays of simple types and should be converted back`() { + fun primitiveToArray(value: Any): Any = + when (value) { + is BooleanArray -> value.toList() + is ByteArray -> value.toList() + is CharArray -> value.toList() + is DoubleArray -> value.toList() + is FloatArray -> value.toList() + is IntArray -> value.toList() + is LongArray -> value.toList() + is ShortArray -> value.toList() + else -> value + } + listOf( - mapOf("a" to "x", "b" to "y", "c" to "z") to mapOf("a" to "x", "b" to "y", "c" to "z"), - mapOf("a" to 1, "b" to 2, "c" to 3) to mapOf("a" to 1L, "b" to 2L, "c" to 3L)) + listOf(1, 2, 3) to + listOf( + Struct(propertyType).put(LONG, 1L), + Struct(propertyType).put(LONG, 2L), + Struct(propertyType).put(LONG, 3L)), + listOf("a", "b", "c") to + listOf( + Struct(propertyType).put(STRING, "a"), + Struct(propertyType).put(STRING, "b"), + Struct(propertyType).put(STRING, "c")), + setOf(true, false) to + listOf( + Struct(propertyType).put(BOOLEAN, true), + Struct(propertyType).put(BOOLEAN, false))) + .forEach { (value, expected) -> + withClue(value) { + val schema = DynamicTypes.toConnectSchema(value, false) + val converted = DynamicTypes.toConnectValue(schema, value) + + converted shouldBe expected + + val reverted = DynamicTypes.fromConnectValue(schema, converted) + reverted shouldBe primitiveToArray(value) + } + } + } + + @Test + fun `maps should be returned as maps of structs and should be converted back`() { + listOf( + mapOf("a" to "x", "b" to "y", "c" to "z") to + mapOf( + "a" to Struct(propertyType).put(STRING, "x"), + "b" to Struct(propertyType).put(STRING, "y"), + "c" to Struct(propertyType).put(STRING, "z")), + mapOf("a" to 1, "b" to 2, "c" to 3) to + mapOf( + "a" to Struct(propertyType).put(LONG, 1L), + "b" to Struct(propertyType).put(LONG, 2L), + "c" to Struct(propertyType).put(LONG, 3L))) .forEach { (value, expected) -> withClue(value) { val schema = DynamicTypes.toConnectSchema(value, false) @@ -625,12 +615,15 @@ class DynamicTypesTest { val converted = DynamicTypes.toConnectValue(schema, point) converted shouldBe - Struct(schema) - .put("dimension", 2.toByte()) - .put("srid", point.srid()) - .put("x", point.x()) - .put("y", point.y()) - .put("z", null) + Struct(propertyType) + .put( + POINT, + Struct(pointSchema) + .put("dimension", 2.toByte()) + .put("srid", point.srid()) + .put("x", point.x()) + .put("y", point.y()) + .put("z", null)) val reverted = DynamicTypes.fromConnectValue(schema, converted) reverted shouldBe point @@ -643,12 +636,15 @@ class DynamicTypesTest { val converted = DynamicTypes.toConnectValue(schema, point) converted shouldBe - Struct(schema) - .put("dimension", 3.toByte()) - .put("srid", point.srid()) - .put("x", point.x()) - .put("y", point.y()) - .put("z", point.z()) + Struct(propertyType) + .put( + POINT, + Struct(pointSchema) + .put("dimension", 3.toByte()) + .put("srid", point.srid()) + .put("x", point.x()) + .put("y", point.y()) + .put("z", point.z())) val reverted = DynamicTypes.fromConnectValue(schema, converted) reverted shouldBe point @@ -666,10 +662,10 @@ class DynamicTypesTest { converted shouldBe Struct(schema) - .put("", 0L) - .put("", listOf("Person", "Employee")) - .put("name", "john") - .put("surname", "doe") + .put("", Struct(propertyType).put(LONG, 0L)) + .put("", Struct(propertyType).put(STRING_LIST, listOf("Person", "Employee"))) + .put("name", Struct(propertyType).put(STRING, "john")) + .put("surname", Struct(propertyType).put(STRING, "doe")) val reverted = DynamicTypes.fromConnectValue(schema, converted) reverted shouldBe @@ -694,12 +690,12 @@ class DynamicTypesTest { converted shouldBe Struct(schema) - .put("", 0L) - .put("", 1L) - .put("", 2L) - .put("", "KNOWS") - .put("name", "john") - .put("surname", "doe") + .put("", Struct(propertyType).put(LONG, 0L)) + .put("", Struct(propertyType).put(LONG, 1L)) + .put("", Struct(propertyType).put(LONG, 2L)) + .put("", Struct(propertyType).put(STRING, "KNOWS")) + .put("name", Struct(propertyType).put(STRING, "john")) + .put("surname", Struct(propertyType).put(STRING, "doe")) val reverted = DynamicTypes.fromConnectValue(schema, converted) reverted shouldBe @@ -713,7 +709,7 @@ class DynamicTypesTest { } @Test - fun `maps with values of different types should be returned as structs and should be converted back`() { + fun `maps with values of different simple types should be returned as map of structs and should be converted back`() { val map = mapOf( "name" to "john", @@ -725,15 +721,14 @@ class DynamicTypesTest { val converted = DynamicTypes.toConnectValue(schema, map) converted shouldBe - Struct(schema) - .put("name", "john") - .put("age", 21L) - .put( - "dob", - Struct(SimpleTypes.LOCALDATE_STRUCT.schema) - .put(EPOCH_DAYS, LocalDate.of(1999, 12, 31).toEpochDay())) - .put("employed", true) - .put("nullable", null) + mapOf( + "name" to Struct(propertyType).put(STRING, "john"), + "age" to Struct(propertyType).put(LONG, 21L), + "dob" to + Struct(propertyType) + .put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(LocalDate.of(1999, 12, 31))), + "employed" to Struct(propertyType).put(BOOLEAN, true), + "nullable" to null) val reverted = DynamicTypes.fromConnectValue(schema, converted) reverted shouldBe map @@ -746,15 +741,13 @@ class DynamicTypesTest { val converted = DynamicTypes.toConnectValue(schema, coll) converted shouldBe - Struct(schema) - .put("e0", "john") - .put("e1", 21L) - .put( - "e2", - Struct(SimpleTypes.LOCALDATE_STRUCT.schema) - .put(EPOCH_DAYS, LocalDate.of(1999, 12, 31).toEpochDay())) - .put("e3", true) - .put("e4", null) + listOf( + Struct(propertyType).put(STRING, "john"), + Struct(propertyType).put(LONG, 21L), + Struct(propertyType) + .put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(LocalDate.of(1999, 12, 31))), + Struct(propertyType).put(BOOLEAN, true), + null) val reverted = DynamicTypes.fromConnectValue(schema, converted) reverted shouldBe coll @@ -767,17 +760,14 @@ class DynamicTypesTest { .field("id", Schema.INT32_SCHEMA) .field("name", Schema.STRING_SCHEMA) .field("last_name", Schema.STRING_SCHEMA) - .field("dob", SimpleTypes.LOCALDATE_STRUCT.schema) + .field("dob", propertyType) .build() val struct = Struct(schema) .put("id", 1) .put("name", "john") .put("last_name", "doe") - .put( - "dob", - DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE_STRUCT.schema, LocalDate.of(2000, 1, 1))) + .put("dob", DynamicTypes.toConnectValue(propertyType, LocalDate.of(2000, 1, 1))) DynamicTypes.fromConnectValue(schema, struct) shouldBe mapOf("id" to 1, "name" to "john", "last_name" to "doe", "dob" to LocalDate.of(2000, 1, 1)) @@ -786,32 +776,35 @@ class DynamicTypesTest { @Test fun `structs with complex values should be returned as maps`() { val addressSchema = - SchemaBuilder.struct() - .field("city", Schema.STRING_SCHEMA) - .field("country", Schema.STRING_SCHEMA) - .build() + SchemaBuilder.struct().field("city", propertyType).field("country", propertyType).build() val schema = SchemaBuilder.struct() - .field("id", Schema.INT32_SCHEMA) - .field("name", Schema.STRING_SCHEMA) - .field("last_name", Schema.STRING_SCHEMA) - .field("dob", SimpleTypes.LOCALDATE_STRUCT.schema) + .field("id", propertyType) + .field("name", propertyType) + .field("last_name", propertyType) + .field("dob", propertyType) .field("address", addressSchema) - .field("years_of_interest", SchemaBuilder.array(Schema.INT32_SCHEMA)) + .field("years_of_interest", propertyType) .field( "events_of_interest", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.STRING_SCHEMA)) .build() val struct = Struct(schema) - .put("id", 1) - .put("name", "john") - .put("last_name", "doe") + .put("id", Struct(propertyType).put(LONG, 1L)) + .put("name", Struct(propertyType).put(STRING, "john")) + .put("last_name", Struct(propertyType).put(STRING, "doe")) .put( "dob", - DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE_STRUCT.schema, LocalDate.of(2000, 1, 1))) - .put("address", Struct(addressSchema).put("city", "london").put("country", "uk")) - .put("years_of_interest", listOf(2000, 2005, 2017)) + Struct(propertyType) + .put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(LocalDate.of(2000, 1, 1)))) + .put( + "address", + Struct(addressSchema) + .put("city", Struct(propertyType).put(STRING, "london")) + .put("country", Struct(propertyType).put(STRING, "uk"))) + .put( + "years_of_interest", + Struct(propertyType).put(LONG_LIST, listOf(2000L, 2005L, 2017L))) .put( "events_of_interest", mapOf("2000" to "birth", "2005" to "school", "2017" to "college")) diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt index 40b368b89..91ef3fcd9 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt @@ -27,6 +27,7 @@ import java.time.OffsetTime import java.time.ZoneId import java.time.ZoneOffset import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter import java.util.stream.Stream import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder @@ -105,127 +106,116 @@ class TypesTest { override fun provideArguments(p0: ExtensionContext?): Stream { return Stream.of( - Arguments.of(Named.of("null", null), SimpleTypes.NULL.schema(), null), - Arguments.of(Named.of("boolean", true), SimpleTypes.BOOLEAN.schema(), true), - Arguments.of(Named.of("long", 1), SimpleTypes.LONG.schema(), 1L), - Arguments.of(Named.of("float", 1.0), SimpleTypes.FLOAT.schema(), 1.0), - Arguments.of(Named.of("string", "a string"), SimpleTypes.STRING.schema(), "a string"), + Arguments.of(Named.of("null", null), propertyType, null), + Arguments.of( + Named.of("boolean", true), propertyType, Struct(propertyType).put(BOOLEAN, true)), + Arguments.of(Named.of("long", 1), propertyType, Struct(propertyType).put(LONG, 1L)), + Arguments.of(Named.of("float", 1.0), propertyType, Struct(propertyType).put(FLOAT, 1.0)), + Arguments.of( + Named.of("string", "a string"), + propertyType, + Struct(propertyType).put(STRING, "a string")), LocalDate.of(1999, 12, 31).let { Arguments.of( Named.of("local date", it), - SimpleTypes.LOCALDATE_STRUCT.schema(), - Struct(SimpleTypes.LOCALDATE_STRUCT.schema).put(EPOCH_DAYS, it.toEpochDay())) + propertyType, + Struct(propertyType).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(it))) }, LocalTime.of(23, 59, 59, 5).let { Arguments.of( Named.of("local time", it), - SimpleTypes.LOCALTIME_STRUCT.schema(), - Struct(SimpleTypes.LOCALTIME_STRUCT.schema).put(NANOS_OF_DAY, it.toNanoOfDay())) + propertyType, + Struct(propertyType).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(it))) }, LocalDateTime.of(1999, 12, 31, 23, 59, 59, 5).let { Arguments.of( Named.of("local date time", it), - SimpleTypes.LOCALDATETIME_STRUCT.schema(), - Struct(SimpleTypes.LOCALDATETIME_STRUCT.schema) - .put(EPOCH_DAYS, it.toLocalDate().toEpochDay()) - .put(NANOS_OF_DAY, it.toLocalTime().toNanoOfDay())) + propertyType, + Struct(propertyType) + .put(LOCAL_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }, OffsetTime.of(23, 59, 59, 5, ZoneOffset.ofHours(1)).let { Arguments.of( Named.of("offset time", it), - SimpleTypes.OFFSETTIME_STRUCT.schema(), - Struct(SimpleTypes.OFFSETTIME_STRUCT.schema) - .put(NANOS_OF_DAY, it.toLocalTime().toNanoOfDay()) - .put(ZONE_ID, it.offset.id)) + propertyType, + Struct(propertyType).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(it))) }, OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 5, ZoneOffset.ofHours(1)).let { Arguments.of( Named.of("offset date time", it), - SimpleTypes.ZONEDDATETIME_STRUCT.schema(), - Struct(SimpleTypes.ZONEDDATETIME_STRUCT.schema) - .put(EPOCH_SECONDS, it.toEpochSecond()) - .put(NANOS_OF_SECOND, it.nano) - .put(ZONE_ID, it.offset.id)) + propertyType, + Struct(propertyType) + .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }, ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 5, ZoneId.of("Europe/Istanbul")).let { Arguments.of( Named.of("offset date time", it), - SimpleTypes.ZONEDDATETIME_STRUCT.schema(), - Struct(SimpleTypes.ZONEDDATETIME_STRUCT.schema) - .put(EPOCH_SECONDS, it.toEpochSecond()) - .put(NANOS_OF_SECOND, it.nano) - .put(ZONE_ID, it.zone.id)) + propertyType, + Struct(propertyType) + .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }, Arguments.of( Named.of("duration", Values.isoDuration(5, 2, 23, 5).asIsoDuration()), - SimpleTypes.DURATION.schema(), - Struct(SimpleTypes.DURATION.schema()) - .put("months", 5L) - .put("days", 2L) - .put("seconds", 23L) - .put("nanoseconds", 5)), + propertyType, + Struct(propertyType) + .put( + DURATION, + Struct(durationSchema) + .put("months", 5L) + .put("days", 2L) + .put("seconds", 23L) + .put("nanoseconds", 5))), Arguments.of( Named.of("point - 2d", Values.point(7203, 2.3, 4.5).asPoint()), - SimpleTypes.POINT.schema(), - Struct(SimpleTypes.POINT.schema()) - .put("dimension", 2.toByte()) - .put("srid", 7203) - .put("x", 2.3) - .put("y", 4.5) - .put("z", null)), + propertyType, + Struct(propertyType) + .put( + POINT, + Struct(pointSchema) + .put("dimension", 2.toByte()) + .put("srid", 7203) + .put("x", 2.3) + .put("y", 4.5) + .put("z", null))), Arguments.of( Named.of("point - 3d", Values.point(4979, 12.78, 56.7, 100.0).asPoint()), - SimpleTypes.POINT.schema(), - Struct(SimpleTypes.POINT.schema()) - .put("dimension", 3.toByte()) - .put("srid", 4979) - .put("x", 12.78) - .put("y", 56.7) - .put("z", 100.0)), + propertyType, + Struct(propertyType) + .put( + POINT, + Struct(pointSchema) + .put("dimension", 3.toByte()) + .put("srid", 4979) + .put("x", 12.78) + .put("y", 56.7) + .put("z", 100.0))), Arguments.of( Named.of("list - uniformly typed elements", (1L..50L).toList()), - SchemaBuilder.array(SimpleTypes.LONG.schema()).build(), - (1L..50L).toList()), + SchemaBuilder.array(propertyType).build(), + (1L..50L).map { Struct(propertyType).put(LONG, it) }.toList()), Arguments.of( Named.of("list - non-uniformly typed elements", listOf(1, true, 2.0, "a string")), - SchemaBuilder.struct() - .field("e0", SimpleTypes.LONG.schema()) - .field("e1", SimpleTypes.BOOLEAN.schema()) - .field("e2", SimpleTypes.FLOAT.schema()) - .field("e3", SimpleTypes.STRING.schema()) - .build(), - Struct( - SchemaBuilder.struct() - .field("e0", SimpleTypes.LONG.schema()) - .field("e1", SimpleTypes.BOOLEAN.schema()) - .field("e2", SimpleTypes.FLOAT.schema()) - .field("e3", SimpleTypes.STRING.schema()) - .build()) - .put("e0", 1L) - .put("e1", true) - .put("e2", 2.0) - .put("e3", "a string")), + SchemaBuilder.array(propertyType).build(), + listOf( + Struct(propertyType).put(LONG, 1L), + Struct(propertyType).put(BOOLEAN, true), + Struct(propertyType).put(FLOAT, 2.0), + Struct(propertyType).put(STRING, "a string"))), Arguments.of( Named.of("map - uniformly typed values", mapOf("a" to 1, "b" to 2, "c" to 3)), - SchemaBuilder.map(SimpleTypes.STRING.schema(), SimpleTypes.LONG.schema()).build(), - mapOf("a" to 1, "b" to 2, "c" to 3)), + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build(), + mapOf( + "a" to Struct(propertyType).put(LONG, 1L), + "b" to Struct(propertyType).put(LONG, 2L), + "c" to Struct(propertyType).put(LONG, 3L))), Arguments.of( Named.of( "map - non-uniformly typed values", mapOf("a" to 1, "b" to true, "c" to 3.0)), - SchemaBuilder.struct() - .field("a", SimpleTypes.LONG.schema()) - .field("b", SimpleTypes.BOOLEAN.schema()) - .field("c", SimpleTypes.FLOAT.schema()) - .build(), - Struct( - SchemaBuilder.struct() - .field("a", SimpleTypes.LONG.schema()) - .field("b", SimpleTypes.BOOLEAN.schema()) - .field("c", SimpleTypes.FLOAT.schema()) - .build()) - .put("a", 1L) - .put("b", true) - .put("c", 3.0))) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build(), + mapOf( + "a" to Struct(propertyType).put(LONG, 1L), + "b" to Struct(propertyType).put(BOOLEAN, true), + "c" to Struct(propertyType).put(FLOAT, 3.0)))) } } @@ -256,23 +246,25 @@ class TypesTest { schemaAndValue(person).also { (schema, converted, reverted) -> schema shouldBe SchemaBuilder.struct() - .field("", SimpleTypes.LONG.schema()) - .field("", SchemaBuilder.array(SimpleTypes.STRING.schema()).build()) - .field("name", SimpleTypes.STRING.schema()) - .field("surname", SimpleTypes.STRING.schema()) - .field("dob", SimpleTypes.LOCALDATE_STRUCT.schema()) + .field("", propertyType) + .field("", propertyType) + .field("name", propertyType) + .field("surname", propertyType) + .field("dob", propertyType) .build() converted shouldBe Struct(schema) - .put("", person.id()) - .put("", person.labels().toList()) - .put("name", "john") - .put("surname", "doe") + .put("", Struct(propertyType).put(LONG, person.id())) + .put("", Struct(propertyType).put(STRING_LIST, person.labels().toList())) + .put("name", Struct(propertyType).put(STRING, "john")) + .put("surname", Struct(propertyType).put(STRING, "doe")) .put( "dob", - Struct(SimpleTypes.LOCALDATE_STRUCT.schema()) - .put(EPOCH_DAYS, LocalDate.of(1999, 12, 31).toEpochDay())) + Struct(propertyType) + .put( + LOCAL_DATE, + DateTimeFormatter.ISO_DATE.format(LocalDate.of(1999, 12, 31)))) reverted shouldBe mapOf( @@ -287,21 +279,23 @@ class TypesTest { schemaAndValue(company).also { (schema, converted, reverted) -> schema shouldBe SchemaBuilder.struct() - .field("", SimpleTypes.LONG.schema()) - .field("", SchemaBuilder.array(SimpleTypes.STRING.schema()).build()) - .field("name", SimpleTypes.STRING.schema()) - .field("est", SimpleTypes.LOCALDATE_STRUCT.schema()) + .field("", propertyType) + .field("", propertyType) + .field("name", propertyType) + .field("est", propertyType) .build() converted shouldBe Struct(schema) - .put("", company.id()) - .put("", company.labels().toList()) - .put("name", "acme corp") + .put("", Struct(propertyType).put(LONG, company.id())) + .put("", Struct(propertyType).put(STRING_LIST, company.labels().toList())) + .put("name", Struct(propertyType).put(STRING, "acme corp")) .put( "est", - Struct(SimpleTypes.LOCALDATE_STRUCT.schema()) - .put(EPOCH_DAYS, LocalDate.of(1980, 1, 1).toEpochDay())) + Struct(propertyType) + .put( + LOCAL_DATE, + DateTimeFormatter.ISO_DATE.format(LocalDate.of(1980, 1, 1)))) reverted shouldBe mapOf( @@ -315,25 +309,27 @@ class TypesTest { schemaAndValue(worksFor).also { (schema, converted, reverted) -> schema shouldBe SchemaBuilder.struct() - .field("", SimpleTypes.LONG.schema()) - .field("", SimpleTypes.STRING.schema()) - .field("", SimpleTypes.LONG.schema()) - .field("", SimpleTypes.LONG.schema()) - .field("contractId", SimpleTypes.LONG.schema()) - .field("since", SimpleTypes.LOCALDATE_STRUCT.schema()) + .field("", propertyType) + .field("", propertyType) + .field("", propertyType) + .field("", propertyType) + .field("contractId", propertyType) + .field("since", propertyType) .build() converted shouldBe Struct(schema) - .put("", worksFor.id()) - .put("", worksFor.type()) - .put("", worksFor.startNodeId()) - .put("", worksFor.endNodeId()) - .put("contractId", 5916L) + .put("", Struct(propertyType).put(LONG, worksFor.id())) + .put("", Struct(propertyType).put(STRING, worksFor.type())) + .put("", Struct(propertyType).put(LONG, worksFor.startNodeId())) + .put("", Struct(propertyType).put(LONG, worksFor.endNodeId())) + .put("contractId", Struct(propertyType).put(LONG, 5916L)) .put( "since", - Struct(SimpleTypes.LOCALDATE_STRUCT.schema()) - .put(EPOCH_DAYS, LocalDate.of(2000, 1, 5).toEpochDay())) + Struct(propertyType) + .put( + LOCAL_DATE, + DateTimeFormatter.ISO_DATE.format(LocalDate.of(2000, 1, 5)))) reverted shouldBe mapOf( @@ -445,38 +441,27 @@ class TypesTest { val expectedSchema = SchemaBuilder.struct() - .field("id", Schema.OPTIONAL_STRING_SCHEMA) + .field("id", propertyType) .field( "data", SchemaBuilder.struct() .field( "arr", SchemaBuilder.array( - SchemaBuilder.map( - Schema.STRING_SCHEMA, Schema.OPTIONAL_STRING_SCHEMA) + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) .optional() .build()) .optional() .build()) .field( "arr_mixed", - SchemaBuilder.struct() - .field( - "e0", - SchemaBuilder.map( - Schema.STRING_SCHEMA, Schema.OPTIONAL_STRING_SCHEMA) - .optional() - .build()) - .field("e1", SimpleTypes.NULL.schema()) - .field( - "e2", - SchemaBuilder.map( - Schema.STRING_SCHEMA, Schema.OPTIONAL_INT64_SCHEMA) + SchemaBuilder.array( + SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) .optional() .build()) .optional() .build()) - .field("id", Schema.OPTIONAL_STRING_SCHEMA) + .field("id", propertyType) .field( "root", SchemaBuilder.array( @@ -484,8 +469,7 @@ class TypesTest { Schema.STRING_SCHEMA, SchemaBuilder.array( SchemaBuilder.map( - Schema.STRING_SCHEMA, - Schema.OPTIONAL_STRING_SCHEMA) + Schema.STRING_SCHEMA, propertyType) .optional() .build()) .optional() @@ -504,23 +488,30 @@ class TypesTest { val converted = DynamicTypes.toConnectValue(schema, returned) converted shouldBe Struct(schema) - .put("id", "ROOT_ID") + .put("id", Struct(propertyType).put(STRING, "ROOT_ID")) .put( "data", Struct(schema.field("data").schema()) - .put("arr", listOf(null, mapOf("foo" to "bar"))) + .put( + "arr", + listOf(null, mapOf("foo" to Struct(propertyType).put(STRING, "bar")))) .put( "arr_mixed", - Struct(schema.field("data").schema().field("arr_mixed").schema()) - .put("e0", mapOf("foo" to "bar")) - .put("e1", null) - .put("e2", mapOf("foo" to 1L))) - .put("id", "ROOT_ID") + listOf( + mapOf("foo" to Struct(propertyType).put(STRING, "bar")), + null, + mapOf("foo" to Struct(propertyType).put(LONG, 1L)))) + .put("id", Struct(propertyType).put(STRING, "ROOT_ID")) .put( "root", listOf( mapOf("children" to listOf()), - mapOf("children" to listOf(mapOf("name" to "child")))))) + mapOf( + "children" to + listOf( + mapOf( + "name" to + Struct(propertyType).put(STRING, "child"))))))) val reverted = DynamicTypes.fromConnectValue(schema, converted) reverted shouldBe returned diff --git a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCudIT.kt b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCudIT.kt index 69625bf2f..b066d11a8 100644 --- a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCudIT.kt +++ b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCudIT.kt @@ -27,7 +27,7 @@ import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct import org.junit.jupiter.api.Test import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.SimpleTypes +import org.neo4j.connectors.kafka.data.propertyType import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.format.KafkaConverter import org.neo4j.connectors.kafka.testing.format.KeyValueConverter @@ -122,8 +122,8 @@ abstract class Neo4jCudIT { SchemaBuilder.struct() .field("id", Schema.INT64_SCHEMA) .field("foo", Schema.STRING_SCHEMA) - .field("dob", SimpleTypes.LOCALDATE_STRUCT.schema) - .field("place", SimpleTypes.POINT.schema) + .field("dob", propertyType) + .field("place", propertyType) .build() val createNodeSchema = SchemaBuilder.struct() @@ -147,12 +147,11 @@ abstract class Neo4jCudIT { .put("foo", "foo-value") .put( "dob", - DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE_STRUCT.schema, LocalDate.of(1995, 1, 1))) + DynamicTypes.toConnectValue(propertyType, LocalDate.of(1995, 1, 1))) .put( "place", DynamicTypes.toConnectValue( - SimpleTypes.POINT.schema, Values.point(7203, 1.0, 2.5).asPoint()))), + propertyType, Values.point(7203, 1.0, 2.5).asPoint()))), ) eventually(30.seconds) { session.run("MATCH (n) RETURN n", emptyMap()).single() } @@ -188,8 +187,8 @@ abstract class Neo4jCudIT { val propertiesSchema = SchemaBuilder.struct() .field("foo", Schema.STRING_SCHEMA) - .field("dob", SimpleTypes.LOCALDATE_STRUCT.schema) - .field("place", SimpleTypes.POINT.schema) + .field("dob", propertyType) + .field("place", propertyType) .build() val updateNodeSchema = SchemaBuilder.struct() @@ -214,12 +213,11 @@ abstract class Neo4jCudIT { .put("foo", "foo-value-updated") .put( "dob", - DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE_STRUCT.schema, LocalDate.of(1995, 1, 1))) + DynamicTypes.toConnectValue(propertyType, LocalDate.of(1995, 1, 1))) .put( "place", DynamicTypes.toConnectValue( - SimpleTypes.POINT.schema, Values.point(7203, 1.0, 2.5).asPoint()))), + propertyType, Values.point(7203, 1.0, 2.5).asPoint()))), ) eventually(30.seconds) { @@ -256,8 +254,8 @@ abstract class Neo4jCudIT { SchemaBuilder.struct() .field("id", Schema.INT64_SCHEMA) .field("foo_new", Schema.STRING_SCHEMA) - .field("dob", SimpleTypes.LOCALDATE_STRUCT.schema) - .field("place", SimpleTypes.POINT.schema) + .field("dob", propertyType) + .field("place", propertyType) .build() val mergeNodeSchema = SchemaBuilder.struct() @@ -283,12 +281,11 @@ abstract class Neo4jCudIT { .put("foo_new", "foo-new-value-merged") .put( "dob", - DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE_STRUCT.schema, LocalDate.of(1995, 1, 1))) + DynamicTypes.toConnectValue(propertyType, LocalDate.of(1995, 1, 1))) .put( "place", DynamicTypes.toConnectValue( - SimpleTypes.POINT.schema, Values.point(7203, 1.0, 2.5).asPoint()))), + propertyType, Values.point(7203, 1.0, 2.5).asPoint()))), ) eventually(30.seconds) { diff --git a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt index 3214971ad..5af5f0798 100644 --- a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt +++ b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt @@ -43,7 +43,7 @@ import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.ArgumentsProvider import org.junit.jupiter.params.provider.ArgumentsSource import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.SimpleTypes +import org.neo4j.connectors.kafka.data.propertyType import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.format.KafkaConverter import org.neo4j.connectors.kafka.testing.format.KeyValueConverter @@ -299,17 +299,7 @@ abstract class Neo4jCypherIT { Arguments.of(Schema.OPTIONAL_BOOLEAN_SCHEMA), Arguments.of(Schema.OPTIONAL_STRING_SCHEMA), Arguments.of(Schema.OPTIONAL_BYTES_SCHEMA), - Arguments.of(SimpleTypes.LOCALDATE.schema(true)), - Arguments.of(SimpleTypes.LOCALTIME.schema(true)), - Arguments.of(SimpleTypes.LOCALDATE.schema(true)), - Arguments.of(SimpleTypes.OFFSETTIME.schema(true)), - Arguments.of(SimpleTypes.ZONEDDATETIME.schema(true)), - Arguments.of(SimpleTypes.LOCALDATE_STRUCT.schema(true)), - Arguments.of(SimpleTypes.LOCALTIME_STRUCT.schema(true)), - Arguments.of(SimpleTypes.LOCALDATE_STRUCT.schema(true)), - Arguments.of(SimpleTypes.OFFSETTIME_STRUCT.schema(true)), - Arguments.of(SimpleTypes.ZONEDDATETIME_STRUCT.schema(true)), - Arguments.of(SimpleTypes.DURATION.schema(true))) + Arguments.of(propertyType)) } } @@ -339,49 +329,37 @@ abstract class Neo4jCypherIT { object KnownTypes : ArgumentsProvider { override fun provideArguments(context: ExtensionContext?): Stream { return Stream.of( - Arguments.of(SimpleTypes.BOOLEAN.schema(false), true), - Arguments.of(SimpleTypes.BOOLEAN.schema(true), false), - Arguments.of(SimpleTypes.LONG.schema(false), Long.MAX_VALUE), - Arguments.of(SimpleTypes.LONG.schema(true), Long.MIN_VALUE), - Arguments.of(SimpleTypes.FLOAT.schema(false), Double.MAX_VALUE), - Arguments.of(SimpleTypes.FLOAT.schema(true), Double.MIN_VALUE), - Arguments.of(SimpleTypes.STRING.schema(false), "a string"), - Arguments.of(SimpleTypes.STRING.schema(true), "another string"), - Arguments.of(SimpleTypes.BYTES.schema(false), "a string".encodeToByteArray()), - Arguments.of(SimpleTypes.BYTES.schema(true), "another string".encodeToByteArray()), - Arguments.of(SimpleTypes.LOCALDATE_STRUCT.schema(false), LocalDate.of(2019, 5, 1)), - Arguments.of(SimpleTypes.LOCALDATE_STRUCT.schema(true), LocalDate.of(2019, 5, 1)), + Arguments.of(propertyType, true), + Arguments.of(propertyType, false), + Arguments.of(propertyType, Long.MAX_VALUE), + Arguments.of(propertyType, Long.MIN_VALUE), + Arguments.of(propertyType, Double.MAX_VALUE), + Arguments.of(propertyType, Double.MIN_VALUE), + Arguments.of(propertyType, "a string"), + Arguments.of(propertyType, "another string"), + Arguments.of(propertyType, "a string".encodeToByteArray()), + Arguments.of(propertyType, "another string".encodeToByteArray()), + Arguments.of(propertyType, LocalDate.of(2019, 5, 1)), + Arguments.of(propertyType, LocalDate.of(2019, 5, 1)), + Arguments.of(propertyType, LocalDateTime.of(2019, 5, 1, 23, 59, 59, 999999999)), + Arguments.of(propertyType, LocalDateTime.of(2019, 5, 1, 23, 59, 59, 999999999)), + Arguments.of(propertyType, LocalTime.of(23, 59, 59, 999999999)), + Arguments.of(propertyType, LocalTime.of(23, 59, 59, 999999999)), Arguments.of( - SimpleTypes.LOCALDATETIME_STRUCT.schema(false), - LocalDateTime.of(2019, 5, 1, 23, 59, 59, 999999999)), - Arguments.of( - SimpleTypes.LOCALDATETIME_STRUCT.schema(true), - LocalDateTime.of(2019, 5, 1, 23, 59, 59, 999999999)), - Arguments.of( - SimpleTypes.LOCALTIME_STRUCT.schema(false), LocalTime.of(23, 59, 59, 999999999)), - Arguments.of( - SimpleTypes.LOCALTIME_STRUCT.schema(true), LocalTime.of(23, 59, 59, 999999999)), - Arguments.of( - SimpleTypes.ZONEDDATETIME_STRUCT.schema(false), + propertyType, ZonedDateTime.of(2019, 5, 1, 23, 59, 59, 999999999, ZoneId.of("Europe/Istanbul"))), Arguments.of( - SimpleTypes.ZONEDDATETIME_STRUCT.schema(true), + propertyType, ZonedDateTime.of(2019, 5, 1, 23, 59, 59, 999999999, ZoneId.of("Europe/Istanbul"))), + Arguments.of(propertyType, OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHours(2))), Arguments.of( - SimpleTypes.OFFSETTIME_STRUCT.schema(false), - OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHours(2))), - Arguments.of( - SimpleTypes.OFFSETTIME_STRUCT.schema(true), - OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHoursMinutes(2, 30))), - Arguments.of( - SimpleTypes.DURATION.schema(false), Values.isoDuration(5, 4, 3, 2).asIsoDuration()), - Arguments.of( - SimpleTypes.DURATION.schema(true), Values.isoDuration(5, 4, 3, 2).asIsoDuration()), - Arguments.of(SimpleTypes.POINT.schema(false), Values.point(7203, 2.3, 4.5).asPoint()), - Arguments.of(SimpleTypes.POINT.schema(true), Values.point(7203, 2.3, 4.5).asPoint()), - Arguments.of( - SimpleTypes.POINT.schema(false), Values.point(4979, 2.3, 4.5, 0.0).asPoint()), - Arguments.of(SimpleTypes.POINT.schema(true), Values.point(4979, 2.3, 4.5, 0.0).asPoint()), + propertyType, OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHoursMinutes(2, 30))), + Arguments.of(propertyType, Values.isoDuration(5, 4, 3, 2).asIsoDuration()), + Arguments.of(propertyType, Values.isoDuration(5, 4, 3, 2).asIsoDuration()), + Arguments.of(propertyType, Values.point(7203, 2.3, 4.5).asPoint()), + Arguments.of(propertyType, Values.point(7203, 2.3, 4.5).asPoint()), + Arguments.of(propertyType, Values.point(4979, 2.3, 4.5, 0.0).asPoint()), + Arguments.of(propertyType, Values.point(4979, 2.3, 4.5, 0.0).asPoint()), ) } } diff --git a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jNodePatternIT.kt b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jNodePatternIT.kt index ac654682f..99be2d8bd 100644 --- a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jNodePatternIT.kt +++ b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jNodePatternIT.kt @@ -30,7 +30,7 @@ import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct import org.junit.jupiter.api.Test import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.SimpleTypes +import org.neo4j.connectors.kafka.data.propertyType import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.format.KafkaConverter import org.neo4j.connectors.kafka.testing.format.KeyValueConverter @@ -88,8 +88,8 @@ abstract class Neo4jNodePatternIT { .field("id", Schema.INT64_SCHEMA) .field("name", Schema.STRING_SCHEMA) .field("surname", Schema.STRING_SCHEMA) - .field("dob", SimpleTypes.LOCALDATE_STRUCT.schema) - .field("place", SimpleTypes.POINT.schema) + .field("dob", propertyType) + .field("place", propertyType) .build() .let { schema -> producer.publish( @@ -101,12 +101,11 @@ abstract class Neo4jNodePatternIT { .put("surname", "doe") .put( "dob", - DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE_STRUCT.schema, LocalDate.of(1995, 1, 1))) + DynamicTypes.toConnectValue(propertyType, LocalDate.of(1995, 1, 1))) .put( "place", DynamicTypes.toConnectValue( - SimpleTypes.POINT.schema, Values.point(7203, 1.0, 2.5).asPoint()))) + propertyType, Values.point(7203, 1.0, 2.5).asPoint()))) } eventually(30.seconds) { session.run("MATCH (n:User) RETURN n", emptyMap()).single() } diff --git a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jRelationshipPatternIT.kt b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jRelationshipPatternIT.kt index 8028a4477..b85d7d4f6 100644 --- a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jRelationshipPatternIT.kt +++ b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jRelationshipPatternIT.kt @@ -29,7 +29,7 @@ import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct import org.junit.jupiter.api.Test import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.SimpleTypes +import org.neo4j.connectors.kafka.data.propertyType import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.format.KafkaConverter import org.neo4j.connectors.kafka.testing.format.KeyValueConverter @@ -68,8 +68,8 @@ abstract class Neo4jRelationshipPatternIT { SchemaBuilder.struct() .field("userId", Schema.INT64_SCHEMA) .field("productId", Schema.INT64_SCHEMA) - .field("at", SimpleTypes.LOCALDATE_STRUCT.schema) - .field("place", SimpleTypes.POINT.schema) + .field("at", propertyType) + .field("place", propertyType) .build() .let { schema -> producer.publish( @@ -79,13 +79,11 @@ abstract class Neo4jRelationshipPatternIT { .put("userId", 1L) .put("productId", 2L) .put( - "at", - DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE_STRUCT.schema, LocalDate.of(1995, 1, 1))) + "at", DynamicTypes.toConnectValue(propertyType, LocalDate.of(1995, 1, 1))) .put( "place", DynamicTypes.toConnectValue( - SimpleTypes.POINT.schema, Values.point(7203, 1.0, 2.5).asPoint()))) + propertyType, Values.point(7203, 1.0, 2.5).asPoint()))) } eventually(30.seconds) { diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/NodePatternHandlerTest.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/NodePatternHandlerTest.kt index fb02d314f..993b2b8dc 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/NodePatternHandlerTest.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/NodePatternHandlerTest.kt @@ -30,7 +30,7 @@ import org.neo4j.connectors.kafka.data.ConstraintData import org.neo4j.connectors.kafka.data.ConstraintEntityType import org.neo4j.connectors.kafka.data.ConstraintType import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.SimpleTypes +import org.neo4j.connectors.kafka.data.propertyType import org.neo4j.connectors.kafka.exceptions.InvalidDataException import org.neo4j.connectors.kafka.sink.ChangeQuery import org.neo4j.cypherdsl.core.renderer.Renderer @@ -295,7 +295,7 @@ class NodePatternHandlerTest : HandlerTest() { .field("id", Schema.INT32_SCHEMA) .field("name", Schema.STRING_SCHEMA) .field("surname", Schema.STRING_SCHEMA) - .field("dob", SimpleTypes.LOCALDATE_STRUCT.schema) + .field("dob", propertyType) .build() assertQueryAndParameters( @@ -306,10 +306,7 @@ class NodePatternHandlerTest : HandlerTest() { .put("id", 1) .put("name", "john") .put("surname", "doe") - .put( - "dob", - DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE_STRUCT.schema, LocalDate.of(2000, 1, 1))), + .put("dob", DynamicTypes.toConnectValue(propertyType, LocalDate.of(2000, 1, 1))), expected = listOf( listOf( diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt index e3bc871e0..676a3eabb 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt @@ -37,8 +37,8 @@ import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.connectors.kafka.connect.ConnectHeader import org.neo4j.connectors.kafka.data.DynamicTypes import org.neo4j.connectors.kafka.data.Headers -import org.neo4j.connectors.kafka.data.SimpleTypes import org.neo4j.connectors.kafka.data.TemporalDataSchemaType +import org.neo4j.connectors.kafka.data.propertyType import org.neo4j.connectors.kafka.testing.assertions.TopicVerifier import org.neo4j.connectors.kafka.testing.format.KafkaConverter.AVRO import org.neo4j.connectors.kafka.testing.format.KafkaConverter.JSON_SCHEMA @@ -238,37 +238,37 @@ abstract class Neo4jCdcSourceIT { properties.getStruct("localDate") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE_STRUCT.schema, + propertyType, LocalDate.of(2024, 1, 1), ) as Struct properties.getStruct("localDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATETIME_STRUCT.schema, + propertyType, LocalDateTime.of(2024, 1, 1, 12, 0, 0), ) as Struct properties.getStruct("localTime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.LOCALTIME_STRUCT.schema, + propertyType, LocalTime.of(12, 0, 0), ) as Struct properties.getStruct("zonedDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.ZONEDDATETIME_STRUCT.schema, + propertyType, ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), ) as Struct properties.getStruct("offsetDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.ZONEDDATETIME_STRUCT.schema, + propertyType, OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), ) as Struct properties.getStruct("offsetTime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.OFFSETTIME_STRUCT.schema, + propertyType, OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), ) as Struct } @@ -314,37 +314,37 @@ abstract class Neo4jCdcSourceIT { properties.getString("localDate") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE.schema, + propertyType, LocalDate.of(2024, 1, 1), ) properties.getString("localDatetime") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATETIME.schema, + propertyType, LocalDateTime.of(2024, 1, 1, 12, 0, 0), ) properties.getString("localTime") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.LOCALTIME.schema, + propertyType, LocalTime.of(12, 0, 0), ) properties.getString("zonedDatetime") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.ZONEDDATETIME.schema, + propertyType, ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), ) properties.getString("offsetDatetime") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.ZONEDDATETIME.schema, + propertyType, OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), ) properties.getString("offsetTime") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.OFFSETTIME.schema, + propertyType, OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), ) } diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt index fa5b4c0df..1617f2e37 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt @@ -30,8 +30,8 @@ import java.time.ZonedDateTime import org.apache.kafka.connect.data.Struct import org.junit.jupiter.api.Test import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.SimpleTypes import org.neo4j.connectors.kafka.data.TemporalDataSchemaType +import org.neo4j.connectors.kafka.data.propertyType import org.neo4j.connectors.kafka.testing.MapSupport.excludingKeys import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.assertions.TopicVerifier @@ -215,37 +215,37 @@ abstract class Neo4jSourceQueryIT { .assertMessageValue { value -> value.getStruct("localDate") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE_STRUCT.schema, + propertyType, LocalDate.of(2024, 1, 1), ) as Struct value.getStruct("localDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATETIME_STRUCT.schema, + propertyType, LocalDateTime.of(2024, 1, 1, 12, 0, 0), ) as Struct value.getStruct("localTime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.LOCALTIME_STRUCT.schema, + propertyType, LocalTime.of(12, 0, 0), ) as Struct value.getStruct("zonedDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.ZONEDDATETIME_STRUCT.schema, + propertyType, ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), ) as Struct value.getStruct("offsetDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.ZONEDDATETIME_STRUCT.schema, + propertyType, OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), ) as Struct value.getStruct("offsetTime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - SimpleTypes.OFFSETTIME_STRUCT.schema, + propertyType, OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), ) as Struct } @@ -288,37 +288,37 @@ abstract class Neo4jSourceQueryIT { .assertMessageValue { value -> value.getString("localDate") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATE.schema, + propertyType, LocalDate.of(2024, 1, 1), ) value.getString("localDatetime") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.LOCALDATETIME.schema, + propertyType, LocalDateTime.of(2024, 1, 1, 12, 0, 0), ) value.getString("localTime") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.LOCALTIME.schema, + propertyType, LocalTime.of(12, 0, 0), ) value.getString("zonedDatetime") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.ZONEDDATETIME.schema, + propertyType, ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), ) value.getString("offsetDatetime") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.ZONEDDATETIME.schema, + propertyType, OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), ) value.getString("offsetTime") shouldBe DynamicTypes.toConnectValue( - SimpleTypes.OFFSETTIME.schema, + propertyType, OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), ) } From a42738a7bc3eeddf874da6931e4f96ab1c1d5c7c Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Fri, 26 Jul 2024 23:42:07 +0100 Subject: [PATCH 2/9] refactor: second pass --- .../kafka/data/ChangeEventExtensions.kt | 26 +- .../org/neo4j/connectors/kafka/data/Types.kt | 754 +++++++++--------- .../kafka/data/ChangeEventExtensionsTest.kt | 240 +++--- .../connectors/kafka/data/DynamicTypesTest.kt | 347 ++++---- .../neo4j/connectors/kafka/data/TypesTest.kt | 199 +++-- .../neo4j/connectors/kafka/sink/Neo4jCudIT.kt | 30 +- .../connectors/kafka/sink/Neo4jCypherIT.kt | 59 +- .../kafka/sink/Neo4jNodePatternIT.kt | 12 +- .../kafka/sink/Neo4jRelationshipPatternIT.kt | 13 +- .../sink/strategy/NodePatternHandlerTest.kt | 9 +- .../kafka/source/Neo4jCdcSourceIT.kt | 27 +- .../kafka/source/Neo4jSourceQueryIT.kt | 27 +- 12 files changed, 867 insertions(+), 876 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt index 9d8b77775..25b667a90 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt @@ -71,8 +71,8 @@ class ChangeEventConverter( .field("connectionServer", Schema.OPTIONAL_STRING_SCHEMA) .field("serverId", Schema.STRING_SCHEMA) .field("captureMode", Schema.STRING_SCHEMA) - .field("txStartTime", propertyType) - .field("txCommitTime", propertyType) + .field("txStartTime", PropertyType.schema) + .field("txCommitTime", PropertyType.schema) .field( "txMetadata", toConnectSchema( @@ -102,8 +102,10 @@ class ChangeEventConverter( it.put("connectionServer", metadata.connectionServer) it.put("serverId", metadata.serverId) it.put("captureMode", metadata.captureMode.name) - it.put("txStartTime", DynamicTypes.toConnectValue(propertyType, metadata.txStartTime)) - it.put("txCommitTime", DynamicTypes.toConnectValue(propertyType, metadata.txCommitTime)) + it.put( + "txStartTime", DynamicTypes.toConnectValue(PropertyType.schema, metadata.txStartTime)) + it.put( + "txCommitTime", DynamicTypes.toConnectValue(PropertyType.schema, metadata.txCommitTime)) it.put( "txMetadata", DynamicTypes.toConnectValue(schema.field("txMetadata").schema(), metadata.txMetadata)) @@ -209,7 +211,7 @@ class ChangeEventConverter( } private fun schemaForKeys(): Schema { - return SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + return SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) .optional() .build() } @@ -220,7 +222,8 @@ class ChangeEventConverter( .apply { this.field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) this.field( - "properties", SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + "properties", + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) } .optional() .build() @@ -238,7 +241,7 @@ class ChangeEventConverter( it.put( "properties", before.properties.mapValues { e -> - DynamicTypes.toConnectValue(propertyType, e.value) + DynamicTypes.toConnectValue(PropertyType.schema, e.value) }) }) } @@ -251,7 +254,7 @@ class ChangeEventConverter( it.put( "properties", after.properties.mapValues { e -> - DynamicTypes.toConnectValue(propertyType, e.value) + DynamicTypes.toConnectValue(PropertyType.schema, e.value) }) }) } @@ -265,7 +268,8 @@ class ChangeEventConverter( SchemaBuilder.struct() .apply { this.field( - "properties", SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + "properties", + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) } .optional() .build() @@ -286,7 +290,7 @@ class ChangeEventConverter( it.put( "properties", before.properties.mapValues { e -> - DynamicTypes.toConnectValue(propertyType, e.value) + DynamicTypes.toConnectValue(PropertyType.schema, e.value) }) }) } @@ -298,7 +302,7 @@ class ChangeEventConverter( it.put( "properties", after.properties.mapValues { e -> - DynamicTypes.toConnectValue(propertyType, e.value) + DynamicTypes.toConnectValue(PropertyType.schema, e.value) }) }) } diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt index 73139e6de..6bcd8addd 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt @@ -42,94 +42,325 @@ internal fun Schema.id(): String = this.name().orEmpty().ifEmpty { this.type().n internal fun Schema.shortId(): String = this.id().split('.').last() -const val EPOCH_DAYS = "epochDays" -const val NANOS_OF_DAY = "nanosOfDay" -const val EPOCH_SECONDS = "epochSeconds" -const val NANOS_OF_SECOND = "nanosOfSecond" -const val ZONE_ID = "zoneId" -const val MONTHS = "months" -const val DAYS = "days" -const val SECONDS = "seconds" -const val NANOS = "nanoseconds" -const val SR_ID = "srid" -const val X = "x" -const val Y = "y" -const val Z = "z" -const val DIMENSION = "dimension" -const val TWO_D: Byte = 2 -const val THREE_D: Byte = 3 - -val durationSchema: Schema = - SchemaBuilder(Schema.Type.STRUCT) - .field(MONTHS, Schema.INT64_SCHEMA) - .field(DAYS, Schema.INT64_SCHEMA) - .field(SECONDS, Schema.INT64_SCHEMA) - .field(NANOS, Schema.INT32_SCHEMA) - .optional() - .build() - -val pointSchema: Schema = - SchemaBuilder(Schema.Type.STRUCT) - .field(DIMENSION, Schema.INT8_SCHEMA) - .field(SR_ID, Schema.INT32_SCHEMA) - .field(X, Schema.FLOAT64_SCHEMA) - .field(Y, Schema.FLOAT64_SCHEMA) - .field(Z, Schema.OPTIONAL_FLOAT64_SCHEMA) - .optional() - .build() - -const val BOOLEAN = "B" -const val BOOLEAN_LIST = "LB" -const val LONG = "I64" -const val LONG_LIST = "LI64" -const val FLOAT = "F64" -const val FLOAT_LIST = "LF64" -const val STRING = "S" -const val STRING_LIST = "LS" -const val BYTES = "BA" -const val LOCAL_DATE = "TLD" -const val LOCAL_DATE_LIST = "LTLD" -const val LOCAL_DATE_TIME = "TLDT" -const val LOCAL_DATE_TIME_LIST = "LTLDT" -const val LOCAL_TIME = "TLT" -const val LOCAL_TIME_LIST = "LTLT" -const val ZONED_DATE_TIME = "TZDT" -const val ZONED_DATE_TIME_LIST = "LZDT" -const val OFFSET_TIME = "TOT" -const val OFFSET_TIME_LIST = "LTOT" -const val DURATION = "TD" -const val DURATION_LIST = "LTD" -const val POINT = "SP" -const val POINT_LIST = "LSP" - -val propertyType: Schema = - SchemaBuilder.struct() - .namespaced("Neo4jSimpleType") - .field(BOOLEAN, Schema.OPTIONAL_BOOLEAN_SCHEMA) - .field(LONG, Schema.OPTIONAL_INT64_SCHEMA) - .field(FLOAT, Schema.OPTIONAL_FLOAT64_SCHEMA) - .field(STRING, Schema.OPTIONAL_STRING_SCHEMA) - .field(BYTES, Schema.OPTIONAL_BYTES_SCHEMA) - .field(LOCAL_DATE, Schema.OPTIONAL_STRING_SCHEMA) - .field(LOCAL_DATE_TIME, Schema.OPTIONAL_STRING_SCHEMA) - .field(LOCAL_TIME, Schema.OPTIONAL_STRING_SCHEMA) - .field(ZONED_DATE_TIME, Schema.OPTIONAL_STRING_SCHEMA) - .field(OFFSET_TIME, Schema.OPTIONAL_STRING_SCHEMA) - .field(DURATION, durationSchema) - .field(POINT, pointSchema) - .field(BOOLEAN_LIST, SchemaBuilder.array(Schema.BOOLEAN_SCHEMA).optional().build()) - .field(LONG_LIST, SchemaBuilder.array(Schema.INT64_SCHEMA).optional().build()) - .field(FLOAT_LIST, SchemaBuilder.array(Schema.FLOAT64_SCHEMA).optional().build()) - .field(STRING_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(LOCAL_DATE_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(LOCAL_DATE_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(LOCAL_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(ZONED_DATE_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(OFFSET_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(DURATION_LIST, SchemaBuilder.array(durationSchema).optional().build()) - .field(POINT_LIST, SchemaBuilder.array(pointSchema).optional().build()) - .optional() - .build() +@Suppress("UNCHECKED_CAST") +object PropertyType { + const val MONTHS = "months" + const val DAYS = "days" + const val SECONDS = "seconds" + const val NANOS = "nanoseconds" + const val SR_ID = "srid" + const val X = "x" + const val Y = "y" + const val Z = "z" + const val DIMENSION = "dimension" + const val TWO_D: Byte = 2 + const val THREE_D: Byte = 3 + + const val BOOLEAN = "B" + const val BOOLEAN_LIST = "LB" + const val LONG = "I64" + const val LONG_LIST = "LI64" + const val FLOAT = "F64" + const val FLOAT_LIST = "LF64" + const val STRING = "S" + const val STRING_LIST = "LS" + const val BYTES = "BA" + const val LOCAL_DATE = "TLD" + const val LOCAL_DATE_LIST = "LTLD" + const val LOCAL_DATE_TIME = "TLDT" + const val LOCAL_DATE_TIME_LIST = "LTLDT" + const val LOCAL_TIME = "TLT" + const val LOCAL_TIME_LIST = "LTLT" + const val ZONED_DATE_TIME = "TZDT" + const val ZONED_DATE_TIME_LIST = "LZDT" + const val OFFSET_TIME = "TOT" + const val OFFSET_TIME_LIST = "LTOT" + const val DURATION = "TD" + const val DURATION_LIST = "LTD" + const val POINT = "SP" + const val POINT_LIST = "LSP" + + val durationSchema: Schema = + SchemaBuilder(Schema.Type.STRUCT) + .field(MONTHS, Schema.INT64_SCHEMA) + .field(DAYS, Schema.INT64_SCHEMA) + .field(SECONDS, Schema.INT64_SCHEMA) + .field(NANOS, Schema.INT32_SCHEMA) + .optional() + .build() + + val pointSchema: Schema = + SchemaBuilder(Schema.Type.STRUCT) + .field(DIMENSION, Schema.INT8_SCHEMA) + .field(SR_ID, Schema.INT32_SCHEMA) + .field(X, Schema.FLOAT64_SCHEMA) + .field(Y, Schema.FLOAT64_SCHEMA) + .field(Z, Schema.OPTIONAL_FLOAT64_SCHEMA) + .optional() + .build() + + val schema: Schema = + SchemaBuilder.struct() + .namespaced("Neo4jPropertyType") + .field(BOOLEAN, Schema.OPTIONAL_BOOLEAN_SCHEMA) + .field(LONG, Schema.OPTIONAL_INT64_SCHEMA) + .field(FLOAT, Schema.OPTIONAL_FLOAT64_SCHEMA) + .field(STRING, Schema.OPTIONAL_STRING_SCHEMA) + .field(BYTES, Schema.OPTIONAL_BYTES_SCHEMA) + .field(LOCAL_DATE, Schema.OPTIONAL_STRING_SCHEMA) + .field(LOCAL_DATE_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(LOCAL_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(ZONED_DATE_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(OFFSET_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(DURATION, durationSchema) + .field(POINT, pointSchema) + .field(BOOLEAN_LIST, SchemaBuilder.array(Schema.BOOLEAN_SCHEMA).optional().build()) + .field(LONG_LIST, SchemaBuilder.array(Schema.INT64_SCHEMA).optional().build()) + .field(FLOAT_LIST, SchemaBuilder.array(Schema.FLOAT64_SCHEMA).optional().build()) + .field(STRING_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(LOCAL_DATE_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(LOCAL_DATE_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(LOCAL_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(ZONED_DATE_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(OFFSET_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(DURATION_LIST, SchemaBuilder.array(durationSchema).optional().build()) + .field(POINT_LIST, SchemaBuilder.array(pointSchema).optional().build()) + .optional() + .build() + + fun toConnectValue(value: Any?): Struct? { + return when (value) { + is Boolean -> Struct(schema).put(BOOLEAN, value) + is Float -> Struct(schema).put(FLOAT, value.toDouble()) + is Double -> Struct(schema).put(FLOAT, value) + is Number -> Struct(schema).put(LONG, value.toLong()) + is String -> Struct(schema).put(STRING, value) + is Char -> Struct(schema).put(STRING, value.toString()) + is CharArray -> Struct(schema).put(STRING, String(value)) + is ByteArray -> Struct(schema).put(BYTES, value) + is ByteBuffer -> Struct(schema).put(BYTES, value.array()) + is LocalDate -> Struct(schema).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(value)) + is LocalDateTime -> + Struct(schema).put(LOCAL_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) + is LocalTime -> Struct(schema).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(value)) + is OffsetDateTime -> + Struct(schema).put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) + is ZonedDateTime -> + Struct(schema).put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) + is OffsetTime -> Struct(schema).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(value)) + is IsoDuration -> + Struct(schema) + .put( + DURATION, + Struct(durationSchema) + .put(MONTHS, value.months()) + .put(DAYS, value.days()) + .put(SECONDS, value.seconds()) + .put(NANOS, value.nanoseconds())) + is Point -> + Struct(schema) + .put( + POINT, + Struct(pointSchema) + .put(SR_ID, value.srid()) + .put(X, value.x()) + .put(Y, value.y()) + .also { + it.put(DIMENSION, if (value.z().isNaN()) TWO_D else THREE_D) + if (!value.z().isNaN()) { + it.put(Z, value.z()) + } + }) + is ShortArray -> Struct(schema).put(LONG_LIST, value.map { s -> s.toLong() }.toList()) + is IntArray -> Struct(schema).put(LONG_LIST, value.map { s -> s.toLong() }.toList()) + is LongArray -> Struct(schema).put(LONG_LIST, value.toList()) + is FloatArray -> Struct(schema).put(FLOAT_LIST, value.map { s -> s.toDouble() }.toList()) + is DoubleArray -> Struct(schema).put(FLOAT_LIST, value.toList()) + is BooleanArray -> Struct(schema).put(BOOLEAN_LIST, value.toList()) + is Array<*> -> + when (val componentType = value::class.java.componentType.kotlin) { + Boolean::class -> Struct(schema).put(BOOLEAN_LIST, value.toList()) + Byte::class -> Struct(schema).put(BYTES, (value as Array).toByteArray()) + Short::class -> + Struct(schema) + .put(LONG_LIST, (value as Array).map { s -> s.toLong() }.toList()) + Int::class -> + Struct(schema) + .put(LONG_LIST, (value as Array).map { s -> s.toLong() }.toList()) + Long::class -> Struct(schema).put(LONG_LIST, (value as Array).toList()) + Float::class -> + Struct(schema) + .put(FLOAT_LIST, (value as Array).map { s -> s.toDouble() }.toList()) + Double::class -> Struct(schema).put(FLOAT_LIST, (value as Array).toList()) + String::class -> Struct(schema).put(STRING_LIST, value.toList()) + LocalDate::class -> + Struct(schema) + .put( + LOCAL_DATE_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_DATE.format(s) } + .toList()) + LocalDateTime::class -> + Struct(schema) + .put( + LOCAL_DATE_TIME_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } + .toList()) + LocalTime::class -> + Struct(schema) + .put( + LOCAL_TIME_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_TIME.format(s) } + .toList()) + OffsetDateTime::class -> + Struct(schema) + .put( + ZONED_DATE_TIME_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } + .toList()) + ZonedDateTime::class -> + Struct(schema) + .put( + ZONED_DATE_TIME_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } + .toList()) + OffsetTime::class -> + Struct(schema) + .put( + OFFSET_TIME_LIST, + (value as Array) + .map { s -> DateTimeFormatter.ISO_TIME.format(s) } + .toList()) + else -> + if (IsoDuration::class.java.isAssignableFrom(componentType.java)) { + Struct(schema) + .put( + DURATION_LIST, + value + .map { s -> s as IsoDuration } + .map { + Struct(durationSchema) + .put(MONTHS, it.months()) + .put(DAYS, it.days()) + .put(SECONDS, it.seconds()) + .put(NANOS, it.nanoseconds()) + } + .toList()) + } else if (Point::class.java.isAssignableFrom(componentType.java)) { + Struct(schema) + .put( + POINT_LIST, + value + .map { s -> s as Point } + .map { s -> + Struct(pointSchema) + .put(SR_ID, s.srid()) + .put(X, s.x()) + .put(Y, s.y()) + .also { + it.put(DIMENSION, if (s.z().isNaN()) TWO_D else THREE_D) + if (!s.z().isNaN()) { + it.put(Z, s.z()) + } + } + } + .toList()) + } else { + throw IllegalArgumentException( + "unsupported array type: array of ${value.javaClass.componentType.name}") + } + } + else -> throw IllegalArgumentException("unsupported property type: ${value?.javaClass?.name}") + } + } + + fun fromConnectValue(value: Struct?): Any? { + return value?.let { + for (f in it.schema().fields()) { + if (it.getWithoutDefault(f.name()) == null) { + continue + } + + return when (f.name()) { + BOOLEAN -> it.get(f) as Boolean? + BOOLEAN_LIST -> it.get(f) as List<*>? + LONG -> it.get(f) as Long? + LONG_LIST -> it.get(f) as List<*>? + FLOAT -> it.get(f) as Double? + FLOAT_LIST -> it.get(f) as List<*>? + STRING -> it.get(f) as String? + STRING_LIST -> it.get(f) as List<*>? + BYTES -> it.get(f) as ByteArray? + LOCAL_DATE -> + (it.get(f) as String?)?.let { s -> + DateTimeFormatter.ISO_DATE.parse(s) { parsed -> LocalDate.from(parsed) } + } + LOCAL_DATE_LIST -> it.get(f) as List<*>? + LOCAL_TIME -> + (it.get(f) as String?)?.let { s -> + DateTimeFormatter.ISO_TIME.parse(s) { parsed -> LocalTime.from(parsed) } + } + LOCAL_TIME_LIST -> it.get(f) as List<*>? + LOCAL_DATE_TIME -> + (it.get(f) as String?)?.let { s -> + DateTimeFormatter.ISO_DATE_TIME.parse(s) { parsed -> LocalDateTime.from(parsed) } + } + LOCAL_DATE_TIME_LIST -> it.get(f) as List<*>? + ZONED_DATE_TIME -> + (it.get(f) as String?)?.let { s -> + DateTimeFormatter.ISO_DATE_TIME.parse(s) { parsed -> + val zoneId = parsed.query(TemporalQueries.zone()) + + if (zoneId is ZoneOffset) { + OffsetDateTime.from(parsed) + } else { + ZonedDateTime.from(parsed) + } + } + } + ZONED_DATE_TIME_LIST -> it.get(f) as List<*>? + OFFSET_TIME -> + (it.get(f) as String?)?.let { s -> + DateTimeFormatter.ISO_TIME.parse(s) { parsed -> OffsetTime.from(parsed) } + } + OFFSET_TIME_LIST -> it.get(f) as List<*>? + DURATION -> + (it.get(f) as Struct?) + ?.let { s -> + Values.isoDuration( + s.getInt64(MONTHS), + s.getInt64(DAYS), + s.getInt64(SECONDS), + s.getInt32(NANOS)) + } + ?.asIsoDuration() + DURATION_LIST -> it.get(f) as List<*>? + POINT -> + (it.get(f) as Struct?) + ?.let { s -> + when (val dimension = s.getInt8(DIMENSION)) { + TWO_D -> Values.point(s.getInt32(SR_ID), s.getFloat64(X), s.getFloat64(Y)) + THREE_D -> + Values.point( + s.getInt32(SR_ID), s.getFloat64(X), s.getFloat64(Y), s.getFloat64(Z)) + else -> + throw IllegalArgumentException("unsupported dimension value ${dimension}") + } + } + ?.asPoint() + POINT_LIST -> it.get(f) as List<*>? + else -> throw IllegalArgumentException("unsupported neo4j type: ${f.name()}") + } + } + + return null + } + } +} fun Schema.matches(other: Schema): Boolean { return this.id() == other.id() || this.shortId() == other.shortId() @@ -143,159 +374,8 @@ object DynamicTypes { return null } - if (schema == propertyType) { - return when (value) { - is Boolean -> Struct(propertyType).put(BOOLEAN, value) - is Float -> Struct(propertyType).put(FLOAT, value.toDouble()) - is Double -> Struct(propertyType).put(FLOAT, value) - is Number -> Struct(propertyType).put(LONG, value.toLong()) - is String -> Struct(propertyType).put(STRING, value) - is Char -> Struct(propertyType).put(STRING, value.toString()) - is CharArray -> Struct(propertyType).put(STRING, String(value)) - is ByteArray -> Struct(propertyType).put(BYTES, value) - is ByteBuffer -> Struct(propertyType).put(BYTES, value.array()) - is LocalDate -> - Struct(propertyType).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(value)) - is LocalDateTime -> - Struct(propertyType).put(LOCAL_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) - is LocalTime -> - Struct(propertyType).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(value)) - is OffsetDateTime -> - Struct(propertyType).put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) - is ZonedDateTime -> - Struct(propertyType).put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) - is OffsetTime -> - Struct(propertyType).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(value)) - is IsoDuration -> - Struct(propertyType) - .put( - DURATION, - Struct(durationSchema) - .put(MONTHS, value.months()) - .put(DAYS, value.days()) - .put(SECONDS, value.seconds()) - .put(NANOS, value.nanoseconds())) - is Point -> - Struct(propertyType) - .put( - POINT, - Struct(pointSchema) - .put(SR_ID, value.srid()) - .put(X, value.x()) - .put(Y, value.y()) - .also { - it.put(DIMENSION, if (value.z().isNaN()) TWO_D else THREE_D) - if (!value.z().isNaN()) { - it.put(Z, value.z()) - } - }) - is ShortArray -> Struct(propertyType).put(LONG_LIST, value.map { s -> s.toLong() }.toList()) - is IntArray -> Struct(propertyType).put(LONG_LIST, value.map { s -> s.toLong() }.toList()) - is LongArray -> Struct(propertyType).put(LONG_LIST, value.toList()) - is FloatArray -> - Struct(propertyType).put(FLOAT_LIST, value.map { s -> s.toDouble() }.toList()) - is DoubleArray -> Struct(propertyType).put(FLOAT_LIST, value.toList()) - is BooleanArray -> Struct(propertyType).put(BOOLEAN_LIST, value.toList()) - is Array<*> -> - when (val componentType = value::class.java.componentType.kotlin) { - Boolean::class -> Struct(propertyType).put(BOOLEAN_LIST, value.toList()) - Byte::class -> Struct(propertyType).put(BYTES, (value as Array).toByteArray()) - Short::class -> - Struct(propertyType) - .put(LONG_LIST, (value as Array).map { s -> s.toLong() }.toList()) - Int::class -> - Struct(propertyType) - .put(LONG_LIST, (value as Array).map { s -> s.toLong() }.toList()) - Long::class -> Struct(propertyType).put(LONG_LIST, (value as Array).toList()) - Float::class -> - Struct(propertyType) - .put(FLOAT_LIST, (value as Array).map { s -> s.toDouble() }.toList()) - Double::class -> - Struct(propertyType).put(FLOAT_LIST, (value as Array).toList()) - String::class -> Struct(propertyType).put(STRING_LIST, value.toList()) - LocalDate::class -> - Struct(propertyType) - .put( - LOCAL_DATE_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_DATE.format(s) } - .toList()) - LocalDateTime::class -> - Struct(propertyType) - .put( - LOCAL_DATE_TIME_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } - .toList()) - LocalTime::class -> - Struct(propertyType) - .put( - LOCAL_TIME_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_TIME.format(s) } - .toList()) - OffsetDateTime::class -> - Struct(propertyType) - .put( - ZONED_DATE_TIME_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } - .toList()) - ZonedDateTime::class -> - Struct(propertyType) - .put( - ZONED_DATE_TIME_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } - .toList()) - OffsetTime::class -> - Struct(propertyType) - .put( - OFFSET_TIME_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_TIME.format(s) } - .toList()) - else -> - if (IsoDuration::class.java.isAssignableFrom(componentType.java)) { - Struct(propertyType) - .put( - DURATION_LIST, - value - .map { s -> s as IsoDuration } - .map { - Struct(durationSchema) - .put(MONTHS, it.months()) - .put(DAYS, it.days()) - .put(SECONDS, it.seconds()) - .put(NANOS, it.nanoseconds()) - } - .toList()) - } else if (Point::class.java.isAssignableFrom(componentType.java)) { - Struct(propertyType) - .put( - POINT_LIST, - value - .map { s -> s as Point } - .map { s -> - Struct(pointSchema) - .put(SR_ID, s.srid()) - .put(X, s.x()) - .put(Y, s.y()) - .also { - it.put(DIMENSION, if (s.z().isNaN()) TWO_D else THREE_D) - if (!s.z().isNaN()) { - it.put(Z, s.z()) - } - } - } - .toList()) - } else { - throw IllegalArgumentException( - "unsupported array type: array of ${value.javaClass.componentType.name}") - } - } - else -> throw IllegalArgumentException("unsupported property type: ${value.javaClass.name}") - } + if (schema == PropertyType.schema) { + return PropertyType.toConnectValue(value) } return when (schema.type()) { @@ -313,29 +393,23 @@ object DynamicTypes { when (value) { is Node -> Struct(schema).apply { - put("", toConnectValue(propertyType, value.id())) - put( - "", - toConnectValue(propertyType, value.labels().toList().toTypedArray())) + put("", value.id()) + put("", value.labels().toList()) value .asMap { it.asObject() } - .forEach { e -> - put(e.key, toConnectValue(schema.field(e.key).schema(), e.value)) - } + .forEach { e -> put(e.key, PropertyType.toConnectValue(e.value)) } } is Relationship -> Struct(schema).apply { - put("", toConnectValue(propertyType, value.id())) - put("", toConnectValue(propertyType, value.type())) - put("", toConnectValue(propertyType, value.startNodeId())) - put("", toConnectValue(propertyType, value.endNodeId())) + put("", value.id()) + put("", value.type()) + put("", value.startNodeId()) + put("", value.endNodeId()) value .asMap { it.asObject() } - .forEach { e -> - put(e.key, toConnectValue(schema.field(e.key).schema(), e.value)) - } + .forEach { e -> put(e.key, PropertyType.toConnectValue(e.value)) } } is Map<*, *> -> Struct(schema).apply { @@ -388,97 +462,7 @@ object DynamicTypes { } Schema.Type.STRUCT -> when { - propertyType.matches(schema) -> - (value as Struct?)?.let { - for (f in it.schema().fields()) { - if (it.getWithoutDefault(f.name()) == null) { - continue - } - - return when (f.name()) { - BOOLEAN -> it.get(f) as Boolean? - BOOLEAN_LIST -> it.get(f) as List<*>? - LONG -> it.get(f) as Long? - LONG_LIST -> it.get(f) as List<*>? - FLOAT -> it.get(f) as Double? - FLOAT_LIST -> it.get(f) as List<*>? - STRING -> it.get(f) as String? - STRING_LIST -> it.get(f) as List<*>? - BYTES -> it.get(f) as ByteArray? - LOCAL_DATE -> - (it.get(f) as String?)?.let { s -> - DateTimeFormatter.ISO_DATE.parse(s) { parsed -> LocalDate.from(parsed) } - } - LOCAL_DATE_LIST -> it.get(f) as List<*>? - LOCAL_TIME -> - (it.get(f) as String?)?.let { s -> - DateTimeFormatter.ISO_TIME.parse(s) { parsed -> LocalTime.from(parsed) } - } - LOCAL_TIME_LIST -> it.get(f) as List<*>? - LOCAL_DATE_TIME -> - (it.get(f) as String?)?.let { s -> - DateTimeFormatter.ISO_DATE_TIME.parse(s) { parsed -> - LocalDateTime.from(parsed) - } - } - LOCAL_DATE_TIME_LIST -> it.get(f) as List<*>? - ZONED_DATE_TIME -> - (it.get(f) as String?)?.let { s -> - DateTimeFormatter.ISO_DATE_TIME.parse(s) { parsed -> - val zoneId = parsed.query(TemporalQueries.zone()) - - if (zoneId is ZoneOffset) { - OffsetDateTime.from(parsed) - } else { - ZonedDateTime.from(parsed) - } - } - } - ZONED_DATE_TIME_LIST -> it.get(f) as List<*>? - OFFSET_TIME -> - (it.get(f) as String?)?.let { s -> - DateTimeFormatter.ISO_TIME.parse(s) { parsed -> - OffsetTime.from(parsed) - } - } - OFFSET_TIME_LIST -> it.get(f) as List<*>? - DURATION -> - (it.get(f) as Struct?) - ?.let { s -> - Values.isoDuration( - s.getInt64(MONTHS), - s.getInt64(DAYS), - s.getInt64(SECONDS), - s.getInt32(NANOS)) - } - ?.asIsoDuration() - DURATION_LIST -> it.get(f) as List<*>? - POINT -> - (it.get(f) as Struct?) - ?.let { s -> - when (val dimension = s.getInt8(DIMENSION)) { - TWO_D -> - Values.point( - s.getInt32(SR_ID), s.getFloat64(X), s.getFloat64(Y)) - THREE_D -> - Values.point( - s.getInt32(SR_ID), - s.getFloat64(X), - s.getFloat64(Y), - s.getFloat64(Z)) - else -> - throw IllegalArgumentException( - "unsupported dimension value ${dimension}") - } - } - ?.asPoint() - POINT_LIST -> it.get(f) as List<*>? - else -> throw IllegalArgumentException("unsupported neo4j type: ${f.name()}") - } - } - - return null - } + PropertyType.schema.matches(schema) -> PropertyType.fromConnectValue(value as Struct?) else -> { val result = mutableMapOf() val struct = value as Struct @@ -555,22 +539,22 @@ object DynamicTypes { temporalDataSchemaType: TemporalDataSchemaType = TemporalDataSchemaType.STRUCT, ): Schema = when (value) { - null -> propertyType - is Boolean -> propertyType + null -> PropertyType.schema + is Boolean, is Float, - is Double -> propertyType - is Number -> propertyType + is Double, + is Number, is Char, is CharArray, - is CharSequence -> propertyType + is CharSequence, is ByteBuffer, - is ByteArray -> propertyType + is ByteArray, is ShortArray, is IntArray, - is LongArray -> propertyType + is LongArray, is FloatArray, - is DoubleArray -> propertyType - is BooleanArray -> propertyType + is DoubleArray, + is BooleanArray -> PropertyType.schema is Array<*> -> { when (val componentType = value::class.java.componentType.kotlin) { Boolean::class, @@ -586,12 +570,12 @@ object DynamicTypes { LocalTime::class, OffsetDateTime::class, ZonedDateTime::class, - OffsetTime::class -> propertyType + OffsetTime::class -> PropertyType.schema else -> if (IsoDuration::class.java.isAssignableFrom(componentType.java)) { - propertyType + PropertyType.schema } else if (Point::class.java.isAssignableFrom(componentType.java)) { - propertyType + PropertyType.schema } else { val first = value.firstOrNull { it.notNullOrEmpty() } val schema = @@ -600,29 +584,21 @@ object DynamicTypes { } } } - is LocalDate -> propertyType - is LocalDateTime -> propertyType - is LocalTime -> propertyType - is OffsetDateTime -> propertyType - is ZonedDateTime -> propertyType - is OffsetTime -> propertyType - is IsoDuration -> propertyType - is Point -> propertyType + is LocalDate, + is LocalDateTime, + is LocalTime, + is OffsetDateTime, + is ZonedDateTime, + is OffsetTime, + is IsoDuration, + is Point -> PropertyType.schema is Node -> SchemaBuilder.struct() .apply { - field("", propertyType) - field("", propertyType) - - value.keys().forEach { - field( - it, - toConnectSchema( - value.get(it).asObject(), - optional, - forceMapsAsStruct, - temporalDataSchemaType)) - } + field("", Schema.INT64_SCHEMA) + field("", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) + + value.keys().forEach { field(it, PropertyType.schema) } if (optional) optional() } @@ -630,20 +606,12 @@ object DynamicTypes { is Relationship -> SchemaBuilder.struct() .apply { - field("", propertyType) - field("", propertyType) - field("", propertyType) - field("", propertyType) - - value.keys().forEach { - field( - it, - toConnectSchema( - value.get(it).asObject(), - optional, - forceMapsAsStruct, - temporalDataSchemaType)) - } + field("", Schema.INT64_SCHEMA) + field("", Schema.STRING_SCHEMA) + field("", Schema.INT64_SCHEMA) + field("", Schema.INT64_SCHEMA) + + value.keys().forEach { field(it, PropertyType.schema) } if (optional) optional() } @@ -655,7 +623,7 @@ object DynamicTypes { .map { toConnectSchema(it, optional, forceMapsAsStruct, temporalDataSchemaType) } when (nonEmptyElementTypes.toSet().size) { - 0 -> SchemaBuilder.array(propertyType).apply { if (optional) optional() }.build() + 0 -> SchemaBuilder.array(PropertyType.schema).apply { if (optional) optional() }.build() 1 -> SchemaBuilder.array(nonEmptyElementTypes.first()) .apply { if (optional) optional() } diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt index 3bc33d206..20ad31a33 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt @@ -19,7 +19,6 @@ package org.neo4j.connectors.kafka.data import io.kotest.matchers.shouldBe import java.time.LocalDate import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter import org.apache.kafka.connect.data.Schema import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct @@ -66,13 +65,13 @@ class ChangeEventExtensionsTest { .field("connectionServer", Schema.OPTIONAL_STRING_SCHEMA) .field("serverId", Schema.STRING_SCHEMA) .field("captureMode", Schema.STRING_SCHEMA) - .field("txStartTime", propertyType) - .field("txCommitTime", propertyType) + .field("txStartTime", PropertyType.schema) + .field("txCommitTime", PropertyType.schema) .field( "txMetadata", SchemaBuilder.struct() - .field("user", propertyType) - .field("app", propertyType) + .field("user", PropertyType.schema) + .field("app", PropertyType.schema) .optional() .build()) .build() @@ -89,23 +88,15 @@ class ChangeEventExtensionsTest { .put("connectionServer", change.metadata.connectionServer) .put("serverId", change.metadata.serverId) .put("captureMode", change.metadata.captureMode.name) - .put( - "txStartTime", - change.metadata.txStartTime.let { - Struct(propertyType) - .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it)) - }) + .put("txStartTime", change.metadata.txStartTime.let { PropertyType.toConnectValue(it) }) .put( "txCommitTime", - change.metadata.txCommitTime.let { - Struct(propertyType) - .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it)) - }) + change.metadata.txCommitTime.let { PropertyType.toConnectValue(it) }) .put( "txMetadata", Struct(schema.nestedSchema("metadata.txMetadata")) - .put("user", Struct(propertyType).put(STRING, "app_user")) - .put("app", Struct(propertyType).put(STRING, "hr"))) + .put("user", PropertyType.toConnectValue("app_user")) + .put("app", PropertyType.toConnectValue("hr"))) } @Test @@ -136,13 +127,15 @@ class ChangeEventExtensionsTest { .field( "Label1", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .field( "Label2", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .optional() @@ -156,7 +149,8 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .field( @@ -165,7 +159,8 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .build()) @@ -184,9 +179,9 @@ class ChangeEventExtensionsTest { "Label1", listOf( mapOf( - "name" to Struct(propertyType).put(STRING, "john"), - "surname" to Struct(propertyType).put(STRING, "doe")))) - .put("Label2", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L))))) + "name" to PropertyType.toConnectValue("john"), + "surname" to PropertyType.toConnectValue("doe")))) + .put("Label2", listOf(mapOf("id" to PropertyType.toConnectValue(5L))))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -197,9 +192,9 @@ class ChangeEventExtensionsTest { .put( "properties", mapOf( - "id" to Struct(propertyType).put(LONG, 5L), - "name" to Struct(propertyType).put(STRING, "john"), - "surname" to Struct(propertyType).put(STRING, "doe"))))) + "id" to PropertyType.toConnectValue(5L), + "name" to PropertyType.toConnectValue("john"), + "surname" to PropertyType.toConnectValue("doe"))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -235,13 +230,15 @@ class ChangeEventExtensionsTest { .field( "Label1", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .field( "Label2", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .optional() @@ -255,7 +252,8 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .field( @@ -264,7 +262,8 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .build()) @@ -283,9 +282,9 @@ class ChangeEventExtensionsTest { "Label1", listOf( mapOf( - "name" to Struct(propertyType).put(STRING, "john"), - "surname" to Struct(propertyType).put(STRING, "doe")))) - .put("Label2", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L))))) + "name" to PropertyType.toConnectValue("john"), + "surname" to PropertyType.toConnectValue("doe")))) + .put("Label2", listOf(mapOf("id" to PropertyType.toConnectValue(5L))))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -296,9 +295,9 @@ class ChangeEventExtensionsTest { .put( "properties", mapOf( - "id" to Struct(propertyType).put(LONG, 5L), - "name" to Struct(propertyType).put(STRING, "john"), - "surname" to Struct(propertyType).put(STRING, "doe")))) + "id" to PropertyType.toConnectValue(5L), + "name" to PropertyType.toConnectValue("john"), + "surname" to PropertyType.toConnectValue("doe")))) .put( "after", Struct(schema.nestedSchema("event.state.after")) @@ -306,10 +305,10 @@ class ChangeEventExtensionsTest { .put( "properties", mapOf( - "id" to Struct(propertyType).put(LONG, 5L), - "name" to Struct(propertyType).put(STRING, "john"), - "surname" to Struct(propertyType).put(STRING, "doe"), - "age" to Struct(propertyType).put(LONG, 25L))))) + "id" to PropertyType.toConnectValue(5L), + "name" to PropertyType.toConnectValue("john"), + "surname" to PropertyType.toConnectValue("doe"), + "age" to PropertyType.toConnectValue(25L))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -343,13 +342,15 @@ class ChangeEventExtensionsTest { .field( "Label1", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .field( "Label2", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .optional() @@ -363,7 +364,8 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .field( @@ -372,7 +374,8 @@ class ChangeEventExtensionsTest { .field("labels", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .build()) @@ -391,9 +394,9 @@ class ChangeEventExtensionsTest { "Label1", listOf( mapOf( - "name" to Struct(propertyType).put(STRING, "john"), - "surname" to Struct(propertyType).put(STRING, "doe")))) - .put("Label2", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L))))) + "name" to PropertyType.toConnectValue("john"), + "surname" to PropertyType.toConnectValue("doe")))) + .put("Label2", listOf(mapOf("id" to PropertyType.toConnectValue(5L))))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -404,10 +407,10 @@ class ChangeEventExtensionsTest { .put( "properties", mapOf( - "id" to Struct(propertyType).put(LONG, 5L), - "name" to Struct(propertyType).put(STRING, "john"), - "surname" to Struct(propertyType).put(STRING, "doe"), - "age" to Struct(propertyType).put(LONG, 25L))))) + "id" to PropertyType.toConnectValue(5L), + "name" to PropertyType.toConnectValue("john"), + "surname" to PropertyType.toConnectValue("doe"), + "age" to PropertyType.toConnectValue(25L))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -448,7 +451,7 @@ class ChangeEventExtensionsTest { .field( "Person", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) .build()) .optional() .schema()) @@ -466,7 +469,7 @@ class ChangeEventExtensionsTest { .field( "Company", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) .build()) .optional() .schema()) @@ -475,7 +478,8 @@ class ChangeEventExtensionsTest { .build()) .field( "keys", - SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.array( + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) .optional() .build()) .field( @@ -486,7 +490,8 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .field( @@ -494,7 +499,8 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .build()) @@ -516,7 +522,7 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.start.keys")) .put( "Person", - listOf(mapOf("name" to Struct(propertyType).put(STRING, "john")))))) + listOf(mapOf("name" to PropertyType.toConnectValue("john")))))) .put( "end", Struct(schema.nestedSchema("event.end")) @@ -527,10 +533,8 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.end.keys")) .put( "Company", - listOf( - mapOf( - "name" to Struct(propertyType).put(STRING, "acme corp")))))) - .put("keys", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L)))) + listOf(mapOf("name" to PropertyType.toConnectValue("acme corp")))))) + .put("keys", listOf(mapOf("id" to PropertyType.toConnectValue(5L)))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -540,13 +544,9 @@ class ChangeEventExtensionsTest { .put( "properties", mapOf( - "id" to Struct(propertyType).put(LONG, 5L), + "id" to PropertyType.toConnectValue(5L), "since" to - Struct(propertyType) - .put( - LOCAL_DATE, - DateTimeFormatter.ISO_DATE.format( - LocalDate.of(1999, 12, 31))))))) + PropertyType.toConnectValue(LocalDate.of(1999, 12, 31)))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -587,7 +587,7 @@ class ChangeEventExtensionsTest { .field( "Person", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) .build()) .optional() .schema()) @@ -605,7 +605,7 @@ class ChangeEventExtensionsTest { .field( "Company", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) .build()) .optional() .schema()) @@ -614,7 +614,8 @@ class ChangeEventExtensionsTest { .build()) .field( "keys", - SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.array( + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) .optional() .build()) .field( @@ -625,7 +626,8 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .field( @@ -633,7 +635,8 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .build()) @@ -655,7 +658,7 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.start.keys")) .put( "Person", - listOf(mapOf("name" to Struct(propertyType).put(STRING, "john")))))) + listOf(mapOf("name" to PropertyType.toConnectValue("john")))))) .put( "end", Struct(schema.nestedSchema("event.end")) @@ -666,10 +669,8 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.end.keys")) .put( "Company", - listOf( - mapOf( - "name" to Struct(propertyType).put(STRING, "acme corp")))))) - .put("keys", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L)))) + listOf(mapOf("name" to PropertyType.toConnectValue("acme corp")))))) + .put("keys", listOf(mapOf("id" to PropertyType.toConnectValue(5L)))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -679,26 +680,18 @@ class ChangeEventExtensionsTest { .put( "properties", mapOf( - "id" to Struct(propertyType).put(LONG, 5L), + "id" to PropertyType.toConnectValue(5L), "since" to - Struct(propertyType) - .put( - LOCAL_DATE, - DateTimeFormatter.ISO_DATE.format( - LocalDate.of(1999, 12, 31)))))) + PropertyType.toConnectValue(LocalDate.of(1999, 12, 31))))) .put( "after", Struct(schema.nestedSchema("event.state.after")) .put( "properties", mapOf( - "id" to Struct(propertyType).put(LONG, 5L), + "id" to PropertyType.toConnectValue(5L), "since" to - Struct(propertyType) - .put( - LOCAL_DATE, - DateTimeFormatter.ISO_DATE.format( - LocalDate.of(2000, 1, 1))))))) + PropertyType.toConnectValue(LocalDate.of(2000, 1, 1)))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -739,7 +732,7 @@ class ChangeEventExtensionsTest { .field( "Person", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) .build()) .optional() .build()) @@ -757,7 +750,7 @@ class ChangeEventExtensionsTest { .field( "Company", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) .build()) .optional() .build()) @@ -766,7 +759,8 @@ class ChangeEventExtensionsTest { .build()) .field( "keys", - SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.array( + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) .optional() .schema()) .field( @@ -777,7 +771,8 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .field( @@ -785,7 +780,8 @@ class ChangeEventExtensionsTest { SchemaBuilder.struct() .field( "properties", - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + .build()) .optional() .build()) .build()) @@ -807,7 +803,7 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.start.keys")) .put( "Person", - listOf(mapOf("name" to Struct(propertyType).put(STRING, "john")))))) + listOf(mapOf("name" to PropertyType.toConnectValue("john")))))) .put( "end", Struct(schema.nestedSchema("event.end")) @@ -818,10 +814,8 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.end.keys")) .put( "Company", - listOf( - mapOf( - "name" to Struct(propertyType).put(STRING, "acme corp")))))) - .put("keys", listOf(mapOf("id" to Struct(propertyType).put(LONG, 5L)))) + listOf(mapOf("name" to PropertyType.toConnectValue("acme corp")))))) + .put("keys", listOf(mapOf("id" to PropertyType.toConnectValue(5L)))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -831,13 +825,9 @@ class ChangeEventExtensionsTest { .put( "properties", mapOf( - "id" to Struct(propertyType).put(LONG, 5L), + "id" to PropertyType.toConnectValue(5L), "since" to - Struct(propertyType) - .put( - LOCAL_DATE, - DateTimeFormatter.ISO_DATE.format( - LocalDate.of(2000, 1, 1))))))) + PropertyType.toConnectValue(LocalDate.of(2000, 1, 1)))))) val reverted = value.toChangeEvent() reverted shouldBe change @@ -875,7 +865,7 @@ class ChangeEventExtensionsTest { null)) val expectedKeySchema = - SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build()) + SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) .optional() .build() schema.nestedSchema("event.keys") shouldBe expectedKeySchema @@ -913,30 +903,20 @@ class ChangeEventExtensionsTest { .put("connectionType", "bolt") .put("connectionClient", "127.0.0.1:32000") .put("connectionServer", "127.0.0.1:7687") - .put( - "txStartTime", - startTime.let { - Struct(propertyType) - .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it)) - }) - .put( - "txCommitTime", - commitTime.let { - Struct(propertyType) - .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it)) - }) + .put("txStartTime", PropertyType.toConnectValue(startTime)) + .put("txCommitTime", PropertyType.toConnectValue(commitTime)) .put( "txMetadata", Struct(schema.nestedSchema("txMetadata").schema()) - .put("user", Struct(propertyType).put(STRING, "app_user")) - .put("app", Struct(propertyType).put(STRING, "hr")) + .put("user", PropertyType.toConnectValue("app_user")) + .put("app", PropertyType.toConnectValue("hr")) .put( "xyz", Struct(schema.nestedSchema("txMetadata.xyz")) - .put("a", Struct(propertyType).put(LONG, 1L)) - .put("b", Struct(propertyType).put(LONG, 2L)))) - .put("new_field", Struct(propertyType).put(STRING, "abc")) - .put("another_field", Struct(propertyType).put(LONG, 1L)) + .put("a", PropertyType.toConnectValue(1L)) + .put("b", PropertyType.toConnectValue(2L)))) + .put("new_field", PropertyType.toConnectValue("abc")) + .put("another_field", PropertyType.toConnectValue(1L)) val reverted = converted.toMetadata() reverted shouldBe metadata @@ -1061,16 +1041,16 @@ class ChangeEventExtensionsTest { .put( "Person", listOf( - mapOf("id" to Struct(propertyType).put(LONG, 1L)), + mapOf("id" to PropertyType.toConnectValue(1L)), mapOf( - "name" to Struct(propertyType).put(STRING, "john"), - "surname" to Struct(propertyType).put(STRING, "doe")))) + "name" to PropertyType.toConnectValue("john"), + "surname" to PropertyType.toConnectValue("doe")))) .put( "Employee", listOf( mapOf( - "id" to Struct(propertyType).put(LONG, 5L), - "company_id" to Struct(propertyType).put(LONG, 7L))))) + "id" to PropertyType.toConnectValue(5L), + "company_id" to PropertyType.toConnectValue(7L))))) val reverted = converted.toNode() reverted shouldBe node diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt index 418afafd9..7704ef679 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt @@ -41,6 +41,20 @@ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.ArgumentsProvider import org.junit.jupiter.params.provider.ArgumentsSource +import org.neo4j.connectors.kafka.data.PropertyType.BOOLEAN +import org.neo4j.connectors.kafka.data.PropertyType.BOOLEAN_LIST +import org.neo4j.connectors.kafka.data.PropertyType.BYTES +import org.neo4j.connectors.kafka.data.PropertyType.DURATION +import org.neo4j.connectors.kafka.data.PropertyType.FLOAT +import org.neo4j.connectors.kafka.data.PropertyType.FLOAT_LIST +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE_TIME +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_TIME +import org.neo4j.connectors.kafka.data.PropertyType.LONG_LIST +import org.neo4j.connectors.kafka.data.PropertyType.OFFSET_TIME +import org.neo4j.connectors.kafka.data.PropertyType.POINT +import org.neo4j.connectors.kafka.data.PropertyType.STRING_LIST +import org.neo4j.connectors.kafka.data.PropertyType.ZONED_DATE_TIME import org.neo4j.driver.Value import org.neo4j.driver.Values import org.neo4j.driver.types.Node @@ -51,22 +65,22 @@ class DynamicTypesTest { @Test fun `should derive schema for simple types correctly`() { // NULL - DynamicTypes.toConnectSchema(null, false) shouldBe propertyType - DynamicTypes.toConnectSchema(null, true) shouldBe propertyType + DynamicTypes.toConnectSchema(null, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(null, true) shouldBe PropertyType.schema // Integer, Long, etc. listOf(8.toByte(), 8.toShort(), 8.toInt(), 8.toLong()).forEach { number -> withClue(number) { - DynamicTypes.toConnectSchema(number, false) shouldBe propertyType - DynamicTypes.toConnectSchema(number, true) shouldBe propertyType + DynamicTypes.toConnectSchema(number, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(number, true) shouldBe PropertyType.schema } } // Float, Double listOf(8.toFloat(), 8.toDouble()).forEach { number -> withClue(number) { - DynamicTypes.toConnectSchema(number, false) shouldBe propertyType - DynamicTypes.toConnectSchema(number, true) shouldBe propertyType + DynamicTypes.toConnectSchema(number, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(number, true) shouldBe PropertyType.schema } } @@ -88,40 +102,40 @@ class DynamicTypesTest { }) .forEach { string -> withClue(string) { - DynamicTypes.toConnectSchema(string, false) shouldBe propertyType - DynamicTypes.toConnectSchema(string, true) shouldBe propertyType + DynamicTypes.toConnectSchema(string, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(string, true) shouldBe PropertyType.schema } } // Byte Array listOf(ByteArray(0), ByteBuffer.allocate(0)).forEach { bytes -> withClue(bytes) { - DynamicTypes.toConnectSchema(bytes, false) shouldBe propertyType - DynamicTypes.toConnectSchema(bytes, true) shouldBe propertyType + DynamicTypes.toConnectSchema(bytes, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(bytes, true) shouldBe PropertyType.schema } } // Boolean Array (boolean[]) listOf(BooleanArray(0), BooleanArray(1) { true }).forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe propertyType - DynamicTypes.toConnectSchema(array, true) shouldBe propertyType + DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema } } // Array of Boolean (Boolean[]) listOf(Array(1) { true }).forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe propertyType - DynamicTypes.toConnectSchema(array, true) shouldBe propertyType + DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema } } // Int Arrays (short[], int[], long[]) listOf(ShortArray(1), IntArray(1), LongArray(1)).forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe propertyType - DynamicTypes.toConnectSchema(array, true) shouldBe propertyType + DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema } } @@ -129,131 +143,136 @@ class DynamicTypesTest { listOf(Array(1) { i -> i }, Array(1) { i -> i.toShort() }, Array(1) { i -> i.toLong() }) .forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe propertyType - DynamicTypes.toConnectSchema(array, true) shouldBe propertyType + DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema } } // Float Arrays (float[], double[]) listOf(FloatArray(1), DoubleArray(1)).forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe propertyType - DynamicTypes.toConnectSchema(array, true) shouldBe propertyType + DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema } } // Float Arrays (Float[], Double[]) listOf(Array(1) { i -> i.toFloat() }, Array(1) { i -> i.toDouble() }).forEach { array -> withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe propertyType - DynamicTypes.toConnectSchema(array, true) shouldBe propertyType + DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema } } // String Array - DynamicTypes.toConnectSchema(Array(1) { "a" }, false) shouldBe propertyType - DynamicTypes.toConnectSchema(Array(1) { "a" }, true) shouldBe propertyType + DynamicTypes.toConnectSchema(Array(1) { "a" }, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(Array(1) { "a" }, true) shouldBe PropertyType.schema // Temporal Types - DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), false) shouldBe propertyType - DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), true) shouldBe propertyType + DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), true) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( LocalDate.of(1999, 12, 31), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( LocalDate.of(1999, 12, 31), optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), false) shouldBe propertyType - DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), true) shouldBe propertyType + DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), true) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( LocalTime.of(23, 59, 59), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( LocalTime.of(23, 59, 59), optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema(LocalDateTime.of(1999, 12, 31, 23, 59, 59), false) shouldBe - propertyType + PropertyType.schema DynamicTypes.toConnectSchema(LocalDateTime.of(1999, 12, 31, 23, 59, 59), true) shouldBe - propertyType + PropertyType.schema DynamicTypes.toConnectSchema( LocalDateTime.of(1999, 12, 31, 23, 59, 59), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( LocalDateTime.of(1999, 12, 31, 23, 59, 59), optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema(OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), false) shouldBe - propertyType + PropertyType.schema DynamicTypes.toConnectSchema(OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), true) shouldBe - propertyType + PropertyType.schema DynamicTypes.toConnectSchema( OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), false) shouldBe propertyType + OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), false) shouldBe + PropertyType.schema DynamicTypes.toConnectSchema( - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), true) shouldBe propertyType + OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), true) shouldBe + PropertyType.schema DynamicTypes.toConnectSchema( OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), false) shouldBe - propertyType + PropertyType.schema DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), true) shouldBe - propertyType + PropertyType.schema DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe propertyType + temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( - Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), false) shouldBe propertyType + Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), false) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( - Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), true) shouldBe propertyType + Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), true) shouldBe PropertyType.schema // Point listOf(Values.point(4326, 1.0, 2.0).asPoint(), Values.point(4326, 1.0, 2.0, 3.0).asPoint()) .forEach { point -> withClue(point) { - DynamicTypes.toConnectSchema(point, false) shouldBe propertyType - DynamicTypes.toConnectSchema(point, true) shouldBe propertyType + DynamicTypes.toConnectSchema(point, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(point, true) shouldBe PropertyType.schema } } // Node DynamicTypes.toConnectSchema(TestNode(0, emptyList(), emptyMap()), false) shouldBe - SchemaBuilder.struct().field("", propertyType).field("", propertyType).build() + SchemaBuilder.struct() + .field("", Schema.INT64_SCHEMA) + .field("", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) + .build() DynamicTypes.toConnectSchema( TestNode( @@ -262,19 +281,19 @@ class DynamicTypesTest { mapOf("name" to Values.value("john"), "surname" to Values.value("doe"))), false) shouldBe SchemaBuilder.struct() - .field("", propertyType) - .field("", propertyType) - .field("name", propertyType) - .field("surname", propertyType) + .field("", Schema.INT64_SCHEMA) + .field("", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) + .field("name", PropertyType.schema) + .field("surname", PropertyType.schema) .build() // Relationship DynamicTypes.toConnectSchema(TestRelationship(0, 1, 2, "KNOWS", emptyMap()), false) shouldBe SchemaBuilder.struct() - .field("", propertyType) - .field("", propertyType) - .field("", propertyType) - .field("", propertyType) + .field("", Schema.INT64_SCHEMA) + .field("", Schema.STRING_SCHEMA) + .field("", Schema.INT64_SCHEMA) + .field("", Schema.INT64_SCHEMA) .build() DynamicTypes.toConnectSchema( TestRelationship( @@ -285,12 +304,12 @@ class DynamicTypesTest { mapOf("name" to Values.value("john"), "surname" to Values.value("doe"))), false) shouldBe SchemaBuilder.struct() - .field("", propertyType) - .field("", propertyType) - .field("", propertyType) - .field("", propertyType) - .field("name", propertyType) - .field("surname", propertyType) + .field("", Schema.INT64_SCHEMA) + .field("", Schema.STRING_SCHEMA) + .field("", Schema.INT64_SCHEMA) + .field("", Schema.INT64_SCHEMA) + .field("name", PropertyType.schema) + .field("surname", PropertyType.schema) .build() } @@ -299,9 +318,9 @@ class DynamicTypesTest { listOf(listOf(), setOf(), arrayOf()).forEach { collection -> withClue(collection) { DynamicTypes.toConnectSchema(collection, false) shouldBe - SchemaBuilder.array(propertyType).build() + SchemaBuilder.array(PropertyType.schema).build() DynamicTypes.toConnectSchema(collection, true) shouldBe - SchemaBuilder.array(propertyType).optional().build() + SchemaBuilder.array(PropertyType.schema).optional().build() } } } @@ -311,9 +330,9 @@ class DynamicTypesTest { listOf(listOf(1, 2, 3), listOf("a", "b", "c"), setOf(true)).forEach { collection -> withClue(collection) { DynamicTypes.toConnectSchema(collection, false) shouldBe - SchemaBuilder.array(propertyType).build() + SchemaBuilder.array(PropertyType.schema).build() DynamicTypes.toConnectSchema(collection, true) shouldBe - SchemaBuilder.array(propertyType).optional().build() + SchemaBuilder.array(PropertyType.schema).optional().build() } } } @@ -321,10 +340,10 @@ class DynamicTypesTest { @Test fun `collections with elements of different types should map to a struct schema`() { DynamicTypes.toConnectSchema(listOf(1, true, "a", 5.toFloat()), false) shouldBe - SchemaBuilder.array(propertyType).build() + SchemaBuilder.array(PropertyType.schema).build() DynamicTypes.toConnectSchema(listOf(1, true, "a", 5.toFloat()), true) shouldBe - SchemaBuilder.array(propertyType).optional().build() + SchemaBuilder.array(PropertyType.schema).optional().build() } @Test @@ -345,9 +364,9 @@ class DynamicTypesTest { @Test fun `maps with simple typed values should map to a map schema`() { listOf( - mapOf("a" to 1, "b" to 2, "c" to 3) to propertyType, - mapOf("a" to "a", "b" to "b", "c" to "c") to propertyType, - mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong()) to propertyType) + mapOf("a" to 1, "b" to 2, "c" to 3) to PropertyType.schema, + mapOf("a" to "a", "b" to "b", "c" to "c") to PropertyType.schema, + mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong()) to PropertyType.schema) .forEach { (map, valueSchema) -> withClue("not optional: $map") { DynamicTypes.toConnectSchema(map, false) shouldBe @@ -356,9 +375,9 @@ class DynamicTypesTest { } listOf( - mapOf("a" to 1, "b" to 2, "c" to 3) to propertyType, - mapOf("a" to "a", "b" to "b", "c" to "c") to propertyType, - mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong()) to propertyType) + mapOf("a" to 1, "b" to 2, "c" to 3) to PropertyType.schema, + mapOf("a" to "a", "b" to "b", "c" to "c") to PropertyType.schema, + mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong()) to PropertyType.schema) .forEach { (map, valueSchema) -> withClue("optional: $map") { DynamicTypes.toConnectSchema(map, true) shouldBe @@ -371,11 +390,11 @@ class DynamicTypesTest { fun `maps with values of different types should map to a map of struct schema`() { DynamicTypes.toConnectSchema( mapOf("a" to 1, "b" to true, "c" to "string", "d" to 5.toFloat()), false) shouldBe - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build() + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build() DynamicTypes.toConnectSchema( mapOf("a" to 1, "b" to true, "c" to "string", "d" to 5.toFloat()), true) shouldBe - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).optional().build() + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).optional().build() } @Test @@ -391,19 +410,20 @@ class DynamicTypesTest { @Test fun `simple types should be converted to themselves and should be converted back`() { listOf( - Triple(true, Struct(propertyType).put(BOOLEAN, true), true), - Triple(false, Struct(propertyType).put(BOOLEAN, false), false), - Triple(1.toShort(), Struct(propertyType).put(LONG, 1.toLong()), 1L), - Triple(2, Struct(propertyType).put(LONG, 2.toLong()), 2L), - Triple(3.toLong(), Struct(propertyType).put(LONG, 3.toLong()), 3L), - Triple(4.toFloat(), Struct(propertyType).put(FLOAT, 4.toDouble()), 4.toDouble()), - Triple(5.toDouble(), Struct(propertyType).put(FLOAT, 5.toDouble()), 5.toDouble()), - Triple('c', Struct(propertyType).put(STRING, "c"), "c"), - Triple("string", Struct(propertyType).put(STRING, "string"), "string"), - Triple("string".toCharArray(), Struct(propertyType).put(STRING, "string"), "string"), + Triple(true, Struct(PropertyType.schema).put(BOOLEAN, true), true), + Triple(false, Struct(PropertyType.schema).put(BOOLEAN, false), false), + Triple(1.toShort(), PropertyType.toConnectValue(1.toLong()), 1L), + Triple(2, PropertyType.toConnectValue(2.toLong()), 2L), + Triple(3.toLong(), PropertyType.toConnectValue(3.toLong()), 3L), + Triple(4.toFloat(), Struct(PropertyType.schema).put(FLOAT, 4.toDouble()), 4.toDouble()), + Triple( + 5.toDouble(), Struct(PropertyType.schema).put(FLOAT, 5.toDouble()), 5.toDouble()), + Triple('c', PropertyType.toConnectValue("c"), "c"), + Triple("string", PropertyType.toConnectValue("string"), "string"), + Triple("string".toCharArray(), PropertyType.toConnectValue("string"), "string"), Triple( "string".toByteArray(), - Struct(propertyType).put(BYTES, "string".toByteArray()), + Struct(PropertyType.schema).put(BYTES, "string".toByteArray()), "string".toByteArray())) .forEach { (value, expected, expectedValue) -> withClue(value) { @@ -437,32 +457,35 @@ class DynamicTypesTest { return Stream.of( LocalDate.of(1999, 12, 31).let { Arguments.of( - it, Struct(propertyType).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(it))) + it, + Struct(PropertyType.schema).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(it))) }, LocalTime.of(23, 59, 59, 9999).let { Arguments.of( - it, Struct(propertyType).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(it))) + it, + Struct(PropertyType.schema).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(it))) }, LocalDateTime.of(1999, 12, 31, 23, 59, 59, 9999).let { Arguments.of( it, - Struct(propertyType) + Struct(PropertyType.schema) .put(LOCAL_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }, OffsetTime.of(23, 59, 59, 9999, ZoneOffset.UTC).let { Arguments.of( - it, Struct(propertyType).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(it))) + it, + Struct(PropertyType.schema).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(it))) }, OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 9999, ZoneOffset.ofHours(1)).let { Arguments.of( it, - Struct(propertyType) + Struct(PropertyType.schema) .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }, ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 9999, ZoneId.of("Europe/Istanbul")).let { Arguments.of( it, - Struct(propertyType) + Struct(PropertyType.schema) .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }) } @@ -472,10 +495,10 @@ class DynamicTypesTest { fun `duration types should be returned as structs and should be converted back`() { listOf( Values.isoDuration(5, 2, 0, 9999).asIsoDuration() to - Struct(propertyType) + Struct(PropertyType.schema) .put( DURATION, - Struct(durationSchema) + Struct(PropertyType.durationSchema) .put("months", 5L) .put("days", 2L) .put("seconds", 0L) @@ -510,21 +533,22 @@ class DynamicTypesTest { listOf( ShortArray(1) { 1 } to - Struct(propertyType).put(LONG_LIST, LongArray(1) { 1.toLong() }.toList()), + Struct(PropertyType.schema).put(LONG_LIST, LongArray(1) { 1.toLong() }.toList()), IntArray(1) { 1 } to - Struct(propertyType).put(LONG_LIST, LongArray(1) { 1.toLong() }.toList()), - LongArray(1) { 1 } to Struct(propertyType).put(LONG_LIST, LongArray(1) { 1 }.toList()), + Struct(PropertyType.schema).put(LONG_LIST, LongArray(1) { 1.toLong() }.toList()), + LongArray(1) { 1 } to + Struct(PropertyType.schema).put(LONG_LIST, LongArray(1) { 1 }.toList()), FloatArray(1) { 1F } to - Struct(propertyType).put(FLOAT_LIST, DoubleArray(1) { 1.0 }.toList()), + Struct(PropertyType.schema).put(FLOAT_LIST, DoubleArray(1) { 1.0 }.toList()), DoubleArray(1) { 1.0 } to - Struct(propertyType).put(FLOAT_LIST, DoubleArray(1) { 1.0 }.toList()), + Struct(PropertyType.schema).put(FLOAT_LIST, DoubleArray(1) { 1.0 }.toList()), BooleanArray(1) { true } to - Struct(propertyType).put(BOOLEAN_LIST, BooleanArray(1) { true }.toList()), - Array(1) { 1 } to Struct(propertyType).put(LONG_LIST, Array(1) { 1L }.toList()), + Struct(PropertyType.schema).put(BOOLEAN_LIST, BooleanArray(1) { true }.toList()), + Array(1) { 1 } to Struct(PropertyType.schema).put(LONG_LIST, Array(1) { 1L }.toList()), Array(1) { 1.toShort() } to - Struct(propertyType).put(LONG_LIST, Array(1) { 1L }.toList()), + Struct(PropertyType.schema).put(LONG_LIST, Array(1) { 1L }.toList()), Array(1) { "string" } to - Struct(propertyType).put(STRING_LIST, Array(1) { "string" }.toList())) + Struct(PropertyType.schema).put(STRING_LIST, Array(1) { "string" }.toList())) .forEach { (value, expected) -> withClue(value) { val schema = DynamicTypes.toConnectSchema(value, false) @@ -556,18 +580,18 @@ class DynamicTypesTest { listOf( listOf(1, 2, 3) to listOf( - Struct(propertyType).put(LONG, 1L), - Struct(propertyType).put(LONG, 2L), - Struct(propertyType).put(LONG, 3L)), + PropertyType.toConnectValue(1L), + PropertyType.toConnectValue(2L), + PropertyType.toConnectValue(3L)), listOf("a", "b", "c") to listOf( - Struct(propertyType).put(STRING, "a"), - Struct(propertyType).put(STRING, "b"), - Struct(propertyType).put(STRING, "c")), + PropertyType.toConnectValue("a"), + PropertyType.toConnectValue("b"), + PropertyType.toConnectValue("c")), setOf(true, false) to listOf( - Struct(propertyType).put(BOOLEAN, true), - Struct(propertyType).put(BOOLEAN, false))) + Struct(PropertyType.schema).put(BOOLEAN, true), + Struct(PropertyType.schema).put(BOOLEAN, false))) .forEach { (value, expected) -> withClue(value) { val schema = DynamicTypes.toConnectSchema(value, false) @@ -586,14 +610,14 @@ class DynamicTypesTest { listOf( mapOf("a" to "x", "b" to "y", "c" to "z") to mapOf( - "a" to Struct(propertyType).put(STRING, "x"), - "b" to Struct(propertyType).put(STRING, "y"), - "c" to Struct(propertyType).put(STRING, "z")), + "a" to PropertyType.toConnectValue("x"), + "b" to PropertyType.toConnectValue("y"), + "c" to PropertyType.toConnectValue("z")), mapOf("a" to 1, "b" to 2, "c" to 3) to mapOf( - "a" to Struct(propertyType).put(LONG, 1L), - "b" to Struct(propertyType).put(LONG, 2L), - "c" to Struct(propertyType).put(LONG, 3L))) + "a" to PropertyType.toConnectValue(1L), + "b" to PropertyType.toConnectValue(2L), + "c" to PropertyType.toConnectValue(3L))) .forEach { (value, expected) -> withClue(value) { val schema = DynamicTypes.toConnectSchema(value, false) @@ -615,10 +639,10 @@ class DynamicTypesTest { val converted = DynamicTypes.toConnectValue(schema, point) converted shouldBe - Struct(propertyType) + Struct(PropertyType.schema) .put( POINT, - Struct(pointSchema) + Struct(PropertyType.pointSchema) .put("dimension", 2.toByte()) .put("srid", point.srid()) .put("x", point.x()) @@ -636,10 +660,10 @@ class DynamicTypesTest { val converted = DynamicTypes.toConnectValue(schema, point) converted shouldBe - Struct(propertyType) + Struct(PropertyType.schema) .put( POINT, - Struct(pointSchema) + Struct(PropertyType.pointSchema) .put("dimension", 3.toByte()) .put("srid", point.srid()) .put("x", point.x()) @@ -662,10 +686,10 @@ class DynamicTypesTest { converted shouldBe Struct(schema) - .put("", Struct(propertyType).put(LONG, 0L)) - .put("", Struct(propertyType).put(STRING_LIST, listOf("Person", "Employee"))) - .put("name", Struct(propertyType).put(STRING, "john")) - .put("surname", Struct(propertyType).put(STRING, "doe")) + .put("", 0L) + .put("", listOf("Person", "Employee")) + .put("name", PropertyType.toConnectValue("john")) + .put("surname", PropertyType.toConnectValue("doe")) val reverted = DynamicTypes.fromConnectValue(schema, converted) reverted shouldBe @@ -690,12 +714,12 @@ class DynamicTypesTest { converted shouldBe Struct(schema) - .put("", Struct(propertyType).put(LONG, 0L)) - .put("", Struct(propertyType).put(LONG, 1L)) - .put("", Struct(propertyType).put(LONG, 2L)) - .put("", Struct(propertyType).put(STRING, "KNOWS")) - .put("name", Struct(propertyType).put(STRING, "john")) - .put("surname", Struct(propertyType).put(STRING, "doe")) + .put("", 0L) + .put("", 1L) + .put("", 2L) + .put("", "KNOWS") + .put("name", PropertyType.toConnectValue("john")) + .put("surname", PropertyType.toConnectValue("doe")) val reverted = DynamicTypes.fromConnectValue(schema, converted) reverted shouldBe @@ -722,12 +746,12 @@ class DynamicTypesTest { converted shouldBe mapOf( - "name" to Struct(propertyType).put(STRING, "john"), - "age" to Struct(propertyType).put(LONG, 21L), + "name" to PropertyType.toConnectValue("john"), + "age" to PropertyType.toConnectValue(21L), "dob" to - Struct(propertyType) + Struct(PropertyType.schema) .put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(LocalDate.of(1999, 12, 31))), - "employed" to Struct(propertyType).put(BOOLEAN, true), + "employed" to Struct(PropertyType.schema).put(BOOLEAN, true), "nullable" to null) val reverted = DynamicTypes.fromConnectValue(schema, converted) @@ -742,11 +766,11 @@ class DynamicTypesTest { converted shouldBe listOf( - Struct(propertyType).put(STRING, "john"), - Struct(propertyType).put(LONG, 21L), - Struct(propertyType) + PropertyType.toConnectValue("john"), + PropertyType.toConnectValue(21L), + Struct(PropertyType.schema) .put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(LocalDate.of(1999, 12, 31))), - Struct(propertyType).put(BOOLEAN, true), + Struct(PropertyType.schema).put(BOOLEAN, true), null) val reverted = DynamicTypes.fromConnectValue(schema, converted) @@ -760,14 +784,14 @@ class DynamicTypesTest { .field("id", Schema.INT32_SCHEMA) .field("name", Schema.STRING_SCHEMA) .field("last_name", Schema.STRING_SCHEMA) - .field("dob", propertyType) + .field("dob", PropertyType.schema) .build() val struct = Struct(schema) .put("id", 1) .put("name", "john") .put("last_name", "doe") - .put("dob", DynamicTypes.toConnectValue(propertyType, LocalDate.of(2000, 1, 1))) + .put("dob", DynamicTypes.toConnectValue(PropertyType.schema, LocalDate.of(2000, 1, 1))) DynamicTypes.fromConnectValue(schema, struct) shouldBe mapOf("id" to 1, "name" to "john", "last_name" to "doe", "dob" to LocalDate.of(2000, 1, 1)) @@ -776,35 +800,38 @@ class DynamicTypesTest { @Test fun `structs with complex values should be returned as maps`() { val addressSchema = - SchemaBuilder.struct().field("city", propertyType).field("country", propertyType).build() + SchemaBuilder.struct() + .field("city", PropertyType.schema) + .field("country", PropertyType.schema) + .build() val schema = SchemaBuilder.struct() - .field("id", propertyType) - .field("name", propertyType) - .field("last_name", propertyType) - .field("dob", propertyType) + .field("id", PropertyType.schema) + .field("name", PropertyType.schema) + .field("last_name", PropertyType.schema) + .field("dob", PropertyType.schema) .field("address", addressSchema) - .field("years_of_interest", propertyType) + .field("years_of_interest", PropertyType.schema) .field( "events_of_interest", SchemaBuilder.map(Schema.STRING_SCHEMA, Schema.STRING_SCHEMA)) .build() val struct = Struct(schema) - .put("id", Struct(propertyType).put(LONG, 1L)) - .put("name", Struct(propertyType).put(STRING, "john")) - .put("last_name", Struct(propertyType).put(STRING, "doe")) + .put("id", PropertyType.toConnectValue(1L)) + .put("name", PropertyType.toConnectValue("john")) + .put("last_name", PropertyType.toConnectValue("doe")) .put( "dob", - Struct(propertyType) + Struct(PropertyType.schema) .put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(LocalDate.of(2000, 1, 1)))) .put( "address", Struct(addressSchema) - .put("city", Struct(propertyType).put(STRING, "london")) - .put("country", Struct(propertyType).put(STRING, "uk"))) + .put("city", PropertyType.toConnectValue("london")) + .put("country", PropertyType.toConnectValue("uk"))) .put( "years_of_interest", - Struct(propertyType).put(LONG_LIST, listOf(2000L, 2005L, 2017L))) + Struct(PropertyType.schema).put(LONG_LIST, listOf(2000L, 2005L, 2017L))) .put( "events_of_interest", mapOf("2000" to "birth", "2005" to "school", "2017" to "college")) diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt index 91ef3fcd9..b004df6e9 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt @@ -43,6 +43,15 @@ import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.ArgumentsProvider import org.junit.jupiter.params.provider.ArgumentsSource import org.neo4j.cdc.client.CDCClient +import org.neo4j.connectors.kafka.data.PropertyType.BOOLEAN +import org.neo4j.connectors.kafka.data.PropertyType.DURATION +import org.neo4j.connectors.kafka.data.PropertyType.FLOAT +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE_TIME +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_TIME +import org.neo4j.connectors.kafka.data.PropertyType.OFFSET_TIME +import org.neo4j.connectors.kafka.data.PropertyType.POINT +import org.neo4j.connectors.kafka.data.PropertyType.ZONED_DATE_TIME import org.neo4j.driver.AuthTokens import org.neo4j.driver.Driver import org.neo4j.driver.GraphDatabase @@ -106,72 +115,77 @@ class TypesTest { override fun provideArguments(p0: ExtensionContext?): Stream { return Stream.of( - Arguments.of(Named.of("null", null), propertyType, null), + Arguments.of(Named.of("null", null), PropertyType.schema, null), Arguments.of( - Named.of("boolean", true), propertyType, Struct(propertyType).put(BOOLEAN, true)), - Arguments.of(Named.of("long", 1), propertyType, Struct(propertyType).put(LONG, 1L)), - Arguments.of(Named.of("float", 1.0), propertyType, Struct(propertyType).put(FLOAT, 1.0)), + Named.of("boolean", true), + PropertyType.schema, + Struct(PropertyType.schema).put(BOOLEAN, true)), + Arguments.of(Named.of("long", 1), PropertyType.schema, PropertyType.toConnectValue(1L)), + Arguments.of( + Named.of("float", 1.0), + PropertyType.schema, + Struct(PropertyType.schema).put(FLOAT, 1.0)), Arguments.of( Named.of("string", "a string"), - propertyType, - Struct(propertyType).put(STRING, "a string")), + PropertyType.schema, + PropertyType.toConnectValue("a string")), LocalDate.of(1999, 12, 31).let { Arguments.of( Named.of("local date", it), - propertyType, - Struct(propertyType).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(it))) + PropertyType.schema, + Struct(PropertyType.schema).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(it))) }, LocalTime.of(23, 59, 59, 5).let { Arguments.of( Named.of("local time", it), - propertyType, - Struct(propertyType).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(it))) + PropertyType.schema, + Struct(PropertyType.schema).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(it))) }, LocalDateTime.of(1999, 12, 31, 23, 59, 59, 5).let { Arguments.of( Named.of("local date time", it), - propertyType, - Struct(propertyType) + PropertyType.schema, + Struct(PropertyType.schema) .put(LOCAL_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }, OffsetTime.of(23, 59, 59, 5, ZoneOffset.ofHours(1)).let { Arguments.of( Named.of("offset time", it), - propertyType, - Struct(propertyType).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(it))) + PropertyType.schema, + Struct(PropertyType.schema).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(it))) }, OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 5, ZoneOffset.ofHours(1)).let { Arguments.of( Named.of("offset date time", it), - propertyType, - Struct(propertyType) + PropertyType.schema, + Struct(PropertyType.schema) .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }, ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 5, ZoneId.of("Europe/Istanbul")).let { Arguments.of( Named.of("offset date time", it), - propertyType, - Struct(propertyType) + PropertyType.schema, + Struct(PropertyType.schema) .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) }, Arguments.of( Named.of("duration", Values.isoDuration(5, 2, 23, 5).asIsoDuration()), - propertyType, - Struct(propertyType) + PropertyType.schema, + Struct(PropertyType.schema) .put( DURATION, - Struct(durationSchema) + Struct(PropertyType.durationSchema) .put("months", 5L) .put("days", 2L) .put("seconds", 23L) .put("nanoseconds", 5))), Arguments.of( Named.of("point - 2d", Values.point(7203, 2.3, 4.5).asPoint()), - propertyType, - Struct(propertyType) + PropertyType.schema, + Struct(PropertyType.schema) .put( POINT, - Struct(pointSchema) + Struct(PropertyType.pointSchema) .put("dimension", 2.toByte()) .put("srid", 7203) .put("x", 2.3) @@ -179,11 +193,11 @@ class TypesTest { .put("z", null))), Arguments.of( Named.of("point - 3d", Values.point(4979, 12.78, 56.7, 100.0).asPoint()), - propertyType, - Struct(propertyType) + PropertyType.schema, + Struct(PropertyType.schema) .put( POINT, - Struct(pointSchema) + Struct(PropertyType.pointSchema) .put("dimension", 3.toByte()) .put("srid", 4979) .put("x", 12.78) @@ -191,31 +205,31 @@ class TypesTest { .put("z", 100.0))), Arguments.of( Named.of("list - uniformly typed elements", (1L..50L).toList()), - SchemaBuilder.array(propertyType).build(), - (1L..50L).map { Struct(propertyType).put(LONG, it) }.toList()), + SchemaBuilder.array(PropertyType.schema).build(), + (1L..50L).map { PropertyType.toConnectValue(it) }.toList()), Arguments.of( Named.of("list - non-uniformly typed elements", listOf(1, true, 2.0, "a string")), - SchemaBuilder.array(propertyType).build(), + SchemaBuilder.array(PropertyType.schema).build(), listOf( - Struct(propertyType).put(LONG, 1L), - Struct(propertyType).put(BOOLEAN, true), - Struct(propertyType).put(FLOAT, 2.0), - Struct(propertyType).put(STRING, "a string"))), + PropertyType.toConnectValue(1L), + Struct(PropertyType.schema).put(BOOLEAN, true), + Struct(PropertyType.schema).put(FLOAT, 2.0), + PropertyType.toConnectValue("a string"))), Arguments.of( Named.of("map - uniformly typed values", mapOf("a" to 1, "b" to 2, "c" to 3)), - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build(), + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build(), mapOf( - "a" to Struct(propertyType).put(LONG, 1L), - "b" to Struct(propertyType).put(LONG, 2L), - "c" to Struct(propertyType).put(LONG, 3L))), + "a" to PropertyType.toConnectValue(1L), + "b" to PropertyType.toConnectValue(2L), + "c" to PropertyType.toConnectValue(3L))), Arguments.of( Named.of( "map - non-uniformly typed values", mapOf("a" to 1, "b" to true, "c" to 3.0)), - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType).build(), + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build(), mapOf( - "a" to Struct(propertyType).put(LONG, 1L), - "b" to Struct(propertyType).put(BOOLEAN, true), - "c" to Struct(propertyType).put(FLOAT, 3.0)))) + "a" to PropertyType.toConnectValue(1L), + "b" to Struct(PropertyType.schema).put(BOOLEAN, true), + "c" to Struct(PropertyType.schema).put(FLOAT, 3.0)))) } } @@ -246,25 +260,20 @@ class TypesTest { schemaAndValue(person).also { (schema, converted, reverted) -> schema shouldBe SchemaBuilder.struct() - .field("", propertyType) - .field("", propertyType) - .field("name", propertyType) - .field("surname", propertyType) - .field("dob", propertyType) + .field("", Schema.INT64_SCHEMA) + .field("", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) + .field("name", PropertyType.schema) + .field("surname", PropertyType.schema) + .field("dob", PropertyType.schema) .build() converted shouldBe Struct(schema) - .put("", Struct(propertyType).put(LONG, person.id())) - .put("", Struct(propertyType).put(STRING_LIST, person.labels().toList())) - .put("name", Struct(propertyType).put(STRING, "john")) - .put("surname", Struct(propertyType).put(STRING, "doe")) - .put( - "dob", - Struct(propertyType) - .put( - LOCAL_DATE, - DateTimeFormatter.ISO_DATE.format(LocalDate.of(1999, 12, 31)))) + .put("", person.id()) + .put("", person.labels().toList()) + .put("name", PropertyType.toConnectValue("john")) + .put("surname", PropertyType.toConnectValue("doe")) + .put("dob", PropertyType.toConnectValue(LocalDate.of(1999, 12, 31))) reverted shouldBe mapOf( @@ -279,23 +288,18 @@ class TypesTest { schemaAndValue(company).also { (schema, converted, reverted) -> schema shouldBe SchemaBuilder.struct() - .field("", propertyType) - .field("", propertyType) - .field("name", propertyType) - .field("est", propertyType) + .field("", Schema.INT64_SCHEMA) + .field("", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) + .field("name", PropertyType.schema) + .field("est", PropertyType.schema) .build() converted shouldBe Struct(schema) - .put("", Struct(propertyType).put(LONG, company.id())) - .put("", Struct(propertyType).put(STRING_LIST, company.labels().toList())) - .put("name", Struct(propertyType).put(STRING, "acme corp")) - .put( - "est", - Struct(propertyType) - .put( - LOCAL_DATE, - DateTimeFormatter.ISO_DATE.format(LocalDate.of(1980, 1, 1)))) + .put("", company.id()) + .put("", company.labels().toList()) + .put("name", PropertyType.toConnectValue("acme corp")) + .put("est", PropertyType.toConnectValue(LocalDate.of(1980, 1, 1))) reverted shouldBe mapOf( @@ -309,27 +313,22 @@ class TypesTest { schemaAndValue(worksFor).also { (schema, converted, reverted) -> schema shouldBe SchemaBuilder.struct() - .field("", propertyType) - .field("", propertyType) - .field("", propertyType) - .field("", propertyType) - .field("contractId", propertyType) - .field("since", propertyType) + .field("", Schema.INT64_SCHEMA) + .field("", Schema.STRING_SCHEMA) + .field("", Schema.INT64_SCHEMA) + .field("", Schema.INT64_SCHEMA) + .field("contractId", PropertyType.schema) + .field("since", PropertyType.schema) .build() converted shouldBe Struct(schema) - .put("", Struct(propertyType).put(LONG, worksFor.id())) - .put("", Struct(propertyType).put(STRING, worksFor.type())) - .put("", Struct(propertyType).put(LONG, worksFor.startNodeId())) - .put("", Struct(propertyType).put(LONG, worksFor.endNodeId())) - .put("contractId", Struct(propertyType).put(LONG, 5916L)) - .put( - "since", - Struct(propertyType) - .put( - LOCAL_DATE, - DateTimeFormatter.ISO_DATE.format(LocalDate.of(2000, 1, 5)))) + .put("", worksFor.id()) + .put("", worksFor.type()) + .put("", worksFor.startNodeId()) + .put("", worksFor.endNodeId()) + .put("contractId", PropertyType.toConnectValue(5916L)) + .put("since", PropertyType.toConnectValue(LocalDate.of(2000, 1, 5))) reverted shouldBe mapOf( @@ -441,14 +440,14 @@ class TypesTest { val expectedSchema = SchemaBuilder.struct() - .field("id", propertyType) + .field("id", PropertyType.schema) .field( "data", SchemaBuilder.struct() .field( "arr", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) .optional() .build()) .optional() @@ -456,12 +455,12 @@ class TypesTest { .field( "arr_mixed", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, propertyType) + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) .optional() .build()) .optional() .build()) - .field("id", propertyType) + .field("id", PropertyType.schema) .field( "root", SchemaBuilder.array( @@ -469,7 +468,7 @@ class TypesTest { Schema.STRING_SCHEMA, SchemaBuilder.array( SchemaBuilder.map( - Schema.STRING_SCHEMA, propertyType) + Schema.STRING_SCHEMA, PropertyType.schema) .optional() .build()) .optional() @@ -488,20 +487,18 @@ class TypesTest { val converted = DynamicTypes.toConnectValue(schema, returned) converted shouldBe Struct(schema) - .put("id", Struct(propertyType).put(STRING, "ROOT_ID")) + .put("id", PropertyType.toConnectValue("ROOT_ID")) .put( "data", Struct(schema.field("data").schema()) - .put( - "arr", - listOf(null, mapOf("foo" to Struct(propertyType).put(STRING, "bar")))) + .put("arr", listOf(null, mapOf("foo" to PropertyType.toConnectValue("bar")))) .put( "arr_mixed", listOf( - mapOf("foo" to Struct(propertyType).put(STRING, "bar")), + mapOf("foo" to PropertyType.toConnectValue("bar")), null, - mapOf("foo" to Struct(propertyType).put(LONG, 1L)))) - .put("id", Struct(propertyType).put(STRING, "ROOT_ID")) + mapOf("foo" to PropertyType.toConnectValue(1L)))) + .put("id", PropertyType.toConnectValue("ROOT_ID")) .put( "root", listOf( @@ -509,9 +506,7 @@ class TypesTest { mapOf( "children" to listOf( - mapOf( - "name" to - Struct(propertyType).put(STRING, "child"))))))) + mapOf("name" to PropertyType.toConnectValue("child"))))))) val reverted = DynamicTypes.fromConnectValue(schema, converted) reverted shouldBe returned diff --git a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCudIT.kt b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCudIT.kt index b066d11a8..1e1e3a614 100644 --- a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCudIT.kt +++ b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCudIT.kt @@ -27,7 +27,8 @@ import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct import org.junit.jupiter.api.Test import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.propertyType +import org.neo4j.connectors.kafka.data.PropertyType +import org.neo4j.connectors.kafka.data.PropertyType.schema import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.format.KafkaConverter import org.neo4j.connectors.kafka.testing.format.KeyValueConverter @@ -122,8 +123,8 @@ abstract class Neo4jCudIT { SchemaBuilder.struct() .field("id", Schema.INT64_SCHEMA) .field("foo", Schema.STRING_SCHEMA) - .field("dob", propertyType) - .field("place", propertyType) + .field("dob", PropertyType.schema) + .field("place", PropertyType.schema) .build() val createNodeSchema = SchemaBuilder.struct() @@ -147,11 +148,12 @@ abstract class Neo4jCudIT { .put("foo", "foo-value") .put( "dob", - DynamicTypes.toConnectValue(propertyType, LocalDate.of(1995, 1, 1))) + DynamicTypes.toConnectValue( + PropertyType.schema, LocalDate.of(1995, 1, 1))) .put( "place", DynamicTypes.toConnectValue( - propertyType, Values.point(7203, 1.0, 2.5).asPoint()))), + PropertyType.schema, Values.point(7203, 1.0, 2.5).asPoint()))), ) eventually(30.seconds) { session.run("MATCH (n) RETURN n", emptyMap()).single() } @@ -187,8 +189,8 @@ abstract class Neo4jCudIT { val propertiesSchema = SchemaBuilder.struct() .field("foo", Schema.STRING_SCHEMA) - .field("dob", propertyType) - .field("place", propertyType) + .field("dob", PropertyType.schema) + .field("place", PropertyType.schema) .build() val updateNodeSchema = SchemaBuilder.struct() @@ -213,11 +215,12 @@ abstract class Neo4jCudIT { .put("foo", "foo-value-updated") .put( "dob", - DynamicTypes.toConnectValue(propertyType, LocalDate.of(1995, 1, 1))) + DynamicTypes.toConnectValue( + PropertyType.schema, LocalDate.of(1995, 1, 1))) .put( "place", DynamicTypes.toConnectValue( - propertyType, Values.point(7203, 1.0, 2.5).asPoint()))), + PropertyType.schema, Values.point(7203, 1.0, 2.5).asPoint()))), ) eventually(30.seconds) { @@ -254,8 +257,8 @@ abstract class Neo4jCudIT { SchemaBuilder.struct() .field("id", Schema.INT64_SCHEMA) .field("foo_new", Schema.STRING_SCHEMA) - .field("dob", propertyType) - .field("place", propertyType) + .field("dob", PropertyType.schema) + .field("place", PropertyType.schema) .build() val mergeNodeSchema = SchemaBuilder.struct() @@ -281,11 +284,12 @@ abstract class Neo4jCudIT { .put("foo_new", "foo-new-value-merged") .put( "dob", - DynamicTypes.toConnectValue(propertyType, LocalDate.of(1995, 1, 1))) + DynamicTypes.toConnectValue( + PropertyType.schema, LocalDate.of(1995, 1, 1))) .put( "place", DynamicTypes.toConnectValue( - propertyType, Values.point(7203, 1.0, 2.5).asPoint()))), + PropertyType.schema, Values.point(7203, 1.0, 2.5).asPoint()))), ) eventually(30.seconds) { diff --git a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt index 5af5f0798..a89d6fabc 100644 --- a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt +++ b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt @@ -43,7 +43,8 @@ import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.ArgumentsProvider import org.junit.jupiter.params.provider.ArgumentsSource import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.propertyType +import org.neo4j.connectors.kafka.data.PropertyType +import org.neo4j.connectors.kafka.data.PropertyType.schema import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.format.KafkaConverter import org.neo4j.connectors.kafka.testing.format.KeyValueConverter @@ -299,7 +300,7 @@ abstract class Neo4jCypherIT { Arguments.of(Schema.OPTIONAL_BOOLEAN_SCHEMA), Arguments.of(Schema.OPTIONAL_STRING_SCHEMA), Arguments.of(Schema.OPTIONAL_BYTES_SCHEMA), - Arguments.of(propertyType)) + Arguments.of(PropertyType.schema)) } } @@ -329,37 +330,39 @@ abstract class Neo4jCypherIT { object KnownTypes : ArgumentsProvider { override fun provideArguments(context: ExtensionContext?): Stream { return Stream.of( - Arguments.of(propertyType, true), - Arguments.of(propertyType, false), - Arguments.of(propertyType, Long.MAX_VALUE), - Arguments.of(propertyType, Long.MIN_VALUE), - Arguments.of(propertyType, Double.MAX_VALUE), - Arguments.of(propertyType, Double.MIN_VALUE), - Arguments.of(propertyType, "a string"), - Arguments.of(propertyType, "another string"), - Arguments.of(propertyType, "a string".encodeToByteArray()), - Arguments.of(propertyType, "another string".encodeToByteArray()), - Arguments.of(propertyType, LocalDate.of(2019, 5, 1)), - Arguments.of(propertyType, LocalDate.of(2019, 5, 1)), - Arguments.of(propertyType, LocalDateTime.of(2019, 5, 1, 23, 59, 59, 999999999)), - Arguments.of(propertyType, LocalDateTime.of(2019, 5, 1, 23, 59, 59, 999999999)), - Arguments.of(propertyType, LocalTime.of(23, 59, 59, 999999999)), - Arguments.of(propertyType, LocalTime.of(23, 59, 59, 999999999)), + Arguments.of(PropertyType.schema, true), + Arguments.of(PropertyType.schema, false), + Arguments.of(PropertyType.schema, Long.MAX_VALUE), + Arguments.of(PropertyType.schema, Long.MIN_VALUE), + Arguments.of(PropertyType.schema, Double.MAX_VALUE), + Arguments.of(PropertyType.schema, Double.MIN_VALUE), + Arguments.of(PropertyType.schema, "a string"), + Arguments.of(PropertyType.schema, "another string"), + Arguments.of(PropertyType.schema, "a string".encodeToByteArray()), + Arguments.of(PropertyType.schema, "another string".encodeToByteArray()), + Arguments.of(PropertyType.schema, LocalDate.of(2019, 5, 1)), + Arguments.of(PropertyType.schema, LocalDate.of(2019, 5, 1)), + Arguments.of(PropertyType.schema, LocalDateTime.of(2019, 5, 1, 23, 59, 59, 999999999)), + Arguments.of(PropertyType.schema, LocalDateTime.of(2019, 5, 1, 23, 59, 59, 999999999)), + Arguments.of(PropertyType.schema, LocalTime.of(23, 59, 59, 999999999)), + Arguments.of(PropertyType.schema, LocalTime.of(23, 59, 59, 999999999)), Arguments.of( - propertyType, + PropertyType.schema, ZonedDateTime.of(2019, 5, 1, 23, 59, 59, 999999999, ZoneId.of("Europe/Istanbul"))), Arguments.of( - propertyType, + PropertyType.schema, ZonedDateTime.of(2019, 5, 1, 23, 59, 59, 999999999, ZoneId.of("Europe/Istanbul"))), - Arguments.of(propertyType, OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHours(2))), Arguments.of( - propertyType, OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHoursMinutes(2, 30))), - Arguments.of(propertyType, Values.isoDuration(5, 4, 3, 2).asIsoDuration()), - Arguments.of(propertyType, Values.isoDuration(5, 4, 3, 2).asIsoDuration()), - Arguments.of(propertyType, Values.point(7203, 2.3, 4.5).asPoint()), - Arguments.of(propertyType, Values.point(7203, 2.3, 4.5).asPoint()), - Arguments.of(propertyType, Values.point(4979, 2.3, 4.5, 0.0).asPoint()), - Arguments.of(propertyType, Values.point(4979, 2.3, 4.5, 0.0).asPoint()), + PropertyType.schema, OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHours(2))), + Arguments.of( + PropertyType.schema, + OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHoursMinutes(2, 30))), + Arguments.of(PropertyType.schema, Values.isoDuration(5, 4, 3, 2).asIsoDuration()), + Arguments.of(PropertyType.schema, Values.isoDuration(5, 4, 3, 2).asIsoDuration()), + Arguments.of(PropertyType.schema, Values.point(7203, 2.3, 4.5).asPoint()), + Arguments.of(PropertyType.schema, Values.point(7203, 2.3, 4.5).asPoint()), + Arguments.of(PropertyType.schema, Values.point(4979, 2.3, 4.5, 0.0).asPoint()), + Arguments.of(PropertyType.schema, Values.point(4979, 2.3, 4.5, 0.0).asPoint()), ) } } diff --git a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jNodePatternIT.kt b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jNodePatternIT.kt index 99be2d8bd..1659a322e 100644 --- a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jNodePatternIT.kt +++ b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jNodePatternIT.kt @@ -30,7 +30,8 @@ import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct import org.junit.jupiter.api.Test import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.propertyType +import org.neo4j.connectors.kafka.data.PropertyType +import org.neo4j.connectors.kafka.data.PropertyType.schema import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.format.KafkaConverter import org.neo4j.connectors.kafka.testing.format.KeyValueConverter @@ -88,8 +89,8 @@ abstract class Neo4jNodePatternIT { .field("id", Schema.INT64_SCHEMA) .field("name", Schema.STRING_SCHEMA) .field("surname", Schema.STRING_SCHEMA) - .field("dob", propertyType) - .field("place", propertyType) + .field("dob", PropertyType.schema) + .field("place", PropertyType.schema) .build() .let { schema -> producer.publish( @@ -101,11 +102,12 @@ abstract class Neo4jNodePatternIT { .put("surname", "doe") .put( "dob", - DynamicTypes.toConnectValue(propertyType, LocalDate.of(1995, 1, 1))) + DynamicTypes.toConnectValue( + PropertyType.schema, LocalDate.of(1995, 1, 1))) .put( "place", DynamicTypes.toConnectValue( - propertyType, Values.point(7203, 1.0, 2.5).asPoint()))) + PropertyType.schema, Values.point(7203, 1.0, 2.5).asPoint()))) } eventually(30.seconds) { session.run("MATCH (n:User) RETURN n", emptyMap()).single() } diff --git a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jRelationshipPatternIT.kt b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jRelationshipPatternIT.kt index b85d7d4f6..d7813ccb5 100644 --- a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jRelationshipPatternIT.kt +++ b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jRelationshipPatternIT.kt @@ -29,7 +29,8 @@ import org.apache.kafka.connect.data.SchemaBuilder import org.apache.kafka.connect.data.Struct import org.junit.jupiter.api.Test import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.propertyType +import org.neo4j.connectors.kafka.data.PropertyType +import org.neo4j.connectors.kafka.data.PropertyType.schema import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.format.KafkaConverter import org.neo4j.connectors.kafka.testing.format.KeyValueConverter @@ -68,8 +69,8 @@ abstract class Neo4jRelationshipPatternIT { SchemaBuilder.struct() .field("userId", Schema.INT64_SCHEMA) .field("productId", Schema.INT64_SCHEMA) - .field("at", propertyType) - .field("place", propertyType) + .field("at", PropertyType.schema) + .field("place", PropertyType.schema) .build() .let { schema -> producer.publish( @@ -79,11 +80,13 @@ abstract class Neo4jRelationshipPatternIT { .put("userId", 1L) .put("productId", 2L) .put( - "at", DynamicTypes.toConnectValue(propertyType, LocalDate.of(1995, 1, 1))) + "at", + DynamicTypes.toConnectValue( + PropertyType.schema, LocalDate.of(1995, 1, 1))) .put( "place", DynamicTypes.toConnectValue( - propertyType, Values.point(7203, 1.0, 2.5).asPoint()))) + PropertyType.schema, Values.point(7203, 1.0, 2.5).asPoint()))) } eventually(30.seconds) { diff --git a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/NodePatternHandlerTest.kt b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/NodePatternHandlerTest.kt index 993b2b8dc..b6125f960 100644 --- a/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/NodePatternHandlerTest.kt +++ b/sink/src/test/kotlin/org/neo4j/connectors/kafka/sink/strategy/NodePatternHandlerTest.kt @@ -30,7 +30,8 @@ import org.neo4j.connectors.kafka.data.ConstraintData import org.neo4j.connectors.kafka.data.ConstraintEntityType import org.neo4j.connectors.kafka.data.ConstraintType import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.propertyType +import org.neo4j.connectors.kafka.data.PropertyType +import org.neo4j.connectors.kafka.data.PropertyType.schema import org.neo4j.connectors.kafka.exceptions.InvalidDataException import org.neo4j.connectors.kafka.sink.ChangeQuery import org.neo4j.cypherdsl.core.renderer.Renderer @@ -295,7 +296,7 @@ class NodePatternHandlerTest : HandlerTest() { .field("id", Schema.INT32_SCHEMA) .field("name", Schema.STRING_SCHEMA) .field("surname", Schema.STRING_SCHEMA) - .field("dob", propertyType) + .field("dob", PropertyType.schema) .build() assertQueryAndParameters( @@ -306,7 +307,9 @@ class NodePatternHandlerTest : HandlerTest() { .put("id", 1) .put("name", "john") .put("surname", "doe") - .put("dob", DynamicTypes.toConnectValue(propertyType, LocalDate.of(2000, 1, 1))), + .put( + "dob", + DynamicTypes.toConnectValue(PropertyType.schema, LocalDate.of(2000, 1, 1))), expected = listOf( listOf( diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt index 676a3eabb..0c776278e 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt @@ -37,8 +37,9 @@ import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.connectors.kafka.connect.ConnectHeader import org.neo4j.connectors.kafka.data.DynamicTypes import org.neo4j.connectors.kafka.data.Headers +import org.neo4j.connectors.kafka.data.PropertyType +import org.neo4j.connectors.kafka.data.PropertyType.schema import org.neo4j.connectors.kafka.data.TemporalDataSchemaType -import org.neo4j.connectors.kafka.data.propertyType import org.neo4j.connectors.kafka.testing.assertions.TopicVerifier import org.neo4j.connectors.kafka.testing.format.KafkaConverter.AVRO import org.neo4j.connectors.kafka.testing.format.KafkaConverter.JSON_SCHEMA @@ -238,37 +239,37 @@ abstract class Neo4jCdcSourceIT { properties.getStruct("localDate") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalDate.of(2024, 1, 1), ) as Struct properties.getStruct("localDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalDateTime.of(2024, 1, 1, 12, 0, 0), ) as Struct properties.getStruct("localTime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalTime.of(12, 0, 0), ) as Struct properties.getStruct("zonedDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), ) as Struct properties.getStruct("offsetDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), ) as Struct properties.getStruct("offsetTime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), ) as Struct } @@ -314,37 +315,37 @@ abstract class Neo4jCdcSourceIT { properties.getString("localDate") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalDate.of(2024, 1, 1), ) properties.getString("localDatetime") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalDateTime.of(2024, 1, 1, 12, 0, 0), ) properties.getString("localTime") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalTime.of(12, 0, 0), ) properties.getString("zonedDatetime") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), ) properties.getString("offsetDatetime") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), ) properties.getString("offsetTime") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), ) } diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt index 1617f2e37..9701b1b45 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt @@ -30,8 +30,9 @@ import java.time.ZonedDateTime import org.apache.kafka.connect.data.Struct import org.junit.jupiter.api.Test import org.neo4j.connectors.kafka.data.DynamicTypes +import org.neo4j.connectors.kafka.data.PropertyType +import org.neo4j.connectors.kafka.data.PropertyType.schema import org.neo4j.connectors.kafka.data.TemporalDataSchemaType -import org.neo4j.connectors.kafka.data.propertyType import org.neo4j.connectors.kafka.testing.MapSupport.excludingKeys import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.assertions.TopicVerifier @@ -215,37 +216,37 @@ abstract class Neo4jSourceQueryIT { .assertMessageValue { value -> value.getStruct("localDate") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalDate.of(2024, 1, 1), ) as Struct value.getStruct("localDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalDateTime.of(2024, 1, 1, 12, 0, 0), ) as Struct value.getStruct("localTime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalTime.of(12, 0, 0), ) as Struct value.getStruct("zonedDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), ) as Struct value.getStruct("offsetDatetime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), ) as Struct value.getStruct("offsetTime") shouldBeEqualToComparingFields DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), ) as Struct } @@ -288,37 +289,37 @@ abstract class Neo4jSourceQueryIT { .assertMessageValue { value -> value.getString("localDate") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalDate.of(2024, 1, 1), ) value.getString("localDatetime") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalDateTime.of(2024, 1, 1, 12, 0, 0), ) value.getString("localTime") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, LocalTime.of(12, 0, 0), ) value.getString("zonedDatetime") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), ) value.getString("offsetDatetime") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), ) value.getString("offsetTime") shouldBe DynamicTypes.toConnectValue( - propertyType, + PropertyType.schema, OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), ) } From 72fde24cfdb0998788b0bc731fccead054867170 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Fri, 26 Jul 2024 23:44:47 +0100 Subject: [PATCH 3/9] refactor: remove temporal data schema type parameter --- .../kafka/data/ChangeEventExtensions.kt | 13 +---- .../org/neo4j/connectors/kafka/data/Types.kt | 20 ++----- .../connectors/kafka/data/DynamicTypesTest.kt | 56 ++++++------------- .../connectors/kafka/sink/Neo4jCypherIT.kt | 1 - .../connectors/kafka/source/Neo4jQueryTask.kt | 6 +- 5 files changed, 27 insertions(+), 69 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt index 25b667a90..1ba90cb2c 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt @@ -75,20 +75,11 @@ class ChangeEventConverter( .field("txCommitTime", PropertyType.schema) .field( "txMetadata", - toConnectSchema( - metadata.txMetadata, - optional = true, - forceMapsAsStruct = true, - temporalDataSchemaType = temporalDataSchemaType) + toConnectSchema(metadata.txMetadata, optional = true, forceMapsAsStruct = true) .schema()) .also { metadata.additionalEntries.forEach { entry -> - it.field( - entry.key, - toConnectSchema( - entry.value, - optional = true, - temporalDataSchemaType = temporalDataSchemaType)) + it.field(entry.key, toConnectSchema(entry.value, optional = true)) } } .build() diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt index 6bcd8addd..a57d244af 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt @@ -536,7 +536,6 @@ object DynamicTypes { value: Any?, optional: Boolean = false, forceMapsAsStruct: Boolean = false, - temporalDataSchemaType: TemporalDataSchemaType = TemporalDataSchemaType.STRUCT, ): Schema = when (value) { null -> PropertyType.schema @@ -578,8 +577,7 @@ object DynamicTypes { PropertyType.schema } else { val first = value.firstOrNull { it.notNullOrEmpty() } - val schema = - toConnectSchema(first, optional, forceMapsAsStruct, temporalDataSchemaType) + val schema = toConnectSchema(first, optional, forceMapsAsStruct) SchemaBuilder.array(schema).apply { if (optional) optional() }.build() } } @@ -620,7 +618,7 @@ object DynamicTypes { val nonEmptyElementTypes = value .filter { it.notNullOrEmpty() } - .map { toConnectSchema(it, optional, forceMapsAsStruct, temporalDataSchemaType) } + .map { toConnectSchema(it, optional, forceMapsAsStruct) } when (nonEmptyElementTypes.toSet().size) { 0 -> SchemaBuilder.array(PropertyType.schema).apply { if (optional) optional() }.build() @@ -632,9 +630,7 @@ object DynamicTypes { SchemaBuilder.struct() .apply { value.forEachIndexed { i, v -> - this.field( - "e${i}", - toConnectSchema(v, optional, forceMapsAsStruct, temporalDataSchemaType)) + this.field("e${i}", toConnectSchema(v, optional, forceMapsAsStruct)) } } .apply { if (optional) optional() } @@ -653,9 +649,7 @@ object DynamicTypes { } } .filter { e -> e.value.notNullOrEmpty() } - .mapValues { e -> - toConnectSchema(e.value, optional, forceMapsAsStruct, temporalDataSchemaType) - } + .mapValues { e -> toConnectSchema(e.value, optional, forceMapsAsStruct) } val valueSet = elementTypes.values.toSet() when { @@ -665,8 +659,7 @@ object DynamicTypes { value.forEach { this.field( it.key as String, - toConnectSchema( - it.value, optional, forceMapsAsStruct, temporalDataSchemaType)) + toConnectSchema(it.value, optional, forceMapsAsStruct)) } } .apply { if (optional) optional() } @@ -681,8 +674,7 @@ object DynamicTypes { value.forEach { this.field( it.key as String, - toConnectSchema( - it.value, optional, forceMapsAsStruct, temporalDataSchemaType)) + toConnectSchema(it.value, optional, forceMapsAsStruct)) } } .apply { if (optional) optional() } diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt index 7704ef679..d4ba6dfa0 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt @@ -172,26 +172,18 @@ class DynamicTypesTest { DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), false) shouldBe PropertyType.schema DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), true) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema( - LocalDate.of(1999, 12, 31), - optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema( - LocalDate.of(1999, 12, 31), - optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), optional = false) shouldBe + PropertyType.schema + DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), optional = true) shouldBe + PropertyType.schema DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), false) shouldBe PropertyType.schema DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), true) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema( - LocalTime.of(23, 59, 59), - optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema( - LocalTime.of(23, 59, 59), - optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), optional = false) shouldBe + PropertyType.schema + DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), optional = true) shouldBe + PropertyType.schema DynamicTypes.toConnectSchema(LocalDateTime.of(1999, 12, 31, 23, 59, 59), false) shouldBe PropertyType.schema @@ -199,13 +191,9 @@ class DynamicTypesTest { PropertyType.schema DynamicTypes.toConnectSchema( - LocalDateTime.of(1999, 12, 31, 23, 59, 59), - optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema + LocalDateTime.of(1999, 12, 31, 23, 59, 59), optional = false) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( - LocalDateTime.of(1999, 12, 31, 23, 59, 59), - optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema + LocalDateTime.of(1999, 12, 31, 23, 59, 59), optional = true) shouldBe PropertyType.schema DynamicTypes.toConnectSchema(OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), false) shouldBe PropertyType.schema @@ -213,13 +201,9 @@ class DynamicTypesTest { PropertyType.schema DynamicTypes.toConnectSchema( - OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), - optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema + OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), optional = false) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( - OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), - optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema + OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), optional = true) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), false) shouldBe @@ -229,13 +213,11 @@ class DynamicTypesTest { PropertyType.schema DynamicTypes.toConnectSchema( - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), - optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema + OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), optional = false) shouldBe + PropertyType.schema DynamicTypes.toConnectSchema( - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), - true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema + OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), true) shouldBe + PropertyType.schema DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), false) shouldBe @@ -246,12 +228,10 @@ class DynamicTypesTest { DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), - optional = false, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema + optional = false) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), - optional = true, - temporalDataSchemaType = TemporalDataSchemaType.STRING) shouldBe PropertyType.schema + optional = true) shouldBe PropertyType.schema DynamicTypes.toConnectSchema( Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), false) shouldBe PropertyType.schema diff --git a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt index a89d6fabc..37c2bf104 100644 --- a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt +++ b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt @@ -44,7 +44,6 @@ import org.junit.jupiter.params.provider.ArgumentsProvider import org.junit.jupiter.params.provider.ArgumentsSource import org.neo4j.connectors.kafka.data.DynamicTypes import org.neo4j.connectors.kafka.data.PropertyType -import org.neo4j.connectors.kafka.data.PropertyType.schema import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.format.KafkaConverter import org.neo4j.connectors.kafka.testing.format.KeyValueConverter diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jQueryTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jQueryTask.kt index dd6947f69..8b340be23 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jQueryTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jQueryTask.kt @@ -96,11 +96,7 @@ class Neo4jQueryTask : SourceTask() { private fun build(record: Record): SourceRecord { val recordAsMap = record.asMap() val schema = - DynamicTypes.toConnectSchema( - recordAsMap, - optional = true, - forceMapsAsStruct = true, - temporalDataSchemaType = config.temporalDataSchemaType) + DynamicTypes.toConnectSchema(recordAsMap, optional = true, forceMapsAsStruct = true) val value = DynamicTypes.toConnectValue(schema, recordAsMap) return SourceRecord( From 6775fbccbe22cdde4db8731bcf9c14f70ff33b86 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Sun, 28 Jul 2024 01:46:43 +0100 Subject: [PATCH 4/9] refactor: third pass --- .../connectors/kafka/data/DynamicTypes.kt | 382 ++++++++++ .../connectors/kafka/data/PropertyType.kt | 359 +++++++++ .../org/neo4j/connectors/kafka/data/Types.kt | 695 ----------------- .../connectors/kafka/data/DynamicTypesTest.kt | 700 +++++------------- .../connectors/kafka/data/PropertyTypeTest.kt | 499 +++++++++++++ .../neo4j/connectors/kafka/data/TypesTest.kt | 7 +- 6 files changed, 1446 insertions(+), 1196 deletions(-) create mode 100644 common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt create mode 100644 common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt delete mode 100644 common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt create mode 100644 common/src/test/kotlin/org/neo4j/connectors/kafka/data/PropertyTypeTest.kt diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt new file mode 100644 index 000000000..0c99ed7ee --- /dev/null +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt @@ -0,0 +1,382 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.neo4j.connectors.kafka.data + +import java.nio.ByteBuffer +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.time.OffsetDateTime +import java.time.OffsetTime +import java.time.ZonedDateTime +import kotlin.reflect.KClass +import org.apache.kafka.connect.data.Schema +import org.apache.kafka.connect.data.SchemaBuilder +import org.apache.kafka.connect.data.Struct +import org.neo4j.driver.types.IsoDuration +import org.neo4j.driver.types.Node +import org.neo4j.driver.types.Point +import org.neo4j.driver.types.Relationship + +internal fun SchemaBuilder.namespaced(vararg paths: String): SchemaBuilder = + this.name("org.neo4j.connectors.kafka." + paths.joinToString(".")) + +internal fun Schema.id(): String = this.name().orEmpty().ifEmpty { this.type().name } + +internal fun Schema.shortId(): String = this.id().split('.').last() + +fun Schema.matches(other: Schema): Boolean { + return this.id() == other.id() || this.shortId() == other.shortId() +} + +object DynamicTypes { + + fun toConnectValue(schema: Schema, value: Any?): Any? { + if (value == null) { + return null + } + + if (schema == PropertyType.schema) { + return PropertyType.toConnectValue(value) + } + + return when (schema.type()) { + Schema.Type.ARRAY -> + when (value) { + is Collection<*> -> value.map { toConnectValue(schema.valueSchema(), it) } + else -> throw IllegalArgumentException("unsupported array type ${value.javaClass.name}") + } + Schema.Type.MAP -> + when (value) { + is Map<*, *> -> value.mapValues { toConnectValue(schema.valueSchema(), it.value) } + else -> throw IllegalArgumentException("unsupported map type ${value.javaClass.name}") + } + Schema.Type.STRUCT -> + when (value) { + is Node -> + Struct(schema).apply { + put("", value.id()) + put("", value.labels().toList()) + + value + .asMap { it.asObject() } + .forEach { e -> put(e.key, PropertyType.toConnectValue(e.value)) } + } + is Relationship -> + Struct(schema).apply { + put("", value.id()) + put("", value.type()) + put("", value.startNodeId()) + put("", value.endNodeId()) + + value + .asMap { it.asObject() } + .forEach { e -> put(e.key, PropertyType.toConnectValue(e.value)) } + } + is Map<*, *> -> + Struct(schema).apply { + schema.fields().forEach { + put(it.name(), toConnectValue(it.schema(), value[it.name()])) + } + } + is Collection<*> -> + Struct(schema).apply { + schema.fields().forEach { + put( + it.name(), + toConnectValue( + it.schema(), value.elementAt(it.name().substring(1).toInt()))) + } + } + else -> + throw IllegalArgumentException("unsupported struct type ${value.javaClass.name}") + } + else -> value + } + } + + fun fromConnectValue(schema: Schema, value: Any?, skipNullValuesInMaps: Boolean = false): Any? { + if (value == null) { + return null + } + + return when (schema.type()) { + Schema.Type.BOOLEAN -> value as Boolean? + Schema.Type.INT8 -> value as Byte? + Schema.Type.INT16 -> value as Short? + Schema.Type.INT32 -> value as Int? + Schema.Type.INT64 -> value as Long? + Schema.Type.FLOAT32 -> value as Float? + Schema.Type.FLOAT64 -> value as Double? + Schema.Type.BYTES -> + when (value) { + is ByteArray -> value + is ByteBuffer -> value.array() + else -> throw IllegalArgumentException("unsupported bytes type ${value.javaClass.name}") + } + Schema.Type.STRING -> + when (value) { + is Char -> value.toString() + is CharArray -> value.toString() + is CharSequence -> value.toString() + else -> + throw IllegalArgumentException("unsupported string type ${value.javaClass.name}") + } + Schema.Type.STRUCT -> + when { + PropertyType.schema.matches(schema) -> PropertyType.fromConnectValue(value as Struct?) + else -> { + val result = mutableMapOf() + val struct = value as Struct + + for (field in schema.fields()) { + val fieldValue = + fromConnectValue(field.schema(), struct.get(field), skipNullValuesInMaps) + + if (fieldValue != null || !skipNullValuesInMaps) { + result[field.name()] = fieldValue + } + } + + if (result.isNotEmpty() && + result.keys.all { it.startsWith("e") && it.substring(1).toIntOrNull() != null }) { + result + .mapKeys { it.key.substring(1).toInt() } + .entries + .sortedBy { it.key } + .map { it.value } + .toList() + } else { + result + } + } + } + Schema.Type.ARRAY -> { + val result = mutableListOf() + + when { + value.javaClass.isArray -> + for (i in 0 ..< java.lang.reflect.Array.getLength(value)) { + result.add( + fromConnectValue( + schema.valueSchema(), + java.lang.reflect.Array.get(value, i), + skipNullValuesInMaps)) + } + value is Iterable<*> -> + for (element in value) { + result.add(fromConnectValue(schema.valueSchema(), element, skipNullValuesInMaps)) + } + else -> throw IllegalArgumentException("unsupported array type ${value.javaClass.name}") + } + + result.toList() + } + Schema.Type.MAP -> { + val result = mutableMapOf() + val map = value as Map<*, *> + + for (entry in map.entries) { + if (entry.key !is String) { + throw IllegalArgumentException( + "invalid key type (${entry.key?.javaClass?.name} in map value") + } + + result[entry.key as String] = + fromConnectValue(schema.valueSchema(), entry.value, skipNullValuesInMaps) + } + + result + } + else -> + throw IllegalArgumentException( + "unsupported schema ($schema) and value type (${value.javaClass.name})") + } + } + + fun toConnectSchema( + value: Any?, + optional: Boolean = false, + forceMapsAsStruct: Boolean = false, + ): Schema { + return when (value) { + null -> PropertyType.schema + is Boolean, + is Float, + is Double, + is Number, + is Char, + is LocalDate, + is LocalDateTime, + is LocalTime, + is OffsetDateTime, + is ZonedDateTime, + is OffsetTime, + is IsoDuration, + is Point, + is CharArray, + is CharSequence, + is ByteBuffer, + is ByteArray, + is ShortArray, + is IntArray, + is LongArray, + is FloatArray, + is DoubleArray, + is BooleanArray -> PropertyType.schema + is Array<*> -> + if (isSimplePropertyType(value::class.java.componentType.kotlin)) { + PropertyType.schema + } else { + val first = value.firstOrNull { it.notNullOrEmpty() } + val schema = toConnectSchema(first, optional, forceMapsAsStruct) + SchemaBuilder.array(schema).apply { if (optional) optional() }.build() + } + is Node -> + SchemaBuilder.struct() + .apply { + field("", Schema.INT64_SCHEMA) + field("", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) + + value.keys().forEach { field(it, PropertyType.schema) } + + if (optional) optional() + } + .build() + is Relationship -> + SchemaBuilder.struct() + .apply { + field("", Schema.INT64_SCHEMA) + field("", Schema.STRING_SCHEMA) + field("", Schema.INT64_SCHEMA) + field("", Schema.INT64_SCHEMA) + + value.keys().forEach { field(it, PropertyType.schema) } + + if (optional) optional() + } + .build() + is Collection<*> -> { + val elementTypes = value.map { it?.javaClass?.kotlin }.toSet() + val elementType = elementTypes.singleOrNull() + if (elementType != null && isSimplePropertyType(elementType)) { + return PropertyType.schema + } + + val nonEmptyElementTypes = + value + .filter { it.notNullOrEmpty() } + .map { toConnectSchema(it, optional, forceMapsAsStruct) } + + when (nonEmptyElementTypes.toSet().size) { + 0 -> SchemaBuilder.array(PropertyType.schema).apply { if (optional) optional() }.build() + 1 -> + SchemaBuilder.array(nonEmptyElementTypes.first()) + .apply { if (optional) optional() } + .build() + else -> + SchemaBuilder.struct() + .apply { + value.forEachIndexed { i, v -> + this.field("e${i}", toConnectSchema(v, optional, forceMapsAsStruct)) + } + } + .apply { if (optional) optional() } + .build() + } + } + is Map<*, *> -> { + val elementTypes = + value + .mapKeys { + when (val key = it.key) { + is String -> key + else -> + throw IllegalArgumentException( + "unsupported map key type ${key?.javaClass?.name}") + } + } + .filter { e -> e.value.notNullOrEmpty() } + .mapValues { e -> toConnectSchema(e.value, optional, forceMapsAsStruct) } + + val valueSet = elementTypes.values.toSet() + when { + valueSet.isEmpty() -> + SchemaBuilder.struct() + .apply { + value.forEach { + this.field( + it.key as String, toConnectSchema(it.value, optional, forceMapsAsStruct)) + } + } + .apply { if (optional) optional() } + .build() + valueSet.singleOrNull() != null && !forceMapsAsStruct -> + SchemaBuilder.map(Schema.STRING_SCHEMA, elementTypes.values.first()) + .apply { if (optional) optional() } + .build() + else -> + SchemaBuilder.struct() + .apply { + value.forEach { + this.field( + it.key as String, toConnectSchema(it.value, optional, forceMapsAsStruct)) + } + } + .apply { if (optional) optional() } + .build() + } + } + else -> throw IllegalArgumentException("unsupported type ${value.javaClass.name}") + } + } + + private fun isSimplePropertyType(cls: KClass<*>): Boolean = + when (cls) { + Boolean::class, + Byte::class, + Short::class, + Int::class, + Long::class, + Float::class, + Double::class, + String::class, + LocalDate::class, + LocalDateTime::class, + LocalTime::class, + OffsetDateTime::class, + ZonedDateTime::class, + OffsetTime::class -> true + else -> + if (IsoDuration::class.java.isAssignableFrom(cls.java)) { + true + } else if (Point::class.java.isAssignableFrom(cls.java)) { + true + } else { + false + } + } + + private fun Any?.notNullOrEmpty(): Boolean = + when (val value = this) { + null -> false + is Collection<*> -> value.isNotEmpty() && value.any { it.notNullOrEmpty() } + is Array<*> -> value.isNotEmpty() && value.any { it.notNullOrEmpty() } + is Map<*, *> -> value.isNotEmpty() && value.values.any { it.notNullOrEmpty() } + else -> true + } +} diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt new file mode 100644 index 000000000..3f43c7cc9 --- /dev/null +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt @@ -0,0 +1,359 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.neo4j.connectors.kafka.data + +import java.nio.ByteBuffer +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.time.OffsetDateTime +import java.time.OffsetTime +import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.time.temporal.TemporalQueries +import kotlin.reflect.KClass +import org.apache.kafka.connect.data.Schema +import org.apache.kafka.connect.data.SchemaBuilder +import org.apache.kafka.connect.data.Struct +import org.neo4j.driver.Values +import org.neo4j.driver.types.IsoDuration +import org.neo4j.driver.types.Point + +@Suppress("UNCHECKED_CAST") +object PropertyType { + internal const val MONTHS = "months" + internal const val DAYS = "days" + internal const val SECONDS = "seconds" + internal const val NANOS = "nanoseconds" + internal const val SR_ID = "srid" + internal const val X = "x" + internal const val Y = "y" + internal const val Z = "z" + internal const val DIMENSION = "dimension" + internal const val TWO_D: Byte = 2 + internal const val THREE_D: Byte = 3 + + internal const val BOOLEAN = "B" + internal const val BOOLEAN_LIST = "LB" + internal const val LONG = "I64" + internal const val LONG_LIST = "LI64" + internal const val FLOAT = "F64" + internal const val FLOAT_LIST = "LF64" + internal const val STRING = "S" + internal const val STRING_LIST = "LS" + internal const val BYTES = "BA" + internal const val LOCAL_DATE = "TLD" + internal const val LOCAL_DATE_LIST = "LTLD" + internal const val LOCAL_DATE_TIME = "TLDT" + internal const val LOCAL_DATE_TIME_LIST = "LTLDT" + internal const val LOCAL_TIME = "TLT" + internal const val LOCAL_TIME_LIST = "LTLT" + internal const val ZONED_DATE_TIME = "TZDT" + internal const val ZONED_DATE_TIME_LIST = "LZDT" + internal const val OFFSET_TIME = "TOT" + internal const val OFFSET_TIME_LIST = "LTOT" + internal const val DURATION = "TD" + internal const val DURATION_LIST = "LTD" + internal const val POINT = "SP" + internal const val POINT_LIST = "LSP" + + internal val durationSchema: Schema = + SchemaBuilder(Schema.Type.STRUCT) + .field(MONTHS, Schema.INT64_SCHEMA) + .field(DAYS, Schema.INT64_SCHEMA) + .field(SECONDS, Schema.INT64_SCHEMA) + .field(NANOS, Schema.INT32_SCHEMA) + .optional() + .build() + + internal val pointSchema: Schema = + SchemaBuilder(Schema.Type.STRUCT) + .field(DIMENSION, Schema.INT8_SCHEMA) + .field(SR_ID, Schema.INT32_SCHEMA) + .field(X, Schema.FLOAT64_SCHEMA) + .field(Y, Schema.FLOAT64_SCHEMA) + .field(Z, Schema.OPTIONAL_FLOAT64_SCHEMA) + .optional() + .build() + + val schema: Schema = + SchemaBuilder.struct() + .namespaced("Neo4jPropertyType") + .field(BOOLEAN, Schema.OPTIONAL_BOOLEAN_SCHEMA) + .field(LONG, Schema.OPTIONAL_INT64_SCHEMA) + .field(FLOAT, Schema.OPTIONAL_FLOAT64_SCHEMA) + .field(STRING, Schema.OPTIONAL_STRING_SCHEMA) + .field(BYTES, Schema.OPTIONAL_BYTES_SCHEMA) + .field(LOCAL_DATE, Schema.OPTIONAL_STRING_SCHEMA) + .field(LOCAL_DATE_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(LOCAL_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(ZONED_DATE_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(OFFSET_TIME, Schema.OPTIONAL_STRING_SCHEMA) + .field(DURATION, durationSchema) + .field(POINT, pointSchema) + .field(BOOLEAN_LIST, SchemaBuilder.array(Schema.BOOLEAN_SCHEMA).optional().build()) + .field(LONG_LIST, SchemaBuilder.array(Schema.INT64_SCHEMA).optional().build()) + .field(FLOAT_LIST, SchemaBuilder.array(Schema.FLOAT64_SCHEMA).optional().build()) + .field(STRING_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(LOCAL_DATE_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(LOCAL_DATE_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(LOCAL_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(ZONED_DATE_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(OFFSET_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) + .field(DURATION_LIST, SchemaBuilder.array(durationSchema).optional().build()) + .field(POINT_LIST, SchemaBuilder.array(pointSchema).optional().build()) + .optional() + .build() + + fun toConnectValue(value: Any?): Struct? { + return when (value) { + null -> null + is Boolean -> Struct(schema).put(BOOLEAN, value) + is Float -> Struct(schema).put(FLOAT, value.toDouble()) + is Double -> Struct(schema).put(FLOAT, value) + is Number -> Struct(schema).put(LONG, value.toLong()) + is String -> Struct(schema).put(STRING, value) + is Char -> Struct(schema).put(STRING, value.toString()) + is CharArray -> Struct(schema).put(STRING, String(value)) + is CharSequence -> + Struct(schema).put(STRING, value.codePoints().toArray().let { String(it, 0, it.size) }) + is ByteArray -> Struct(schema).put(BYTES, value) + is ByteBuffer -> Struct(schema).put(BYTES, value.array()) + is LocalDate -> Struct(schema).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(value)) + is LocalDateTime -> + Struct(schema).put(LOCAL_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) + is LocalTime -> Struct(schema).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(value)) + is OffsetDateTime -> + Struct(schema).put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) + is ZonedDateTime -> + Struct(schema).put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) + is OffsetTime -> Struct(schema).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(value)) + is IsoDuration -> + Struct(schema) + .put( + DURATION, + Struct(durationSchema) + .put(MONTHS, value.months()) + .put(DAYS, value.days()) + .put(SECONDS, value.seconds()) + .put(NANOS, value.nanoseconds())) + is Point -> + Struct(schema) + .put( + POINT, + Struct(pointSchema) + .put(SR_ID, value.srid()) + .put(X, value.x()) + .put(Y, value.y()) + .also { + it.put(DIMENSION, if (value.z().isNaN()) TWO_D else THREE_D) + if (!value.z().isNaN()) { + it.put(Z, value.z()) + } + }) + is ShortArray -> Struct(schema).put(LONG_LIST, value.map { s -> s.toLong() }.toList()) + is IntArray -> Struct(schema).put(LONG_LIST, value.map { s -> s.toLong() }.toList()) + is LongArray -> Struct(schema).put(LONG_LIST, value.toList()) + is FloatArray -> Struct(schema).put(FLOAT_LIST, value.map { s -> s.toDouble() }.toList()) + is DoubleArray -> Struct(schema).put(FLOAT_LIST, value.toList()) + is BooleanArray -> Struct(schema).put(BOOLEAN_LIST, value.toList()) + is Array<*> -> asList(value.toList(), value::class.java.componentType.kotlin) + is Iterable<*> -> { + val elementTypes = value.map { it?.javaClass?.kotlin }.toSet() + val elementType = elementTypes.singleOrNull() + if (elementType != null) { + return asList(value, elementType) + } + + throw IllegalArgumentException( + "collections with multiple element types are not supported: ${elementTypes.joinToString { it?.java?.name ?: "null" }}") + } + else -> throw IllegalArgumentException("unsupported property type: ${value.javaClass.name}") + } + } + + private fun asList(value: Iterable<*>, componentType: KClass<*>): Struct? = + when (componentType) { + Boolean::class -> Struct(schema).put(BOOLEAN_LIST, value) + Byte::class -> Struct(schema).put(BYTES, (value as List).toByteArray()) + Short::class -> + Struct(schema).put(LONG_LIST, (value as List).map { s -> s.toLong() }) + Int::class -> Struct(schema).put(LONG_LIST, (value as List).map { s -> s.toLong() }) + Long::class -> Struct(schema).put(LONG_LIST, value) + Float::class -> + Struct(schema).put(FLOAT_LIST, (value as List).map { s -> s.toDouble() }) + Double::class -> Struct(schema).put(FLOAT_LIST, value) + String::class -> Struct(schema).put(STRING_LIST, value) + LocalDate::class -> + Struct(schema) + .put( + LOCAL_DATE_LIST, + (value as List).map { s -> DateTimeFormatter.ISO_DATE.format(s) }) + LocalDateTime::class -> + Struct(schema) + .put( + LOCAL_DATE_TIME_LIST, + (value as List).map { s -> + DateTimeFormatter.ISO_DATE_TIME.format(s) + }) + LocalTime::class -> + Struct(schema) + .put( + LOCAL_TIME_LIST, + (value as List).map { s -> DateTimeFormatter.ISO_TIME.format(s) }) + OffsetDateTime::class -> + Struct(schema) + .put( + ZONED_DATE_TIME_LIST, + (value as List).map { s -> + DateTimeFormatter.ISO_DATE_TIME.format(s) + }) + ZonedDateTime::class -> + Struct(schema) + .put( + ZONED_DATE_TIME_LIST, + (value as List).map { s -> + DateTimeFormatter.ISO_DATE_TIME.format(s) + }) + OffsetTime::class -> + Struct(schema) + .put( + OFFSET_TIME_LIST, + (value as List).map { s -> DateTimeFormatter.ISO_TIME.format(s) }) + else -> + if (IsoDuration::class.java.isAssignableFrom(componentType.java)) { + Struct(schema) + .put( + DURATION_LIST, + value + .map { s -> s as IsoDuration } + .map { + Struct(durationSchema) + .put(MONTHS, it.months()) + .put(DAYS, it.days()) + .put(SECONDS, it.seconds()) + .put(NANOS, it.nanoseconds()) + }) + } else if (Point::class.java.isAssignableFrom(componentType.java)) { + Struct(schema) + .put( + POINT_LIST, + value + .map { s -> s as Point } + .map { s -> + Struct(pointSchema) + .put(SR_ID, s.srid()) + .put(X, s.x()) + .put(Y, s.y()) + .also { + it.put(DIMENSION, if (s.z().isNaN()) TWO_D else THREE_D) + if (!s.z().isNaN()) { + it.put(Z, s.z()) + } + } + }) + } else { + throw IllegalArgumentException( + "unsupported array type: array of ${componentType.java.name}") + } + } + + fun fromConnectValue(value: Struct?): Any? { + return value?.let { + for (f in it.schema().fields()) { + if (it.getWithoutDefault(f.name()) == null) { + continue + } + + return when (f.name()) { + BOOLEAN -> it.get(f) as Boolean + BOOLEAN_LIST -> it.get(f) as List<*> + LONG -> it.get(f) as Long + LONG_LIST -> it.get(f) as List<*> + FLOAT -> it.get(f) as Double + FLOAT_LIST -> it.get(f) as List<*> + STRING -> it.get(f) as String + STRING_LIST -> it.get(f) as List<*> + BYTES -> it.get(f) as ByteArray + LOCAL_DATE -> parseLocalDate((it.get(f) as String)) + LOCAL_DATE_LIST -> (it.get(f) as List).map { s -> parseLocalDate(s) } + LOCAL_TIME -> parseLocalTime((it.get(f) as String)) + LOCAL_TIME_LIST -> (it.get(f) as List).map { s -> parseLocalTime(s) } + LOCAL_DATE_TIME -> parseLocalDateTime((it.get(f) as String)) + LOCAL_DATE_TIME_LIST -> (it.get(f) as List).map { s -> parseLocalDateTime(s) } + ZONED_DATE_TIME -> parseZonedDateTime((it.get(f) as String)) + ZONED_DATE_TIME_LIST -> (it.get(f) as List).map { s -> parseZonedDateTime(s) } + OFFSET_TIME -> parseOffsetTime((it.get(f) as String)) + OFFSET_TIME_LIST -> (it.get(f) as List).map { s -> parseOffsetTime(s) } + DURATION -> toDuration((it.get(f) as Struct)) + DURATION_LIST -> (it.get(f) as List).map { s -> toDuration(s) } + POINT -> toPoint((it.get(f) as Struct)) + POINT_LIST -> (it.get(f) as List).map { s -> toPoint(s) } + else -> throw IllegalArgumentException("unsupported neo4j type: ${f.name()}") + } + } + + return null + } + } + + private fun parseLocalDate(s: String): LocalDate = + DateTimeFormatter.ISO_DATE.parse(s) { parsed -> LocalDate.from(parsed) } + + private fun parseLocalTime(s: String): LocalTime = + DateTimeFormatter.ISO_TIME.parse(s) { parsed -> LocalTime.from(parsed) } + + private fun parseLocalDateTime(s: String): LocalDateTime = + DateTimeFormatter.ISO_DATE_TIME.parse(s) { parsed -> LocalDateTime.from(parsed) } + + private fun parseZonedDateTime(s: String) = + DateTimeFormatter.ISO_DATE_TIME.parse(s) { parsed -> + val zoneId = parsed.query(TemporalQueries.zone()) + + if (zoneId is ZoneOffset) { + OffsetDateTime.from(parsed) + } else { + ZonedDateTime.from(parsed) + } + } + + private fun parseOffsetTime(s: String): OffsetTime = + DateTimeFormatter.ISO_TIME.parse(s) { parsed -> OffsetTime.from(parsed) } + + private fun toDuration(s: Struct): IsoDuration = + Values.isoDuration( + s.getInt64(MONTHS), + s.getInt64(DAYS), + s.getInt64(SECONDS), + s.getInt32(NANOS), + ) + .asIsoDuration() + + private fun toPoint(s: Struct): Point = + when (val dimension = s.getInt8(DIMENSION)) { + TWO_D -> Values.point(s.getInt32(SR_ID), s.getFloat64(X), s.getFloat64(Y)) + THREE_D -> + Values.point( + s.getInt32(SR_ID), + s.getFloat64(X), + s.getFloat64(Y), + s.getFloat64(Z), + ) + else -> throw IllegalArgumentException("unsupported dimension value ${dimension}") + }.asPoint() +} diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt deleted file mode 100644 index a57d244af..000000000 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/Types.kt +++ /dev/null @@ -1,695 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.neo4j.connectors.kafka.data - -import java.nio.ByteBuffer -import java.time.LocalDate -import java.time.LocalDateTime -import java.time.LocalTime -import java.time.OffsetDateTime -import java.time.OffsetTime -import java.time.ZoneOffset -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter -import java.time.temporal.TemporalQueries -import org.apache.kafka.connect.data.Schema -import org.apache.kafka.connect.data.SchemaBuilder -import org.apache.kafka.connect.data.Struct -import org.neo4j.driver.Values -import org.neo4j.driver.types.IsoDuration -import org.neo4j.driver.types.Node -import org.neo4j.driver.types.Point -import org.neo4j.driver.types.Relationship - -internal fun SchemaBuilder.namespaced(vararg paths: String): SchemaBuilder = - this.name("org.neo4j.connectors.kafka." + paths.joinToString(".")) - -internal fun Schema.id(): String = this.name().orEmpty().ifEmpty { this.type().name } - -internal fun Schema.shortId(): String = this.id().split('.').last() - -@Suppress("UNCHECKED_CAST") -object PropertyType { - const val MONTHS = "months" - const val DAYS = "days" - const val SECONDS = "seconds" - const val NANOS = "nanoseconds" - const val SR_ID = "srid" - const val X = "x" - const val Y = "y" - const val Z = "z" - const val DIMENSION = "dimension" - const val TWO_D: Byte = 2 - const val THREE_D: Byte = 3 - - const val BOOLEAN = "B" - const val BOOLEAN_LIST = "LB" - const val LONG = "I64" - const val LONG_LIST = "LI64" - const val FLOAT = "F64" - const val FLOAT_LIST = "LF64" - const val STRING = "S" - const val STRING_LIST = "LS" - const val BYTES = "BA" - const val LOCAL_DATE = "TLD" - const val LOCAL_DATE_LIST = "LTLD" - const val LOCAL_DATE_TIME = "TLDT" - const val LOCAL_DATE_TIME_LIST = "LTLDT" - const val LOCAL_TIME = "TLT" - const val LOCAL_TIME_LIST = "LTLT" - const val ZONED_DATE_TIME = "TZDT" - const val ZONED_DATE_TIME_LIST = "LZDT" - const val OFFSET_TIME = "TOT" - const val OFFSET_TIME_LIST = "LTOT" - const val DURATION = "TD" - const val DURATION_LIST = "LTD" - const val POINT = "SP" - const val POINT_LIST = "LSP" - - val durationSchema: Schema = - SchemaBuilder(Schema.Type.STRUCT) - .field(MONTHS, Schema.INT64_SCHEMA) - .field(DAYS, Schema.INT64_SCHEMA) - .field(SECONDS, Schema.INT64_SCHEMA) - .field(NANOS, Schema.INT32_SCHEMA) - .optional() - .build() - - val pointSchema: Schema = - SchemaBuilder(Schema.Type.STRUCT) - .field(DIMENSION, Schema.INT8_SCHEMA) - .field(SR_ID, Schema.INT32_SCHEMA) - .field(X, Schema.FLOAT64_SCHEMA) - .field(Y, Schema.FLOAT64_SCHEMA) - .field(Z, Schema.OPTIONAL_FLOAT64_SCHEMA) - .optional() - .build() - - val schema: Schema = - SchemaBuilder.struct() - .namespaced("Neo4jPropertyType") - .field(BOOLEAN, Schema.OPTIONAL_BOOLEAN_SCHEMA) - .field(LONG, Schema.OPTIONAL_INT64_SCHEMA) - .field(FLOAT, Schema.OPTIONAL_FLOAT64_SCHEMA) - .field(STRING, Schema.OPTIONAL_STRING_SCHEMA) - .field(BYTES, Schema.OPTIONAL_BYTES_SCHEMA) - .field(LOCAL_DATE, Schema.OPTIONAL_STRING_SCHEMA) - .field(LOCAL_DATE_TIME, Schema.OPTIONAL_STRING_SCHEMA) - .field(LOCAL_TIME, Schema.OPTIONAL_STRING_SCHEMA) - .field(ZONED_DATE_TIME, Schema.OPTIONAL_STRING_SCHEMA) - .field(OFFSET_TIME, Schema.OPTIONAL_STRING_SCHEMA) - .field(DURATION, durationSchema) - .field(POINT, pointSchema) - .field(BOOLEAN_LIST, SchemaBuilder.array(Schema.BOOLEAN_SCHEMA).optional().build()) - .field(LONG_LIST, SchemaBuilder.array(Schema.INT64_SCHEMA).optional().build()) - .field(FLOAT_LIST, SchemaBuilder.array(Schema.FLOAT64_SCHEMA).optional().build()) - .field(STRING_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(LOCAL_DATE_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(LOCAL_DATE_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(LOCAL_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(ZONED_DATE_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(OFFSET_TIME_LIST, SchemaBuilder.array(Schema.STRING_SCHEMA).optional().build()) - .field(DURATION_LIST, SchemaBuilder.array(durationSchema).optional().build()) - .field(POINT_LIST, SchemaBuilder.array(pointSchema).optional().build()) - .optional() - .build() - - fun toConnectValue(value: Any?): Struct? { - return when (value) { - is Boolean -> Struct(schema).put(BOOLEAN, value) - is Float -> Struct(schema).put(FLOAT, value.toDouble()) - is Double -> Struct(schema).put(FLOAT, value) - is Number -> Struct(schema).put(LONG, value.toLong()) - is String -> Struct(schema).put(STRING, value) - is Char -> Struct(schema).put(STRING, value.toString()) - is CharArray -> Struct(schema).put(STRING, String(value)) - is ByteArray -> Struct(schema).put(BYTES, value) - is ByteBuffer -> Struct(schema).put(BYTES, value.array()) - is LocalDate -> Struct(schema).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(value)) - is LocalDateTime -> - Struct(schema).put(LOCAL_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) - is LocalTime -> Struct(schema).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(value)) - is OffsetDateTime -> - Struct(schema).put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) - is ZonedDateTime -> - Struct(schema).put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(value)) - is OffsetTime -> Struct(schema).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(value)) - is IsoDuration -> - Struct(schema) - .put( - DURATION, - Struct(durationSchema) - .put(MONTHS, value.months()) - .put(DAYS, value.days()) - .put(SECONDS, value.seconds()) - .put(NANOS, value.nanoseconds())) - is Point -> - Struct(schema) - .put( - POINT, - Struct(pointSchema) - .put(SR_ID, value.srid()) - .put(X, value.x()) - .put(Y, value.y()) - .also { - it.put(DIMENSION, if (value.z().isNaN()) TWO_D else THREE_D) - if (!value.z().isNaN()) { - it.put(Z, value.z()) - } - }) - is ShortArray -> Struct(schema).put(LONG_LIST, value.map { s -> s.toLong() }.toList()) - is IntArray -> Struct(schema).put(LONG_LIST, value.map { s -> s.toLong() }.toList()) - is LongArray -> Struct(schema).put(LONG_LIST, value.toList()) - is FloatArray -> Struct(schema).put(FLOAT_LIST, value.map { s -> s.toDouble() }.toList()) - is DoubleArray -> Struct(schema).put(FLOAT_LIST, value.toList()) - is BooleanArray -> Struct(schema).put(BOOLEAN_LIST, value.toList()) - is Array<*> -> - when (val componentType = value::class.java.componentType.kotlin) { - Boolean::class -> Struct(schema).put(BOOLEAN_LIST, value.toList()) - Byte::class -> Struct(schema).put(BYTES, (value as Array).toByteArray()) - Short::class -> - Struct(schema) - .put(LONG_LIST, (value as Array).map { s -> s.toLong() }.toList()) - Int::class -> - Struct(schema) - .put(LONG_LIST, (value as Array).map { s -> s.toLong() }.toList()) - Long::class -> Struct(schema).put(LONG_LIST, (value as Array).toList()) - Float::class -> - Struct(schema) - .put(FLOAT_LIST, (value as Array).map { s -> s.toDouble() }.toList()) - Double::class -> Struct(schema).put(FLOAT_LIST, (value as Array).toList()) - String::class -> Struct(schema).put(STRING_LIST, value.toList()) - LocalDate::class -> - Struct(schema) - .put( - LOCAL_DATE_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_DATE.format(s) } - .toList()) - LocalDateTime::class -> - Struct(schema) - .put( - LOCAL_DATE_TIME_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } - .toList()) - LocalTime::class -> - Struct(schema) - .put( - LOCAL_TIME_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_TIME.format(s) } - .toList()) - OffsetDateTime::class -> - Struct(schema) - .put( - ZONED_DATE_TIME_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } - .toList()) - ZonedDateTime::class -> - Struct(schema) - .put( - ZONED_DATE_TIME_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_DATE_TIME.format(s) } - .toList()) - OffsetTime::class -> - Struct(schema) - .put( - OFFSET_TIME_LIST, - (value as Array) - .map { s -> DateTimeFormatter.ISO_TIME.format(s) } - .toList()) - else -> - if (IsoDuration::class.java.isAssignableFrom(componentType.java)) { - Struct(schema) - .put( - DURATION_LIST, - value - .map { s -> s as IsoDuration } - .map { - Struct(durationSchema) - .put(MONTHS, it.months()) - .put(DAYS, it.days()) - .put(SECONDS, it.seconds()) - .put(NANOS, it.nanoseconds()) - } - .toList()) - } else if (Point::class.java.isAssignableFrom(componentType.java)) { - Struct(schema) - .put( - POINT_LIST, - value - .map { s -> s as Point } - .map { s -> - Struct(pointSchema) - .put(SR_ID, s.srid()) - .put(X, s.x()) - .put(Y, s.y()) - .also { - it.put(DIMENSION, if (s.z().isNaN()) TWO_D else THREE_D) - if (!s.z().isNaN()) { - it.put(Z, s.z()) - } - } - } - .toList()) - } else { - throw IllegalArgumentException( - "unsupported array type: array of ${value.javaClass.componentType.name}") - } - } - else -> throw IllegalArgumentException("unsupported property type: ${value?.javaClass?.name}") - } - } - - fun fromConnectValue(value: Struct?): Any? { - return value?.let { - for (f in it.schema().fields()) { - if (it.getWithoutDefault(f.name()) == null) { - continue - } - - return when (f.name()) { - BOOLEAN -> it.get(f) as Boolean? - BOOLEAN_LIST -> it.get(f) as List<*>? - LONG -> it.get(f) as Long? - LONG_LIST -> it.get(f) as List<*>? - FLOAT -> it.get(f) as Double? - FLOAT_LIST -> it.get(f) as List<*>? - STRING -> it.get(f) as String? - STRING_LIST -> it.get(f) as List<*>? - BYTES -> it.get(f) as ByteArray? - LOCAL_DATE -> - (it.get(f) as String?)?.let { s -> - DateTimeFormatter.ISO_DATE.parse(s) { parsed -> LocalDate.from(parsed) } - } - LOCAL_DATE_LIST -> it.get(f) as List<*>? - LOCAL_TIME -> - (it.get(f) as String?)?.let { s -> - DateTimeFormatter.ISO_TIME.parse(s) { parsed -> LocalTime.from(parsed) } - } - LOCAL_TIME_LIST -> it.get(f) as List<*>? - LOCAL_DATE_TIME -> - (it.get(f) as String?)?.let { s -> - DateTimeFormatter.ISO_DATE_TIME.parse(s) { parsed -> LocalDateTime.from(parsed) } - } - LOCAL_DATE_TIME_LIST -> it.get(f) as List<*>? - ZONED_DATE_TIME -> - (it.get(f) as String?)?.let { s -> - DateTimeFormatter.ISO_DATE_TIME.parse(s) { parsed -> - val zoneId = parsed.query(TemporalQueries.zone()) - - if (zoneId is ZoneOffset) { - OffsetDateTime.from(parsed) - } else { - ZonedDateTime.from(parsed) - } - } - } - ZONED_DATE_TIME_LIST -> it.get(f) as List<*>? - OFFSET_TIME -> - (it.get(f) as String?)?.let { s -> - DateTimeFormatter.ISO_TIME.parse(s) { parsed -> OffsetTime.from(parsed) } - } - OFFSET_TIME_LIST -> it.get(f) as List<*>? - DURATION -> - (it.get(f) as Struct?) - ?.let { s -> - Values.isoDuration( - s.getInt64(MONTHS), - s.getInt64(DAYS), - s.getInt64(SECONDS), - s.getInt32(NANOS)) - } - ?.asIsoDuration() - DURATION_LIST -> it.get(f) as List<*>? - POINT -> - (it.get(f) as Struct?) - ?.let { s -> - when (val dimension = s.getInt8(DIMENSION)) { - TWO_D -> Values.point(s.getInt32(SR_ID), s.getFloat64(X), s.getFloat64(Y)) - THREE_D -> - Values.point( - s.getInt32(SR_ID), s.getFloat64(X), s.getFloat64(Y), s.getFloat64(Z)) - else -> - throw IllegalArgumentException("unsupported dimension value ${dimension}") - } - } - ?.asPoint() - POINT_LIST -> it.get(f) as List<*>? - else -> throw IllegalArgumentException("unsupported neo4j type: ${f.name()}") - } - } - - return null - } - } -} - -fun Schema.matches(other: Schema): Boolean { - return this.id() == other.id() || this.shortId() == other.shortId() -} - -object DynamicTypes { - - @Suppress("UNCHECKED_CAST") - fun toConnectValue(schema: Schema, value: Any?): Any? { - if (value == null) { - return null - } - - if (schema == PropertyType.schema) { - return PropertyType.toConnectValue(value) - } - - return when (schema.type()) { - Schema.Type.ARRAY -> - when (value) { - is Collection<*> -> value.map { toConnectValue(schema.valueSchema(), it) } - else -> throw IllegalArgumentException("unsupported array type ${value.javaClass.name}") - } - Schema.Type.MAP -> - when (value) { - is Map<*, *> -> value.mapValues { toConnectValue(schema.valueSchema(), it.value) } - else -> throw IllegalArgumentException("unsupported map type ${value.javaClass.name}") - } - Schema.Type.STRUCT -> - when (value) { - is Node -> - Struct(schema).apply { - put("", value.id()) - put("", value.labels().toList()) - - value - .asMap { it.asObject() } - .forEach { e -> put(e.key, PropertyType.toConnectValue(e.value)) } - } - is Relationship -> - Struct(schema).apply { - put("", value.id()) - put("", value.type()) - put("", value.startNodeId()) - put("", value.endNodeId()) - - value - .asMap { it.asObject() } - .forEach { e -> put(e.key, PropertyType.toConnectValue(e.value)) } - } - is Map<*, *> -> - Struct(schema).apply { - schema.fields().forEach { - put(it.name(), toConnectValue(it.schema(), value[it.name()])) - } - } - is Collection<*> -> - Struct(schema).apply { - schema.fields().forEach { - put( - it.name(), - toConnectValue( - it.schema(), value.elementAt(it.name().substring(1).toInt()))) - } - } - else -> - throw IllegalArgumentException("unsupported struct type ${value.javaClass.name}") - } - else -> value - } - } - - fun fromConnectValue(schema: Schema, value: Any?, skipNullValuesInMaps: Boolean = false): Any? { - if (value == null) { - return null - } - - return when (schema.type()) { - Schema.Type.BOOLEAN -> value as Boolean? - Schema.Type.INT8 -> value as Byte? - Schema.Type.INT16 -> value as Short? - Schema.Type.INT32 -> value as Int? - Schema.Type.INT64 -> value as Long? - Schema.Type.FLOAT32 -> value as Float? - Schema.Type.FLOAT64 -> value as Double? - Schema.Type.BYTES -> - when (value) { - is ByteArray -> value - is ByteBuffer -> value.array() - else -> throw IllegalArgumentException("unsupported bytes type ${value.javaClass.name}") - } - Schema.Type.STRING -> - when (value) { - is Char -> value.toString() - is CharArray -> value.toString() - is CharSequence -> value.toString() - else -> - throw IllegalArgumentException("unsupported string type ${value.javaClass.name}") - } - Schema.Type.STRUCT -> - when { - PropertyType.schema.matches(schema) -> PropertyType.fromConnectValue(value as Struct?) - else -> { - val result = mutableMapOf() - val struct = value as Struct - - for (field in schema.fields()) { - val fieldValue = - fromConnectValue(field.schema(), struct.get(field), skipNullValuesInMaps) - - if (fieldValue != null || !skipNullValuesInMaps) { - result[field.name()] = fieldValue - } - } - - if (result.isNotEmpty() && - result.keys.all { it.startsWith("e") && it.substring(1).toIntOrNull() != null }) { - result - .mapKeys { it.key.substring(1).toInt() } - .entries - .sortedBy { it.key } - .map { it.value } - .toList() - } else { - result - } - } - } - Schema.Type.ARRAY -> { - val result = mutableListOf() - - when { - value.javaClass.isArray -> - for (i in 0 ..< java.lang.reflect.Array.getLength(value)) { - result.add( - fromConnectValue( - schema.valueSchema(), - java.lang.reflect.Array.get(value, i), - skipNullValuesInMaps)) - } - value is Iterable<*> -> - for (element in value) { - result.add(fromConnectValue(schema.valueSchema(), element, skipNullValuesInMaps)) - } - else -> throw IllegalArgumentException("unsupported array type ${value.javaClass.name}") - } - - result.toList() - } - Schema.Type.MAP -> { - val result = mutableMapOf() - val map = value as Map<*, *> - - for (entry in map.entries) { - if (entry.key !is String) { - throw IllegalArgumentException( - "invalid key type (${entry.key?.javaClass?.name} in map value") - } - - result[entry.key as String] = - fromConnectValue(schema.valueSchema(), entry.value, skipNullValuesInMaps) - } - - result - } - else -> - throw IllegalArgumentException( - "unsupported schema ($schema) and value type (${value.javaClass.name})") - } - } - - fun toConnectSchema( - value: Any?, - optional: Boolean = false, - forceMapsAsStruct: Boolean = false, - ): Schema = - when (value) { - null -> PropertyType.schema - is Boolean, - is Float, - is Double, - is Number, - is Char, - is CharArray, - is CharSequence, - is ByteBuffer, - is ByteArray, - is ShortArray, - is IntArray, - is LongArray, - is FloatArray, - is DoubleArray, - is BooleanArray -> PropertyType.schema - is Array<*> -> { - when (val componentType = value::class.java.componentType.kotlin) { - Boolean::class, - Byte::class, - Short::class, - Int::class, - Long::class, - Float::class, - Double::class, - String::class, - LocalDate::class, - LocalDateTime::class, - LocalTime::class, - OffsetDateTime::class, - ZonedDateTime::class, - OffsetTime::class -> PropertyType.schema - else -> - if (IsoDuration::class.java.isAssignableFrom(componentType.java)) { - PropertyType.schema - } else if (Point::class.java.isAssignableFrom(componentType.java)) { - PropertyType.schema - } else { - val first = value.firstOrNull { it.notNullOrEmpty() } - val schema = toConnectSchema(first, optional, forceMapsAsStruct) - SchemaBuilder.array(schema).apply { if (optional) optional() }.build() - } - } - } - is LocalDate, - is LocalDateTime, - is LocalTime, - is OffsetDateTime, - is ZonedDateTime, - is OffsetTime, - is IsoDuration, - is Point -> PropertyType.schema - is Node -> - SchemaBuilder.struct() - .apply { - field("", Schema.INT64_SCHEMA) - field("", SchemaBuilder.array(Schema.STRING_SCHEMA).build()) - - value.keys().forEach { field(it, PropertyType.schema) } - - if (optional) optional() - } - .build() - is Relationship -> - SchemaBuilder.struct() - .apply { - field("", Schema.INT64_SCHEMA) - field("", Schema.STRING_SCHEMA) - field("", Schema.INT64_SCHEMA) - field("", Schema.INT64_SCHEMA) - - value.keys().forEach { field(it, PropertyType.schema) } - - if (optional) optional() - } - .build() - is Collection<*> -> { - val nonEmptyElementTypes = - value - .filter { it.notNullOrEmpty() } - .map { toConnectSchema(it, optional, forceMapsAsStruct) } - - when (nonEmptyElementTypes.toSet().size) { - 0 -> SchemaBuilder.array(PropertyType.schema).apply { if (optional) optional() }.build() - 1 -> - SchemaBuilder.array(nonEmptyElementTypes.first()) - .apply { if (optional) optional() } - .build() - else -> - SchemaBuilder.struct() - .apply { - value.forEachIndexed { i, v -> - this.field("e${i}", toConnectSchema(v, optional, forceMapsAsStruct)) - } - } - .apply { if (optional) optional() } - .build() - } - } - is Map<*, *> -> { - val elementTypes = - value - .mapKeys { - when (val key = it.key) { - is String -> key - else -> - throw IllegalArgumentException( - "unsupported map key type ${key?.javaClass?.name}") - } - } - .filter { e -> e.value.notNullOrEmpty() } - .mapValues { e -> toConnectSchema(e.value, optional, forceMapsAsStruct) } - - val valueSet = elementTypes.values.toSet() - when { - valueSet.isEmpty() -> - SchemaBuilder.struct() - .apply { - value.forEach { - this.field( - it.key as String, - toConnectSchema(it.value, optional, forceMapsAsStruct)) - } - } - .apply { if (optional) optional() } - .build() - valueSet.singleOrNull() != null && !forceMapsAsStruct -> - SchemaBuilder.map(Schema.STRING_SCHEMA, elementTypes.values.first()) - .apply { if (optional) optional() } - .build() - else -> - SchemaBuilder.struct() - .apply { - value.forEach { - this.field( - it.key as String, - toConnectSchema(it.value, optional, forceMapsAsStruct)) - } - } - .apply { if (optional) optional() } - .build() - } - } - else -> throw IllegalArgumentException("unsupported type ${value.javaClass.name}") - } - - private fun Any?.notNullOrEmpty(): Boolean = - when (val value = this) { - null -> false - is Collection<*> -> value.isNotEmpty() && value.any { it.notNullOrEmpty() } - is Array<*> -> value.isNotEmpty() && value.any { it.notNullOrEmpty() } - is Map<*, *> -> value.isNotEmpty() && value.values.any { it.notNullOrEmpty() } - else -> true - } -} diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt index d4ba6dfa0..68ba6cbe7 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt @@ -41,20 +41,7 @@ import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.provider.Arguments import org.junit.jupiter.params.provider.ArgumentsProvider import org.junit.jupiter.params.provider.ArgumentsSource -import org.neo4j.connectors.kafka.data.PropertyType.BOOLEAN -import org.neo4j.connectors.kafka.data.PropertyType.BOOLEAN_LIST -import org.neo4j.connectors.kafka.data.PropertyType.BYTES -import org.neo4j.connectors.kafka.data.PropertyType.DURATION -import org.neo4j.connectors.kafka.data.PropertyType.FLOAT -import org.neo4j.connectors.kafka.data.PropertyType.FLOAT_LIST import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE -import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE_TIME -import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_TIME -import org.neo4j.connectors.kafka.data.PropertyType.LONG_LIST -import org.neo4j.connectors.kafka.data.PropertyType.OFFSET_TIME -import org.neo4j.connectors.kafka.data.PropertyType.POINT -import org.neo4j.connectors.kafka.data.PropertyType.STRING_LIST -import org.neo4j.connectors.kafka.data.PropertyType.ZONED_DATE_TIME import org.neo4j.driver.Value import org.neo4j.driver.Values import org.neo4j.driver.types.Node @@ -62,191 +49,149 @@ import org.neo4j.driver.types.Relationship class DynamicTypesTest { - @Test - fun `should derive schema for simple types correctly`() { - // NULL - DynamicTypes.toConnectSchema(null, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(null, true) shouldBe PropertyType.schema - - // Integer, Long, etc. - listOf(8.toByte(), 8.toShort(), 8.toInt(), 8.toLong()).forEach { number -> - withClue(number) { - DynamicTypes.toConnectSchema(number, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(number, true) shouldBe PropertyType.schema - } - } - - // Float, Double - listOf(8.toFloat(), 8.toDouble()).forEach { number -> - withClue(number) { - DynamicTypes.toConnectSchema(number, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(number, true) shouldBe PropertyType.schema - } - } - - // String - listOf( - "a string", - "a char array".toCharArray(), - StringBuilder("a string builder"), - StringBuffer("a string buffer"), - object : CharSequence { - private val value = "a char sequence" - override val length: Int - get() = value.length - - override fun get(index: Int): Char = value[index] - - override fun subSequence(startIndex: Int, endIndex: Int): CharSequence = - value.subSequence(startIndex, endIndex) - }) - .forEach { string -> - withClue(string) { - DynamicTypes.toConnectSchema(string, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(string, true) shouldBe PropertyType.schema - } - } - - // Byte Array - listOf(ByteArray(0), ByteBuffer.allocate(0)).forEach { bytes -> - withClue(bytes) { - DynamicTypes.toConnectSchema(bytes, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(bytes, true) shouldBe PropertyType.schema - } - } - - // Boolean Array (boolean[]) - listOf(BooleanArray(0), BooleanArray(1) { true }).forEach { array -> - withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema - } - } - - // Array of Boolean (Boolean[]) - listOf(Array(1) { true }).forEach { array -> - withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema - } - } - - // Int Arrays (short[], int[], long[]) - listOf(ShortArray(1), IntArray(1), LongArray(1)).forEach { array -> - withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema - } - } + @ParameterizedTest(name = "{0}") + @ArgumentsSource(PropertyTypedValueProvider::class) + fun `should derive schema for property typed values and convert them back and forth`( + name: String, + value: Any?, + expectedIfDifferent: Any? + ) { + DynamicTypes.toConnectSchema(value, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(value, true) shouldBe PropertyType.schema - // Array of Integer (Short[], Integer[], Long[]) - listOf(Array(1) { i -> i }, Array(1) { i -> i.toShort() }, Array(1) { i -> i.toLong() }) - .forEach { array -> - withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema - } - } + val converted = DynamicTypes.toConnectValue(PropertyType.schema, value) + converted shouldBe PropertyType.toConnectValue(value) - // Float Arrays (float[], double[]) - listOf(FloatArray(1), DoubleArray(1)).forEach { array -> - withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema - } - } + val reverted = DynamicTypes.fromConnectValue(PropertyType.schema, converted) + reverted shouldBe (expectedIfDifferent ?: value) + } - // Float Arrays (Float[], Double[]) - listOf(Array(1) { i -> i.toFloat() }, Array(1) { i -> i.toDouble() }).forEach { array -> - withClue(array) { - DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema - } + object PropertyTypedValueProvider : ArgumentsProvider { + override fun provideArguments(ctx: ExtensionContext?): Stream { + return Stream.of( + Arguments.of("null", null, null), + Arguments.of("byte", 8.toByte(), 8L), + Arguments.of("short", 8.toShort(), 8L), + Arguments.of("int", 8, 8L), + Arguments.of("long", 8.toLong(), null), + Arguments.of("float", 8.toFloat(), 8.toDouble()), + Arguments.of("double", 8.toDouble(), null), + Arguments.of("string", "a string", null), + Arguments.of("char array", "a char array".toCharArray(), "a char array"), + Arguments.of("string builder", StringBuilder("a string builder"), "a string builder"), + Arguments.of("string buffer", StringBuilder("a string buffer"), "a string buffer"), + Arguments.of( + "char sequence", + object : CharSequence { + private val value = "a char sequence" + override val length: Int + get() = value.length + + override fun get(index: Int): Char = value[index] + + override fun subSequence(startIndex: Int, endIndex: Int): CharSequence = + value.subSequence(startIndex, endIndex) + }, + "a char sequence"), + Arguments.of("local date", LocalDate.of(1999, 12, 31), null), + Arguments.of("local time", LocalTime.of(23, 59, 59), null), + Arguments.of("local date time", LocalDateTime.of(1999, 12, 31, 23, 59, 59), null), + Arguments.of("offset time", OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), null), + Arguments.of( + "offset date time", + OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), + null), + Arguments.of( + "zoned date time", + ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), + null), + Arguments.of("duration", Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), null), + Arguments.of("point (2d)", Values.point(4326, 1.0, 2.0).asPoint(), null), + Arguments.of("point (3d)", Values.point(4326, 1.0, 2.0, 3.0).asPoint(), null), + Arguments.of("byte array", ByteArray(0), null), + Arguments.of("byte buffer", ByteBuffer.allocate(0), ByteArray(0)), + Arguments.of("array (byte)", Array(1) { 1.toByte() }, null), + Arguments.of("bool array (empty)", BooleanArray(0), null), + Arguments.of("bool array", BooleanArray(1) { true }, null), + Arguments.of("array (bool)", Array(1) { true }, null), + Arguments.of("list (bool)", listOf(true), null), + Arguments.of("short array (empty)", ShortArray(0), null), + Arguments.of("short array", ShortArray(1) { 1.toShort() }, listOf(1L)), + Arguments.of("array (short)", Array(1) { 1.toShort() }, listOf(1L)), + Arguments.of("list (short)", listOf(1.toShort()), listOf(1L)), + Arguments.of("int array (empty)", IntArray(0), null), + Arguments.of("int array", IntArray(1) { 1 }, listOf(1L)), + Arguments.of("array (int)", Array(1) { 1 }, listOf(1L)), + Arguments.of("list (int)", listOf(1), listOf(1L)), + Arguments.of("long array (empty)", LongArray(0), null), + Arguments.of("long array", LongArray(1) { 1.toLong() }, null), + Arguments.of("array (long)", Array(1) { 1.toLong() }, null), + Arguments.of("list (long)", listOf(1L), null), + Arguments.of("float array (empty)", FloatArray(0), null), + Arguments.of("float array", FloatArray(1) { 1.toFloat() }, listOf(1.toDouble())), + Arguments.of("array (float)", Array(1) { 1.toFloat() }, null), + Arguments.of("list (float)", listOf(1.toFloat()), null), + Arguments.of("double array (empty)", DoubleArray(0), null), + Arguments.of("double array", DoubleArray(1) { 1.toDouble() }, null), + Arguments.of("array (double)", Array(1) { 1.toDouble() }, null), + Arguments.of("list (double)", listOf(1.toDouble()), null), + Arguments.of("array (string)", Array(1) { "a" }, null), + Arguments.of("list (string)", listOf("a"), null), + Arguments.of("array (local date)", Array(1) { LocalDate.of(1999, 12, 31) }, null), + Arguments.of("list (local date)", listOf(LocalDate.of(1999, 12, 31)), null), + Arguments.of("array (local time)", Array(1) { LocalTime.of(23, 59, 59) }, null), + Arguments.of("list (local time)", listOf(LocalTime.of(23, 59, 59)), null), + Arguments.of( + "array (local date time)", + Array(1) { LocalDateTime.of(1999, 12, 31, 23, 59, 59) }, + null), + Arguments.of( + "list (local date time)", listOf(LocalDateTime.of(1999, 12, 31, 23, 59, 59)), null), + Arguments.of( + "array (offset time)", + Array(1) { OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC) }, + null), + Arguments.of( + "list (offset time)", listOf(OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC)), null), + Arguments.of( + "array (offset date time)", + Array(1) { OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC) }, + null), + Arguments.of( + "list (offset date time)", + listOf(OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC)), + null), + Arguments.of( + "array (zoned date time)", + Array(1) { + ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")) + }, + null), + Arguments.of( + "list (zoned date time)", + listOf(ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London"))), + null), + Arguments.of( + "array (duration)", + Array(1) { Values.isoDuration(12, 12, 59, 1230).asIsoDuration() }, + null), + Arguments.of( + "list (duration)", + listOf(Values.isoDuration(12, 12, 59, 1230).asIsoDuration()), + null), + Arguments.of( + "array (point (2d))", Array(1) { Values.point(4326, 1.0, 2.0).asPoint() }, null), + Arguments.of("list (point (2d))", listOf(Values.point(4326, 1.0, 2.0).asPoint()), null), + Arguments.of( + "array (point (3d))", Array(1) { Values.point(4326, 1.0, 2.0, 3.0).asPoint() }, null), + Arguments.of( + "list (point (3d))", listOf(Values.point(4326, 1.0, 2.0, 3.0).asPoint()), null), + ) } + } - // String Array - DynamicTypes.toConnectSchema(Array(1) { "a" }, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(Array(1) { "a" }, true) shouldBe PropertyType.schema - - // Temporal Types - DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), true) shouldBe PropertyType.schema - - DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), optional = false) shouldBe - PropertyType.schema - DynamicTypes.toConnectSchema(LocalDate.of(1999, 12, 31), optional = true) shouldBe - PropertyType.schema - - DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), true) shouldBe PropertyType.schema - - DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), optional = false) shouldBe - PropertyType.schema - DynamicTypes.toConnectSchema(LocalTime.of(23, 59, 59), optional = true) shouldBe - PropertyType.schema - - DynamicTypes.toConnectSchema(LocalDateTime.of(1999, 12, 31, 23, 59, 59), false) shouldBe - PropertyType.schema - DynamicTypes.toConnectSchema(LocalDateTime.of(1999, 12, 31, 23, 59, 59), true) shouldBe - PropertyType.schema - - DynamicTypes.toConnectSchema( - LocalDateTime.of(1999, 12, 31, 23, 59, 59), optional = false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema( - LocalDateTime.of(1999, 12, 31, 23, 59, 59), optional = true) shouldBe PropertyType.schema - - DynamicTypes.toConnectSchema(OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), false) shouldBe - PropertyType.schema - DynamicTypes.toConnectSchema(OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), true) shouldBe - PropertyType.schema - - DynamicTypes.toConnectSchema( - OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), optional = false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema( - OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC), optional = true) shouldBe PropertyType.schema - - DynamicTypes.toConnectSchema( - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), false) shouldBe - PropertyType.schema - DynamicTypes.toConnectSchema( - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), true) shouldBe - PropertyType.schema - - DynamicTypes.toConnectSchema( - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), optional = false) shouldBe - PropertyType.schema - DynamicTypes.toConnectSchema( - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC), true) shouldBe - PropertyType.schema - - DynamicTypes.toConnectSchema( - ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), false) shouldBe - PropertyType.schema - DynamicTypes.toConnectSchema( - ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), true) shouldBe - PropertyType.schema - - DynamicTypes.toConnectSchema( - ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), - optional = false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema( - ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London")), - optional = true) shouldBe PropertyType.schema - - DynamicTypes.toConnectSchema( - Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema( - Values.isoDuration(12, 12, 59, 1230).asIsoDuration(), true) shouldBe PropertyType.schema - - // Point - listOf(Values.point(4326, 1.0, 2.0).asPoint(), Values.point(4326, 1.0, 2.0, 3.0).asPoint()) - .forEach { point -> - withClue(point) { - DynamicTypes.toConnectSchema(point, false) shouldBe PropertyType.schema - DynamicTypes.toConnectSchema(point, true) shouldBe PropertyType.schema - } - } - + @Test + fun `should derive schema for entity types correctly`() { // Node DynamicTypes.toConnectSchema(TestNode(0, emptyList(), emptyMap()), false) shouldBe SchemaBuilder.struct() @@ -294,7 +239,7 @@ class DynamicTypesTest { } @Test - fun `empty collections or arrays should map to an array schema`() { + fun `empty collections or arrays should map to an array of property type`() { listOf(listOf(), setOf(), arrayOf()).forEach { collection -> withClue(collection) { DynamicTypes.toConnectSchema(collection, false) shouldBe @@ -305,27 +250,37 @@ class DynamicTypesTest { } } - @Test - fun `collections with elements of single type should map to an array schema`() { - listOf(listOf(1, 2, 3), listOf("a", "b", "c"), setOf(true)).forEach { collection -> - withClue(collection) { - DynamicTypes.toConnectSchema(collection, false) shouldBe - SchemaBuilder.array(PropertyType.schema).build() - DynamicTypes.toConnectSchema(collection, true) shouldBe - SchemaBuilder.array(PropertyType.schema).optional().build() - } - } - } - - @Test - fun `collections with elements of different types should map to a struct schema`() { - DynamicTypes.toConnectSchema(listOf(1, true, "a", 5.toFloat()), false) shouldBe + @ParameterizedTest(name = "{0}") + @ArgumentsSource(PropertyTypedCollectionProvider::class) + fun `collections with elements of property types should map to an array schema`( + name: String, + value: Any? + ) { + DynamicTypes.toConnectSchema(value, false) shouldBe SchemaBuilder.array(PropertyType.schema).build() - - DynamicTypes.toConnectSchema(listOf(1, true, "a", 5.toFloat()), true) shouldBe + DynamicTypes.toConnectSchema(value, true) shouldBe SchemaBuilder.array(PropertyType.schema).optional().build() } + object PropertyTypedCollectionProvider : ArgumentsProvider { + override fun provideArguments(ctx: ExtensionContext?): Stream { + return Stream.of( + Arguments.of( + "list of mixed simple types", + listOf(1, true, "a", 5.toFloat(), LocalDate.of(1999, 1, 1))), + Arguments.of( + "list of mixed types", + listOf( + 1, + true, + "a", + 5.toFloat(), + LocalDate.of(1999, 1, 1), + IntArray(1) { 1 }, + Array(1) { LocalTime.of(23, 59, 59) }))) + } + } + @Test fun `empty maps should map to an empty struct schema`() { DynamicTypes.toConnectSchema(mapOf(), false) shouldBe @@ -341,248 +296,45 @@ class DynamicTypesTest { } shouldHaveMessage ("unsupported map key type java.lang.Integer") } - @Test - fun `maps with simple typed values should map to a map schema`() { - listOf( - mapOf("a" to 1, "b" to 2, "c" to 3) to PropertyType.schema, - mapOf("a" to "a", "b" to "b", "c" to "c") to PropertyType.schema, - mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong()) to PropertyType.schema) - .forEach { (map, valueSchema) -> - withClue("not optional: $map") { - DynamicTypes.toConnectSchema(map, false) shouldBe - SchemaBuilder.map(Schema.STRING_SCHEMA, valueSchema).build() - } - } - - listOf( - mapOf("a" to 1, "b" to 2, "c" to 3) to PropertyType.schema, - mapOf("a" to "a", "b" to "b", "c" to "c") to PropertyType.schema, - mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong()) to PropertyType.schema) - .forEach { (map, valueSchema) -> - withClue("optional: $map") { - DynamicTypes.toConnectSchema(map, true) shouldBe - SchemaBuilder.map(Schema.STRING_SCHEMA, valueSchema).optional().build() - } - } - } - - @Test - fun `maps with values of different types should map to a map of struct schema`() { - DynamicTypes.toConnectSchema( - mapOf("a" to 1, "b" to true, "c" to "string", "d" to 5.toFloat()), false) shouldBe + @ParameterizedTest(name = "{0}") + @ArgumentsSource(PropertyTypedMapProvider::class) + fun `maps with property typed values should map to a map schema`(name: String, value: Any?) { + DynamicTypes.toConnectSchema(value, false) shouldBe SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build() - - DynamicTypes.toConnectSchema( - mapOf("a" to 1, "b" to true, "c" to "string", "d" to 5.toFloat()), true) shouldBe + DynamicTypes.toConnectSchema(value, true) shouldBe SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).optional().build() } - @Test - fun `unsupported types should throw`() { - data class Test(val a: String) - - listOf(object {}, java.sql.Date(0), object : Entity(emptyMap()) {}, Test("a string")).forEach { - value -> - shouldThrow { DynamicTypes.toConnectSchema(value, false) } - } - } - - @Test - fun `simple types should be converted to themselves and should be converted back`() { - listOf( - Triple(true, Struct(PropertyType.schema).put(BOOLEAN, true), true), - Triple(false, Struct(PropertyType.schema).put(BOOLEAN, false), false), - Triple(1.toShort(), PropertyType.toConnectValue(1.toLong()), 1L), - Triple(2, PropertyType.toConnectValue(2.toLong()), 2L), - Triple(3.toLong(), PropertyType.toConnectValue(3.toLong()), 3L), - Triple(4.toFloat(), Struct(PropertyType.schema).put(FLOAT, 4.toDouble()), 4.toDouble()), - Triple( - 5.toDouble(), Struct(PropertyType.schema).put(FLOAT, 5.toDouble()), 5.toDouble()), - Triple('c', PropertyType.toConnectValue("c"), "c"), - Triple("string", PropertyType.toConnectValue("string"), "string"), - Triple("string".toCharArray(), PropertyType.toConnectValue("string"), "string"), - Triple( - "string".toByteArray(), - Struct(PropertyType.schema).put(BYTES, "string".toByteArray()), - "string".toByteArray())) - .forEach { (value, expected, expectedValue) -> - withClue(value) { - val schema = DynamicTypes.toConnectSchema(value, false) - val converted = DynamicTypes.toConnectValue(schema, value) - val reverted = DynamicTypes.fromConnectValue(schema, converted) - - converted shouldBe expected - reverted shouldBe expectedValue - } - } - } - - @ParameterizedTest - @ArgumentsSource(TemporalTypes::class) - fun `temporal types should be returned as structs and should be converted back`( - value: Any, - expected: Any - ) { - val schema = DynamicTypes.toConnectSchema(value, false) - val converted = DynamicTypes.toConnectValue(schema, value) - - converted shouldBe expected - - val reverted = DynamicTypes.fromConnectValue(schema, converted) - reverted shouldBe value - } - - object TemporalTypes : ArgumentsProvider { - override fun provideArguments(context: ExtensionContext?): Stream { + object PropertyTypedMapProvider : ArgumentsProvider { + override fun provideArguments(ctx: ExtensionContext?): Stream { return Stream.of( - LocalDate.of(1999, 12, 31).let { - Arguments.of( - it, - Struct(PropertyType.schema).put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(it))) - }, - LocalTime.of(23, 59, 59, 9999).let { - Arguments.of( - it, - Struct(PropertyType.schema).put(LOCAL_TIME, DateTimeFormatter.ISO_TIME.format(it))) - }, - LocalDateTime.of(1999, 12, 31, 23, 59, 59, 9999).let { - Arguments.of( - it, - Struct(PropertyType.schema) - .put(LOCAL_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) - }, - OffsetTime.of(23, 59, 59, 9999, ZoneOffset.UTC).let { - Arguments.of( - it, - Struct(PropertyType.schema).put(OFFSET_TIME, DateTimeFormatter.ISO_TIME.format(it))) - }, - OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 9999, ZoneOffset.ofHours(1)).let { - Arguments.of( - it, - Struct(PropertyType.schema) - .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) - }, - ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 9999, ZoneId.of("Europe/Istanbul")).let { - Arguments.of( - it, - Struct(PropertyType.schema) - .put(ZONED_DATE_TIME, DateTimeFormatter.ISO_DATE_TIME.format(it))) - }) + Arguments.of("string to int", mapOf("a" to 1, "b" to 2, "c" to 3)), + Arguments.of("string to string", mapOf("a" to "a", "b" to "b", "c" to "c")), + Arguments.of( + "string to numeric", + mapOf("a" to 1, "b" to 2.toShort(), "c" to 3.toLong(), "d" to 4.toFloat())), + Arguments.of( + "string to mixed simple type", + mapOf("a" to 1, "b" to true, "c" to "string", "d" to 4.toFloat())), + Arguments.of( + "string to mixed", + mapOf( + "a" to 1, + "b" to true, + "c" to "string", + "d" to 4.toFloat(), + "e" to Array(1) { LocalDate.of(1999, 1, 1) })), + ) } } @Test - fun `duration types should be returned as structs and should be converted back`() { - listOf( - Values.isoDuration(5, 2, 0, 9999).asIsoDuration() to - Struct(PropertyType.schema) - .put( - DURATION, - Struct(PropertyType.durationSchema) - .put("months", 5L) - .put("days", 2L) - .put("seconds", 0L) - .put("nanoseconds", 9999))) - .forEach { (value, expected) -> - withClue(value) { - val schema = DynamicTypes.toConnectSchema(value, false) - val converted = DynamicTypes.toConnectValue(schema, value) - - converted shouldBe expected - - val reverted = DynamicTypes.fromConnectValue(schema, converted) - reverted shouldBe value - } - } - } - - @Test - fun `arrays should be returned as list of simple types and should be converted back`() { - fun primitiveToArray(value: Any): Any = - when (value) { - is BooleanArray -> value.toList() - is ByteArray -> value.toList() - is CharArray -> value.toList() - is DoubleArray -> value.toList() - is FloatArray -> value.toList() - is IntArray -> value.toList() - is LongArray -> value.toList() - is ShortArray -> value.toList() - else -> value - } - - listOf( - ShortArray(1) { 1 } to - Struct(PropertyType.schema).put(LONG_LIST, LongArray(1) { 1.toLong() }.toList()), - IntArray(1) { 1 } to - Struct(PropertyType.schema).put(LONG_LIST, LongArray(1) { 1.toLong() }.toList()), - LongArray(1) { 1 } to - Struct(PropertyType.schema).put(LONG_LIST, LongArray(1) { 1 }.toList()), - FloatArray(1) { 1F } to - Struct(PropertyType.schema).put(FLOAT_LIST, DoubleArray(1) { 1.0 }.toList()), - DoubleArray(1) { 1.0 } to - Struct(PropertyType.schema).put(FLOAT_LIST, DoubleArray(1) { 1.0 }.toList()), - BooleanArray(1) { true } to - Struct(PropertyType.schema).put(BOOLEAN_LIST, BooleanArray(1) { true }.toList()), - Array(1) { 1 } to Struct(PropertyType.schema).put(LONG_LIST, Array(1) { 1L }.toList()), - Array(1) { 1.toShort() } to - Struct(PropertyType.schema).put(LONG_LIST, Array(1) { 1L }.toList()), - Array(1) { "string" } to - Struct(PropertyType.schema).put(STRING_LIST, Array(1) { "string" }.toList())) - .forEach { (value, expected) -> - withClue(value) { - val schema = DynamicTypes.toConnectSchema(value, false) - val converted = DynamicTypes.toConnectValue(schema, value) - - converted shouldBe expected - - val reverted = DynamicTypes.fromConnectValue(schema, converted) - reverted shouldBe primitiveToArray(value) - } - } - } - - @Test - fun `collections should be returned as arrays of simple types and should be converted back`() { - fun primitiveToArray(value: Any): Any = - when (value) { - is BooleanArray -> value.toList() - is ByteArray -> value.toList() - is CharArray -> value.toList() - is DoubleArray -> value.toList() - is FloatArray -> value.toList() - is IntArray -> value.toList() - is LongArray -> value.toList() - is ShortArray -> value.toList() - else -> value - } - - listOf( - listOf(1, 2, 3) to - listOf( - PropertyType.toConnectValue(1L), - PropertyType.toConnectValue(2L), - PropertyType.toConnectValue(3L)), - listOf("a", "b", "c") to - listOf( - PropertyType.toConnectValue("a"), - PropertyType.toConnectValue("b"), - PropertyType.toConnectValue("c")), - setOf(true, false) to - listOf( - Struct(PropertyType.schema).put(BOOLEAN, true), - Struct(PropertyType.schema).put(BOOLEAN, false))) - .forEach { (value, expected) -> - withClue(value) { - val schema = DynamicTypes.toConnectSchema(value, false) - val converted = DynamicTypes.toConnectValue(schema, value) - - converted shouldBe expected + fun `unsupported types should throw`() { + data class Test(val a: String) - val reverted = DynamicTypes.fromConnectValue(schema, converted) - reverted shouldBe primitiveToArray(value) - } - } + listOf(object {}, java.sql.Date(0), object : Entity(emptyMap()) {}, Test("a string")).forEach { + shouldThrow { DynamicTypes.toConnectSchema(it, false) } + } } @Test @@ -611,49 +363,6 @@ class DynamicTypesTest { } } - @Test - fun `2d points should be returned as structs and should be converted back`() { - // listOf(, Values.point(4326, 1.0, 2.0, 3.0).asPoint()) - val point = Values.point(4326, 1.0, 2.0).asPoint() - val schema = DynamicTypes.toConnectSchema(point, false) - val converted = DynamicTypes.toConnectValue(schema, point) - - converted shouldBe - Struct(PropertyType.schema) - .put( - POINT, - Struct(PropertyType.pointSchema) - .put("dimension", 2.toByte()) - .put("srid", point.srid()) - .put("x", point.x()) - .put("y", point.y()) - .put("z", null)) - - val reverted = DynamicTypes.fromConnectValue(schema, converted) - reverted shouldBe point - } - - @Test - fun `3d points should be returned as structs and should be converted back`() { - val point = Values.point(4326, 1.0, 2.0, 3.0).asPoint() - val schema = DynamicTypes.toConnectSchema(point, false) - val converted = DynamicTypes.toConnectValue(schema, point) - - converted shouldBe - Struct(PropertyType.schema) - .put( - POINT, - Struct(PropertyType.pointSchema) - .put("dimension", 3.toByte()) - .put("srid", point.srid()) - .put("x", point.x()) - .put("y", point.y()) - .put("z", point.z())) - - val reverted = DynamicTypes.fromConnectValue(schema, converted) - reverted shouldBe point - } - @Test fun `nodes should be returned as structs and should be converted back as maps`() { val node = @@ -713,7 +422,7 @@ class DynamicTypesTest { } @Test - fun `maps with values of different simple types should be returned as map of structs and should be converted back`() { + fun `maps with values of different simple types should be returned as map of property types and should be converted back`() { val map = mapOf( "name" to "john", @@ -731,7 +440,7 @@ class DynamicTypesTest { "dob" to Struct(PropertyType.schema) .put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(LocalDate.of(1999, 12, 31))), - "employed" to Struct(PropertyType.schema).put(BOOLEAN, true), + "employed" to PropertyType.toConnectValue(true), "nullable" to null) val reverted = DynamicTypes.fromConnectValue(schema, converted) @@ -739,7 +448,7 @@ class DynamicTypesTest { } @Test - fun `collections with elements of different types should be returned as struct and should be converted back`() { + fun `collections with elements of different types should be returned as list of property types and should be converted back`() { val coll = listOf("john", 21, LocalDate.of(1999, 12, 31), true, null) val schema = DynamicTypes.toConnectSchema(coll, false) val converted = DynamicTypes.toConnectValue(schema, coll) @@ -750,7 +459,7 @@ class DynamicTypesTest { PropertyType.toConnectValue(21L), Struct(PropertyType.schema) .put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(LocalDate.of(1999, 12, 31))), - Struct(PropertyType.schema).put(BOOLEAN, true), + PropertyType.toConnectValue(true), null) val reverted = DynamicTypes.fromConnectValue(schema, converted) @@ -771,7 +480,7 @@ class DynamicTypesTest { .put("id", 1) .put("name", "john") .put("last_name", "doe") - .put("dob", DynamicTypes.toConnectValue(PropertyType.schema, LocalDate.of(2000, 1, 1))) + .put("dob", PropertyType.toConnectValue(LocalDate.of(2000, 1, 1))) DynamicTypes.fromConnectValue(schema, struct) shouldBe mapOf("id" to 1, "name" to "john", "last_name" to "doe", "dob" to LocalDate.of(2000, 1, 1)) @@ -800,18 +509,13 @@ class DynamicTypesTest { .put("id", PropertyType.toConnectValue(1L)) .put("name", PropertyType.toConnectValue("john")) .put("last_name", PropertyType.toConnectValue("doe")) - .put( - "dob", - Struct(PropertyType.schema) - .put(LOCAL_DATE, DateTimeFormatter.ISO_DATE.format(LocalDate.of(2000, 1, 1)))) + .put("dob", PropertyType.toConnectValue(LocalDate.of(2000, 1, 1))) .put( "address", Struct(addressSchema) .put("city", PropertyType.toConnectValue("london")) .put("country", PropertyType.toConnectValue("uk"))) - .put( - "years_of_interest", - Struct(PropertyType.schema).put(LONG_LIST, listOf(2000L, 2005L, 2017L))) + .put("years_of_interest", PropertyType.toConnectValue(listOf(2000L, 2005L, 2017L))) .put( "events_of_interest", mapOf("2000" to "birth", "2005" to "school", "2017" to "college")) diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/PropertyTypeTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/PropertyTypeTest.kt new file mode 100644 index 000000000..d9dec3327 --- /dev/null +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/PropertyTypeTest.kt @@ -0,0 +1,499 @@ +/* + * Copyright (c) "Neo4j" + * Neo4j Sweden AB [https://neo4j.com] + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.neo4j.connectors.kafka.data + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.matchers.shouldBe +import io.kotest.matchers.throwable.shouldHaveMessage +import java.nio.ByteBuffer +import java.time.Instant +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.time.OffsetDateTime +import java.time.OffsetTime +import java.time.ZoneId +import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.util.stream.Stream +import org.apache.kafka.connect.data.Struct +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.extension.ExtensionContext +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.ArgumentsProvider +import org.junit.jupiter.params.provider.ArgumentsSource +import org.neo4j.connectors.kafka.data.PropertyType.BOOLEAN +import org.neo4j.connectors.kafka.data.PropertyType.BYTES +import org.neo4j.connectors.kafka.data.PropertyType.DAYS +import org.neo4j.connectors.kafka.data.PropertyType.DIMENSION +import org.neo4j.connectors.kafka.data.PropertyType.DURATION +import org.neo4j.connectors.kafka.data.PropertyType.DURATION_LIST +import org.neo4j.connectors.kafka.data.PropertyType.FLOAT +import org.neo4j.connectors.kafka.data.PropertyType.FLOAT_LIST +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE_LIST +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE_TIME +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE_TIME_LIST +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_TIME +import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_TIME_LIST +import org.neo4j.connectors.kafka.data.PropertyType.LONG +import org.neo4j.connectors.kafka.data.PropertyType.LONG_LIST +import org.neo4j.connectors.kafka.data.PropertyType.MONTHS +import org.neo4j.connectors.kafka.data.PropertyType.NANOS +import org.neo4j.connectors.kafka.data.PropertyType.OFFSET_TIME_LIST +import org.neo4j.connectors.kafka.data.PropertyType.POINT +import org.neo4j.connectors.kafka.data.PropertyType.POINT_LIST +import org.neo4j.connectors.kafka.data.PropertyType.SECONDS +import org.neo4j.connectors.kafka.data.PropertyType.SR_ID +import org.neo4j.connectors.kafka.data.PropertyType.STRING +import org.neo4j.connectors.kafka.data.PropertyType.STRING_LIST +import org.neo4j.connectors.kafka.data.PropertyType.THREE_D +import org.neo4j.connectors.kafka.data.PropertyType.TWO_D +import org.neo4j.connectors.kafka.data.PropertyType.X +import org.neo4j.connectors.kafka.data.PropertyType.Y +import org.neo4j.connectors.kafka.data.PropertyType.Z +import org.neo4j.connectors.kafka.data.PropertyType.ZONED_DATE_TIME +import org.neo4j.connectors.kafka.data.PropertyType.ZONED_DATE_TIME_LIST +import org.neo4j.driver.Values + +class PropertyTypeTest { + + @ParameterizedTest(name = "{0}") + @ArgumentsSource(PropertyTypedValues::class) + fun `simple types should be converted back and forth`( + name: String, + value: Any?, + expectedConverted: Any?, + expectedReverted: Any? + ) { + val converted = PropertyType.toConnectValue(value) + converted shouldBe expectedConverted + + val reverted = PropertyType.fromConnectValue(converted) + reverted shouldBe expectedReverted + } + + object PropertyTypedValues : ArgumentsProvider { + override fun provideArguments(p0: ExtensionContext?): Stream { + return Stream.of( + Arguments.of("null", null, null, null), + Arguments.of("boolean", true, Struct(PropertyType.schema).put(BOOLEAN, true), true), + Arguments.of("byte", 1.toByte(), Struct(PropertyType.schema).put(LONG, 1L), 1L), + Arguments.of("short", 1.toShort(), Struct(PropertyType.schema).put(LONG, 1L), 1L), + Arguments.of("int", 1, Struct(PropertyType.schema).put(LONG, 1L), 1L), + Arguments.of("long", 1L, Struct(PropertyType.schema).put(LONG, 1L), 1L), + Arguments.of( + "float", + 1.toFloat(), + Struct(PropertyType.schema).put(FLOAT, 1.toDouble()), + 1.toDouble()), + Arguments.of( + "double", + 1.toDouble(), + Struct(PropertyType.schema).put(FLOAT, 1.toDouble()), + 1.toDouble()), + Arguments.of("char", 'c', Struct(PropertyType.schema).put(STRING, "c"), "c"), + Arguments.of( + "string", "string", Struct(PropertyType.schema).put(STRING, "string"), "string"), + Arguments.of( + "char array", + "string".toCharArray(), + Struct(PropertyType.schema).put(STRING, "string"), + "string"), + Arguments.of( + "string builder", + StringBuilder().append("string"), + Struct(PropertyType.schema).put(STRING, "string"), + "string"), + Arguments.of( + "string buffer", + StringBuffer().append("string"), + Struct(PropertyType.schema).put(STRING, "string"), + "string"), + Arguments.of( + "local date", + LocalDate.of(1999, 1, 1), + Struct(PropertyType.schema).put(LOCAL_DATE, "1999-01-01"), + LocalDate.of(1999, 1, 1)), + Arguments.of( + "local time", + LocalTime.of(23, 59, 59, 999999999), + Struct(PropertyType.schema).put(LOCAL_TIME, "23:59:59.999999999"), + LocalTime.of(23, 59, 59, 999999999)), + Arguments.of( + "local date time", + LocalDateTime.of(1999, 1, 1, 23, 59, 59, 999999999), + Struct(PropertyType.schema).put(LOCAL_DATE_TIME, "1999-01-01T23:59:59.999999999"), + LocalDateTime.of(1999, 1, 1, 23, 59, 59, 999999999)), + Arguments.of( + "offset date time", + OffsetDateTime.of(1999, 1, 1, 23, 59, 59, 999999999, ZoneOffset.ofHours(1)), + Struct(PropertyType.schema) + .put(ZONED_DATE_TIME, "1999-01-01T23:59:59.999999999+01:00"), + OffsetDateTime.of(1999, 1, 1, 23, 59, 59, 999999999, ZoneOffset.ofHours(1))), + Arguments.of( + "zoned date time", + ZonedDateTime.of(1999, 1, 1, 23, 59, 59, 999999999, ZoneId.of("Europe/Istanbul")), + Struct(PropertyType.schema) + .put(ZONED_DATE_TIME, "1999-01-01T23:59:59.999999999+02:00[Europe/Istanbul]"), + ZonedDateTime.of(1999, 1, 1, 23, 59, 59, 999999999, ZoneId.of("Europe/Istanbul"))), + Arguments.of( + "duration", + Values.isoDuration(5, 20, 10000, 999999999).asIsoDuration(), + Struct(PropertyType.schema) + .put( + DURATION, + Struct(PropertyType.durationSchema) + .put(MONTHS, 5L) + .put(DAYS, 20L) + .put(SECONDS, 10000L) + .put(NANOS, 999999999)), + Values.isoDuration(5, 20, 10000, 999999999).asIsoDuration()), + Arguments.of( + "point (2d)", + Values.point(4326, 1.0, 2.0).asPoint(), + Struct(PropertyType.schema) + .put( + POINT, + Struct(PropertyType.pointSchema) + .put(SR_ID, 4326) + .put(X, 1.0) + .put(Y, 2.0) + .put(DIMENSION, TWO_D)), + Values.point(4326, 1.0, 2.0).asPoint()), + Arguments.of( + "point (3d)", + Values.point(4326, 1.0, 2.0, 3.0).asPoint(), + Struct(PropertyType.schema) + .put( + POINT, + Struct(PropertyType.pointSchema) + .put(SR_ID, 4326) + .put(X, 1.0) + .put(Y, 2.0) + .put(Z, 3.0) + .put(DIMENSION, THREE_D)), + Values.point(4326, 1.0, 2.0, 3.0).asPoint()), + Arguments.of( + "byte array", + "a string".toByteArray(), + Struct(PropertyType.schema).put(BYTES, "a string".toByteArray()), + "a string".toByteArray()), + Arguments.of( + "byte buffer", + ByteBuffer.allocate(1).put(1), + Struct(PropertyType.schema).put(BYTES, ByteArray(1) { 1 }), + ByteArray(1) { 1 }), + Arguments.of( + "array (byte)", + Array(1) { 1.toByte() }, + Struct(PropertyType.schema).put(BYTES, ByteArray(1) { 1 }), + ByteArray(1) { 1 }), + Arguments.of( + "list (byte)", + listOf(1.toByte(), 1.toByte()), + Struct(PropertyType.schema).put(BYTES, ByteArray(2) { 1 }), + ByteArray(2) { 1.toByte() }), + Arguments.of( + "short array (empty)", + ShortArray(0), + Struct(PropertyType.schema).put(LONG_LIST, emptyList()), + emptyList()), + Arguments.of( + "short array", + ShortArray(1) { 1.toShort() }, + Struct(PropertyType.schema).put(LONG_LIST, listOf(1L)), + listOf(1L)), + Arguments.of( + "array (short)", + Array(1) { 1.toShort() }, + Struct(PropertyType.schema).put(LONG_LIST, listOf(1L)), + listOf(1L)), + Arguments.of( + "list (short)", + listOf(1.toShort(), 2.toShort()), + Struct(PropertyType.schema).put(LONG_LIST, listOf(1L, 2L)), + listOf(1L, 2L)), + Arguments.of( + "int array (empty)", + IntArray(0), + Struct(PropertyType.schema).put(LONG_LIST, emptyList()), + emptyList()), + Arguments.of( + "int array", + IntArray(1) { 1 }, + Struct(PropertyType.schema).put(LONG_LIST, listOf(1L)), + listOf(1L)), + Arguments.of( + "array (int)", + Array(1) { 1 }, + Struct(PropertyType.schema).put(LONG_LIST, listOf(1L)), + listOf(1L)), + Arguments.of( + "list (int)", + listOf(1, 2), + Struct(PropertyType.schema).put(LONG_LIST, listOf(1L, 2L)), + listOf(1L, 2L)), + Arguments.of( + "long array (empty)", + LongArray(0), + Struct(PropertyType.schema).put(LONG_LIST, emptyList()), + emptyList()), + Arguments.of( + "long array", + LongArray(1) { 1.toLong() }, + Struct(PropertyType.schema).put(LONG_LIST, listOf(1L)), + listOf(1L)), + Arguments.of( + "array (long)", + Array(1) { 1.toLong() }, + Struct(PropertyType.schema).put(LONG_LIST, listOf(1L)), + listOf(1L)), + Arguments.of( + "list (long)", + listOf(1.toLong(), 2.toLong()), + Struct(PropertyType.schema).put(LONG_LIST, listOf(1L, 2L)), + listOf(1L, 2L)), + Arguments.of( + "float array (empty)", + FloatArray(0), + Struct(PropertyType.schema).put(FLOAT_LIST, emptyList()), + emptyList()), + Arguments.of( + "float array", + FloatArray(1) { 1.toFloat() }, + Struct(PropertyType.schema).put(FLOAT_LIST, listOf(1.toDouble())), + listOf(1.toDouble())), + Arguments.of( + "array (float)", + Array(1) { 1.toFloat() }, + Struct(PropertyType.schema).put(FLOAT_LIST, listOf(1.toDouble())), + listOf(1.toDouble())), + Arguments.of( + "list (float)", + listOf(1.toFloat(), 2.toFloat()), + Struct(PropertyType.schema).put(FLOAT_LIST, listOf(1.toDouble(), 2.toDouble())), + listOf(1.toDouble(), 2.toDouble())), + Arguments.of( + "double array (empty)", + DoubleArray(0), + Struct(PropertyType.schema).put(FLOAT_LIST, emptyList()), + emptyList()), + Arguments.of( + "double array", + DoubleArray(1) { 1.toDouble() }, + Struct(PropertyType.schema).put(FLOAT_LIST, listOf(1.toDouble())), + listOf(1.toDouble())), + Arguments.of( + "array (double)", + Array(1) { 1.toDouble() }, + Struct(PropertyType.schema).put(FLOAT_LIST, listOf(1.toDouble())), + listOf(1.toDouble())), + Arguments.of( + "list (double)", + listOf(1.toDouble(), 2.toDouble()), + Struct(PropertyType.schema).put(FLOAT_LIST, listOf(1.toDouble(), 2.toDouble())), + listOf(1.toDouble(), 2.toDouble())), + Arguments.of( + "array (string)", + Array(1) { "a" }, + Struct(PropertyType.schema).put(STRING_LIST, listOf("a")), + listOf("a")), + Arguments.of( + "list (string)", + listOf("a", "b"), + Struct(PropertyType.schema).put(STRING_LIST, listOf("a", "b")), + listOf("a", "b")), + Arguments.of( + "array (local date)", + Array(1) { LocalDate.of(1999, 1, 1) }, + Struct(PropertyType.schema).put(LOCAL_DATE_LIST, listOf("1999-01-01")), + listOf(LocalDate.of(1999, 1, 1))), + Arguments.of( + "list (local date)", + listOf(LocalDate.of(1999, 1, 1)), + Struct(PropertyType.schema).put(LOCAL_DATE_LIST, listOf("1999-01-01")), + listOf(LocalDate.of(1999, 1, 1))), + Arguments.of( + "array (local time)", + Array(1) { LocalTime.of(23, 59, 59, 999999999) }, + Struct(PropertyType.schema).put(LOCAL_TIME_LIST, listOf("23:59:59.999999999")), + listOf(LocalTime.of(23, 59, 59, 999999999))), + Arguments.of( + "list (local time)", + listOf(LocalTime.of(23, 59, 59, 999999999)), + Struct(PropertyType.schema).put(LOCAL_TIME_LIST, listOf("23:59:59.999999999")), + listOf(LocalTime.of(23, 59, 59, 999999999))), + Arguments.of( + "array (local date time)", + Array(1) { LocalDateTime.of(1999, 1, 1, 23, 59, 59, 999999999) }, + Struct(PropertyType.schema) + .put(LOCAL_DATE_TIME_LIST, listOf("1999-01-01T23:59:59.999999999")), + listOf(LocalDateTime.of(1999, 1, 1, 23, 59, 59, 999999999))), + Arguments.of( + "list (local date time)", + listOf(LocalDateTime.of(1999, 1, 1, 23, 59, 59, 999999999)), + Struct(PropertyType.schema) + .put(LOCAL_DATE_TIME_LIST, listOf("1999-01-01T23:59:59.999999999")), + listOf(LocalDateTime.of(1999, 1, 1, 23, 59, 59, 999999999))), + Arguments.of( + "array (offset date time)", + Array(1) { + OffsetDateTime.of(1999, 1, 1, 23, 59, 59, 999999999, ZoneOffset.ofHours(2)) + }, + Struct(PropertyType.schema) + .put(ZONED_DATE_TIME_LIST, listOf("1999-01-01T23:59:59.999999999+02:00")), + listOf(OffsetDateTime.of(1999, 1, 1, 23, 59, 59, 999999999, ZoneOffset.ofHours(2)))), + Arguments.of( + "list (offset date time)", + listOf(OffsetDateTime.of(1999, 1, 1, 23, 59, 59, 999999999, ZoneOffset.ofHours(2))), + Struct(PropertyType.schema) + .put(ZONED_DATE_TIME_LIST, listOf("1999-01-01T23:59:59.999999999+02:00")), + listOf(OffsetDateTime.of(1999, 1, 1, 23, 59, 59, 999999999, ZoneOffset.ofHours(2)))), + Arguments.of( + "array (zoned date time)", + Array(1) { + ZonedDateTime.of(1999, 1, 1, 23, 59, 59, 999999999, ZoneId.of("Europe/Istanbul")) + }, + Struct(PropertyType.schema) + .put( + ZONED_DATE_TIME_LIST, + listOf("1999-01-01T23:59:59.999999999+02:00[Europe/Istanbul]")), + listOf( + ZonedDateTime.of( + 1999, 1, 1, 23, 59, 59, 999999999, ZoneId.of("Europe/Istanbul")))), + Arguments.of( + "list (zoned date time)", + listOf( + ZonedDateTime.of( + 1999, 1, 1, 23, 59, 59, 999999999, ZoneId.of("Europe/Istanbul"))), + Struct(PropertyType.schema) + .put( + ZONED_DATE_TIME_LIST, + listOf("1999-01-01T23:59:59.999999999+02:00[Europe/Istanbul]")), + listOf( + ZonedDateTime.of( + 1999, 1, 1, 23, 59, 59, 999999999, ZoneId.of("Europe/Istanbul")))), + Arguments.of( + "array (offset time)", + Array(1) { OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHours(2)) }, + Struct(PropertyType.schema).put(OFFSET_TIME_LIST, listOf("23:59:59.999999999+02:00")), + listOf(OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHours(2)))), + Arguments.of( + "list (offset time)", + listOf(OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHours(2))), + Struct(PropertyType.schema).put(OFFSET_TIME_LIST, listOf("23:59:59.999999999+02:00")), + listOf(OffsetTime.of(23, 59, 59, 999999999, ZoneOffset.ofHours(2)))), + Arguments.of( + "array (duration)", + Array(1) { Values.isoDuration(12, 12, 59, 1230).asIsoDuration() }, + Struct(PropertyType.schema) + .put( + DURATION_LIST, + listOf( + Struct(PropertyType.durationSchema) + .put(MONTHS, 12L) + .put(DAYS, 12L) + .put(SECONDS, 59L) + .put(NANOS, 1230))), + listOf(Values.isoDuration(12, 12, 59, 1230).asIsoDuration())), + Arguments.of( + "list (duration)", + listOf(Values.isoDuration(12, 12, 59, 1230).asIsoDuration()), + Struct(PropertyType.schema) + .put( + DURATION_LIST, + listOf( + Struct(PropertyType.durationSchema) + .put(MONTHS, 12L) + .put(DAYS, 12L) + .put(SECONDS, 59L) + .put(NANOS, 1230))), + listOf(Values.isoDuration(12, 12, 59, 1230).asIsoDuration())), + Arguments.of( + "array (point (2d))", + Array(1) { Values.point(4326, 1.0, 2.0).asPoint() }, + Struct(PropertyType.schema) + .put( + POINT_LIST, + listOf( + Struct(PropertyType.pointSchema) + .put(SR_ID, 4326) + .put(X, 1.0) + .put(Y, 2.0) + .put(DIMENSION, TWO_D))), + listOf(Values.point(4326, 1.0, 2.0).asPoint())), + Arguments.of( + "list (point (2d))", + listOf(Values.point(4326, 1.0, 2.0).asPoint()), + Struct(PropertyType.schema) + .put( + POINT_LIST, + listOf( + Struct(PropertyType.pointSchema) + .put(SR_ID, 4326) + .put(X, 1.0) + .put(Y, 2.0) + .put(DIMENSION, TWO_D))), + listOf(Values.point(4326, 1.0, 2.0).asPoint())), + Arguments.of( + "array (point (3d))", + Array(1) { Values.point(4326, 1.0, 2.0, 3.0).asPoint() }, + Struct(PropertyType.schema) + .put( + POINT_LIST, + listOf( + Struct(PropertyType.pointSchema) + .put(SR_ID, 4326) + .put(X, 1.0) + .put(Y, 2.0) + .put(Z, 3.0) + .put(DIMENSION, THREE_D))), + listOf(Values.point(4326, 1.0, 2.0, 3.0).asPoint())), + Arguments.of( + "list (point (3d))", + listOf(Values.point(4326, 1.0, 2.0, 3.0).asPoint()), + Struct(PropertyType.schema) + .put( + POINT_LIST, + listOf( + Struct(PropertyType.pointSchema) + .put(SR_ID, 4326) + .put(X, 1.0) + .put(Y, 2.0) + .put(Z, 3.0) + .put(DIMENSION, THREE_D))), + listOf(Values.point(4326, 1.0, 2.0, 3.0).asPoint())), + ) + } + } + + @Test + fun `should throw when unsupported type is provided`() { + shouldThrow { + PropertyType.toConnectValue(java.sql.Timestamp.from(Instant.now())) + } shouldHaveMessage "unsupported property type: java.sql.Timestamp" + } + + @Test + fun `should throw when unsupported array type is provided`() { + shouldThrow { + PropertyType.toConnectValue(Array(1) { java.sql.Timestamp.from(Instant.now()) }) + } shouldHaveMessage "unsupported array type: array of java.sql.Timestamp" + } +} diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt index b004df6e9..203dbd490 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt @@ -49,6 +49,7 @@ import org.neo4j.connectors.kafka.data.PropertyType.FLOAT import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE_TIME import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_TIME +import org.neo4j.connectors.kafka.data.PropertyType.LONG_LIST import org.neo4j.connectors.kafka.data.PropertyType.OFFSET_TIME import org.neo4j.connectors.kafka.data.PropertyType.POINT import org.neo4j.connectors.kafka.data.PropertyType.ZONED_DATE_TIME @@ -204,9 +205,9 @@ class TypesTest { .put("y", 56.7) .put("z", 100.0))), Arguments.of( - Named.of("list - uniformly typed elements", (1L..50L).toList()), - SchemaBuilder.array(PropertyType.schema).build(), - (1L..50L).map { PropertyType.toConnectValue(it) }.toList()), + Named.of("list - long", (1L..50L).toList()), + PropertyType.schema, + Struct(PropertyType.schema).put(LONG_LIST, (1L..50L).toList())), Arguments.of( Named.of("list - non-uniformly typed elements", listOf(1, true, 2.0, "a string")), SchemaBuilder.array(PropertyType.schema).build(), From 21b4c79a19c2ba6a2f2b983d1e26990531109d14 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Sun, 28 Jul 2024 02:11:06 +0100 Subject: [PATCH 5/9] test: fix unit tests --- .../kafka/source/Neo4jCdcKeyStrategyTest.kt | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/source/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcKeyStrategyTest.kt b/source/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcKeyStrategyTest.kt index 4ac072478..c1f276e9d 100644 --- a/source/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcKeyStrategyTest.kt +++ b/source/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcKeyStrategyTest.kt @@ -39,6 +39,7 @@ import org.neo4j.cdc.client.model.NodeState import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.cdc.client.model.RelationshipState import org.neo4j.connectors.kafka.data.ChangeEventConverter +import org.neo4j.connectors.kafka.data.PropertyType import org.neo4j.connectors.kafka.source.Neo4jCdcKeyStrategy.ELEMENT_ID import org.neo4j.connectors.kafka.source.Neo4jCdcKeyStrategy.ENTITY_KEYS import org.neo4j.connectors.kafka.source.Neo4jCdcKeyStrategy.SKIP @@ -112,11 +113,7 @@ object TestData { val elementIdSchema: Schema = Schema.STRING_SCHEMA private val propertySchema: Schema = - SchemaBuilder.struct() - .field("foo", Schema.OPTIONAL_STRING_SCHEMA) - .field("bar", Schema.OPTIONAL_INT64_SCHEMA) - .optional() - .build() + SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build() val nodeKeysSchema: Schema = SchemaBuilder.struct() @@ -137,8 +134,9 @@ object TestData { .put( LABEL, listOf( - Struct(propertySchema).put("foo", "fighters").put("bar", 42L), - ), + mapOf( + "foo" to PropertyType.toConnectValue("fighters"), + "bar" to PropertyType.toConnectValue(42L))), )) val relKeysSchema: Schema = @@ -152,8 +150,9 @@ object TestData { .put( "keys", listOf( - Struct(propertySchema).put("foo", "fighters").put("bar", 42L), - )) + mapOf( + "foo" to PropertyType.toConnectValue("fighters"), + "bar" to PropertyType.toConnectValue(42L)))) val nodeChange = ChangeEventConverter() From 9aa347f0e294276a243d8f10763e2fe1f0c54f23 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Mon, 29 Jul 2024 11:23:53 +0100 Subject: [PATCH 6/9] refactor: keys should be serialised as structs but not maps --- .../kafka/data/ChangeEventExtensions.kt | 25 ++- .../connectors/kafka/data/DynamicTypes.kt | 7 +- .../kafka/data/ChangeEventExtensionsTest.kt | 183 +++++++++++++----- .../kafka/source/Neo4jCdcKeyStrategyTest.kt | 20 +- 4 files changed, 169 insertions(+), 66 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt index 1ba90cb2c..461129f19 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt @@ -136,6 +136,7 @@ class ChangeEventConverter( internal fun nodeEventToConnectValue(nodeEvent: NodeEvent, schema: Schema): Struct = Struct(schema).also { val keys = DynamicTypes.toConnectValue(schema.field("keys").schema(), nodeEvent.keys) + it.put("elementId", nodeEvent.elementId) it.put("eventType", nodeEvent.eventType.name) it.put("operation", nodeEvent.operation.name) @@ -154,7 +155,7 @@ class ChangeEventConverter( .field("type", Schema.STRING_SCHEMA) .field("start", nodeToConnectSchema(relationshipEvent.start)) .field("end", nodeToConnectSchema(relationshipEvent.end)) - .field("keys", schemaForKeys()) + .field("keys", schemaForKeys(relationshipEvent.keys)) .field( "state", relationshipStateSchema(relationshipEvent.before, relationshipEvent.after)) .build() @@ -166,6 +167,7 @@ class ChangeEventConverter( Struct(schema).also { val keys = DynamicTypes.toConnectValue(schema.field("keys").schema(), relationshipEvent.keys) + it.put("elementId", relationshipEvent.elementId) it.put("eventType", relationshipEvent.eventType.name) it.put("operation", relationshipEvent.operation.name) @@ -196,13 +198,28 @@ class ChangeEventConverter( private fun schemaForKeysByLabel(keys: Map>>?): Schema { return SchemaBuilder.struct() - .apply { keys?.forEach { field(it.key, schemaForKeys()) } } + .apply { keys?.forEach { field(it.key, schemaForKeys(it.value)) } } .optional() .build() } - private fun schemaForKeys(): Schema { - return SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) + private fun schemaForKeys(keys: List>?): Schema { + return SchemaBuilder.array( + // We need to define a uniform structure of key array elements. Because all elements + // must have identical structure, we list all available keys as optional fields. + SchemaBuilder.struct() + .apply { + keys?.forEach { key -> + key.forEach { + field( + it.key, + DynamicTypes.toConnectSchema( + it.value, optional = true, forceMapsAsStruct = true)) + } + } + } + .optional() + .build()) .optional() .build() } diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt index 0c99ed7ee..0325f1a4e 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt @@ -44,7 +44,6 @@ fun Schema.matches(other: Schema): Boolean { } object DynamicTypes { - fun toConnectValue(schema: Schema, value: Any?): Any? { if (value == null) { return null @@ -313,9 +312,9 @@ object DynamicTypes { .filter { e -> e.value.notNullOrEmpty() } .mapValues { e -> toConnectSchema(e.value, optional, forceMapsAsStruct) } - val valueSet = elementTypes.values.toSet() + val elementValueTypesSet = elementTypes.values.toSet() when { - valueSet.isEmpty() -> + elementValueTypesSet.isEmpty() -> SchemaBuilder.struct() .apply { value.forEach { @@ -325,7 +324,7 @@ object DynamicTypes { } .apply { if (optional) optional() } .build() - valueSet.singleOrNull() != null && !forceMapsAsStruct -> + elementValueTypesSet.singleOrNull() != null && !forceMapsAsStruct -> SchemaBuilder.map(Schema.STRING_SCHEMA, elementTypes.values.first()) .apply { if (optional) optional() } .build() diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt index 20ad31a33..e918cf207 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensionsTest.kt @@ -127,14 +127,19 @@ class ChangeEventExtensionsTest { .field( "Label1", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("name", PropertyType.schema) + .field("surname", PropertyType.schema) + .optional() .build()) .optional() .build()) .field( "Label2", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("id", PropertyType.schema) + .optional() .build()) .optional() .build()) @@ -178,10 +183,14 @@ class ChangeEventExtensionsTest { .put( "Label1", listOf( - mapOf( - "name" to PropertyType.toConnectValue("john"), - "surname" to PropertyType.toConnectValue("doe")))) - .put("Label2", listOf(mapOf("id" to PropertyType.toConnectValue(5L))))) + Struct(schema.nestedSchema("event.keys.Label1").valueSchema()) + .put("name", PropertyType.toConnectValue("john")) + .put("surname", PropertyType.toConnectValue("doe")))) + .put( + "Label2", + listOf( + Struct(schema.nestedSchema("event.keys.Label2").valueSchema()) + .put("id", PropertyType.toConnectValue(5L))))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -230,14 +239,19 @@ class ChangeEventExtensionsTest { .field( "Label1", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("name", PropertyType.schema) + .field("surname", PropertyType.schema) + .optional() .build()) .optional() .build()) .field( "Label2", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("id", PropertyType.schema) + .optional() .build()) .optional() .build()) @@ -281,10 +295,14 @@ class ChangeEventExtensionsTest { .put( "Label1", listOf( - mapOf( - "name" to PropertyType.toConnectValue("john"), - "surname" to PropertyType.toConnectValue("doe")))) - .put("Label2", listOf(mapOf("id" to PropertyType.toConnectValue(5L))))) + Struct(schema.nestedSchema("event.keys.Label1").valueSchema()) + .put("name", PropertyType.toConnectValue("john")) + .put("surname", PropertyType.toConnectValue("doe")))) + .put( + "Label2", + listOf( + Struct(schema.nestedSchema("event.keys.Label2").valueSchema()) + .put("id", PropertyType.toConnectValue(5L))))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -342,14 +360,19 @@ class ChangeEventExtensionsTest { .field( "Label1", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("name", PropertyType.schema) + .field("surname", PropertyType.schema) + .optional() .build()) .optional() .build()) .field( "Label2", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("id", PropertyType.schema) + .optional() .build()) .optional() .build()) @@ -393,10 +416,14 @@ class ChangeEventExtensionsTest { .put( "Label1", listOf( - mapOf( - "name" to PropertyType.toConnectValue("john"), - "surname" to PropertyType.toConnectValue("doe")))) - .put("Label2", listOf(mapOf("id" to PropertyType.toConnectValue(5L))))) + Struct(schema.nestedSchema("event.keys.Label1").valueSchema()) + .put("name", PropertyType.toConnectValue("john")) + .put("surname", PropertyType.toConnectValue("doe")))) + .put( + "Label2", + listOf( + Struct(schema.nestedSchema("event.keys.Label2").valueSchema()) + .put("id", PropertyType.toConnectValue(5L))))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -451,10 +478,12 @@ class ChangeEventExtensionsTest { .field( "Person", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("name", PropertyType.schema) + .optional() .build()) .optional() - .schema()) + .build()) .optional() .build()) .build()) @@ -469,17 +498,19 @@ class ChangeEventExtensionsTest { .field( "Company", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("name", PropertyType.schema) + .optional() .build()) .optional() - .schema()) + .build()) .optional() .build()) .build()) .field( "keys", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) + SchemaBuilder.struct().field("id", PropertyType.schema).optional().build()) .optional() .build()) .field( @@ -522,7 +553,12 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.start.keys")) .put( "Person", - listOf(mapOf("name" to PropertyType.toConnectValue("john")))))) + listOf( + Struct( + schema + .nestedSchema("event.start.keys.Person") + .valueSchema()) + .put("name", PropertyType.toConnectValue("john")))))) .put( "end", Struct(schema.nestedSchema("event.end")) @@ -533,8 +569,17 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.end.keys")) .put( "Company", - listOf(mapOf("name" to PropertyType.toConnectValue("acme corp")))))) - .put("keys", listOf(mapOf("id" to PropertyType.toConnectValue(5L)))) + listOf( + Struct( + schema + .nestedSchema("event.end.keys.Company") + .valueSchema()) + .put("name", PropertyType.toConnectValue("acme corp")))))) + .put( + "keys", + listOf( + Struct(schema.nestedSchema("event.keys").valueSchema()) + .put("id", PropertyType.toConnectValue(5L)))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -587,10 +632,12 @@ class ChangeEventExtensionsTest { .field( "Person", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("name", PropertyType.schema) + .optional() .build()) .optional() - .schema()) + .build()) .optional() .build()) .build()) @@ -605,17 +652,19 @@ class ChangeEventExtensionsTest { .field( "Company", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("name", PropertyType.schema) + .optional() .build()) .optional() - .schema()) + .build()) .optional() .build()) .build()) .field( "keys", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) + SchemaBuilder.struct().field("id", PropertyType.schema).optional().build()) .optional() .build()) .field( @@ -658,7 +707,12 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.start.keys")) .put( "Person", - listOf(mapOf("name" to PropertyType.toConnectValue("john")))))) + listOf( + Struct( + schema + .nestedSchema("event.start.keys.Person") + .valueSchema()) + .put("name", PropertyType.toConnectValue("john")))))) .put( "end", Struct(schema.nestedSchema("event.end")) @@ -669,8 +723,17 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.end.keys")) .put( "Company", - listOf(mapOf("name" to PropertyType.toConnectValue("acme corp")))))) - .put("keys", listOf(mapOf("id" to PropertyType.toConnectValue(5L)))) + listOf( + Struct( + schema + .nestedSchema("event.end.keys.Company") + .valueSchema()) + .put("name", PropertyType.toConnectValue("acme corp")))))) + .put( + "keys", + listOf( + Struct(schema.nestedSchema("event.keys").valueSchema()) + .put("id", PropertyType.toConnectValue(5L)))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -732,7 +795,9 @@ class ChangeEventExtensionsTest { .field( "Person", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("name", PropertyType.schema) + .optional() .build()) .optional() .build()) @@ -750,7 +815,9 @@ class ChangeEventExtensionsTest { .field( "Company", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema) + SchemaBuilder.struct() + .field("name", PropertyType.schema) + .optional() .build()) .optional() .build()) @@ -760,9 +827,9 @@ class ChangeEventExtensionsTest { .field( "keys", SchemaBuilder.array( - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) + SchemaBuilder.struct().field("id", PropertyType.schema).optional().build()) .optional() - .schema()) + .build()) .field( "state", SchemaBuilder.struct() @@ -803,7 +870,12 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.start.keys")) .put( "Person", - listOf(mapOf("name" to PropertyType.toConnectValue("john")))))) + listOf( + Struct( + schema + .nestedSchema("event.start.keys.Person") + .valueSchema()) + .put("name", PropertyType.toConnectValue("john")))))) .put( "end", Struct(schema.nestedSchema("event.end")) @@ -814,8 +886,17 @@ class ChangeEventExtensionsTest { Struct(schema.nestedSchema("event.end.keys")) .put( "Company", - listOf(mapOf("name" to PropertyType.toConnectValue("acme corp")))))) - .put("keys", listOf(mapOf("id" to PropertyType.toConnectValue(5L)))) + listOf( + Struct( + schema + .nestedSchema("event.end.keys.Company") + .valueSchema()) + .put("name", PropertyType.toConnectValue("acme corp")))))) + .put( + "keys", + listOf( + Struct(schema.nestedSchema("event.keys").valueSchema()) + .put("id", PropertyType.toConnectValue(5L)))) .put( "state", Struct(schema.nestedSchema("event.state")) @@ -865,9 +946,8 @@ class ChangeEventExtensionsTest { null)) val expectedKeySchema = - SchemaBuilder.array(SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build()) - .optional() - .build() + SchemaBuilder.array(SchemaBuilder.struct().optional().build()).optional().build() + schema.nestedSchema("event.keys") shouldBe expectedKeySchema value.nestedValue("event.keys") shouldBe emptyList() } @@ -1041,16 +1121,17 @@ class ChangeEventExtensionsTest { .put( "Person", listOf( - mapOf("id" to PropertyType.toConnectValue(1L)), - mapOf( - "name" to PropertyType.toConnectValue("john"), - "surname" to PropertyType.toConnectValue("doe")))) + Struct(schema.nestedSchema("keys.Person").valueSchema()) + .put("id", PropertyType.toConnectValue(1L)), + Struct(schema.nestedSchema("keys.Person").valueSchema()) + .put("name", PropertyType.toConnectValue("john")) + .put("surname", PropertyType.toConnectValue("doe")))) .put( "Employee", listOf( - mapOf( - "id" to PropertyType.toConnectValue(5L), - "company_id" to PropertyType.toConnectValue(7L))))) + Struct(schema.nestedSchema("keys.Employee").valueSchema()) + .put("id", PropertyType.toConnectValue(5L)) + .put("company_id", PropertyType.toConnectValue(7L))))) val reverted = converted.toNode() reverted shouldBe node diff --git a/source/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcKeyStrategyTest.kt b/source/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcKeyStrategyTest.kt index c1f276e9d..e968d30e0 100644 --- a/source/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcKeyStrategyTest.kt +++ b/source/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcKeyStrategyTest.kt @@ -113,7 +113,11 @@ object TestData { val elementIdSchema: Schema = Schema.STRING_SCHEMA private val propertySchema: Schema = - SchemaBuilder.map(Schema.STRING_SCHEMA, PropertyType.schema).build() + SchemaBuilder.struct() + .field("foo", PropertyType.schema) + .field("bar", PropertyType.schema) + .optional() + .build() val nodeKeysSchema: Schema = SchemaBuilder.struct() @@ -134,9 +138,10 @@ object TestData { .put( LABEL, listOf( - mapOf( - "foo" to PropertyType.toConnectValue("fighters"), - "bar" to PropertyType.toConnectValue(42L))), + Struct(propertySchema) + .put("foo", PropertyType.toConnectValue("fighters")) + .put("bar", PropertyType.toConnectValue(42L)), + ), )) val relKeysSchema: Schema = @@ -150,9 +155,10 @@ object TestData { .put( "keys", listOf( - mapOf( - "foo" to PropertyType.toConnectValue("fighters"), - "bar" to PropertyType.toConnectValue(42L)))) + Struct(propertySchema) + .put("foo", PropertyType.toConnectValue("fighters")) + .put("bar", PropertyType.toConnectValue(42L)), + )) val nodeChange = ChangeEventConverter() From c2c2bd5199293d6e3a697ea6706a04055ddde29a Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Mon, 29 Jul 2024 13:13:26 +0100 Subject: [PATCH 7/9] test: make all tests green --- .../kafka/data/ChangeEventExtensions.kt | 4 +- .../connectors/kafka/data/PropertyType.kt | 9 +- .../kafka/data/TemporalDataSchemaType.kt | 22 --- .../neo4j-source-configuration.properties | 4 +- pom.xml | 2 +- .../connectors/kafka/sink/Neo4jCypherIT.kt | 13 +- .../kafka/source/Neo4jCdcSourceIT.kt | 165 ---------------- .../kafka/source/Neo4jCdcSourceNodesIT.kt | 169 ++++++++++++++-- .../source/Neo4jCdcSourceRelationshipsIT.kt | 180 +++++------------- .../kafka/source/Neo4jSourceQueryIT.kt | 159 ---------------- .../connectors/kafka/source/Neo4jCdcTask.kt | 2 +- .../kafka/source/SourceConfiguration.kt | 14 -- .../kafka/source/SourceConfigurationTest.kt | 11 -- .../kafka/testing/format/KafkaConverter.kt | 3 +- .../kafka/testing/source/Neo4jSource.kt | 2 - .../testing/source/Neo4jSourceExtension.kt | 4 +- .../testing/source/Neo4jSourceRegistration.kt | 4 - 17 files changed, 220 insertions(+), 547 deletions(-) delete mode 100644 common/src/main/kotlin/org/neo4j/connectors/kafka/data/TemporalDataSchemaType.kt diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt index 461129f19..ff5a3cf6b 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/ChangeEventExtensions.kt @@ -33,9 +33,7 @@ import org.neo4j.cdc.client.model.RelationshipEvent import org.neo4j.cdc.client.model.RelationshipState import org.neo4j.connectors.kafka.data.DynamicTypes.toConnectSchema -class ChangeEventConverter( - val temporalDataSchemaType: TemporalDataSchemaType = TemporalDataSchemaType.STRUCT, -) { +class ChangeEventConverter() { fun toConnectValue(changeEvent: ChangeEvent): SchemaAndValue { val schema = toConnectSchema(changeEvent) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt index 3f43c7cc9..6061c1248 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt @@ -289,7 +289,14 @@ object PropertyType { FLOAT_LIST -> it.get(f) as List<*> STRING -> it.get(f) as String STRING_LIST -> it.get(f) as List<*> - BYTES -> it.get(f) as ByteArray + BYTES -> + when (val bytes = it.get(f)) { + is ByteArray -> bytes + is ByteBuffer -> bytes.array() + else -> + throw IllegalArgumentException( + "unsupported BYTES value: ${bytes.javaClass.name}") + } LOCAL_DATE -> parseLocalDate((it.get(f) as String)) LOCAL_DATE_LIST -> (it.get(f) as List).map { s -> parseLocalDate(s) } LOCAL_TIME -> parseLocalTime((it.get(f) as String)) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/TemporalDataSchemaType.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/TemporalDataSchemaType.kt deleted file mode 100644 index 0e0ae90f7..000000000 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/TemporalDataSchemaType.kt +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) "Neo4j" - * Neo4j Sweden AB [https://neo4j.com] - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.neo4j.connectors.kafka.data - -enum class TemporalDataSchemaType(val description: String) { - STRUCT("struct"), - STRING("string") -} diff --git a/common/src/main/resources/neo4j-source-configuration.properties b/common/src/main/resources/neo4j-source-configuration.properties index 43576b007..1d4f9fca0 100644 --- a/common/src/main/resources/neo4j-source-configuration.properties +++ b/common/src/main/resources/neo4j-source-configuration.properties @@ -26,6 +26,4 @@ neo4j.query.poll-interval=Type: String;\nDescription: Interval in which the quer neo4j.query.topic=Type: String;\nDescription: Kafka topic to publish change events gathered through provided query. neo4j.query.timeout=Type: Duration;\nDescription: Maximum amount of time query is allowed to run (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). neo4j.cdc.poll-duration=Type: Duration;\nDescription: Maximum amount of time Kafka Connect poll request will wait for a change to be received from the database (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). -neo4j.cdc.poll-interval=Type: Duration;\nDescription: The interval in which the database will be polled for changes (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). -neo4j.temporal-schema.type=Type: Enum;\nDescription: Schema type for temporal data. \ - If set to STRUCT, temporal data is converted into a connector defined struct object. If set to STRING, temporal data is converted into a string using the ISO standard. \ No newline at end of file +neo4j.cdc.poll-interval=Type: Duration;\nDescription: The interval in which the database will be polled for changes (valid units are: `ms`, `s`, `m`, `h` and `d`; default unit is `s`). \ No newline at end of file diff --git a/pom.xml b/pom.xml index 996f63b5c..870aeecdd 100644 --- a/pom.xml +++ b/pom.xml @@ -81,7 +81,7 @@ 2022.9.2 4.4.9 UTF-8 - 3.23.2 + 3.19.6 2023.0.7 1.7.36 4.0.0 diff --git a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt index 37c2bf104..00f073cbf 100644 --- a/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt +++ b/sink-connector/src/test/kotlin/org/neo4j/connectors/kafka/sink/Neo4jCypherIT.kt @@ -455,7 +455,7 @@ abstract class Neo4jCypherIT { CypherStrategy( TOPIC, """ - CREATE (n:Data) SET n = __value + CREATE (n:Data) SET n = __value.map """)]) @Test fun `should support complex maps`( @@ -468,9 +468,14 @@ abstract class Neo4jCypherIT { "lastName" to "doe", "dob" to LocalDate.of(1999, 1, 1), "siblings" to 3) - val schema = DynamicTypes.toConnectSchema(value) - - producer.publish(valueSchema = schema, value = DynamicTypes.toConnectValue(schema, value)) + DynamicTypes.toConnectSchema(value).let { mapSchema -> + // Protobuf does not support top level MAP values, so we are wrapping it inside a struct + SchemaBuilder.struct().field("map", mapSchema).build().let { wrapper -> + producer.publish( + valueSchema = wrapper, + value = Struct(wrapper).put("map", DynamicTypes.toConnectValue(mapSchema, value))) + } + } eventually(30.seconds) { session.run("MATCH (n:Data) RETURN n", emptyMap()).single() } .get(0) diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt index 0c776278e..b60994d1b 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt @@ -18,28 +18,15 @@ package org.neo4j.connectors.kafka.source import io.kotest.matchers.collections.shouldHaveSingleElement import io.kotest.matchers.collections.shouldHaveSize -import io.kotest.matchers.equality.shouldBeEqualToComparingFields -import io.kotest.matchers.shouldBe import java.time.Duration import java.time.LocalDate -import java.time.LocalDateTime -import java.time.LocalTime -import java.time.OffsetDateTime -import java.time.OffsetTime -import java.time.ZoneId -import java.time.ZoneOffset -import java.time.ZonedDateTime import org.apache.kafka.connect.data.Schema -import org.apache.kafka.connect.data.Struct import org.apache.kafka.connect.storage.SimpleHeaderConverter import org.junit.jupiter.api.Test import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.connectors.kafka.connect.ConnectHeader -import org.neo4j.connectors.kafka.data.DynamicTypes import org.neo4j.connectors.kafka.data.Headers -import org.neo4j.connectors.kafka.data.PropertyType import org.neo4j.connectors.kafka.data.PropertyType.schema -import org.neo4j.connectors.kafka.data.TemporalDataSchemaType import org.neo4j.connectors.kafka.testing.assertions.TopicVerifier import org.neo4j.connectors.kafka.testing.format.KafkaConverter.AVRO import org.neo4j.connectors.kafka.testing.format.KafkaConverter.JSON_SCHEMA @@ -199,158 +186,6 @@ abstract class Neo4jCdcSourceIT { } .verifyWithin(Duration.ofSeconds(30)) } - - @Neo4jSource( - startFrom = "EARLIEST", - strategy = CDC, - cdc = - CdcSource( - topics = - arrayOf( - CdcSourceTopic( - topic = "neo4j-cdc-topic", - patterns = - arrayOf( - CdcSourceParam( - "(:TestSource{localDate, localDatetime, localTime, zonedDatetime, offsetDatetime, offsetTime})"))))), - temporalDataSchemaType = TemporalDataSchemaType.STRUCT) - @Test - fun `should return struct temporal types`( - @TopicConsumer(topic = "neo4j-cdc-topic", offset = "earliest") - consumer: ConvertingKafkaConsumer, - session: Session - ) { - session - .run( - "CREATE (:TestSource {" + - "localDate: date('2024-01-01'), " + - "localDatetime: localdatetime('2024-01-01T12:00:00'), " + - "localTime: localtime('12:00:00'), " + - "zonedDatetime: datetime('2024-01-01T12:00:00[Europe/Stockholm]'), " + - "offsetDatetime: datetime('2024-01-01T12:00:00Z'), " + - "offsetTime: time('12:00:00Z'), " + - "timestamp: 0})") - .consume() - - TopicVerifier.create(consumer) - .assertMessageValue { value -> - val properties = - value.getStruct("event").getStruct("state").getStruct("after").getStruct("properties") - - properties.getStruct("localDate") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalDate.of(2024, 1, 1), - ) as Struct - - properties.getStruct("localDatetime") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalDateTime.of(2024, 1, 1, 12, 0, 0), - ) as Struct - - properties.getStruct("localTime") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalTime.of(12, 0, 0), - ) as Struct - - properties.getStruct("zonedDatetime") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), - ) as Struct - - properties.getStruct("offsetDatetime") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), - ) as Struct - - properties.getStruct("offsetTime") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), - ) as Struct - } - .verifyWithin(Duration.ofSeconds(30)) - } - - @Neo4jSource( - startFrom = "EARLIEST", - strategy = CDC, - cdc = - CdcSource( - topics = - arrayOf( - CdcSourceTopic( - topic = "neo4j-cdc-topic", - patterns = - arrayOf( - CdcSourceParam( - "(:TestSource{localDate, localDatetime, localTime, zonedDatetime, offsetDatetime, offsetTime})"))))), - temporalDataSchemaType = TemporalDataSchemaType.STRING) - @Test - fun `should return string temporal types`( - @TopicConsumer(topic = "neo4j-cdc-topic", offset = "earliest") - consumer: ConvertingKafkaConsumer, - session: Session - ) { - session - .run( - "CREATE (:TestSource {" + - "localDate: date('2024-01-01'), " + - "localDatetime: localdatetime('2024-01-01T12:00:00'), " + - "localTime: localtime('12:00:00'), " + - "zonedDatetime: datetime('2024-01-01T12:00:00[Europe/Stockholm]'), " + - "offsetDatetime: datetime('2024-01-01T12:00:00Z'), " + - "offsetTime: time('12:00:00Z'), " + - "timestamp: 0})") - .consume() - - TopicVerifier.create(consumer) - .assertMessageValue { value -> - val properties = - value.getStruct("event").getStruct("state").getStruct("after").getStruct("properties") - - properties.getString("localDate") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalDate.of(2024, 1, 1), - ) - - properties.getString("localDatetime") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalDateTime.of(2024, 1, 1, 12, 0, 0), - ) - - properties.getString("localTime") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalTime.of(12, 0, 0), - ) - - properties.getString("zonedDatetime") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), - ) - - properties.getString("offsetDatetime") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), - ) - - properties.getString("offsetTime") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), - ) - } - .verifyWithin(Duration.ofSeconds(30)) - } } @KeyValueConverter(key = AVRO, value = AVRO) class Neo4jCdcSourceAvroIT : Neo4jCdcSourceIT() diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt index b652e0455..e4dd78f48 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt @@ -17,10 +17,12 @@ package org.neo4j.connectors.kafka.source import java.time.Duration -import org.junit.jupiter.api.Disabled +import java.time.LocalDate +import java.time.LocalDateTime import org.junit.jupiter.api.Test import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.cdc.client.model.EntityOperation +import org.neo4j.cdc.client.model.EntityOperation.CREATE import org.neo4j.cdc.client.model.EntityOperation.UPDATE import org.neo4j.cdc.client.model.EventType.NODE import org.neo4j.connectors.kafka.testing.assertions.ChangeEventAssert.Companion.assertThat @@ -69,7 +71,7 @@ abstract class Neo4jCdcSourceNodesIT { .assertMessageValue { value -> assertThat(value) .hasEventType(NODE) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .labelledAs("TestSource") .hasNoBeforeState() .hasAfterStateProperties(mapOf("name" to "Jane", "surname" to "Doe")) @@ -122,7 +124,7 @@ abstract class Neo4jCdcSourceNodesIT { .assertMessageValue { value -> assertThat(value) .hasEventType(NODE) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .labelledAs("TestSource") .hasNoBeforeState() .hasAfterStateProperties(mapOf("name" to "Jane", "surname" to "Doe", "age" to 42L)) @@ -237,7 +239,7 @@ abstract class Neo4jCdcSourceNodesIT { topic = "neo4j-cdc-create-inc", patterns = arrayOf(CdcSourceParam("(:TestSource)")))))) @Test - open fun `should read changes with different properties using the default topic compatibility mode`( + fun `should read changes with different properties using the default topic compatibility mode`( @TopicConsumer(topic = "neo4j-cdc-create-inc", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -249,7 +251,7 @@ abstract class Neo4jCdcSourceNodesIT { .assertMessageValue { value -> assertThat(value) .hasEventType(NODE) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .labelledAs("TestSource") .hasNoBeforeState() .hasAfterStateProperties(mapOf("name" to "John")) @@ -257,7 +259,7 @@ abstract class Neo4jCdcSourceNodesIT { .assertMessageValue { value -> assertThat(value) .hasEventType(NODE) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .labelledAs("TestSource") .hasNoBeforeState() .hasAfterStateProperties(mapOf("title" to "Neo4j")) @@ -303,7 +305,7 @@ abstract class Neo4jCdcSourceNodesIT { .assertMessageValue { value -> assertThat(value) .hasEventType(NODE) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .labelledAs("TestSource") .hasNoBeforeState() .hasAfterStateProperties(mapOf("name" to "Jane", "surname" to "Doe", "age" to 42L)) @@ -341,6 +343,142 @@ abstract class Neo4jCdcSourceNodesIT { .verifyWithin(Duration.ofSeconds(30)) } + @Neo4jSource( + startFrom = "EARLIEST", + strategy = CDC, + cdc = + CdcSource( + patternsIndexed = true, + topics = + arrayOf( + CdcSourceTopic( + topic = "cdc", patterns = arrayOf(CdcSourceParam("(:TestSource)")))))) + @Test + fun `should publish changes with property type changes`( + @TopicConsumer(topic = "cdc", offset = "earliest") consumer: ConvertingKafkaConsumer, + session: Session + ) { + session + .run( + "CREATE (n:TestSource) SET n = ${'$'}props", + mapOf("props" to mapOf("name" to "Jane", "surname" to "Doe", "age" to 42))) + .consume() + session + .run( + "MATCH (ts:TestSource {name: 'Jane'}) SET ts += ${'$'}props", + mapOf( + "props" to + mapOf( + "surname" to "Smith", + "age" to "42", + "dob" to LocalDateTime.of(1982, 1, 1, 0, 0, 0, 0)))) + .consume() + session + .run( + "MATCH (ts:TestSource {name: 'Jane'}) SET ts += ${'$'}props", + mapOf("props" to mapOf("age" to 42, "dob" to LocalDate.of(1982, 1, 1)))) + .consume() + + TopicVerifier.create(consumer) + .assertMessageValue { value -> + assertThat(value) + .hasEventType(NODE) + .hasOperation(CREATE) + .labelledAs("TestSource") + .hasNoBeforeState() + .hasAfterStateProperties(mapOf("name" to "Jane", "surname" to "Doe", "age" to 42L)) + } + .assertMessageValue { value -> + assertThat(value) + .hasEventType(NODE) + .hasOperation(UPDATE) + .labelledAs("TestSource") + .hasBeforeStateProperties(mapOf("name" to "Jane", "surname" to "Doe", "age" to 42L)) + .hasAfterStateProperties( + mapOf( + "name" to "Jane", + "surname" to "Smith", + "age" to "42", + "dob" to LocalDateTime.of(1982, 1, 1, 0, 0, 0, 0))) + } + .assertMessageValue { value -> + assertThat(value) + .hasEventType(NODE) + .hasOperation(UPDATE) + .labelledAs("TestSource") + .hasBeforeStateProperties( + mapOf( + "name" to "Jane", + "surname" to "Smith", + "age" to "42", + "dob" to LocalDateTime.of(1982, 1, 1, 0, 0, 0, 0))) + .hasAfterStateProperties( + mapOf( + "name" to "Jane", + "surname" to "Smith", + "age" to 42L, + "dob" to LocalDate.of(1982, 1, 1))) + } + .verifyWithin(Duration.ofSeconds(30)) + } + + @Neo4jSource( + startFrom = "EARLIEST", + strategy = CDC, + cdc = + CdcSource( + patternsIndexed = true, + topics = + arrayOf( + CdcSourceTopic( + topic = "cdc", patterns = arrayOf(CdcSourceParam("(:TestSource)")))))) + @Test + fun `should read each operation to a single topic`( + @TopicConsumer(topic = "cdc", offset = "earliest") consumer: ConvertingKafkaConsumer, + session: Session + ) { + session.run("CREATE (:TestSource {name: 'Jane', surname: 'Doe', age: 42})", mapOf()).consume() + session.run("MATCH (ts:TestSource {name: 'Jane'}) SET ts.surname = 'Smith'").consume() + session.run("MATCH (ts:TestSource {name: 'Jane'}) DELETE ts").consume() + + TopicVerifier.create(consumer) + .assertMessageValue { value -> + assertThat(value) + .hasEventType(NODE) + .hasOperation(CREATE) + .labelledAs("TestSource") + .hasNoBeforeState() + .hasAfterStateProperties(mapOf("name" to "Jane", "surname" to "Doe", "age" to 42L)) + } + .assertMessageValue { value -> + assertThat(value) + .hasEventType(NODE) + .hasOperation(UPDATE) + .labelledAs("TestSource") + .hasBeforeStateProperties(mapOf("name" to "Jane", "surname" to "Doe", "age" to 42L)) + .hasAfterStateProperties( + mapOf( + "name" to "Jane", + "surname" to "Smith", + "age" to 42L, + )) + } + .assertMessageValue { value -> + assertThat(value) + .hasEventType(NODE) + .hasOperation(EntityOperation.DELETE) + .labelledAs("TestSource") + .hasBeforeStateProperties( + mapOf( + "name" to "Jane", + "surname" to "Smith", + "age" to 42L, + )) + .hasNoAfterState() + } + .verifyWithin(Duration.ofSeconds(30)) + } + @Neo4jSource( startFrom = "EARLIEST", strategy = CDC, @@ -376,7 +514,7 @@ abstract class Neo4jCdcSourceNodesIT { .assertMessageValue { value -> assertThat(value) .hasEventType(NODE) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .labelledAs("TestSource") .hasNoBeforeState() .hasAfterStateProperties(mapOf("name" to "Alice")) @@ -419,7 +557,7 @@ abstract class Neo4jCdcSourceNodesIT { .assertMessageValue { value -> assertThat(value) .hasEventType(NODE) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .hasLabels(setOf("TestSource", "Employee")) .hasNoBeforeState() .hasAfterStateProperties(mapOf("id" to 1L, "name" to "John", "employeeId" to 456L)) @@ -436,18 +574,7 @@ abstract class Neo4jCdcSourceNodesIT { class Neo4jCdcSourceNodesAvroIT : Neo4jCdcSourceNodesIT() @KeyValueConverter(key = JSON_SCHEMA, value = JSON_SCHEMA) -class Neo4jCdcSourceNodesJsonIT : Neo4jCdcSourceNodesIT() { - - @Disabled("Json schema doesn't tolerate when an optional field changes the name") - override fun `should read changes with different properties using the default topic compatibility mode`( - consumer: ConvertingKafkaConsumer, - session: Session - ) { - super - .`should read changes with different properties using the default topic compatibility mode`( - consumer, session) - } -} +class Neo4jCdcSourceNodesJsonIT : Neo4jCdcSourceNodesIT() @KeyValueConverter(key = PROTOBUF, value = PROTOBUF) class Neo4jCdcSourceNodesProtobufIT : Neo4jCdcSourceNodesIT() diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt index 2feb74110..a173fe429 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt @@ -17,9 +17,7 @@ package org.neo4j.connectors.kafka.source import java.time.Duration -import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test -import org.junit.jupiter.api.TestInfo import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.cdc.client.model.EntityOperation import org.neo4j.cdc.client.model.EntityOperation.DELETE @@ -55,33 +53,23 @@ abstract class Neo4jCdcSourceRelationshipsIT { patterns = arrayOf( CdcSourceParam( - "(:TestSource)-[:RELIES_TO {execId,weight,-rate}]->(:TestSource)")))))) + "(:TestSource)-[:RELIES_TO {weight,-rate}]->(:TestSource)")))))) @Test fun `should read changes caught by patterns`( - testInfo: TestInfo, @TopicConsumer(topic = "neo4j-cdc-rels", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session ) { - val executionId = testInfo.displayName + System.currentTimeMillis() - val params = mapOf("execId" to executionId) session .run( - """CREATE (s:TestSource {name: 'Bob', execId: ${'$'}execId}) - |CREATE (t:TestSource {name: 'Alice', execId: ${'$'}execId}) - |CREATE (s)-[:RELIES_TO {weight: 1, rate: 42, execId: ${'$'}execId}]->(t) + """CREATE (s:TestSource {name: 'Bob'}) + |CREATE (t:TestSource {name: 'Alice'}) + |CREATE (s)-[:RELIES_TO {weight: 1, rate: 42}]->(t) """ - .trimMargin(), - params) - .consume() - session - .run( - "MATCH (:TestSource)-[r:RELIES_TO {execId: \$execId}]-(:TestSource) SET r.weight = 2", - params) - .consume() - session - .run("MATCH (:TestSource)-[r:RELIES_TO {execId: \$execId}]-(:TestSource) DELETE r", params) + .trimMargin()) .consume() + session.run("MATCH (:TestSource)-[r:RELIES_TO]-(:TestSource) SET r.weight = 2").consume() + session.run("MATCH (:TestSource)-[r:RELIES_TO]-(:TestSource) DELETE r").consume() TopicVerifier.create(consumer) .assertMessageValue { value -> @@ -92,7 +80,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .startLabelledAs("TestSource") .endLabelledAs("TestSource") .hasNoBeforeState() - .hasAfterStateProperties(mapOf("weight" to 1L, "execId" to executionId)) + .hasAfterStateProperties(mapOf("weight" to 1L)) } .assertMessageValue { value -> assertThat(value) @@ -101,8 +89,8 @@ abstract class Neo4jCdcSourceRelationshipsIT { .hasType("RELIES_TO") .startLabelledAs("TestSource") .endLabelledAs("TestSource") - .hasBeforeStateProperties(mapOf("weight" to 1L, "execId" to executionId)) - .hasAfterStateProperties(mapOf("weight" to 2L, "execId" to executionId)) + .hasBeforeStateProperties(mapOf("weight" to 1L)) + .hasAfterStateProperties(mapOf("weight" to 2L)) } .assertMessageValue { value -> assertThat(value) @@ -111,7 +99,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .hasType("RELIES_TO") .startLabelledAs("TestSource") .endLabelledAs("TestSource") - .hasBeforeStateProperties(mapOf("weight" to 2L, "execId" to executionId)) + .hasBeforeStateProperties(mapOf("weight" to 2L)) .hasNoAfterState() } .verifyWithin(Duration.ofSeconds(30)) @@ -129,35 +117,23 @@ abstract class Neo4jCdcSourceRelationshipsIT { patterns = arrayOf(CdcSourceParam("()-[:RELIES_TO {}]->()")))))) @Test fun `should read property removal and additions`( - testInfo: TestInfo, @TopicConsumer(topic = "neo4j-cdc-rels-prop-remove-add", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session ) { - val executionId = testInfo.displayName + System.currentTimeMillis() - val params = mapOf("execId" to executionId) session .run( - """CREATE (s:TestSource {name: 'Bob', execId: ${'$'}execId}) - |CREATE (t:TestSource {name: 'Alice', execId: ${'$'}execId}) - |CREATE (s)-[:RELIES_TO {weight: 1, rate: 42, execId: ${'$'}execId}]->(t) + """CREATE (s:TestSource {name: 'Bob'}) + |CREATE (t:TestSource {name: 'Alice'}) + |CREATE (s)-[:RELIES_TO {weight: 1, rate: 42}]->(t) """ - .trimMargin(), - params) - .consume() - session - .run( - "MATCH (:TestSource)-[r:RELIES_TO {execId: \$execId}]-(:TestSource) SET r.weight = 2, r.rate = NULL", - params) - .consume() - session - .run( - "MATCH (:TestSource)-[r:RELIES_TO {execId: \$execId}]-(:TestSource) SET r.rate = 50", - params) + .trimMargin()) .consume() session - .run("MATCH (:TestSource)-[r:RELIES_TO {execId: \$execId}]-(:TestSource) DELETE r", params) + .run("MATCH (:TestSource)-[r:RELIES_TO]-(:TestSource) SET r.weight = 2, r.rate = NULL") .consume() + session.run("MATCH (:TestSource)-[r:RELIES_TO]-(:TestSource) SET r.rate = 50").consume() + session.run("MATCH (:TestSource)-[r:RELIES_TO]-(:TestSource) DELETE r").consume() TopicVerifier.create(consumer) .assertMessageValue { value -> @@ -168,8 +144,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .startLabelledAs("TestSource") .endLabelledAs("TestSource") .hasNoBeforeState() - .hasAfterStateProperties( - mapOf("weight" to 1L, "rate" to 42L, "execId" to executionId)) + .hasAfterStateProperties(mapOf("weight" to 1L, "rate" to 42L)) } .assertMessageValue { value -> assertThat(value) @@ -178,9 +153,8 @@ abstract class Neo4jCdcSourceRelationshipsIT { .hasType("RELIES_TO") .startLabelledAs("TestSource") .endLabelledAs("TestSource") - .hasBeforeStateProperties( - mapOf("weight" to 1L, "rate" to 42L, "execId" to executionId)) - .hasAfterStateProperties(mapOf("weight" to 2L, "execId" to executionId)) + .hasBeforeStateProperties(mapOf("weight" to 1L, "rate" to 42L)) + .hasAfterStateProperties(mapOf("weight" to 2L)) } .assertMessageValue { value -> assertThat(value) @@ -189,9 +163,8 @@ abstract class Neo4jCdcSourceRelationshipsIT { .hasType("RELIES_TO") .startLabelledAs("TestSource") .endLabelledAs("TestSource") - .hasBeforeStateProperties(mapOf("weight" to 2L, "execId" to executionId)) - .hasAfterStateProperties( - mapOf("weight" to 2L, "rate" to 50L, "execId" to executionId)) + .hasBeforeStateProperties(mapOf("weight" to 2L)) + .hasAfterStateProperties(mapOf("weight" to 2L, "rate" to 50L)) } .assertMessageValue { value -> assertThat(value) @@ -200,8 +173,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .hasType("RELIES_TO") .startLabelledAs("TestSource") .endLabelledAs("TestSource") - .hasBeforeStateProperties( - mapOf("weight" to 2L, "rate" to 50L, "execId" to executionId)) + .hasBeforeStateProperties(mapOf("weight" to 2L, "rate" to 50L)) .hasNoAfterState() } .verifyWithin(Duration.ofSeconds(30)) @@ -217,24 +189,16 @@ abstract class Neo4jCdcSourceRelationshipsIT { arrayOf( CdcSourceTopic( topic = "neo4j-cdc-update-rel", - patterns = - arrayOf(CdcSourceParam(value = "(:A)-[:R {a,b,c,execId,-d}]->(:B)")), + patterns = arrayOf(CdcSourceParam(value = "(:A)-[:R {a,b,c,-d}]->(:B)")), operations = arrayOf(CdcSourceParam(value = "UPDATE")), changesTo = arrayOf(CdcSourceParam(value = "a,c")))))) @Test fun `should read only specified field changes on update`( - testInfo: TestInfo, @TopicConsumer(topic = "neo4j-cdc-update-rel", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session ) { - val executionId = testInfo.displayName + System.currentTimeMillis() - val params = mapOf("execId" to executionId) - session - .run( - "CREATE (:A)-[:R {a: 'foo', b: 'bar', c: 'abra', d: 'cadabra', execId: \$execId}]->(:B)", - params) - .consume() + session.run("CREATE (:A)-[:R {a: 'foo', b: 'bar', c: 'abra', d: 'cadabra'}]->(:B)").consume() session.run("MATCH (:A)-[r:R {a: 'foo'}]->(:B) SET r.a = 'mini', r.b = 'midi'").consume() session.run("MATCH (:A)-[r:R {a: 'mini'}]->(:B) SET r.a = 'eni', r.c = 'beni'").consume() session.run("MATCH (:A)-[r:R {a: 'eni'}]->(:B) SET r.a = 'obi', r.c = 'bobi'").consume() @@ -247,10 +211,8 @@ abstract class Neo4jCdcSourceRelationshipsIT { .hasType("R") .startLabelledAs("A") .endLabelledAs("B") - .hasBeforeStateProperties( - mapOf("a" to "mini", "b" to "midi", "c" to "abra", "execId" to executionId)) - .hasAfterStateProperties( - mapOf("a" to "eni", "b" to "midi", "c" to "beni", "execId" to executionId)) + .hasBeforeStateProperties(mapOf("a" to "mini", "b" to "midi", "c" to "abra")) + .hasAfterStateProperties(mapOf("a" to "eni", "b" to "midi", "c" to "beni")) } .assertMessageValue { value -> assertThat(value) @@ -259,10 +221,8 @@ abstract class Neo4jCdcSourceRelationshipsIT { .hasType("R") .startLabelledAs("A") .endLabelledAs("B") - .hasBeforeStateProperties( - mapOf("a" to "eni", "b" to "midi", "c" to "beni", "execId" to executionId)) - .hasAfterStateProperties( - mapOf("a" to "obi", "b" to "midi", "c" to "bobi", "execId" to executionId)) + .hasBeforeStateProperties(mapOf("a" to "eni", "b" to "midi", "c" to "beni")) + .hasAfterStateProperties(mapOf("a" to "obi", "b" to "midi", "c" to "bobi")) } .verifyWithin(Duration.ofSeconds(30)) } @@ -280,21 +240,12 @@ abstract class Neo4jCdcSourceRelationshipsIT { arrayOf(CdcSourceParam("(:Person)-[:IS_EMPLOYEE]->(:Company)")))))) @Test open fun `should read changes with different properties using the default topic compatibility mode`( - testInfo: TestInfo, @TopicConsumer(topic = "neo4j-cdc-create-inc-rel", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session ) { - val executionId = testInfo.displayName + System.currentTimeMillis() - val params = mapOf("execId" to executionId) - session - .run("CREATE (:Person)-[:IS_EMPLOYEE {role: 'SWE', execId: \$execId}]->(:Company)", params) - .consume() - session - .run( - "CREATE (:Person)-[:IS_EMPLOYEE {tribe: 'engineering', execId: \$execId}]->(:Company)", - params) - .consume() + session.run("CREATE (:Person)-[:IS_EMPLOYEE {role: 'SWE'}]->(:Company)").consume() + session.run("CREATE (:Person)-[:IS_EMPLOYEE {tribe: 'engineering'}]->(:Company)").consume() TopicVerifier.create(consumer) .assertMessageValue { value -> @@ -305,7 +256,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .startLabelledAs("Person") .endLabelledAs("Company") .hasNoBeforeState() - .hasAfterStateProperties(mapOf("role" to "SWE", "execId" to executionId)) + .hasAfterStateProperties(mapOf("role" to "SWE")) } .assertMessageValue { value -> assertThat(value) @@ -315,7 +266,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .startLabelledAs("Person") .endLabelledAs("Company") .hasNoBeforeState() - .hasAfterStateProperties(mapOf("tribe" to "engineering", "execId" to executionId)) + .hasAfterStateProperties(mapOf("tribe" to "engineering")) } .verifyWithin(Duration.ofSeconds(30)) } @@ -342,7 +293,6 @@ abstract class Neo4jCdcSourceRelationshipsIT { operations = arrayOf(CdcSourceParam("DELETE")))))) @Test fun `should read each operation to a separate topic`( - testInfo: TestInfo, @TopicConsumer(topic = "cdc-creates-rel", offset = "earliest") createsConsumer: ConvertingKafkaConsumer, @TopicConsumer(topic = "cdc-updates-rel", offset = "earliest") @@ -351,20 +301,9 @@ abstract class Neo4jCdcSourceRelationshipsIT { deletesConsumer: ConvertingKafkaConsumer, session: Session ) { - val executionId = testInfo.displayName + System.currentTimeMillis() - val params = mapOf("execId" to executionId) - session - .run( - "CREATE (:Person)-[:EMPLOYED {execId: \$execId, role: 'SWE'}]->(:Company)", - mapOf("execId" to executionId)) - .consume() - session - .run( - "MATCH (:Person)-[r:EMPLOYED {execId: \$execId}]->(:Company) SET r.role = 'EM'", params) - .consume() - session - .run("MATCH (:Person)-[r:EMPLOYED {execId: \$execId}]->(:Company) DELETE r", params) - .consume() + session.run("CREATE (:Person)-[:EMPLOYED {role: 'SWE'}]->(:Company)").consume() + session.run("MATCH (:Person)-[r:EMPLOYED]->(:Company) SET r.role = 'EM'").consume() + session.run("MATCH (:Person)-[r:EMPLOYED]->(:Company) DELETE r").consume() TopicVerifier.create(createsConsumer) .assertMessageValue { value -> @@ -375,7 +314,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .startLabelledAs("Person") .endLabelledAs("Company") .hasNoBeforeState() - .hasAfterStateProperties(mapOf("role" to "SWE", "execId" to executionId)) + .hasAfterStateProperties(mapOf("role" to "SWE")) } .verifyWithin(Duration.ofSeconds(30)) TopicVerifier.create(updatesConsumer) @@ -386,8 +325,8 @@ abstract class Neo4jCdcSourceRelationshipsIT { .hasType("EMPLOYED") .startLabelledAs("Person") .endLabelledAs("Company") - .hasBeforeStateProperties(mapOf("role" to "SWE", "execId" to executionId)) - .hasAfterStateProperties(mapOf("role" to "EM", "execId" to executionId)) + .hasBeforeStateProperties(mapOf("role" to "SWE")) + .hasAfterStateProperties(mapOf("role" to "EM")) } .verifyWithin(Duration.ofSeconds(30)) TopicVerifier.create(deletesConsumer) @@ -398,7 +337,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .hasType("EMPLOYED") .startLabelledAs("Person") .endLabelledAs("Company") - .hasBeforeStateProperties(mapOf("role" to "EM", "execId" to executionId)) + .hasBeforeStateProperties(mapOf("role" to "EM")) .hasNoAfterState() } .verifyWithin(Duration.ofSeconds(30)) @@ -419,27 +358,20 @@ abstract class Neo4jCdcSourceRelationshipsIT { arrayOf(CdcMetadata(key = "txMetadata.testLabel", value = "B")))))) @Test fun `should read changes marked with specific transaction metadata attribute`( - testInfo: TestInfo, @TopicConsumer(topic = "neo4j-cdc-metadata-rel", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session ) { - val executionId = testInfo.displayName + System.currentTimeMillis() - val params = mapOf("execId" to executionId) val transaction1 = session.beginTransaction( TransactionConfig.builder().withMetadata(mapOf("testLabel" to "A")).build()) - transaction1 - .run("CREATE (:Person)-[:EMPLOYED {execId: \$execId, role: 'SWE'}]->(:Company)", params) - .consume() + transaction1.run("CREATE (:Person)-[:EMPLOYED {role: 'SWE'}]->(:Company)").consume() transaction1.commit() val transaction2 = session.beginTransaction( TransactionConfig.builder().withMetadata(mapOf("testLabel" to "B")).build()) - transaction2 - .run("CREATE (:Person)-[:EMPLOYED {execId: \$execId, role: 'EM'}]->(:Company)", params) - .consume() + transaction2.run("CREATE (:Person)-[:EMPLOYED {role: 'EM'}]->(:Company)").consume() transaction2.commit() TopicVerifier.create(consumer) @@ -451,7 +383,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .startLabelledAs("Person") .endLabelledAs("Company") .hasNoBeforeState() - .hasAfterStateProperties(mapOf("role" to "EM", "execId" to executionId)) + .hasAfterStateProperties(mapOf("role" to "EM")) .hasTxMetadata(mapOf("testLabel" to "B")) } .verifyWithin(Duration.ofSeconds(30)) @@ -470,12 +402,10 @@ abstract class Neo4jCdcSourceRelationshipsIT { arrayOf(CdcSourceParam("(:Person)-[:EMPLOYED]->(:Company)")))))) @Test fun `should read changes containing relationship keys`( - testInfo: TestInfo, @TopicConsumer(topic = "neo4j-cdc-keys-rel", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session ) { - val executionId = testInfo.displayName + System.currentTimeMillis() session .run( "CREATE CONSTRAINT employedId FOR ()-[r:EMPLOYED]->() REQUIRE r.id IS RELATIONSHIP KEY") @@ -485,11 +415,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { "CREATE CONSTRAINT employedRole FOR ()-[r:EMPLOYED]->() REQUIRE r.role IS RELATIONSHIP KEY") .consume() - session - .run( - "CREATE (:Person)-[:EMPLOYED {execId: \$execId, id: 1, role: 'SWE'}]->(:Company)", - mapOf("execId" to executionId)) - .consume() + session.run("CREATE (:Person)-[:EMPLOYED {id: 1, role: 'SWE'}]->(:Company)").consume() TopicVerifier.create(consumer) .assertMessageValue { value -> @@ -500,7 +426,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .startLabelledAs("Person") .endLabelledAs("Company") .hasNoBeforeState() - .hasAfterStateProperties(mapOf("id" to 1L, "role" to "SWE", "execId" to executionId)) + .hasAfterStateProperties(mapOf("id" to 1L, "role" to "SWE")) .hasRelationshipKeys(listOf(mapOf("id" to 1L), mapOf("role" to "SWE"))) } .verifyWithin(Duration.ofSeconds(30)) @@ -511,19 +437,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { class Neo4jCdcSourceRelationshipsAvroIT : Neo4jCdcSourceRelationshipsIT() @KeyValueConverter(key = JSON_SCHEMA, value = JSON_SCHEMA) -class Neo4jCdcSourceRelationshipsJsonIT : Neo4jCdcSourceRelationshipsIT() { - - @Disabled("Json schema doesn't tolerate when an optional field changes the name") - override fun `should read changes with different properties using the default topic compatibility mode`( - testInfo: TestInfo, - consumer: ConvertingKafkaConsumer, - session: Session - ) { - super - .`should read changes with different properties using the default topic compatibility mode`( - testInfo, consumer, session) - } -} +class Neo4jCdcSourceRelationshipsJsonIT : Neo4jCdcSourceRelationshipsIT() @KeyValueConverter(key = PROTOBUF, value = PROTOBUF) class Neo4jCdcSourceRelationshipsProtobufIT : Neo4jCdcSourceRelationshipsIT() diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt index 9701b1b45..c02c38108 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jSourceQueryIT.kt @@ -16,23 +16,11 @@ */ package org.neo4j.connectors.kafka.source -import io.kotest.matchers.equality.shouldBeEqualToComparingFields import io.kotest.matchers.shouldBe import java.time.Duration -import java.time.LocalDate -import java.time.LocalDateTime -import java.time.LocalTime import java.time.OffsetDateTime -import java.time.OffsetTime -import java.time.ZoneId import java.time.ZoneOffset -import java.time.ZonedDateTime -import org.apache.kafka.connect.data.Struct import org.junit.jupiter.api.Test -import org.neo4j.connectors.kafka.data.DynamicTypes -import org.neo4j.connectors.kafka.data.PropertyType -import org.neo4j.connectors.kafka.data.PropertyType.schema -import org.neo4j.connectors.kafka.data.TemporalDataSchemaType import org.neo4j.connectors.kafka.testing.MapSupport.excludingKeys import org.neo4j.connectors.kafka.testing.TestSupport.runTest import org.neo4j.connectors.kafka.testing.assertions.TopicVerifier @@ -178,153 +166,6 @@ abstract class Neo4jSourceQueryIT { } .verifyWithin(Duration.ofSeconds(30)) } - - @Neo4jSource( - topic = TOPIC, - strategy = SourceStrategy.QUERY, - streamingProperty = "timestamp", - startFrom = "EARLIEST", - query = - "MATCH (ts:TestSource) WHERE ts.timestamp > \$lastCheck RETURN " + - "ts.localDate AS localDate, " + - "ts.localDatetime AS localDatetime, " + - "ts.localTime AS localTime, " + - "ts.zonedDatetime AS zonedDatetime, " + - "ts.offsetDatetime AS offsetDatetime, " + - "ts.offsetTime AS offsetTime, " + - "ts.timestamp AS timestamp", - temporalDataSchemaType = TemporalDataSchemaType.STRUCT, - ) - @Test - fun `should return struct temporal types`( - @TopicConsumer(topic = TOPIC, offset = "earliest") consumer: ConvertingKafkaConsumer, - session: Session - ) = runTest { - session - .run( - "CREATE (:TestSource {" + - "localDate: date('2024-01-01'), " + - "localDatetime: localdatetime('2024-01-01T12:00:00'), " + - "localTime: localtime('12:00:00'), " + - "zonedDatetime: datetime('2024-01-01T12:00:00[Europe/Stockholm]'), " + - "offsetDatetime: datetime('2024-01-01T12:00:00Z'), " + - "offsetTime: time('12:00:00Z'), " + - "timestamp: 0})") - .consume() - - TopicVerifier.create(consumer) - .assertMessageValue { value -> - value.getStruct("localDate") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalDate.of(2024, 1, 1), - ) as Struct - - value.getStruct("localDatetime") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalDateTime.of(2024, 1, 1, 12, 0, 0), - ) as Struct - - value.getStruct("localTime") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalTime.of(12, 0, 0), - ) as Struct - - value.getStruct("zonedDatetime") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), - ) as Struct - - value.getStruct("offsetDatetime") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), - ) as Struct - - value.getStruct("offsetTime") shouldBeEqualToComparingFields - DynamicTypes.toConnectValue( - PropertyType.schema, - OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), - ) as Struct - } - .verifyWithin(Duration.ofSeconds(300)) - } - - @Neo4jSource( - topic = TOPIC, - strategy = SourceStrategy.QUERY, - streamingProperty = "timestamp", - startFrom = "EARLIEST", - query = - "MATCH (ts:TestSource) WHERE ts.timestamp > \$lastCheck RETURN " + - "ts.localDate AS localDate, " + - "ts.localDatetime AS localDatetime, " + - "ts.localTime AS localTime, " + - "ts.zonedDatetime AS zonedDatetime, " + - "ts.offsetDatetime AS offsetDatetime, " + - "ts.offsetTime AS offsetTime, " + - "ts.timestamp AS timestamp", - temporalDataSchemaType = TemporalDataSchemaType.STRING) - @Test - fun `should return string temporal types`( - @TopicConsumer(topic = TOPIC, offset = "earliest") consumer: ConvertingKafkaConsumer, - session: Session - ) = runTest { - session - .run( - "CREATE (:TestSource {" + - "localDate: date('2024-01-01'), " + - "localDatetime: localdatetime('2024-01-01T12:00:00'), " + - "localTime: localtime('12:00:00'), " + - "zonedDatetime: datetime('2024-01-01T12:00:00[Europe/Stockholm]'), " + - "offsetDatetime: datetime('2024-01-01T12:00:00Z'), " + - "offsetTime: time('12:00:00Z'), " + - "timestamp: 0})") - .consume() - - TopicVerifier.create(consumer) - .assertMessageValue { value -> - value.getString("localDate") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalDate.of(2024, 1, 1), - ) - - value.getString("localDatetime") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalDateTime.of(2024, 1, 1, 12, 0, 0), - ) - - value.getString("localTime") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - LocalTime.of(12, 0, 0), - ) - - value.getString("zonedDatetime") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - ZonedDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneId.of("Europe/Stockholm")), - ) - - value.getString("offsetDatetime") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - OffsetDateTime.of(2024, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC), - ) - - value.getString("offsetTime") shouldBe - DynamicTypes.toConnectValue( - PropertyType.schema, - OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC), - ) - } - .verifyWithin(Duration.ofSeconds(300)) - } } @KeyValueConverter(key = AVRO, value = AVRO) class Neo4jSourceAvroIT : Neo4jSourceQueryIT() diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt index bb616ee10..c32aed1d4 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcTask.kt @@ -74,7 +74,7 @@ class Neo4jCdcTask : SourceTask() { offset = AtomicReference(resumeFrom(config, cdc)) log.info("resuming from offset: ${offset.get()}") - changeEventConverter = ChangeEventConverter(config.temporalDataSchemaType) + changeEventConverter = ChangeEventConverter() } override fun stop() { diff --git a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt index 8fa1241bd..99eb74c42 100644 --- a/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt +++ b/source/src/main/kotlin/org/neo4j/connectors/kafka/source/SourceConfiguration.kt @@ -41,7 +41,6 @@ import org.neo4j.connectors.kafka.configuration.helpers.Validators import org.neo4j.connectors.kafka.configuration.helpers.Validators.validateNonEmptyIfVisible import org.neo4j.connectors.kafka.configuration.helpers.parseSimpleString import org.neo4j.connectors.kafka.configuration.helpers.toSimpleString -import org.neo4j.connectors.kafka.data.TemporalDataSchemaType import org.neo4j.driver.TransactionConfig enum class SourceType(val description: String) { @@ -91,10 +90,6 @@ class SourceConfiguration(originals: Map<*, *>) : val topic get(): String = getString(QUERY_TOPIC) - val temporalDataSchemaType - get(): TemporalDataSchemaType = - TemporalDataSchemaType.valueOf(getString(TEMPORAL_DATA_SCHEMA_TYPE)) - val partition get(): Map { return when (strategy) { @@ -409,7 +404,6 @@ class SourceConfiguration(originals: Map<*, *>) : const val QUERY_TOPIC = "neo4j.query.topic" const val CDC_POLL_INTERVAL = "neo4j.cdc.poll-interval" const val CDC_POLL_DURATION = "neo4j.cdc.poll-duration" - const val TEMPORAL_DATA_SCHEMA_TYPE = "neo4j.temporal-schema.type" private const val GROUP_NAME_TOPIC = "topic" private const val GROUP_NAME_INDEX = "index" private const val GROUP_NAME_METADATA = "metadata" @@ -609,13 +603,5 @@ class SourceConfiguration(originals: Map<*, *>) : Recommenders.visibleIf(STRATEGY, Predicate.isEqual(SourceType.CDC.name)) validator = Validators.pattern(SIMPLE_DURATION_PATTERN) }) - .define( - ConfigKeyBuilder.of(TEMPORAL_DATA_SCHEMA_TYPE, ConfigDef.Type.STRING) { - importance = ConfigDef.Importance.MEDIUM - defaultValue = TemporalDataSchemaType.STRUCT.name - group = Groups.CONNECTOR.title - validator = Validators.enum(TemporalDataSchemaType::class.java) - recommender = Recommenders.enum(TemporalDataSchemaType::class.java) - }) } } diff --git a/source/src/test/kotlin/org/neo4j/connectors/kafka/source/SourceConfigurationTest.kt b/source/src/test/kotlin/org/neo4j/connectors/kafka/source/SourceConfigurationTest.kt index 1940d3a15..ec239a41a 100644 --- a/source/src/test/kotlin/org/neo4j/connectors/kafka/source/SourceConfigurationTest.kt +++ b/source/src/test/kotlin/org/neo4j/connectors/kafka/source/SourceConfigurationTest.kt @@ -147,17 +147,6 @@ class SourceConfigurationTest { it shouldHaveMessage "Invalid value for configuration neo4j.query.streaming-property: Must not be blank." } - - assertFailsWith(ConfigException::class) { - SourceConfiguration( - mapOf( - Neo4jConfiguration.URI to "neo4j://localhost", - SourceConfiguration.TEMPORAL_DATA_SCHEMA_TYPE to "none")) - } - .also { - it shouldHaveMessage - "Invalid value none for configuration neo4j.temporal-schema.type: Must be one of: 'STRUCT', 'STRING'." - } } @Test diff --git a/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/format/KafkaConverter.kt b/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/format/KafkaConverter.kt index d003af7d6..e9d3768a2 100644 --- a/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/format/KafkaConverter.kt +++ b/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/format/KafkaConverter.kt @@ -32,7 +32,8 @@ import org.neo4j.connectors.kafka.testing.format.json.JsonSchemaSerializer import org.neo4j.connectors.kafka.testing.format.protobuf.ProtobufSerializer import org.neo4j.connectors.kafka.testing.format.string.StringSerializer -private val PROTOBUF_OPTIONS = mapOf("optional.for.nullables" to "true") +private val PROTOBUF_OPTIONS = + mapOf("enhanced.protobuf.schema.support" to "true", "optional.for.nullables" to "true") enum class KafkaConverter( val className: String, diff --git a/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSource.kt b/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSource.kt index b24d102d6..2e70e1828 100644 --- a/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSource.kt +++ b/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSource.kt @@ -17,7 +17,6 @@ package org.neo4j.connectors.kafka.testing.source import org.junit.jupiter.api.extension.ExtendWith -import org.neo4j.connectors.kafka.data.TemporalDataSchemaType import org.neo4j.connectors.kafka.testing.DEFAULT_TO_ENV @Target(AnnotationTarget.FUNCTION) @@ -36,7 +35,6 @@ annotation class Neo4jSource( val startFrom: String = "NOW", val startFromValue: String = "", val strategy: SourceStrategy = SourceStrategy.QUERY, - val temporalDataSchemaType: TemporalDataSchemaType = TemporalDataSchemaType.STRUCT, // QUERY strategy val topic: String = "", diff --git a/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSourceExtension.kt b/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSourceExtension.kt index 70c3432ec..16d05c923 100644 --- a/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSourceExtension.kt +++ b/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSourceExtension.kt @@ -164,8 +164,8 @@ internal class Neo4jSourceExtension( cdcOperations = sourceAnnotation.cdc.paramAsMap(CdcSourceTopic::operations), cdcChangesTo = sourceAnnotation.cdc.paramAsMap(CdcSourceTopic::changesTo), cdcMetadata = sourceAnnotation.cdc.metadataAsMap(), - cdcKeySerializations = sourceAnnotation.cdc.keySerializationsAsMap(), - temporalDataSchemaType = sourceAnnotation.temporalDataSchemaType) + cdcKeySerializations = sourceAnnotation.cdc.keySerializationsAsMap()) + source.register(kafkaConnectExternalUri.resolve(sourceAnnotation)) topicRegistry.log() } diff --git a/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSourceRegistration.kt b/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSourceRegistration.kt index 3e587d0f8..5f227169f 100644 --- a/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSourceRegistration.kt +++ b/testing/src/main/kotlin/org/neo4j/connectors/kafka/testing/source/Neo4jSourceRegistration.kt @@ -18,7 +18,6 @@ package org.neo4j.connectors.kafka.testing.source import java.net.URI import java.time.Duration -import org.neo4j.connectors.kafka.data.TemporalDataSchemaType import org.neo4j.connectors.kafka.testing.RegistrationSupport.randomizedName import org.neo4j.connectors.kafka.testing.RegistrationSupport.registerConnector import org.neo4j.connectors.kafka.testing.RegistrationSupport.unregisterConnector @@ -48,7 +47,6 @@ internal class Neo4jSourceRegistration( cdcChangesTo: Map>, cdcMetadata: Map>>, cdcKeySerializations: Map, - temporalDataSchemaType: TemporalDataSchemaType ) { private val name: String = randomizedName("Neo4jSourceConnector") @@ -80,8 +78,6 @@ internal class Neo4jSourceRegistration( } putAll(valueConverter.additionalProperties.mapKeys { "value.converter.${it.key}" }) - put("neo4j.temporal-schema.type", temporalDataSchemaType.name) - if (strategy == QUERY) { put("neo4j.query.topic", topic) put("neo4j.query", query) From 9f1bb8dfb6ef72ea87c9fc01acff39d1e95f0809 Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Mon, 29 Jul 2024 16:13:24 +0100 Subject: [PATCH 8/9] test: review comments --- .../connectors/kafka/data/PropertyType.kt | 5 +- pom.xml | 1 + .../kafka/source/Neo4jCdcSourceNodesIT.kt | 16 +- .../source/Neo4jCdcSourceRelationshipsIT.kt | 174 +++++++++++++++--- 4 files changed, 166 insertions(+), 30 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt index 6061c1248..a4ff68898 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt @@ -276,7 +276,10 @@ object PropertyType { fun fromConnectValue(value: Struct?): Any? { return value?.let { for (f in it.schema().fields()) { - if (it.getWithoutDefault(f.name()) == null) { + val fieldValue = it.getWithoutDefault(f.name()) + // not set list fields are returned back as empty lists, so we are looking for a non-empty + // field here + if (fieldValue == null || (fieldValue is Collection<*> && fieldValue.isEmpty())) { continue } diff --git a/pom.xml b/pom.xml index 870aeecdd..44b778278 100644 --- a/pom.xml +++ b/pom.xml @@ -81,6 +81,7 @@ 2022.9.2 4.4.9 UTF-8 + 3.19.6 2023.0.7 1.7.36 diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt index e4dd78f48..eb62ebe60 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt @@ -56,7 +56,7 @@ abstract class Neo4jCdcSourceNodesIT { patterns = arrayOf(CdcSourceParam("(:TestSource{name,+surname,-age})")))))) @Test - fun `should read changes caught by patterns`( + fun `should publish changes caught by patterns`( @TopicConsumer(topic = "neo4j-cdc-topic", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -106,7 +106,7 @@ abstract class Neo4jCdcSourceNodesIT { topic = "neo4j-cdc-topic-prop-remove-add", patterns = arrayOf(CdcSourceParam("(:TestSource)")))))) @Test - fun `should read property removal and additions`( + fun `should publish property removal and additions`( @TopicConsumer(topic = "neo4j-cdc-topic-prop-remove-add", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -183,7 +183,7 @@ abstract class Neo4jCdcSourceNodesIT { operations = arrayOf(CdcSourceParam(value = "UPDATE")), changesTo = arrayOf(CdcSourceParam(value = "surname,email")))))) @Test - fun `should read only specified field changes on update`( + fun `should publish only specified field changes on update`( @TopicConsumer(topic = "neo4j-cdc-update-topic", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -239,7 +239,7 @@ abstract class Neo4jCdcSourceNodesIT { topic = "neo4j-cdc-create-inc", patterns = arrayOf(CdcSourceParam("(:TestSource)")))))) @Test - fun `should read changes with different properties using the default topic compatibility mode`( + fun `should publish changes with different properties using the default topic compatibility mode`( @TopicConsumer(topic = "neo4j-cdc-create-inc", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -288,7 +288,7 @@ abstract class Neo4jCdcSourceNodesIT { patterns = arrayOf(CdcSourceParam("(:TestSource)")), operations = arrayOf(CdcSourceParam("DELETE")))))) @Test - fun `should read each operation to a separate topic`( + fun `should publish each operation to a separate topic`( @TopicConsumer(topic = "cdc-creates", offset = "earliest") createsConsumer: ConvertingKafkaConsumer, @TopicConsumer(topic = "cdc-updates", offset = "earliest") @@ -433,7 +433,7 @@ abstract class Neo4jCdcSourceNodesIT { CdcSourceTopic( topic = "cdc", patterns = arrayOf(CdcSourceParam("(:TestSource)")))))) @Test - fun `should read each operation to a single topic`( + fun `should publish each operation to a single topic`( @TopicConsumer(topic = "cdc", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session ) { @@ -493,7 +493,7 @@ abstract class Neo4jCdcSourceNodesIT { metadata = arrayOf(CdcMetadata(key = "txMetadata.testLabel", value = "B")))))) @Test - fun `should read changes marked with specific transaction metadata attribute`( + fun `should publish changes marked with specific transaction metadata attribute`( @TopicConsumer(topic = "neo4j-cdc-metadata", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -534,7 +534,7 @@ abstract class Neo4jCdcSourceNodesIT { topic = "neo4j-cdc-keys", patterns = arrayOf(CdcSourceParam("(:TestSource)")))))) @Test - fun `should read changes containing node keys`( + fun `should publish changes containing node keys`( @TopicConsumer(topic = "neo4j-cdc-keys", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt index a173fe429..fd79d0009 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt @@ -17,10 +17,13 @@ package org.neo4j.connectors.kafka.source import java.time.Duration +import java.time.LocalDate +import java.time.LocalDateTime import org.junit.jupiter.api.Test import org.neo4j.cdc.client.model.ChangeEvent -import org.neo4j.cdc.client.model.EntityOperation +import org.neo4j.cdc.client.model.EntityOperation.CREATE import org.neo4j.cdc.client.model.EntityOperation.DELETE +import org.neo4j.cdc.client.model.EntityOperation.UPDATE import org.neo4j.cdc.client.model.EventType.RELATIONSHIP import org.neo4j.connectors.kafka.testing.assertions.ChangeEventAssert.Companion.assertThat import org.neo4j.connectors.kafka.testing.assertions.TopicVerifier @@ -55,7 +58,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { CdcSourceParam( "(:TestSource)-[:RELIES_TO {weight,-rate}]->(:TestSource)")))))) @Test - fun `should read changes caught by patterns`( + fun `should publish changes caught by patterns`( @TopicConsumer(topic = "neo4j-cdc-rels", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -75,7 +78,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .hasType("RELIES_TO") .startLabelledAs("TestSource") .endLabelledAs("TestSource") @@ -85,7 +88,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.UPDATE) + .hasOperation(UPDATE) .hasType("RELIES_TO") .startLabelledAs("TestSource") .endLabelledAs("TestSource") @@ -116,7 +119,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { topic = "neo4j-cdc-rels-prop-remove-add", patterns = arrayOf(CdcSourceParam("()-[:RELIES_TO {}]->()")))))) @Test - fun `should read property removal and additions`( + fun `should publish property removal and additions`( @TopicConsumer(topic = "neo4j-cdc-rels-prop-remove-add", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -139,7 +142,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .hasType("RELIES_TO") .startLabelledAs("TestSource") .endLabelledAs("TestSource") @@ -149,7 +152,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.UPDATE) + .hasOperation(UPDATE) .hasType("RELIES_TO") .startLabelledAs("TestSource") .endLabelledAs("TestSource") @@ -159,7 +162,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.UPDATE) + .hasOperation(UPDATE) .hasType("RELIES_TO") .startLabelledAs("TestSource") .endLabelledAs("TestSource") @@ -193,7 +196,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { operations = arrayOf(CdcSourceParam(value = "UPDATE")), changesTo = arrayOf(CdcSourceParam(value = "a,c")))))) @Test - fun `should read only specified field changes on update`( + fun `should publish only specified field changes on update`( @TopicConsumer(topic = "neo4j-cdc-update-rel", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -207,7 +210,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.UPDATE) + .hasOperation(UPDATE) .hasType("R") .startLabelledAs("A") .endLabelledAs("B") @@ -217,7 +220,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.UPDATE) + .hasOperation(UPDATE) .hasType("R") .startLabelledAs("A") .endLabelledAs("B") @@ -239,7 +242,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { patterns = arrayOf(CdcSourceParam("(:Person)-[:IS_EMPLOYEE]->(:Company)")))))) @Test - open fun `should read changes with different properties using the default topic compatibility mode`( + open fun `should publish changes with different properties using the default topic compatibility mode`( @TopicConsumer(topic = "neo4j-cdc-create-inc-rel", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -251,7 +254,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .hasType("IS_EMPLOYEE") .startLabelledAs("Person") .endLabelledAs("Company") @@ -261,7 +264,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .hasType("IS_EMPLOYEE") .startLabelledAs("Person") .endLabelledAs("Company") @@ -292,7 +295,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { patterns = arrayOf(CdcSourceParam("(:Person)-[:EMPLOYED]->(:Company)")), operations = arrayOf(CdcSourceParam("DELETE")))))) @Test - fun `should read each operation to a separate topic`( + fun `should publish each operation to a separate topic`( @TopicConsumer(topic = "cdc-creates-rel", offset = "earliest") createsConsumer: ConvertingKafkaConsumer, @TopicConsumer(topic = "cdc-updates-rel", offset = "earliest") @@ -309,7 +312,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .hasType("EMPLOYED") .startLabelledAs("Person") .endLabelledAs("Company") @@ -321,7 +324,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.UPDATE) + .hasOperation(UPDATE) .hasType("EMPLOYED") .startLabelledAs("Person") .endLabelledAs("Company") @@ -343,6 +346,135 @@ abstract class Neo4jCdcSourceRelationshipsIT { .verifyWithin(Duration.ofSeconds(30)) } + @Neo4jSource( + startFrom = "EARLIEST", + strategy = CDC, + cdc = + CdcSource( + patternsIndexed = true, + topics = + arrayOf( + CdcSourceTopic( + topic = "cdc", + patterns = + arrayOf(CdcSourceParam("(:Person)-[:EMPLOYED]->(:Company)")))))) + @Test + fun `should publish changes with property type changes`( + @TopicConsumer(topic = "cdc", offset = "earliest") consumer: ConvertingKafkaConsumer, + session: Session + ) { + session + .run( + "CREATE (:Person)-[r:EMPLOYED]->(:Company) SET r = ${'$'}props", + mapOf("props" to mapOf("role" to "SWE"))) + .consume() + session + .run( + "MATCH (:Person)-[r:EMPLOYED]->(:Company) SET r += ${'$'}props", + mapOf( + "props" to + mapOf("role" to "EM", "since" to LocalDateTime.of(1999, 1, 1, 0, 0, 0, 0)))) + .consume() + session + .run( + "MATCH (:Person)-[r:EMPLOYED]->(:Company) SET r += ${'$'}props", + mapOf( + "props" to + mapOf("role" to listOf("EM", "SWE"), "since" to LocalDate.of(1999, 1, 1)))) + .consume() + + TopicVerifier.create(consumer) + .assertMessageValue { value -> + assertThat(value) + .hasEventType(RELATIONSHIP) + .hasOperation(CREATE) + .hasType("EMPLOYED") + .startLabelledAs("Person") + .endLabelledAs("Company") + .hasNoBeforeState() + .hasAfterStateProperties(mapOf("role" to "SWE")) + } + .assertMessageValue { value -> + assertThat(value) + .hasEventType(RELATIONSHIP) + .hasOperation(UPDATE) + .hasType("EMPLOYED") + .startLabelledAs("Person") + .endLabelledAs("Company") + .hasBeforeStateProperties(mapOf("role" to "SWE")) + .hasAfterStateProperties( + mapOf("role" to "EM", "since" to LocalDateTime.of(1999, 1, 1, 0, 0, 0, 0))) + } + .assertMessageValue { value -> + assertThat(value) + .hasEventType(RELATIONSHIP) + .hasOperation(UPDATE) + .hasType("EMPLOYED") + .startLabelledAs("Person") + .endLabelledAs("Company") + .hasBeforeStateProperties( + mapOf("role" to "EM", "since" to LocalDateTime.of(1999, 1, 1, 0, 0, 0, 0))) + .hasAfterStateProperties( + mapOf("role" to listOf("EM", "SWE"), "since" to LocalDate.of(1999, 1, 1))) + } + .verifyWithin(Duration.ofSeconds(30)) + } + + @Neo4jSource( + startFrom = "EARLIEST", + strategy = CDC, + cdc = + CdcSource( + patternsIndexed = true, + topics = + arrayOf( + CdcSourceTopic( + topic = "cdc", + patterns = + arrayOf(CdcSourceParam("(:Person)-[:EMPLOYED]->(:Company)")))))) + @Test + fun `should publish each operation to a single topic`( + @TopicConsumer(topic = "cdc", offset = "earliest") consumer: ConvertingKafkaConsumer, + session: Session + ) { + session.run("CREATE (:Person)-[:EMPLOYED {role: 'SWE'}]->(:Company)").consume() + session.run("MATCH (:Person)-[r:EMPLOYED]->(:Company) SET r.role = 'EM'").consume() + session.run("MATCH (:Person)-[r:EMPLOYED]->(:Company) DELETE r").consume() + + TopicVerifier.create(consumer) + .assertMessageValue { value -> + assertThat(value) + .hasEventType(RELATIONSHIP) + .hasOperation(CREATE) + .hasType("EMPLOYED") + .startLabelledAs("Person") + .endLabelledAs("Company") + .hasNoBeforeState() + .hasAfterStateProperties(mapOf("role" to "SWE")) + } + .assertMessageValue { value -> + assertThat(value) + .hasEventType(RELATIONSHIP) + .hasOperation(UPDATE) + .hasType("EMPLOYED") + .startLabelledAs("Person") + .endLabelledAs("Company") + .hasBeforeStateProperties(mapOf("role" to "SWE")) + .hasAfterStateProperties(mapOf("role" to "EM")) + } + .assertMessageValue { value -> + assertThat(value) + .hasEventType(RELATIONSHIP) + .hasOperation(DELETE) + .hasType("EMPLOYED") + .startLabelledAs("Person") + .endLabelledAs("Company") + .hasBeforeStateProperties(mapOf("role" to "EM")) + .hasNoAfterState() + } + .verifyWithin(Duration.ofSeconds(30)) + } + @Neo4jSource( startFrom = "EARLIEST", strategy = CDC, @@ -357,7 +489,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { metadata = arrayOf(CdcMetadata(key = "txMetadata.testLabel", value = "B")))))) @Test - fun `should read changes marked with specific transaction metadata attribute`( + fun `should publish changes marked with specific transaction metadata attribute`( @TopicConsumer(topic = "neo4j-cdc-metadata-rel", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -378,7 +510,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .hasType("EMPLOYED") .startLabelledAs("Person") .endLabelledAs("Company") @@ -401,7 +533,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { patterns = arrayOf(CdcSourceParam("(:Person)-[:EMPLOYED]->(:Company)")))))) @Test - fun `should read changes containing relationship keys`( + fun `should publish changes containing relationship keys`( @TopicConsumer(topic = "neo4j-cdc-keys-rel", offset = "earliest") consumer: ConvertingKafkaConsumer, session: Session @@ -421,7 +553,7 @@ abstract class Neo4jCdcSourceRelationshipsIT { .assertMessageValue { value -> assertThat(value) .hasEventType(RELATIONSHIP) - .hasOperation(EntityOperation.CREATE) + .hasOperation(CREATE) .hasType("EMPLOYED") .startLabelledAs("Person") .endLabelledAs("Company") From c6abde4f0aeb28238e974db669b71a4388aa7c3e Mon Sep 17 00:00:00 2001 From: Ali Ince Date: Tue, 30 Jul 2024 12:09:38 +0100 Subject: [PATCH 9/9] test: add tests for empty arrays and lists --- .../connectors/kafka/data/DynamicTypes.kt | 4 + .../connectors/kafka/data/PropertyType.kt | 123 +++++++++++++----- .../connectors/kafka/data/DynamicTypesTest.kt | 51 +++++++- .../connectors/kafka/data/PropertyTypeTest.kt | 10 ++ .../neo4j/connectors/kafka/data/TypesTest.kt | 4 + .../kafka/source/Neo4jCdcSourceIT.kt | 1 - .../kafka/source/Neo4jCdcSourceNodesIT.kt | 45 +++++++ .../source/Neo4jCdcSourceRelationshipsIT.kt | 47 +++++++ 8 files changed, 244 insertions(+), 41 deletions(-) diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt index 0325f1a4e..41dc44d65 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/DynamicTypes.kt @@ -271,6 +271,10 @@ object DynamicTypes { .build() is Collection<*> -> { val elementTypes = value.map { it?.javaClass?.kotlin }.toSet() + if (elementTypes.isEmpty()) { + return PropertyType.schema + } + val elementType = elementTypes.singleOrNull() if (elementType != null && isSimplePropertyType(elementType)) { return PropertyType.schema diff --git a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt index a4ff68898..65aa55472 100644 --- a/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt +++ b/common/src/main/kotlin/org/neo4j/connectors/kafka/data/PropertyType.kt @@ -72,6 +72,34 @@ object PropertyType { internal const val POINT = "SP" internal const val POINT_LIST = "LSP" + private val SIMPLE_TYPE_FIELDS = + listOf( + BOOLEAN, + LONG, + FLOAT, + STRING, + BYTES, + LOCAL_DATE, + LOCAL_DATE_TIME, + LOCAL_TIME, + ZONED_DATE_TIME, + OFFSET_TIME, + DURATION, + POINT) + private val LIST_TYPE_FIELDS = + listOf( + BOOLEAN_LIST, + LONG_LIST, + FLOAT_LIST, + STRING_LIST, + LOCAL_DATE_LIST, + LOCAL_DATE_TIME_LIST, + LOCAL_TIME_LIST, + ZONED_DATE_TIME_LIST, + OFFSET_TIME_LIST, + DURATION_LIST, + POINT_LIST) + internal val durationSchema: Schema = SchemaBuilder(Schema.Type.STRUCT) .field(MONTHS, Schema.INT64_SCHEMA) @@ -175,6 +203,10 @@ object PropertyType { is Array<*> -> asList(value.toList(), value::class.java.componentType.kotlin) is Iterable<*> -> { val elementTypes = value.map { it?.javaClass?.kotlin }.toSet() + if (elementTypes.isEmpty()) { + return asList(value, Int::class) + } + val elementType = elementTypes.singleOrNull() if (elementType != null) { return asList(value, elementType) @@ -275,50 +307,73 @@ object PropertyType { fun fromConnectValue(value: Struct?): Any? { return value?.let { - for (f in it.schema().fields()) { - val fieldValue = it.getWithoutDefault(f.name()) - // not set list fields are returned back as empty lists, so we are looking for a non-empty - // field here - if (fieldValue == null || (fieldValue is Collection<*> && fieldValue.isEmpty())) { - continue - } - - return when (f.name()) { - BOOLEAN -> it.get(f) as Boolean - BOOLEAN_LIST -> it.get(f) as List<*> - LONG -> it.get(f) as Long - LONG_LIST -> it.get(f) as List<*> - FLOAT -> it.get(f) as Double - FLOAT_LIST -> it.get(f) as List<*> - STRING -> it.get(f) as String - STRING_LIST -> it.get(f) as List<*> + val simpleFieldAndValue = + SIMPLE_TYPE_FIELDS.firstNotNullOfOrNull { f -> + val fieldValue = it.getWithoutDefault(f) + if (fieldValue != null) Pair(f, fieldValue) else null + } + if (simpleFieldAndValue != null) { + return when (simpleFieldAndValue.first) { + BOOLEAN -> simpleFieldAndValue.second as Boolean + LONG -> simpleFieldAndValue.second as Long + FLOAT -> simpleFieldAndValue.second as Double + STRING -> simpleFieldAndValue.second as String BYTES -> - when (val bytes = it.get(f)) { + when (val bytes = simpleFieldAndValue.second) { is ByteArray -> bytes is ByteBuffer -> bytes.array() else -> throw IllegalArgumentException( "unsupported BYTES value: ${bytes.javaClass.name}") } - LOCAL_DATE -> parseLocalDate((it.get(f) as String)) - LOCAL_DATE_LIST -> (it.get(f) as List).map { s -> parseLocalDate(s) } - LOCAL_TIME -> parseLocalTime((it.get(f) as String)) - LOCAL_TIME_LIST -> (it.get(f) as List).map { s -> parseLocalTime(s) } - LOCAL_DATE_TIME -> parseLocalDateTime((it.get(f) as String)) - LOCAL_DATE_TIME_LIST -> (it.get(f) as List).map { s -> parseLocalDateTime(s) } - ZONED_DATE_TIME -> parseZonedDateTime((it.get(f) as String)) - ZONED_DATE_TIME_LIST -> (it.get(f) as List).map { s -> parseZonedDateTime(s) } - OFFSET_TIME -> parseOffsetTime((it.get(f) as String)) - OFFSET_TIME_LIST -> (it.get(f) as List).map { s -> parseOffsetTime(s) } - DURATION -> toDuration((it.get(f) as Struct)) - DURATION_LIST -> (it.get(f) as List).map { s -> toDuration(s) } - POINT -> toPoint((it.get(f) as Struct)) - POINT_LIST -> (it.get(f) as List).map { s -> toPoint(s) } - else -> throw IllegalArgumentException("unsupported neo4j type: ${f.name()}") + LOCAL_DATE -> parseLocalDate((simpleFieldAndValue.second as String)) + LOCAL_TIME -> parseLocalTime((simpleFieldAndValue.second as String)) + LOCAL_DATE_TIME -> parseLocalDateTime((simpleFieldAndValue.second as String)) + ZONED_DATE_TIME -> parseZonedDateTime((simpleFieldAndValue.second as String)) + OFFSET_TIME -> parseOffsetTime((simpleFieldAndValue.second as String)) + DURATION -> toDuration((simpleFieldAndValue.second as Struct)) + POINT -> toPoint((simpleFieldAndValue.second as Struct)) + else -> + throw IllegalArgumentException( + "unsupported simple type: ${simpleFieldAndValue.first}: ${simpleFieldAndValue.second.javaClass.name}") + } + } + + val listFieldAndValue = + LIST_TYPE_FIELDS.firstNotNullOfOrNull { f -> + val fieldValue = it.getWithoutDefault(f) + if (fieldValue != null && (fieldValue is Collection<*> && fieldValue.isNotEmpty())) + Pair(f, fieldValue) + else null + } + if (listFieldAndValue != null) { + return when (listFieldAndValue.first) { + BOOLEAN_LIST -> listFieldAndValue.second as List<*> + LONG_LIST -> listFieldAndValue.second as List<*> + FLOAT_LIST -> listFieldAndValue.second as List<*> + STRING_LIST -> listFieldAndValue.second as List<*> + LOCAL_DATE_LIST -> + (listFieldAndValue.second as List).map { s -> parseLocalDate(s) } + LOCAL_TIME_LIST -> + (listFieldAndValue.second as List).map { s -> parseLocalTime(s) } + LOCAL_DATE_TIME_LIST -> + (listFieldAndValue.second as List).map { s -> parseLocalDateTime(s) } + ZONED_DATE_TIME_LIST -> + (listFieldAndValue.second as List).map { s -> parseZonedDateTime(s) } + OFFSET_TIME_LIST -> + (listFieldAndValue.second as List).map { s -> parseOffsetTime(s) } + DURATION_LIST -> (listFieldAndValue.second as List).map { s -> toDuration(s) } + POINT_LIST -> (listFieldAndValue.second as List).map { s -> toPoint(s) } + else -> + throw IllegalArgumentException( + "unsupported list type: ${listFieldAndValue.first}: ${listFieldAndValue.second.javaClass.name}") } } - return null + // Protobuf does not support NULLs in repeated fields, so we always receive LIST typed fields + // as empty lists. If we could not find a simple field and also a non-empty list field, we + // assume the value is an empty list. + return emptyList() } } diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt index 68ba6cbe7..8f73d465f 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/DynamicTypesTest.kt @@ -21,6 +21,7 @@ import io.kotest.assertions.withClue import io.kotest.matchers.shouldBe import io.kotest.matchers.throwable.shouldHaveMessage import java.nio.ByteBuffer +import java.time.Duration import java.time.LocalDate import java.time.LocalDateTime import java.time.LocalTime @@ -45,6 +46,7 @@ import org.neo4j.connectors.kafka.data.PropertyType.LOCAL_DATE import org.neo4j.driver.Value import org.neo4j.driver.Values import org.neo4j.driver.types.Node +import org.neo4j.driver.types.Point import org.neo4j.driver.types.Relationship class DynamicTypesTest { @@ -115,44 +117,55 @@ class DynamicTypesTest { Arguments.of("bool array", BooleanArray(1) { true }, null), Arguments.of("array (bool)", Array(1) { true }, null), Arguments.of("list (bool)", listOf(true), null), + Arguments.of("empty list (bool)", emptyList(), null), Arguments.of("short array (empty)", ShortArray(0), null), Arguments.of("short array", ShortArray(1) { 1.toShort() }, listOf(1L)), Arguments.of("array (short)", Array(1) { 1.toShort() }, listOf(1L)), Arguments.of("list (short)", listOf(1.toShort()), listOf(1L)), + Arguments.of("empty list (short)", emptyList(), null), Arguments.of("int array (empty)", IntArray(0), null), Arguments.of("int array", IntArray(1) { 1 }, listOf(1L)), Arguments.of("array (int)", Array(1) { 1 }, listOf(1L)), Arguments.of("list (int)", listOf(1), listOf(1L)), + Arguments.of("empty list (int)", emptyList(), null), Arguments.of("long array (empty)", LongArray(0), null), Arguments.of("long array", LongArray(1) { 1.toLong() }, null), Arguments.of("array (long)", Array(1) { 1.toLong() }, null), Arguments.of("list (long)", listOf(1L), null), + Arguments.of("empty list (long)", emptyList(), null), Arguments.of("float array (empty)", FloatArray(0), null), Arguments.of("float array", FloatArray(1) { 1.toFloat() }, listOf(1.toDouble())), Arguments.of("array (float)", Array(1) { 1.toFloat() }, null), Arguments.of("list (float)", listOf(1.toFloat()), null), + Arguments.of("empty list (float)", emptyList(), null), Arguments.of("double array (empty)", DoubleArray(0), null), Arguments.of("double array", DoubleArray(1) { 1.toDouble() }, null), Arguments.of("array (double)", Array(1) { 1.toDouble() }, null), Arguments.of("list (double)", listOf(1.toDouble()), null), + Arguments.of("empty list (double)", emptyList(), null), Arguments.of("array (string)", Array(1) { "a" }, null), Arguments.of("list (string)", listOf("a"), null), + Arguments.of("empty list (string)", emptyList(), null), Arguments.of("array (local date)", Array(1) { LocalDate.of(1999, 12, 31) }, null), Arguments.of("list (local date)", listOf(LocalDate.of(1999, 12, 31)), null), + Arguments.of("empty list (local date)", emptyList(), null), Arguments.of("array (local time)", Array(1) { LocalTime.of(23, 59, 59) }, null), Arguments.of("list (local time)", listOf(LocalTime.of(23, 59, 59)), null), + Arguments.of("empty list (local time)", emptyList(), null), Arguments.of( "array (local date time)", Array(1) { LocalDateTime.of(1999, 12, 31, 23, 59, 59) }, null), Arguments.of( "list (local date time)", listOf(LocalDateTime.of(1999, 12, 31, 23, 59, 59)), null), + Arguments.of("empty list (local date time)", emptyList(), null), Arguments.of( "array (offset time)", Array(1) { OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC) }, null), Arguments.of( "list (offset time)", listOf(OffsetTime.of(23, 59, 59, 0, ZoneOffset.UTC)), null), + Arguments.of("empty list (offset time)", emptyList(), null), Arguments.of( "array (offset date time)", Array(1) { OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC) }, @@ -161,6 +174,7 @@ class DynamicTypesTest { "list (offset date time)", listOf(OffsetDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneOffset.UTC)), null), + Arguments.of("empty list (offset date time)", emptyList(), null), Arguments.of( "array (zoned date time)", Array(1) { @@ -171,6 +185,7 @@ class DynamicTypesTest { "list (zoned date time)", listOf(ZonedDateTime.of(1999, 12, 31, 23, 59, 59, 0, ZoneId.of("Europe/London"))), null), + Arguments.of("empty list (zoned date time)", emptyList(), null), Arguments.of( "array (duration)", Array(1) { Values.isoDuration(12, 12, 59, 1230).asIsoDuration() }, @@ -179,6 +194,7 @@ class DynamicTypesTest { "list (duration)", listOf(Values.isoDuration(12, 12, 59, 1230).asIsoDuration()), null), + Arguments.of("empty list (duration)", emptyList(), null), Arguments.of( "array (point (2d))", Array(1) { Values.point(4326, 1.0, 2.0).asPoint() }, null), Arguments.of("list (point (2d))", listOf(Values.point(4326, 1.0, 2.0).asPoint()), null), @@ -186,6 +202,7 @@ class DynamicTypesTest { "array (point (3d))", Array(1) { Values.point(4326, 1.0, 2.0, 3.0).asPoint() }, null), Arguments.of( "list (point (3d))", listOf(Values.point(4326, 1.0, 2.0, 3.0).asPoint()), null), + Arguments.of("empty list (point)", emptyList(), null), ) } } @@ -239,17 +256,39 @@ class DynamicTypesTest { } @Test - fun `empty collections or arrays should map to an array of property type`() { - listOf(listOf(), setOf(), arrayOf()).forEach { collection -> + fun `empty collections should map to property type`() { + listOf(listOf(), setOf()).forEach { collection -> withClue(collection) { - DynamicTypes.toConnectSchema(collection, false) shouldBe - SchemaBuilder.array(PropertyType.schema).build() - DynamicTypes.toConnectSchema(collection, true) shouldBe - SchemaBuilder.array(PropertyType.schema).optional().build() + DynamicTypes.toConnectSchema(collection, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(collection, true) shouldBe PropertyType.schema } } } + @Test + fun `empty arrays should map to an array of property type`() { + DynamicTypes.toConnectSchema(arrayOf(), false) shouldBe + SchemaBuilder.array(PropertyType.schema).build() + DynamicTypes.toConnectSchema(arrayOf(), true) shouldBe + SchemaBuilder.array(PropertyType.schema).optional().build() + } + + @Test + fun `empty arrays of simple types should map to property type`() { + listOf( + arrayOf(), + arrayOf(), + arrayOf(), + arrayOf(), + arrayOf()) + .forEach { array -> + withClue(array) { + DynamicTypes.toConnectSchema(array, false) shouldBe PropertyType.schema + DynamicTypes.toConnectSchema(array, true) shouldBe PropertyType.schema + } + } + } + @ParameterizedTest(name = "{0}") @ArgumentsSource(PropertyTypedCollectionProvider::class) fun `collections with elements of property types should map to an array schema`( diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/PropertyTypeTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/PropertyTypeTest.kt index d9dec3327..0f0432fbb 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/PropertyTypeTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/PropertyTypeTest.kt @@ -479,6 +479,16 @@ class PropertyTypeTest { .put(Z, 3.0) .put(DIMENSION, THREE_D))), listOf(Values.point(4326, 1.0, 2.0, 3.0).asPoint())), + Arguments.of( + "empty list (any)", + emptyList(), + Struct(PropertyType.schema).put(LONG_LIST, emptyList()), + emptyList()), + Arguments.of( + "empty list (typed)", + emptyList(), + Struct(PropertyType.schema).put(LONG_LIST, emptyList()), + emptyList()), ) } } diff --git a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt index 203dbd490..e5a4836d3 100644 --- a/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt +++ b/common/src/test/kotlin/org/neo4j/connectors/kafka/data/TypesTest.kt @@ -204,6 +204,10 @@ class TypesTest { .put("x", 12.78) .put("y", 56.7) .put("z", 100.0))), + Arguments.of( + Named.of("list - empty", emptyList()), + PropertyType.schema, + Struct(PropertyType.schema).put(LONG_LIST, emptyList())), Arguments.of( Named.of("list - long", (1L..50L).toList()), PropertyType.schema, diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt index b60994d1b..98d49447b 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceIT.kt @@ -26,7 +26,6 @@ import org.junit.jupiter.api.Test import org.neo4j.cdc.client.model.ChangeEvent import org.neo4j.connectors.kafka.connect.ConnectHeader import org.neo4j.connectors.kafka.data.Headers -import org.neo4j.connectors.kafka.data.PropertyType.schema import org.neo4j.connectors.kafka.testing.assertions.TopicVerifier import org.neo4j.connectors.kafka.testing.format.KafkaConverter.AVRO import org.neo4j.connectors.kafka.testing.format.KafkaConverter.JSON_SCHEMA diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt index eb62ebe60..f0e1a66fd 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceNodesIT.kt @@ -568,6 +568,51 @@ abstract class Neo4jCdcSourceNodesIT { } .verifyWithin(Duration.ofSeconds(30)) } + + @Neo4jSource( + startFrom = "EARLIEST", + strategy = CDC, + cdc = + CdcSource( + topics = + arrayOf( + CdcSourceTopic( + topic = "cdc", patterns = arrayOf(CdcSourceParam("(:TestSource)")))))) + @Test + fun `should publish changes with arrays`( + @TopicConsumer(topic = "cdc", offset = "earliest") consumer: ConvertingKafkaConsumer, + session: Session + ) { + session + .run( + "CREATE (n:TestSource) SET n = ${'$'}props", + mapOf( + "props" to + mapOf( + "prop1" to arrayOf(1, 2, 3, 4), + "prop2" to arrayOf(LocalDate.of(1999, 1, 1), LocalDate.of(2000, 1, 1)), + "prop3" to listOf("a", "b", "c"), + "prop4" to arrayOf(), + "prop5" to listOf()))) + .consume() + + TopicVerifier.create(consumer) + .assertMessageValue { value -> + assertThat(value) + .hasEventType(NODE) + .hasOperation(CREATE) + .labelledAs("TestSource") + .hasNoBeforeState() + .hasAfterStateProperties( + mapOf( + "prop1" to listOf(1L, 2L, 3L, 4L), + "prop2" to listOf(LocalDate.of(1999, 1, 1), LocalDate.of(2000, 1, 1)), + "prop3" to listOf("a", "b", "c"), + "prop4" to emptyList(), + "prop5" to emptyList())) + } + .verifyWithin(Duration.ofSeconds(30)) + } } @KeyValueConverter(key = AVRO, value = AVRO) diff --git a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt index fd79d0009..813301d1b 100644 --- a/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt +++ b/source-connector/src/test/kotlin/org/neo4j/connectors/kafka/source/Neo4jCdcSourceRelationshipsIT.kt @@ -563,6 +563,53 @@ abstract class Neo4jCdcSourceRelationshipsIT { } .verifyWithin(Duration.ofSeconds(30)) } + + @Neo4jSource( + startFrom = "EARLIEST", + strategy = CDC, + cdc = + CdcSource( + topics = + arrayOf( + CdcSourceTopic( + topic = "cdc", + patterns = + arrayOf(CdcSourceParam("(:Person)-[:EMPLOYED]->(:Company)")))))) + @Test + fun `should publish changes with arrays`( + @TopicConsumer(topic = "cdc", offset = "earliest") consumer: ConvertingKafkaConsumer, + session: Session + ) { + session + .run( + "CREATE (:Person)-[r:EMPLOYED]->(:Company) SET r = ${'$'}props", + mapOf( + "props" to + mapOf( + "prop1" to arrayOf(1, 2, 3, 4), + "prop2" to arrayOf(LocalDate.of(1999, 1, 1), LocalDate.of(2000, 1, 1)), + "prop3" to listOf("a", "b", "c"), + "prop4" to arrayOf(), + "prop5" to listOf()))) + .consume() + + TopicVerifier.create(consumer) + .assertMessageValue { value -> + assertThat(value) + .hasEventType(RELATIONSHIP) + .hasOperation(CREATE) + .hasType("EMPLOYED") + .hasNoBeforeState() + .hasAfterStateProperties( + mapOf( + "prop1" to listOf(1L, 2L, 3L, 4L), + "prop2" to listOf(LocalDate.of(1999, 1, 1), LocalDate.of(2000, 1, 1)), + "prop3" to listOf("a", "b", "c"), + "prop4" to emptyList(), + "prop5" to emptyList())) + } + .verifyWithin(Duration.ofSeconds(30)) + } } @KeyValueConverter(key = AVRO, value = AVRO)