Add Kotlin Native implementation

.circleci/config.yml

@@ -93,7 +93,7 @@ jobs:
 
       - run:
           name: Build conformance test suite
-          command: ./gradlew :conformance:lib:assemble :conformance:jvm:installDist
+          command: ./gradlew :conformance:lib:assemble :conformance:jvm:installDist :conformance:native:installDist
 
       # persist files necessary to run conformance tests
       - persist_to_workspace:
@@ -107,6 +107,9 @@ jobs:
             - build/js
             - conformance/js/failing_tests.txt
             - conformance/js/run.sh
+            # native
+            - conformance/native/build/install/conformance
+            - conformance/native/failing_tests.txt
 
       - save_cache:
           name: Save dependencies cache
@@ -155,6 +158,24 @@ jobs:
             export CONF_TEST_PATH=~/protobuf/conformance/conformance-test-runner
             ./conformance/test-conformance.sh js
 
+  conformance_native:
+    executor:
+      name: native
+
+    steps:
+      - attach_workspace:
+          at: ~/project
+
+      - restore_cache:
+          name: Restore protobuf cache
+          keys:
+            - v3-protobuf-3.10.1
+
+      - run:
+          name: Run Native conformance tests
+          command: |
+            export CONF_TEST_PATH=~/protobuf/conformance/conformance-test-runner
+            ./conformance/test-conformance.sh native
 
 workflows:
   version: 2.1
@@ -167,3 +188,6 @@ workflows:
       - conformance_js:
           requires:
             - build
+      - conformance_native:
+          requires:
+            - build

buildSrc/src/main/kotlin/Versions.kt

@@ -1,7 +1,7 @@
 object Versions {
-    const val kotlin = "1.3.61"
+    const val kotlin = "1.3.71"
     const val kotlinCoroutines = "1.3.5"
-    const val kotlinSerialization = "0.14.0"
+    const val kotlinSerialization = "0.20.0"
     const val protobufJava = "3.11.1"
     const val protobufJs = "6.8.8"
     const val springBootGradlePlugin = "2.2.6.RELEASE"

conformance/lib/build.gradle.kts

@@ -22,44 +22,64 @@ kotlin {
     // For Linux, should be changed to e.g. linuxX64
     // For MacOS, should be changed to e.g. macosX64
     // For Windows, should be changed to e.g. mingwX64
-    // macosX64("macos")
+    macosX64()
 
-    sourceSets["commonMain"].dependencies {
-        implementation(kotlin("stdlib-common"))
-        implementation(project(":runtime"))
-        implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime-common:${Versions.kotlinSerialization}")
-        implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-common:${Versions.kotlinCoroutines}")
-    }
-    sourceSets["commonTest"].dependencies {
-        implementation(kotlin("test-common"))
-        implementation(kotlin("test-annotations-common"))
-    }
+    sourceSets.create("nativeMain")
 
-    sourceSets["jvmMain"].dependencies {
-        implementation(kotlin("stdlib-jdk8"))
-        implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime:${Versions.kotlinSerialization}")
-        implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinCoroutines}")
-    }
+    sourceSets {
+        val commonMain by getting {
+            dependencies {
+                implementation(kotlin("stdlib-common"))
+                implementation(project(":runtime"))
+                implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime-common:${Versions.kotlinSerialization}")
+                implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-common:${Versions.kotlinCoroutines}")
+            }
+        }
 
-    sourceSets["jvmTest"].dependencies {
-        implementation(kotlin("test"))
-        implementation(kotlin("test-junit"))
-        implementation("junit:junit:4.12")
-    }
+        val commonTest by getting {
+            dependencies {
+                implementation(kotlin("test-common"))
+                implementation(kotlin("test-annotations-common"))
+            }
+        }
 
-    sourceSets["jsMain"].dependencies {
-        implementation(kotlin("stdlib-js"))
-        implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime-js:${Versions.kotlinSerialization}")
-        implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-js:${Versions.kotlinCoroutines}")
-    }
-    sourceSets["jsTest"].dependencies {
-        implementation(kotlin("test-js"))
-    }
+        val jvmMain by getting {
+            dependencies {
+                implementation(kotlin("stdlib-jdk8"))
+                implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime:${Versions.kotlinSerialization}")
+                implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${Versions.kotlinCoroutines}")
+            }
+        }
+
+        val jvmTest by getting {
+            dependencies {
+                implementation(kotlin("test"))
+                implementation(kotlin("test-junit"))
+                implementation("junit:junit:4.12")
+            }
+        }
 
-//    sourceSets["macosMain"].dependencies {
-//    }
-//    sourceSets["macosTest"].dependencies {
-//    }
+        val jsMain by getting {
+            dependencies {
+                implementation(kotlin("stdlib-js"))
+                implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime-js:${Versions.kotlinSerialization}")
+                implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-js:${Versions.kotlinCoroutines}")
+            }
+        }
+
+        val jsTest by getting {
+            dependencies {
+                implementation(kotlin("test-js"))
+            }
+        }
+
+        val macosX64Main by getting {
+            dependencies {
+                implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime-native:${Versions.kotlinSerialization}")
+                implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core-native:${Versions.kotlinCoroutines}")
+            }
+        }
+    }
 }
 
 tasks {
@@ -69,4 +89,12 @@ tasks {
         kotlinPackage.set("pbandk.conformance.pb")
         logLevel.set("debug")
     }
+
+    // DCE is now enable by default in Kotlin 1.3.70/71
+    // and it doesn't work well with commonJS modules
+    // Use of commonJs could be removed since default module is now UMD
+    // but would require some code change
+    val processDceJsKotlinJs by getting {
+        enabled = false
+    }
 }

conformance/lib/src/macosX64Main/kotlin/pbandk/conformance/Platform.kt

@@ -0,0 +1,78 @@
+// Copyright 2019 Google Inc.  All rights reserved.
+// https://developers.google.com/protocol-buffers/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+package pbandk.conformance
+
+import platform.posix.*
+import kotlinx.cinterop.cstr
+import kotlinx.cinterop.usePinned
+import kotlinx.cinterop.addressOf
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+
+actual object Platform {
+    actual fun stderrPrintln(str: String) {
+        val strn = str + "\n"
+        val cstr = strn.cstr
+        write(2, cstr, cstr.size.toULong())
+    }
+
+    actual suspend fun stdinReadIntLE() = withContext(Dispatchers.Main) {
+        ByteArray(4).let {
+            if (readBytes(0, it) != 4) null else {
+                it.foldRight(0) { byte, acc ->
+                    (acc shl 8) or (byte.toInt() and 0xff)
+                }
+            }
+        }
+    }
+
+    private fun readBytes(fd: Int, arr: ByteArray): Int {
+        arr.usePinned {
+            return read(fd, it.addressOf(0), arr.size.toULong()).toInt()
+        }
+    }
+
+    actual suspend fun stdinReadFull(size: Int) = withContext(Dispatchers.Main) {
+        ByteArray(size).also {
+            require(readBytes(0, it) == it.size) { "Unable to read full byte array"}
+        }
+    }
+
+    actual fun stdoutWriteIntLE(v: Int) =
+            stdoutWriteFull (ByteArray(4) { (v shr (8 * it)).toByte() })
+
+    actual fun stdoutWriteFull(arr: ByteArray) {
+        arr.usePinned {
+            write(1, it.addressOf(0), arr.size.toULong())
+        }
+    }
+
+    actual inline fun <T> doTry(fn: () -> T, errFn: (Any) -> T) = try { fn() } catch (e: Exception) { errFn(e) }
+}

conformance/native/build.gradle.kts

@@ -0,0 +1,15 @@
+import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+
+plugins {
+    kotlin("multiplatform")
+    application
+}
+
+application {
+    mainClassName = "pbandk.conformance.MainKt"
+    applicationName = "conformance"
+}
+
+dependencies {
+    implementation(project(":conformance:lib"))
+}

conformance/native/failing_tests.txt

@@ -0,0 +1,128 @@
+##
+## We don't fully support JSON yet
+##
+Required.Proto3.JsonInput.Any.ProtobufOutput
+Required.Proto3.JsonInput.AnyNested.ProtobufOutput
+Required.Proto3.JsonInput.AnyUnorderedTypeTag.ProtobufOutput
+Required.Proto3.JsonInput.AnyWithDuration.ProtobufOutput
+Required.Proto3.JsonInput.AnyWithFieldMask.ProtobufOutput
+Required.Proto3.JsonInput.AnyWithInt32ValueWrapper.ProtobufOutput
+Required.Proto3.JsonInput.AnyWithStruct.ProtobufOutput
+Required.Proto3.JsonInput.AnyWithTimestamp.ProtobufOutput
+Required.Proto3.JsonInput.AnyWithValueForInteger.ProtobufOutput
+Required.Proto3.JsonInput.AnyWithValueForJsonObject.ProtobufOutput
+Required.Proto3.JsonInput.BoolFieldTrue.ProtobufOutput
+Required.Proto3.JsonInput.BoolMapEscapedKey.ProtobufOutput
+Required.Proto3.JsonInput.BoolMapField.ProtobufOutput
+Required.Proto3.JsonInput.BytesField.ProtobufOutput
+Required.Proto3.JsonInput.BytesRepeatedField.ProtobufOutput
+Required.Proto3.JsonInput.DoubleFieldInfinity.ProtobufOutput
+Required.Proto3.JsonInput.DoubleFieldMaxNegativeValue.ProtobufOutput
+Required.Proto3.JsonInput.DoubleFieldMaxPositiveValue.ProtobufOutput
+Required.Proto3.JsonInput.DoubleFieldMinNegativeValue.ProtobufOutput
+Required.Proto3.JsonInput.DoubleFieldMinPositiveValue.ProtobufOutput
+Required.Proto3.JsonInput.DoubleFieldNan.ProtobufOutput
+Required.Proto3.JsonInput.DoubleFieldNegativeInfinity.ProtobufOutput
+Required.Proto3.JsonInput.DoubleFieldQuotedValue.ProtobufOutput
+Required.Proto3.JsonInput.DurationMaxValue.ProtobufOutput
+Required.Proto3.JsonInput.DurationMinValue.ProtobufOutput
+Required.Proto3.JsonInput.DurationRepeatedValue.ProtobufOutput
+Required.Proto3.JsonInput.EmptyFieldMask.ProtobufOutput
+Required.Proto3.JsonInput.EnumFieldNumericValueNonZero.ProtobufOutput
+Required.Proto3.JsonInput.EnumFieldWithAlias.ProtobufOutput
+Required.Proto3.JsonInput.EnumFieldWithAliasDifferentCase.ProtobufOutput
+Required.Proto3.JsonInput.EnumFieldWithAliasLowerCase.ProtobufOutput
+Required.Proto3.JsonInput.EnumFieldWithAliasUseAlias.ProtobufOutput
+Required.Proto3.JsonInput.EnumRepeatedField.ProtobufOutput
+Required.Proto3.JsonInput.FieldMask.ProtobufOutput
+Required.Proto3.JsonInput.FieldNameInSnakeCase.ProtobufOutput
+Required.Proto3.JsonInput.FieldNameWithMixedCases.ProtobufOutput
+Required.Proto3.JsonInput.FieldNameWithNumbers.ProtobufOutput
+Required.Proto3.JsonInput.FloatFieldInfinity.ProtobufOutput
+Required.Proto3.JsonInput.FloatFieldMaxNegativeValue.ProtobufOutput
+Required.Proto3.JsonInput.FloatFieldMaxPositiveValue.ProtobufOutput
+Required.Proto3.JsonInput.FloatFieldMinNegativeValue.ProtobufOutput
+Required.Proto3.JsonInput.FloatFieldMinPositiveValue.ProtobufOutput
+Required.Proto3.JsonInput.FloatFieldNan.ProtobufOutput
+Required.Proto3.JsonInput.FloatFieldNegativeInfinity.ProtobufOutput
+Required.Proto3.JsonInput.FloatFieldQuotedValue.ProtobufOutput
+Required.Proto3.JsonInput.HelloWorld.ProtobufOutput
+Required.Proto3.JsonInput.Int32FieldExponentialFormat.ProtobufOutput
+Required.Proto3.JsonInput.Int32FieldFloatTrailingZero.ProtobufOutput
+Required.Proto3.JsonInput.Int32FieldMaxFloatValue.ProtobufOutput
+Required.Proto3.JsonInput.Int32FieldMaxValue.ProtobufOutput
+Required.Proto3.JsonInput.Int32FieldMinFloatValue.ProtobufOutput
+Required.Proto3.JsonInput.Int32FieldMinValue.ProtobufOutput
+Required.Proto3.JsonInput.Int32FieldStringValue.ProtobufOutput
+Required.Proto3.JsonInput.Int32FieldStringValueEscaped.ProtobufOutput
+Required.Proto3.JsonInput.Int32MapEscapedKey.ProtobufOutput
+Required.Proto3.JsonInput.Int32MapField.ProtobufOutput
+Required.Proto3.JsonInput.Int64FieldMaxValue.ProtobufOutput
+Required.Proto3.JsonInput.Int64FieldMaxValueNotQuoted.ProtobufOutput
+Required.Proto3.JsonInput.Int64FieldMinValue.ProtobufOutput
+Required.Proto3.JsonInput.Int64FieldMinValueNotQuoted.ProtobufOutput
+Required.Proto3.JsonInput.Int64MapEscapedKey.ProtobufOutput
+Required.Proto3.JsonInput.Int64MapField.ProtobufOutput
+Required.Proto3.JsonInput.MessageField.ProtobufOutput
+Required.Proto3.JsonInput.MessageMapField.ProtobufOutput
+Required.Proto3.JsonInput.MessageRepeatedField.ProtobufOutput
+Required.Proto3.JsonInput.OptionalBoolWrapper.ProtobufOutput
+Required.Proto3.JsonInput.OptionalBytesWrapper.ProtobufOutput
+Required.Proto3.JsonInput.OptionalDoubleWrapper.ProtobufOutput
+Required.Proto3.JsonInput.OptionalFloatWrapper.ProtobufOutput
+Required.Proto3.JsonInput.OptionalInt32Wrapper.ProtobufOutput
+Required.Proto3.JsonInput.OptionalInt64Wrapper.ProtobufOutput
+Required.Proto3.JsonInput.OptionalStringWrapper.ProtobufOutput
+Required.Proto3.JsonInput.OptionalUint32Wrapper.ProtobufOutput
+Required.Proto3.JsonInput.OptionalUint64Wrapper.ProtobufOutput
+Required.Proto3.JsonInput.OptionalWrapperTypesWithNonDefaultValue.ProtobufOutput
+Required.Proto3.JsonInput.PrimitiveRepeatedField.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedBoolWrapper.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedBytesWrapper.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedDoubleWrapper.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedFloatWrapper.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedInt32Wrapper.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedInt64Wrapper.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedListValue.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedStringWrapper.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedUint32Wrapper.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedUint64Wrapper.ProtobufOutput
+Required.Proto3.JsonInput.RepeatedValue.ProtobufOutput
+Required.Proto3.JsonInput.StringField.ProtobufOutput
+Required.Proto3.JsonInput.StringFieldEscape.ProtobufOutput
+Required.Proto3.JsonInput.StringFieldSurrogatePair.ProtobufOutput
+Required.Proto3.JsonInput.StringFieldUnicode.ProtobufOutput
+Required.Proto3.JsonInput.StringFieldUnicodeEscape.ProtobufOutput
+Required.Proto3.JsonInput.StringFieldUnicodeEscapeWithLowercaseHexLetters.ProtobufOutput
+Required.Proto3.JsonInput.StringRepeatedField.ProtobufOutput
+Required.Proto3.JsonInput.Struct.ProtobufOutput
+Required.Proto3.JsonInput.StructWithEmptyListValue.ProtobufOutput
+Required.Proto3.JsonInput.TimestampLeap.ProtobufOutput
+Required.Proto3.JsonInput.TimestampMaxValue.ProtobufOutput
+Required.Proto3.JsonInput.TimestampMinValue.ProtobufOutput
+Required.Proto3.JsonInput.TimestampRepeatedValue.ProtobufOutput
+Required.Proto3.JsonInput.TimestampWithNegativeOffset.ProtobufOutput
+Required.Proto3.JsonInput.TimestampWithPositiveOffset.ProtobufOutput
+Required.Proto3.JsonInput.Uint32FieldMaxFloatValue.ProtobufOutput
+Required.Proto3.JsonInput.Uint32FieldMaxValue.ProtobufOutput
+Required.Proto3.JsonInput.Uint32MapField.ProtobufOutput
+Required.Proto3.JsonInput.Uint64FieldMaxValue.ProtobufOutput
+Required.Proto3.JsonInput.Uint64FieldMaxValueNotQuoted.ProtobufOutput
+Required.Proto3.JsonInput.Uint64MapField.ProtobufOutput
+Required.Proto3.JsonInput.ValueAcceptBool.ProtobufOutput
+Required.Proto3.JsonInput.ValueAcceptFloat.ProtobufOutput
+Required.Proto3.JsonInput.ValueAcceptInteger.ProtobufOutput
+Required.Proto3.JsonInput.ValueAcceptList.ProtobufOutput
+Required.Proto3.JsonInput.ValueAcceptNull.ProtobufOutput
+Required.Proto3.JsonInput.ValueAcceptObject.ProtobufOutput
+Required.Proto3.JsonInput.ValueAcceptString.ProtobufOutput
+##
+## These failures need to be investigated
+##
+## See https://github.com/streem/pbandk/issues/22
+Required.Proto2.ProtobufInput.RepeatedScalarMessageMerge.ProtobufOutput
+Required.Proto3.ProtobufInput.RepeatedScalarMessageMerge.ProtobufOutput
+## and https://github.com/streem/pbandk/issues/23
+Required.Proto3.ProtobufInput.ValidDataRepeated.BOOL.PackedInput.ProtobufOutput
+Required.Proto3.ProtobufInput.ValidDataRepeated.INT32.PackedInput.ProtobufOutput
+Required.Proto3.ProtobufInput.ValidDataRepeated.UINT32.PackedInput.ProtobufOutput