[vm/ffi] Test truncation and extension on calls and callbacks

This CL adds tests for the truncation and sign extension of the following cases:

* Arguments in registers in callbacks (we already had these for calls).
* Arguments on stack in calls and callbacks.
* Return values in registers in calls and callbacks.

Related CL (replacing FFI pipeline): https://dart-review.googlesource.com/c/sdk/+/129081

Change-Id: I776d103cf72007b686b1fe804d6a3bc6cb0e7122
Cq-Include-Trybots: luci.dart.try:vm-ffi-android-debug-arm-try,vm-ffi-android-debug-arm64-try,app-kernel-linux-debug-x64-try,vm-kernel-linux-debug-ia32-try,vm-kernel-win-debug-x64-try,vm-kernel-win-debug-ia32-try,vm-kernel-precomp-linux-debug-x64-try,vm-dartkb-linux-release-x64-abi-try,vm-kernel-precomp-android-release-arm64-try,vm-kernel-asan-linux-release-x64-try,vm-kernel-linux-release-simarm-try,vm-kernel-linux-release-simarm64-try,vm-kernel-precomp-android-release-arm_x64-try,vm-kernel-precomp-obfuscate-linux-release-x64-try,dart-sdk-linux-try,analyzer-analysis-server-linux-try,analyzer-linux-release-try,front-end-linux-release-x64-try,vm-kernel-precomp-win-release-x64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/132284
Commit-Queue: Daco Harkes <[email protected]>
Reviewed-by: Martin Kustermann <[email protected]>

Author: dcharkes

runtime/bin/ffi_test/ffi_test_functions.cc

@@ -49,8 +49,7 @@ DART_EXPORT int32_t SumPlus42(int32_t a, int32_t b) {
   return retval;
 }
 
-//// Tests for sign and zero extension of arguments and results.
-
+// Tests for sign and zero extension return values when passed to Dart.
 DART_EXPORT uint8_t ReturnMaxUint8() {
   return 0xff;
 }
@@ -75,36 +74,100 @@ DART_EXPORT int32_t ReturnMinInt32() {
   return 0x80000000;
 }
 
+// Test that return values are truncated by callee before passed to Dart.
+DART_EXPORT uint8_t ReturnMaxUint8v2() {
+  uint64_t v = 0xabcff;
+  // Truncated to 8 bits and zero extended, or truncated to 32 bits, depending
+  // on ABI.
+  return v;
+}
+
+DART_EXPORT uint16_t ReturnMaxUint16v2() {
+  uint64_t v = 0xabcffff;
+  return v;
+}
+
+DART_EXPORT uint32_t ReturnMaxUint32v2() {
+  uint64_t v = 0xabcffffffff;
+  return v;
+}
+
+DART_EXPORT int8_t ReturnMinInt8v2() {
+  int64_t v = 0x8abc80;
+  return v;
+}
+
+DART_EXPORT int16_t ReturnMinInt16v2() {
+  int64_t v = 0x8abc8000;
+  return v;
+}
+
+DART_EXPORT int32_t ReturnMinInt32v2() {
+  int64_t v = 0x8abc80000000;
+  return v;
+}
+
+// Test that arguments are truncated correctly.
 DART_EXPORT intptr_t TakeMaxUint8(uint8_t x) {
+  std::cout << "TakeMaxUint8(" << static_cast<int>(x) << ")\n";
   return x == 0xff ? 1 : 0;
 }
 
 DART_EXPORT intptr_t TakeMaxUint16(uint16_t x) {
+  std::cout << "TakeMaxUint16(" << x << ")\n";
   return x == 0xffff ? 1 : 0;
 }
 
 DART_EXPORT intptr_t TakeMaxUint32(uint32_t x) {
+  std::cout << "TakeMaxUint32(" << x << ")\n";
   return x == 0xffffffff ? 1 : 0;
 }
 
 DART_EXPORT intptr_t TakeMinInt8(int8_t x) {
+  std::cout << "TakeMinInt8(" << static_cast<int>(x) << ")\n";
   const int64_t expected = -0x80;
   const int64_t received = x;
   return expected == received ? 1 : 0;
 }
 
 DART_EXPORT intptr_t TakeMinInt16(int16_t x) {
+  std::cout << "TakeMinInt16(" << x << ")\n";
   const int64_t expected = -0x8000;
   const int64_t received = x;
   return expected == received ? 1 : 0;
 }
 
 DART_EXPORT intptr_t TakeMinInt32(int32_t x) {
+  std::cout << "TakeMinInt32(" << x << ")\n";
   const int64_t expected = INT32_MIN;
   const int64_t received = x;
   return expected == received ? 1 : 0;
 }
 
+// Test that arguments are truncated correctly, including stack arguments
+DART_EXPORT intptr_t TakeMaxUint8x10(uint8_t a,
+                                     uint8_t b,
+                                     uint8_t c,
+                                     uint8_t d,
+                                     uint8_t e,
+                                     uint8_t f,
+                                     uint8_t g,
+                                     uint8_t h,
+                                     uint8_t i,
+                                     uint8_t j) {
+  std::cout << "TakeMaxUint8x10(" << static_cast<int>(a) << ", "
+            << static_cast<int>(b) << ", " << static_cast<int>(c) << ", "
+            << static_cast<int>(d) << ", " << static_cast<int>(e) << ", "
+            << static_cast<int>(f) << ", " << static_cast<int>(g) << ", "
+            << static_cast<int>(h) << ", " << static_cast<int>(i) << ", "
+            << static_cast<int>(j) << ", "
+            << ")\n";
+  return (a == 0xff && b == 0xff && c == 0xff && d == 0xff && e == 0xff &&
+          f == 0xff && g == 0xff && h == 0xff && i == 0xff && j == 0xff)
+             ? 1
+             : 0;
+}
+
 // Performs some computation on various sized signed ints.
 // Used for testing value ranges for signed ints.
 DART_EXPORT int64_t IntComputation(int8_t a, int16_t b, int32_t c, int64_t d) {
@@ -656,6 +719,29 @@ DART_EXPORT int TestThrowException(int (*fn)()) {
   return 0;
 }
 
+DART_EXPORT int TestTakeMaxUint8x10(intptr_t (*fn)(uint8_t,
+                                                   uint8_t,
+                                                   uint8_t,
+                                                   uint8_t,
+                                                   uint8_t,
+                                                   uint8_t,
+                                                   uint8_t,
+                                                   uint8_t,
+                                                   uint8_t,
+                                                   uint8_t)) {
+  CHECK_EQ(1, fn(0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF));
+  // Check the argument values are properly truncated.
+  uint64_t v = 0xabcFF;
+  CHECK_EQ(1, fn(v, v, v, v, v, v, v, v, v, v));
+  return 0;
+}
+
+DART_EXPORT int TestReturnMaxUint8(uint8_t (*fn)()) {
+  std::cout << "TestReturnMaxUint8(fn): " << static_cast<int>(fn()) << "\n";
+  CHECK_EQ(0xFF, fn());
+  return 0;
+}
+
 // Receives some pointer (Pointer<NativeType> in Dart) and writes some bits.
 DART_EXPORT void NativeTypePointerParam(void* p) {
   uint8_t* p2 = reinterpret_cast<uint8_t*>(p);

tests/ffi/function_callbacks_test.dart

@@ -189,6 +189,34 @@ Pointer<Void> throwExceptionPointer() {
   throw "Exception.";
 }
 
+typedef TakeMaxUint8x10Type = IntPtr Function(
+    Uint8, Uint8, Uint8, Uint8, Uint8, Uint8, Uint8, Uint8, Uint8, Uint8);
+int takeMaxUint8x10(
+    int a, int b, int c, int d, int e, int f, int g, int h, int i, int j) {
+  print("takeMaxUint8x10($a, $b, $c, $d, $e, $f, $g, $h, $i, $j)");
+  return a == 0xff &&
+          b == 0xff &&
+          c == 0xff &&
+          d == 0xff &&
+          e == 0xff &&
+          f == 0xff &&
+          g == 0xff &&
+          h == 0xff &&
+          i == 0xff &&
+          j == 0xff
+      ? 1
+      : 0;
+}
+
+typedef ReturnMaxUint8Type = Uint8 Function();
+int returnMaxUint8() {
+  return 0xff;
+}
+
+int returnMaxUint8v2() {
+  return 0xabcff;
+}
+
 final List<Test> testcases = [
   Test("SimpleAddition",
       Pointer.fromFunction<SimpleAdditionType>(simpleAddition, 0)),
@@ -213,6 +241,12 @@ final List<Test> testcases = [
       Pointer.fromFunction<ThrowExceptionPointer>(throwExceptionPointer)),
   Test("ThrowException",
       Pointer.fromFunction<ThrowExceptionInt>(throwExceptionInt, 42)),
+  Test("TakeMaxUint8x10",
+      Pointer.fromFunction<TakeMaxUint8x10Type>(takeMaxUint8x10, 0)),
+  Test("ReturnMaxUint8",
+      Pointer.fromFunction<ReturnMaxUint8Type>(returnMaxUint8, 0)),
+  Test("ReturnMaxUint8",
+      Pointer.fromFunction<ReturnMaxUint8Type>(returnMaxUint8v2, 0)),
 ];
 
 void main() {

tests/ffi/function_test.dart

@@ -87,36 +87,60 @@ int Function() returnMaxUint8 = ffiTestFunctions
     .lookup("ReturnMaxUint8")
     .cast<NativeFunction<NativeReturnMaxUint8>>()
     .asFunction();
+int Function() returnMaxUint8v2 = ffiTestFunctions
+    .lookup("ReturnMaxUint8v2")
+    .cast<NativeFunction<NativeReturnMaxUint8>>()
+    .asFunction();
 
 typedef NativeReturnMaxUint16 = Uint16 Function();
 int Function() returnMaxUint16 = ffiTestFunctions
     .lookup("ReturnMaxUint16")
     .cast<NativeFunction<NativeReturnMaxUint16>>()
     .asFunction();
+int Function() returnMaxUint16v2 = ffiTestFunctions
+    .lookup("ReturnMaxUint16v2")
+    .cast<NativeFunction<NativeReturnMaxUint16>>()
+    .asFunction();
 
 typedef NativeReturnMaxUint32 = Uint32 Function();
 int Function() returnMaxUint32 = ffiTestFunctions
     .lookup("ReturnMaxUint32")
     .cast<NativeFunction<NativeReturnMaxUint32>>()
     .asFunction();
+int Function() returnMaxUint32v2 = ffiTestFunctions
+    .lookup("ReturnMaxUint32v2")
+    .cast<NativeFunction<NativeReturnMaxUint32>>()
+    .asFunction();
 
 typedef NativeReturnMinInt8 = Int8 Function();
 int Function() returnMinInt8 = ffiTestFunctions
     .lookup("ReturnMinInt8")
     .cast<NativeFunction<NativeReturnMinInt8>>()
     .asFunction();
+int Function() returnMinInt8v2 = ffiTestFunctions
+    .lookup("ReturnMinInt8v2")
+    .cast<NativeFunction<NativeReturnMinInt8>>()
+    .asFunction();
 
 typedef NativeReturnMinInt16 = Int16 Function();
 int Function() returnMinInt16 = ffiTestFunctions
     .lookup("ReturnMinInt16")
     .cast<NativeFunction<NativeReturnMinInt16>>()
     .asFunction();
+int Function() returnMinInt16v2 = ffiTestFunctions
+    .lookup("ReturnMinInt16v2")
+    .cast<NativeFunction<NativeReturnMinInt16>>()
+    .asFunction();
 
 typedef NativeReturnMinInt32 = Int32 Function();
 int Function() returnMinInt32 = ffiTestFunctions
     .lookup("ReturnMinInt32")
     .cast<NativeFunction<NativeReturnMinInt32>>()
     .asFunction();
+int Function() returnMinInt32v2 = ffiTestFunctions
+    .lookup("ReturnMinInt32v2")
+    .cast<NativeFunction<NativeReturnMinInt32>>()
+    .asFunction();
 
 typedef NativeTakeMaxUint8 = IntPtr Function(Uint8);
 int Function(int) takeMaxUint8 = ffiTestFunctions
@@ -154,20 +178,52 @@ int Function(int) takeMinInt32 = ffiTestFunctions
     .cast<NativeFunction<NativeTakeMinInt32>>()
     .asFunction();
 
+typedef NativeTakeMaxUint8x10 = IntPtr Function(
+    Uint8, Uint8, Uint8, Uint8, Uint8, Uint8, Uint8, Uint8, Uint8, Uint8);
+int Function(int, int, int, int, int, int, int, int, int, int) takeMaxUint8x10 =
+    ffiTestFunctions
+        .lookup("TakeMaxUint8x10")
+        .cast<NativeFunction<NativeTakeMaxUint8x10>>()
+        .asFunction();
+
 void testExtension() {
-  Expect.equals(returnMaxUint8(), 0xff);
-  Expect.equals(returnMaxUint16(), 0xffff);
-  Expect.equals(returnMaxUint32(), 0xffffffff);
-  Expect.equals(returnMinInt8(), -0x80);
-  Expect.equals(returnMinInt16(), -0x8000);
-  Expect.equals(returnMinInt32(), -0x80000000);
-
-  Expect.equals(takeMaxUint8(0xff), 1);
-  Expect.equals(takeMaxUint16(0xffff), 1);
-  Expect.equals(takeMaxUint32(0xffffffff), 1);
-  Expect.equals(takeMinInt8(0x80), 1);
-  Expect.equals(takeMinInt16(0x8000), 1);
-  Expect.equals(takeMinInt32(0x80000000), 1);
+  // Sign extension on the way back to Dart.
+  Expect.equals(0xff, returnMaxUint8());
+  Expect.equals(0xffff, returnMaxUint16());
+  Expect.equals(0xffffffff, returnMaxUint32());
+  Expect.equals(-0x80, returnMinInt8());
+  Expect.equals(-0x8000, returnMinInt16());
+  Expect.equals(-0x80000000, returnMinInt32());
+  // Truncation in C, and sign extension back to Dart.
+  Expect.equals(0xff, returnMaxUint8v2());
+  Expect.equals(0xffff, returnMaxUint16v2());
+  Expect.equals(0xffffffff, returnMaxUint32v2());
+  Expect.equals(-0x80, returnMinInt8v2());
+  Expect.equals(-0x8000, returnMinInt16v2());
+  Expect.equals(-0x80000000, returnMinInt32v2());
+
+  // Upper bits propper, should work without truncation.
+  Expect.equals(1, takeMaxUint8(0xff));
+  Expect.equals(1, takeMaxUint16(0xffff));
+  Expect.equals(1, takeMaxUint32(0xffffffff));
+  Expect.equals(1, takeMinInt8(-0x80));
+  Expect.equals(1, takeMinInt16(-0x8000));
+  Expect.equals(1, takeMinInt32(-0x80000000));
+  Expect.equals(
+      1,
+      takeMaxUint8x10(
+          0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff));
+  // Upper bits garbage, needs to truncate.
+  Expect.equals(1, takeMaxUint8(0xabcff));
+  Expect.equals(1, takeMaxUint16(0xabcffff));
+  Expect.equals(1, takeMaxUint32(0xabcffffffff));
+  Expect.equals(1, takeMinInt8(0x8abc80));
+  Expect.equals(1, takeMinInt16(0x8abc8000));
+  Expect.equals(1, takeMinInt32(0x8abc80000000));
+  Expect.equals(
+      1,
+      takeMaxUint8x10(0xabcff, 0xabcff, 0xabcff, 0xabcff, 0xabcff, 0xabcff,
+          0xabcff, 0xabcff, 0xabcff, 0xabcff));
 }
 
 QuadOp uintComputation = ffiTestFunctions