[vm/ffi] Support nested structs

This CL adds support for nested structs in FFI calls, callbacks, and
memory loads and stores through the Struct classes itself.

Nesting empty structs and nesting a structs in themselves (directly or
indirectly) is reported as error.

This feature is almost fully implemented in the CFE transformation.

Because structs depend on the sizes of their nested structs, the structs
now need to be processed in topological order.

Field access to nested structs branches at runtime on making a derived
Pointer if the backing memory of the outer struct was a Pointer or
making a TypedDataView if the backing memory of the outer struct was
a TypedData.

Assigning to a nested struct is a byte for byte copy from the source.

The only changes in the VM are contained in the native calling
convention calculation which now recursively needs to reason about
fundamental types instead of just 1 struct deep.

Because of the amount of corner cases in the calling conventions that
need to be covered, the tests are generated, rather than hand-written.

ABIs tested on CQ: x64 (Linux, MacOS, Windows), ia32 (Linux, Windows),
arm (Android softFP, Linux hardFP), arm64 Android.
ABIs tested locally through Flutter: arm64 iOS.
ABIs not tested: ia32 Android (emulator), x64 iOS (simulator), arm iOS.
TEST=runtime/bin/ffi_test/ffi_test_functions_generated.cc
TEST=runtime/bin/ffi_test/ffi_test_functions.cc
TEST=tests/{ffi,ffi_2}/function_structs_by_value_generated_test.dart
TEST=tests/{ffi,ffi_2}/function_callbacks_structs_by_value_generated_tes
TEST=tests/{ffi,ffi_2}/function_callbacks_structs_by_value_test.dart
TEST=tests/{ffi,ffi_2}/vmspecific_static_checks_test.dart

Closes #37271.

Contains a temporary workaround for
#44454.

Change-Id: I5e5d10e09e5c3fc209f5f7e997efe17bd362214d
Cq-Include-Trybots: luci.dart.try:dart-sdk-linux-try,dart-sdk-mac-try,dart-sdk-win-try,vm-ffi-android-debug-arm-try,vm-ffi-android-debug-arm64-try,vm-kernel-asan-linux-release-x64-try,vm-kernel-mac-debug-x64-try,vm-kernel-linux-debug-ia32-try,vm-kernel-linux-debug-x64-try,vm-kernel-nnbd-linux-debug-x64-try,vm-kernel-nnbd-linux-debug-ia32-try,vm-kernel-nnbd-mac-release-x64-try,vm-kernel-nnbd-win-debug-x64-try,vm-kernel-precomp-linux-debug-x64-try,vm-kernel-precomp-linux-debug-simarm_x64-try,vm-kernel-precomp-nnbd-linux-debug-x64-try,vm-kernel-precomp-win-release-x64-try,vm-kernel-reload-linux-debug-x64-try,vm-kernel-reload-rollback-linux-debug-x64-try,vm-kernel-win-debug-x64-try,vm-kernel-win-debug-ia32-try,vm-precomp-ffi-qemu-linux-release-arm-try,vm-kernel-precomp-obfuscate-linux-release-x64-try,vm-kernel-msan-linux-release-x64-try,vm-kernel-precomp-msan-linux-release-x64-try,vm-kernel-precomp-android-release-arm_x64-try,analyzer-analysis-server-linux-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/169221
Commit-Queue: Daco Harkes <[email protected]>
Reviewed-by: Martin Kustermann <[email protected]>
Reviewed-by: Dmitry Stefantsov <[email protected]>

Author: dcharkes

pkg/_fe_analyzer_shared/lib/src/messages/codes_generated.dart

@@ -3788,6 +3788,31 @@ Message _withArgumentsFfiFieldAnnotation(String name) {
       arguments: {'name': name});
 }
 
+// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE.
+const Template<Message Function(String name, List<String> _names)>
+    templateFfiFieldCyclic =
+    const Template<Message Function(String name, List<String> _names)>(
+        messageTemplate: r"""Struct '#name' contains itself. Cycle elements:
+#names""", withArguments: _withArgumentsFfiFieldCyclic);
+
+// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE.
+const Code<Message Function(String name, List<String> _names)>
+    codeFfiFieldCyclic =
+    const Code<Message Function(String name, List<String> _names)>(
+  "FfiFieldCyclic",
+);
+
+// DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE.
+Message _withArgumentsFfiFieldCyclic(String name, List<String> _names) {
+  if (name.isEmpty) throw 'No name provided';
+  name = demangleMixinApplicationName(name);
+  if (_names.isEmpty) throw 'No names provided';
+  String names = itemizeNames(_names);
+  return new Message(codeFfiFieldCyclic,
+      message: """Struct '${name}' contains itself. Cycle elements:
+${names}""", arguments: {'name': name, 'names': _names});
+}
+
 // DO NOT EDIT. THIS FILE IS GENERATED. SEE TOP OF FILE.
 const Template<
     Message Function(String name)> templateFfiFieldInitializer = const Template<

pkg/analyzer/lib/error/error.dart

@@ -450,6 +450,7 @@ const List<ErrorCode> errorCodeValues = [
   CompileTimeErrorCode.YIELD_IN_NON_GENERATOR,
   CompileTimeErrorCode.YIELD_OF_INVALID_TYPE,
   FfiCode.ANNOTATION_ON_POINTER_FIELD,
+  FfiCode.EMPTY_STRUCT,
   FfiCode.EXTRA_ANNOTATION_ON_STRUCT_FIELD,
   FfiCode.FIELD_IN_STRUCT_WITH_INITIALIZER,
   FfiCode.FIELD_INITIALIZER_IN_STRUCT,

pkg/analyzer/lib/src/dart/error/ffi_code.dart

@@ -22,6 +22,15 @@ class FfiCode extends AnalyzerErrorCode {
           "any annotations.",
       correction: "Try removing the annotation.");
 
+  /**
+   * Parameters:
+   * 0: the name of the struct class
+   */
+  static const FfiCode EMPTY_STRUCT = FfiCode(
+      name: 'EMPTY_STRUCT',
+      message: "Struct '{0}' is empty. Empty structs are undefined behavior.",
+      correction: "Try adding a field to '{0}' or use a different Struct.");
+
   /**
    * No parameters.
    */
@@ -76,8 +85,9 @@ class FfiCode extends AnalyzerErrorCode {
       name: 'INVALID_FIELD_TYPE_IN_STRUCT',
       message:
           "Fields in struct classes can't have the type '{0}'. They can only "
-          "be declared as 'int', 'double' or 'Pointer'.",
-      correction: "Try using 'int', 'double' or 'Pointer'.");
+          "be declared as 'int', 'double', 'Pointer', or subtype of 'Struct'.",
+      correction:
+          "Try using 'int', 'double', 'Pointer', or subtype of 'Struct'.");
 
   /**
    * No parameters.

pkg/analyzer/lib/src/generated/ffi_verifier.dart

@@ -174,6 +174,8 @@ class FfiVerifier extends RecursiveAstVisitor<void> {
         structFieldCount++;
       } else if (_isPointer(declaredType.element)) {
         structFieldCount++;
+      } else if (_isStructClass(declaredType)) {
+        structFieldCount++;
       }
     }
     return structFieldCount == 0;
@@ -228,7 +230,11 @@ class FfiVerifier extends RecursiveAstVisitor<void> {
   /// Returns `true` iff [nativeType] is a struct type.
   bool _isStructClass(DartType nativeType) {
     if (nativeType is InterfaceType) {
-      final superClassElement = nativeType.element.supertype.element;
+      final superType = nativeType.element.supertype;
+      if (superType == null) {
+        return false;
+      }
+      final superClassElement = superType.element;
       if (superClassElement.library.name == 'dart.ffi') {
         return superClassElement.name == 'Struct' &&
             nativeType.typeArguments?.isEmpty == true;
@@ -518,6 +524,12 @@ class FfiVerifier extends RecursiveAstVisitor<void> {
         _validateAnnotations(fieldType, annotations, _PrimitiveDartType.double);
       } else if (_isPointer(declaredType.element)) {
         _validateNoAnnotations(annotations);
+      } else if (_isStructClass(declaredType)) {
+        final clazz = (declaredType as InterfaceType).element;
+        if (_isEmptyStruct(clazz)) {
+          _errorReporter
+              .reportErrorForNode(FfiCode.EMPTY_STRUCT, node, [clazz.name]);
+        }
       } else {
         _errorReporter.reportErrorForNode(FfiCode.INVALID_FIELD_TYPE_IN_STRUCT,
             fieldType, [fieldType.toSource()]);

pkg/front_end/lib/src/api_unstable/vm.dart

@@ -52,6 +52,7 @@ export '../fasta/fasta_codes.dart'
         templateFfiExpectedNoExceptionalReturn,
         templateFfiExtendsOrImplementsSealedClass,
         templateFfiFieldAnnotation,
+        templateFfiFieldCyclic,
         templateFfiFieldInitializer,
         templateFfiFieldNoAnnotation,
         templateFfiNotStatic,

pkg/front_end/messages.status

@@ -322,6 +322,7 @@ FfiExpectedExceptionalReturn/analyzerCode: Fail
 FfiExpectedNoExceptionalReturn/analyzerCode: Fail
 FfiExtendsOrImplementsSealedClass/analyzerCode: Fail
 FfiFieldAnnotation/analyzerCode: Fail
+FfiFieldCyclic/analyzerCode: Fail
 FfiFieldInitializer/analyzerCode: Fail
 FfiFieldNoAnnotation/analyzerCode: Fail
 FfiNotStatic/analyzerCode: Fail

pkg/front_end/messages.yaml

@@ -4253,6 +4253,13 @@ FfiFieldNoAnnotation:
   template: "Field '#name' requires no annotation to declare its native type, it is a Pointer which is represented by the same type in Dart and native code."
   external: test/ffi_test.dart
 
+FfiFieldCyclic:
+  # Used by dart:ffi
+  template: |
+    Struct '#name' contains itself. Cycle elements:
+    #names
+  external: test/ffi_test.dart
+
 FfiNotStatic:
   # Used by dart:ffi
   template: "#name expects a static function as parameter. dart:ffi only supports calling static Dart functions from native code."

pkg/front_end/testcases/general/ffi_sample.dart.strong.transformed.expect

@@ -13,7 +13,7 @@ class Coordinate extends ffi::Struct {
   @#C3
   static final field core::int* #sizeOf = (#C11).{core::List::[]}(ffi::_abi());
   @#C3
-  constructor #fromPointer(dynamic #pointer) → dynamic
+  constructor #fromTypedDataBase(dynamic #pointer) → dynamic
     : super ffi::Struct::_fromPointer(#pointer)
     ;
   static factory allocate(core::double* x, core::double* y, ffi::Pointer<self::Coordinate*>* next) → self::Coordinate* {

pkg/front_end/testcases/general_nnbd_opt_out/ffi_sample.dart.strong.transformed.expect

@@ -12,7 +12,7 @@ class Coordinate extends ffi::Struct {
   @#C3
   static final field core::int* #sizeOf = (#C6).{core::List::[]}(ffi::_abi());
   @#C3
-  constructor #fromPointer(dynamic #pointer) → dynamic
+  constructor #fromTypedDataBase(dynamic #pointer) → dynamic
     : super ffi::Struct::_fromPointer(#pointer)
     ;
   static factory allocate(core::double* x, core::double* y, ffi::Pointer<self::Coordinate*>* next) → self::Coordinate* {

pkg/front_end/testcases/general_nnbd_opt_out/ffi_sample.dart.weak.transformed.expect

@@ -13,7 +13,7 @@ class Coordinate extends ffi::Struct {
   @#C3
   static final field core::int* #sizeOf = (#C11).{core::List::[]}(ffi::_abi());
   @#C3
-  constructor #fromPointer(dynamic #pointer) → dynamic
+  constructor #fromTypedDataBase(dynamic #pointer) → dynamic
     : super ffi::Struct::_fromPointer(#pointer)
     ;
   static factory allocate(core::double* x, core::double* y, ffi::Pointer<self::Coordinate*>* next) → self::Coordinate* {

pkg/front_end/testcases/incremental_initialize_from_dill/ffi_01.yaml.world.1.expect

@@ -9,7 +9,7 @@ library from "org-dartlang-test:///lib.dart" as lib {
     @#C3
     static final field dart.core::int* #sizeOf = (#C11).{dart.core::List::[]}(dart.ffi::_abi());
     @#C3
-    constructor #fromPointer(dynamic #pointer) → dynamic
+    constructor #fromTypedDataBase(dynamic #pointer) → dynamic
       : super dart.ffi::Struct::_fromPointer(#pointer)
       ;
     static factory allocate(dart.core::double* x, dart.core::double* y, dart.ffi::Pointer<lib::Coordinate*>* next) → lib::Coordinate* {

pkg/front_end/testcases/incremental_initialize_from_dill/ffi_01.yaml.world.2.expect

@@ -9,7 +9,7 @@ library from "org-dartlang-test:///lib.dart" as lib {
     @#C3
     static final field dart.core::int* #sizeOf = (#C11).{dart.core::List::[]}(dart.ffi::_abi());
     @#C3
-    constructor #fromPointer(dynamic #pointer) → dynamic
+    constructor #fromTypedDataBase(dynamic #pointer) → dynamic
       : super dart.ffi::Struct::_fromPointer(#pointer)
       ;
     static factory allocate(dart.core::double* x, dart.core::double* y, dart.ffi::Pointer<lib::Coordinate*>* next) → lib::Coordinate* {

pkg/front_end/testcases/incremental_initialize_from_dill/ffi_02.yaml.world.1.expect

@@ -9,7 +9,7 @@ library from "org-dartlang-test:///lib.dart" as lib {
     @#C3
     static final field dart.core::int* #sizeOf = (#C11).{dart.core::List::[]}(dart.ffi::_abi());
     @#C3
-    constructor #fromPointer(dynamic #pointer) → dynamic
+    constructor #fromTypedDataBase(dynamic #pointer) → dynamic
       : super dart.ffi::Struct::_fromPointer(#pointer)
       ;
     static factory allocate(dart.core::double* x, dart.core::double* y, dart.ffi::Pointer<lib::Coordinate*>* next) → lib::Coordinate* {

pkg/front_end/testcases/incremental_initialize_from_dill/no_outline_change_35.yaml.world.1.expect

@@ -9,7 +9,7 @@ library from "org-dartlang-test:///lib.dart" as lib {
     @#C3
     static final field dart.core::int* #sizeOf = (#C11).{dart.core::List::[]}(dart.ffi::_abi());
     @#C3
-    constructor #fromPointer(dynamic #pointer) → dynamic
+    constructor #fromTypedDataBase(dynamic #pointer) → dynamic
       : super dart.ffi::Struct::_fromPointer(#pointer)
       ;
     static factory allocate(dart.core::double* x, dart.core::double* y, dart.ffi::Pointer<lib::Coordinate*>* next) → lib::Coordinate* {

pkg/front_end/testcases/incremental_initialize_from_dill/no_outline_change_35.yaml.world.2.expect

@@ -9,7 +9,7 @@ library from "org-dartlang-test:///lib.dart" as lib {
     @#C3
     static final field dart.core::int* #sizeOf = (#C11).{dart.core::List::[]}(dart.ffi::_abi());
     @#C3
-    constructor #fromPointer(dynamic #pointer) → dynamic
+    constructor #fromTypedDataBase(dynamic #pointer) → dynamic
       : super dart.ffi::Struct::_fromPointer(#pointer)
       ;
     static factory allocate(dart.core::double* x, dart.core::double* y, dart.ffi::Pointer<lib::Coordinate*>* next) → lib::Coordinate* {

pkg/front_end/testcases/incremental_initialize_from_dill/no_outline_change_35.yaml.world.3.expect

@@ -9,7 +9,7 @@ library from "org-dartlang-test:///lib.dart" as lib {
     @#C3
     static final field dart.core::int* #sizeOf = (#C11).{dart.core::List::[]}(dart.ffi::_abi());
     @#C3
-    constructor #fromPointer(dynamic #pointer) → dynamic
+    constructor #fromTypedDataBase(dynamic #pointer) → dynamic
       : super dart.ffi::Struct::_fromPointer(#pointer)
       ;
     static factory allocate(dart.core::double* x, dart.core::double* y, dart.ffi::Pointer<lib::Coordinate*>* next) → lib::Coordinate* {

pkg/vm/lib/transformations/ffi.dart

@@ -193,10 +193,16 @@ class FfiTransformer extends Transformer {
   final Class doubleClass;
   final Class listClass;
   final Class typeClass;
+  final Procedure unsafeCastMethod;
+  final Class typedDataClass;
+  final Procedure typedDataBufferGetter;
+  final Procedure typedDataOffsetInBytesGetter;
+  final Procedure byteBufferAsUint8List;
   final Class pragmaClass;
   final Field pragmaName;
   final Field pragmaOptions;
   final Procedure listElementAt;
+  final Procedure numAddition;
 
   final Library ffiLibrary;
   final Class nativeFunctionClass;
@@ -221,6 +227,7 @@ class FfiTransformer extends Transformer {
   final Map<NativeType, Procedure> storeMethods;
   final Map<NativeType, Procedure> elementAtMethods;
   final Procedure loadStructMethod;
+  final Procedure memCopy;
   final Procedure asFunctionTearoff;
   final Procedure lookupFunctionTearoff;
 
@@ -235,10 +242,20 @@ class FfiTransformer extends Transformer {
         doubleClass = coreTypes.doubleClass,
         listClass = coreTypes.listClass,
         typeClass = coreTypes.typeClass,
+        unsafeCastMethod =
+            index.getTopLevelMember('dart:_internal', 'unsafeCast'),
+        typedDataClass = index.getClass('dart:typed_data', 'TypedData'),
+        typedDataBufferGetter =
+            index.getMember('dart:typed_data', 'TypedData', 'get:buffer'),
+        typedDataOffsetInBytesGetter = index.getMember(
+            'dart:typed_data', 'TypedData', 'get:offsetInBytes'),
+        byteBufferAsUint8List =
+            index.getMember('dart:typed_data', 'ByteBuffer', 'asUint8List'),
         pragmaClass = coreTypes.pragmaClass,
         pragmaName = coreTypes.pragmaName,
         pragmaOptions = coreTypes.pragmaOptions,
         listElementAt = coreTypes.index.getMember('dart:core', 'List', '[]'),
+        numAddition = coreTypes.index.getMember('dart:core', 'num', '+'),
         ffiLibrary = index.getLibrary('dart:ffi'),
         nativeFunctionClass = index.getClass('dart:ffi', 'NativeFunction'),
         pointerClass = index.getClass('dart:ffi', 'Pointer'),
@@ -283,6 +300,7 @@ class FfiTransformer extends Transformer {
           return index.getTopLevelMember('dart:ffi', "_elementAt$name");
         }),
         loadStructMethod = index.getTopLevelMember('dart:ffi', '_loadStruct'),
+        memCopy = index.getTopLevelMember('dart:ffi', '_memCopy'),
         asFunctionTearoff = index.getMember('dart:ffi', 'NativeFunctionPointer',
             LibraryIndex.tearoffPrefix + 'asFunction'),
         lookupFunctionTearoff = index.getMember(