[pkg][vm] Native Assets builder

This package contains the logic for building native assets.

This package is the backend that invokes toplevel `build.dart` scripts.
For more info on these scripts see https://github.com/dart-lang/native.

This is a separate package so that dartdev and flutter_tools can reuse
the same logic without flutter_tools having to import dartdev.

Some design decisions:

* We don't yet have `build_dependencies`, so we use the ordinary
  dependency graph for ordering of native assets builds. (If there is
  a cycle we refuse to run.)
  Bug: dart-lang/pub#3794
* Builds are cached based on all the configuration provided by the
  caller. Environment variables are ignored in caching. This CL also
  contains a unit test that invokes the build by not passing through
  environment variables. However, for Windows we need to pass through
  at least `SYSTEMROOT` for MSVC to run correctly. So we might need
  to further explore if we can/want to lock env variables down.
  Bug: dart-lang/native#32
  Bug: dart-lang/native#33

Run tests:
```
dart tools/generate_package_config.dart && \
tools/test.py -n unittest-asserts-release-linux pkg/native_assets_builder
```

Bug: #50565
Change-Id: I133052d7195373e87d20924d61e1e96e3d34ce8f
Cq-Include-Trybots: luci.dart.try:pkg-linux-debug-try,pkg-linux-release-try,pkg-mac-release-arm64-try,pkg-mac-release-try,pkg-win-release-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/300203
Reviewed-by: Liam Appelbe <[email protected]>
Commit-Queue: Daco Harkes <[email protected]>
Reviewed-by: Martin Kustermann <[email protected]>
Reviewed-by: Hossein Yousefi <[email protected]>

Author: dcharkes

.gitignore

@@ -75,6 +75,9 @@ client/tests/drt
 packages
 pubspec.lock
 
+# Files for native assets auto-generated per checkout.
+/.dart_tool/native_assets_builder/
+
 # The top level package file (this is auto-generated per checkout).
 /.dart_tool/package_config.json
 

DEPS

@@ -152,7 +152,7 @@ vars = {
   "matcher_rev": "4dfd9adfdcd3de95e6964fc295075a32635e5b37",
   "mime_rev": "eb9d54b8f6fab9442b2b733e19137dc25fb0b868",
   "mockito_rev": "51a7728927c505a907384860f4a407f8b7219004",
-  "native_rev": "64aa5b500fd71b51a34bf4ef0dc05ec24379be00",
+  "native_rev": "e01aa63e851ce971390bc45fc17d16240b8d164a",
   "package_config_rev": "f41f92cd4d2c539d910f2ef041ba6f6bd8469e78",
   "path_rev": "f8d15c2775835fd61ea5e06bb0ab99232f5ec446",
   "pool_rev": "86b4f4328a32a9e9eb4f1601c8296e22d3e50552",

pkg/front_end/lib/src/api_prototype/experimental_flags_generated.dart

@@ -858,6 +858,9 @@ const AllowedExperimentalFlags defaultAllowedExperimentalFlags =
   "meta": {
     ExperimentalFlag.nonNullable,
   },
+  "native_assets_builder": {
+    ExperimentalFlag.nonNullable,
+  },
   "native_stack_traces": {
     ExperimentalFlag.nonNullable,
   },

pkg/front_end/test/spell_checking_list_common.txt

@@ -204,6 +204,8 @@ asserted
 assertion
 assertions
 asserts
+asset
+assets
 assign
 assignability
 assignable
@@ -3114,6 +3116,7 @@ tokens
 too
 took
 tool
+toolchain
 tools
 top
 topologically

pkg/native_assets_builder/.gitignore

@@ -0,0 +1,48 @@
+# Miscellaneous
+*.class
+*.log
+*.pyc
+*.swp
+.DS_Store
+.atom/
+.buildlog/
+.history
+.svn/
+
+# IntelliJ related
+*.iml
+*.ipr
+*.iws
+.idea/
+
+# The .vscode folder contains launch configuration and tasks you configure in
+# VS Code which you may wish to be included in version control, so this line
+# is commented out by default.
+#.vscode/
+
+# Flutter/Dart/Pub related
+**/doc/api/
+**/ios/Flutter/.last_build_id
+.dart_tool/
+.flutter-plugins
+.flutter-plugins-dependencies
+.packages
+.pub-cache/
+.pub/
+/build/
+
+# Web related
+lib/generated_plugin_registrant.dart
+
+# Symbolication related
+app.*.symbols
+
+# Obfuscation related
+app.*.map.json
+
+# Android Studio will place build artifacts here
+/android/app/debug
+/android/app/profile
+/android/app/release
+
+pubspec.lock

pkg/native_assets_builder/AUTHORS

@@ -0,0 +1,6 @@
+# Below is a list of people and organizations that have contributed
+# to the Dart project. Names should be added to the list like so:
+#
+#   Name/Organization <email address>
+
+Google LLC

pkg/native_assets_builder/LICENSE

@@ -0,0 +1,27 @@
+Copyright 2022, the Dart project authors.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google LLC nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pkg/native_assets_builder/OWNERS

@@ -0,0 +1 @@
+file:/tools/OWNERS_VM

pkg/native_assets_builder/README.md

@@ -0,0 +1,7 @@
+This package contains the logic for building native assets.
+
+This package is the backend that invokes toplevel `build.dart` scripts.
+For more info on these scripts see https://github.com/dart-lang/native.
+
+This is a separate package so that dartdev and flutter_tools can reuse
+the same logic with flutter_tools having to import dartdev.

pkg/native_assets_builder/analysis_options.yaml

@@ -0,0 +1,27 @@
+include: package:lints/recommended.yaml
+
+analyzer:
+  language:
+    strict-raw-types: true
+    strict-inference: true
+  # The test projects do _not_ have resolved dependencies through
+  # `dart tools/package_deps/bin/package_deps.dart`.
+  # So don't analyze them in the IDE or on the CI.
+  exclude:
+    - test/test_projects/**.dart
+
+linter:
+  rules:
+    - always_declare_return_types
+    - avoid_dynamic_calls
+    - camel_case_types
+    - depend_on_referenced_packages
+    - directives_ordering
+    - prefer_const_declarations
+    - prefer_expression_function_bodies
+    - prefer_final_in_for_each
+    - prefer_final_locals
+    - prefer_relative_imports
+    - prefer_single_quotes
+    - sort_pub_dependencies
+    - unawaited_futures

pkg/native_assets_builder/lib/native_assets_builder.dart

@@ -0,0 +1,6 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+export 'package:native_assets_builder/src/build_runner/build_runner.dart';
+export 'package:native_assets_builder/src/package_layout/package_layout.dart';

pkg/native_assets_builder/lib/src/build_runner/build_planner.dart

@@ -0,0 +1,102 @@
+// Copyright (c) 2023, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:kernel/util/graph.dart' as graph;
+import 'package:package_config/package_config.dart';
+
+class NativeAssetsBuildPlanner {
+  final PackageGraph packageGraph;
+  final List<Package> packagesWithNativeAssets;
+  final Uri dartExecutable;
+
+  NativeAssetsBuildPlanner({
+    required this.packageGraph,
+    required this.packagesWithNativeAssets,
+    required this.dartExecutable,
+  });
+
+  static Future<NativeAssetsBuildPlanner> fromRootPackageRoot({
+    required Uri rootPackageRoot,
+    required List<Package> packagesWithNativeAssets,
+    required Uri dartExecutable,
+  }) async {
+    final result = await Process.run(
+      dartExecutable.toFilePath(),
+      [
+        'pub',
+        'deps',
+        '--json',
+      ],
+      workingDirectory: rootPackageRoot.toFilePath(),
+    );
+    final packageGraph = PackageGraph.fromPubDepsJsonString(result.stdout);
+    return NativeAssetsBuildPlanner(
+      packageGraph: packageGraph,
+      packagesWithNativeAssets: packagesWithNativeAssets,
+      dartExecutable: dartExecutable,
+    );
+  }
+
+  List<Package> plan() {
+    final packageMap = {
+      for (final package in packagesWithNativeAssets) package.name: package
+    };
+    final packagesToBuild = packageMap.keys.toSet();
+    final stronglyConnectedComponents = packageGraph.computeStrongComponents();
+    final result = <Package>[];
+    for (final stronglyConnectedComponent in stronglyConnectedComponents) {
+      final stronglyConnectedComponentWithNativeAssets = [
+        for (final packageName in stronglyConnectedComponent)
+          if (packagesToBuild.contains(packageName)) packageName
+      ];
+      if (stronglyConnectedComponentWithNativeAssets.length > 1) {
+        throw Exception(
+          'Cyclic dependency for native asset builds in the following '
+          'packages: $stronglyConnectedComponent.',
+        );
+      } else if (stronglyConnectedComponentWithNativeAssets.length == 1) {
+        result.add(
+            packageMap[stronglyConnectedComponentWithNativeAssets.single]!);
+      }
+    }
+    return result;
+  }
+}
+
+class PackageGraph implements graph.Graph<String> {
+  final Map<String, List<String>> map;
+
+  PackageGraph(this.map);
+
+  /// Construct a graph from the JSON produced by `dart pub deps --json`.
+  factory PackageGraph.fromPubDepsJsonString(String json) =>
+      PackageGraph.fromPubDepsJson(jsonDecode(json) as Map<dynamic, dynamic>);
+
+  /// Construct a graph from the JSON produced by `dart pub deps --json`.
+  factory PackageGraph.fromPubDepsJson(Map<dynamic, dynamic> map) {
+    final result = <String, List<String>>{};
+    final packages = map['packages'] as List<dynamic>;
+    for (final package in packages) {
+      final package_ = package as Map<dynamic, dynamic>;
+      final name = package_['name'] as String;
+      final dependencies = (package_['dependencies'] as List<dynamic>)
+          .whereType<String>()
+          .toList();
+      result[name] = dependencies;
+    }
+    return PackageGraph(result);
+  }
+
+  @override
+  Iterable<String> neighborsOf(String vertex) => map[vertex] ?? [];
+
+  @override
+  Iterable<String> get vertices => map.keys;
+
+  List<List<String>> computeStrongComponents() =>
+      graph.computeStrongComponents(this);
+}