Revert "Revert "Use the frontend server to compile pub executables (#2968)" (#3006)" (#3008)

This reverts commit d2ad13d.

Author: jonasfj

.github/workflows/test.yaml

@@ -70,5 +70,5 @@ jobs:
           cat "$_TESTS_FILE.${{ matrix.shard }}"
         shell: bash
       - name: Run tests
-        run: dart test --preset travis $(cat $_TESTS_FILE.${{ matrix.shard }})
+        run: dart test --use-data-isolate-strategy --preset travis $(cat $_TESTS_FILE.${{ matrix.shard }})
         shell: bash

lib/src/dart.dart

@@ -18,6 +18,8 @@ import 'package:analyzer/dart/analysis/session.dart';
 import 'package:analyzer/dart/ast/ast.dart';
 import 'package:analyzer/error/error.dart';
 import 'package:cli_util/cli_util.dart';
+import 'package:frontend_server_client/frontend_server_client.dart';
+import 'package:meta/meta.dart';
 import 'package:path/path.dart' as p;
 
 import 'exceptions.dart';
@@ -38,41 +40,6 @@ bool isEntrypoint(CompilationUnit dart) {
   });
 }
 
-/// Snapshots the Dart executable at [executablePath] to a snapshot at
-/// [snapshotPath].
-///
-/// If [packageConfigFile] is passed, it's used to resolve `package:` URIs in
-/// the executable. Otherwise, a `packages/` directory or a package spec is
-/// inferred from the executable's location.
-///
-/// If [name] is passed, it is used to describe the executable in logs and error
-/// messages.
-Future snapshot(
-  String executablePath,
-  String snapshotPath, {
-  String packageConfigFile,
-  String name,
-}) async {
-  final args = [
-    if (packageConfigFile != null) '--packages=$packageConfigFile',
-    '--snapshot=$snapshotPath',
-    p.toUri(executablePath).toString()
-  ];
-
-  final result = await runProcess(Platform.executable, args);
-  final highlightedName = name = log.bold(name ?? executablePath.toString());
-  if (result.success) {
-    log.message('Precompiled $highlightedName.');
-  } else {
-    // Don't leave partial results.
-    deleteEntry(snapshotPath);
-
-    throw ApplicationException(
-        log.yellow('Failed to precompile $highlightedName:\n') +
-            result.stderr.join('\n'));
-  }
-}
-
 class AnalysisContextManager {
   /// The map from a context root directory to to the context.
   final Map<String, AnalysisContext> _contexts = {};
@@ -181,3 +148,52 @@ class AnalyzerErrorGroup implements Exception {
   @override
   String toString() => errors.join('\n');
 }
+
+/// Precompiles the Dart executable at [executablePath] to a kernel file at
+/// [outputPath].
+///
+/// This file is also cached at [incrementalDillOutputPath] which is used to
+/// initialize the compiler on future runs.
+///
+/// The [packageConfigPath] should point at the package config file to be used
+/// for `package:` uri resolution.
+///
+/// The [name] is used to describe the executable in logs and error messages.
+Future<void> precompile({
+  @required String executablePath,
+  @required String incrementalDillOutputPath,
+  @required String name,
+  @required String outputPath,
+  @required String packageConfigPath,
+}) async {
+  ensureDir(p.dirname(outputPath));
+  ensureDir(p.dirname(incrementalDillOutputPath));
+  const platformDill = 'lib/_internal/vm_platform_strong.dill';
+  final sdkRoot = p.relative(p.dirname(p.dirname(Platform.resolvedExecutable)));
+  var client = await FrontendServerClient.start(
+    executablePath,
+    incrementalDillOutputPath,
+    platformDill,
+    sdkRoot: sdkRoot,
+    packagesJson: packageConfigPath,
+    printIncrementalDependencies: false,
+  );
+  try {
+    var result = await client.compile();
+
+    final highlightedName = log.bold(name);
+    if (result?.errorCount == 0) {
+      log.message('Precompiled $highlightedName.');
+      await File(incrementalDillOutputPath).copy(outputPath);
+    } else {
+      // Don't leave partial results.
+      deleteEntry(outputPath);
+
+      throw ApplicationException(
+          log.yellow('Failed to precompile $highlightedName:\n') +
+              (result?.compilerOutputLines?.join('\n') ?? ''));
+    }
+  } finally {
+    client.kill();
+  }
+}

lib/src/entrypoint.dart

@@ -171,6 +171,10 @@ class Entrypoint {
   /// The path to the directory containing dependency executable snapshots.
   String get _snapshotPath => p.join(cachePath, 'bin');
 
+  /// The path to the directory containing previous dill files for incremental
+  /// builds.
+  String get _incrementalDillsPath => p.join(cachePath, 'incremental');
+
   /// Loads the entrypoint from a package at [rootDir].
   Entrypoint(String rootDir, this.cache)
       : root = Package.load(null, rootDir, cache.sources),
@@ -334,7 +338,7 @@ class Entrypoint {
         ensureDir(_snapshotPath);
       }
       return waitAndPrintErrors(executables.map((executable) {
-        var dir = p.dirname(snapshotPathOfExecutable(executable));
+        var dir = p.dirname(pathOfExecutable(executable));
         cleanDir(dir);
         return _precompileExecutable(executable);
       }));
@@ -344,16 +348,18 @@ class Entrypoint {
   /// Precompiles executable .dart file at [path] to a snapshot.
   Future<void> precompileExecutable(Executable executable) async {
     return await log.progress('Precompiling executable', () async {
-      ensureDir(p.dirname(snapshotPathOfExecutable(executable)));
+      ensureDir(p.dirname(pathOfExecutable(executable)));
       return waitAndPrintErrors([_precompileExecutable(executable)]);
     });
   }
 
   Future<void> _precompileExecutable(Executable executable) async {
     final package = executable.package;
-    await dart.snapshot(
-        resolveExecutable(executable), snapshotPathOfExecutable(executable),
-        packageConfigFile: packageConfigFile,
+    await dart.precompile(
+        executablePath: resolveExecutable(executable),
+        outputPath: pathOfExecutable(executable),
+        incrementalDillOutputPath: incrementalDillPathOfExecutable(executable),
+        packageConfigPath: packageConfigFile,
         name:
             '$package:${p.basenameWithoutExtension(executable.relativePath)}');
   }
@@ -365,7 +371,7 @@ class Entrypoint {
   /// different sdk.
   ///
   /// [path] must be relative.
-  String snapshotPathOfExecutable(Executable executable) {
+  String pathOfExecutable(Executable executable) {
     assert(p.isRelative(executable.relativePath));
     final versionSuffix = sdk.version;
     return isGlobal
@@ -375,6 +381,15 @@ class Entrypoint {
             '${p.basename(executable.relativePath)}-$versionSuffix.snapshot');
   }
 
+  String incrementalDillPathOfExecutable(Executable executable) {
+    assert(p.isRelative(executable.relativePath));
+    return isGlobal
+        ? p.join(_incrementalDillsPath,
+            '${p.basename(executable.relativePath)}.incremental.dill')
+        : p.join(_incrementalDillsPath, executable.package,
+            '${p.basename(executable.relativePath)}.incremental.dill');
+  }
+
   /// The absolute path of [executable] resolved relative to [this].
   String resolveExecutable(Executable executable) {
     return p.join(

lib/src/executable.dart

@@ -72,7 +72,7 @@ Future<int> runExecutable(
 
   entrypoint.migrateCache();
 
-  var snapshotPath = entrypoint.snapshotPathOfExecutable(executable);
+  var snapshotPath = entrypoint.pathOfExecutable(executable);
 
   // Don't compile snapshots for mutable packages, since their code may
   // change later on.
@@ -327,7 +327,7 @@ Future<String> getExecutableForCommand(
     if (!allowSnapshot || entrypoint.packageGraph.isPackageMutable(package)) {
       return p.relative(path, from: root);
     } else {
-      final snapshotPath = entrypoint.snapshotPathOfExecutable(executable);
+      final snapshotPath = entrypoint.pathOfExecutable(executable);
       if (!fileExists(snapshotPath)) {
         await warningsOnlyUnlessTerminal(
           () => entrypoint.precompileExecutable(executable),

lib/src/global_packages.dart

@@ -590,8 +590,7 @@ To recompile executables, first run `global deactivate ${dep.name}`.
         deleteEntry(file);
         _createBinStub(
             entrypoint.root, p.basenameWithoutExtension(file), binStubScript,
-            overwrite: true,
-            snapshot: entrypoint.snapshotPathOfExecutable(executable));
+            overwrite: true, snapshot: entrypoint.pathOfExecutable(executable));
       }
     }
   }
@@ -641,7 +640,7 @@ To recompile executables, first run `global deactivate ${dep.name}`.
         executable,
         script,
         overwrite: overwriteBinStubs,
-        snapshot: entrypoint.snapshotPathOfExecutable(
+        snapshot: entrypoint.pathOfExecutable(
           exec.Executable.adaptProgramName(package.name, script),
         ),
       );

pubspec.yaml

@@ -12,6 +12,7 @@ dependencies:
   cli_util: ^0.3.0
   collection: ^1.15.0
   crypto: ^3.0.1
+  frontend_server_client: ^2.0.0
   http: ^0.13.3
   http_multi_server: ^3.0.1
   meta: ^1.3.0