[fasta] incremental compiler can initialize from dill

As requested in #31691 the incremental compiler should be able to
initialize its state from a previously generated dill file.

This CL introduces that functionally.
It is tested, but actual usage (e.g. proper invalidation of old files)
should live outside of the front-end (i.e. it's the callers responsibility).
One option would be for the caller to load the dill file and use the
included sources to invalidate changed files.

On my machine a from-scratch compile of dart2js takes ~5 seconds,
one initialized from such an output and with a single file invalidated
(though it actually hasn't changed) it takes ~1 second and the resulting
dill file is bit-perfect compared to the from-scratch compiled one.

Closes #31691.

Change-Id: I07f5efca5f2684d73f6c252f2dbc2ad04e9b5cd0
Reviewed-on: https://dart-review.googlesource.com/37260
Commit-Queue: Jens Johansen <[email protected]>
Reviewed-by: Kevin Millikin <[email protected]>

Author: jensjoha

pkg/front_end/lib/src/api_prototype/incremental_kernel_generator.dart

@@ -15,9 +15,11 @@ import '../fasta/incremental_compiler.dart' show IncrementalCompiler;
 import 'compiler_options.dart' show CompilerOptions;
 
 abstract class IncrementalKernelGenerator {
-  factory IncrementalKernelGenerator(CompilerOptions options, Uri entryPoint) {
-    return new IncrementalCompiler(new CompilerContext(
-        new ProcessedOptions(options, false, [entryPoint])));
+  factory IncrementalKernelGenerator(CompilerOptions options, Uri entryPoint,
+      [Uri bootstrapDill]) {
+    return new IncrementalCompiler(
+        new CompilerContext(new ProcessedOptions(options, false, [entryPoint])),
+        bootstrapDill);
   }
 
   /// Returns a component (nee program) whose libraries are the recompiled

pkg/front_end/lib/src/fasta/builder_graph.dart

@@ -0,0 +1,75 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+library fasta.builder_graph;
+
+import 'builder/builder.dart' show LibraryBuilder;
+
+import 'export.dart' show Export;
+
+import 'graph/graph.dart' show Graph;
+
+import 'import.dart' show Import;
+
+import 'dill/dill_library_builder.dart' show DillLibraryBuilder;
+
+import 'source/source_library_builder.dart' show SourceLibraryBuilder;
+
+class BuilderGraph implements Graph<Uri> {
+  final Map<Uri, LibraryBuilder> builders;
+
+  BuilderGraph(this.builders);
+
+  Iterable<Uri> get vertices => builders.keys;
+
+  Iterable<Uri> neighborsOf(Uri vertex) sync* {
+    LibraryBuilder library = builders[vertex];
+    if (library == null) {
+      throw "Library not found: $vertex";
+    }
+    if (library is SourceLibraryBuilder) {
+      for (Import import in library.imports) {
+        Uri uri = import.imported.uri;
+        if (builders.containsKey(uri)) {
+          yield uri;
+        }
+      }
+      for (Export export in library.exports) {
+        Uri uri = export.exported.uri;
+        if (builders.containsKey(uri)) {
+          yield uri;
+        }
+      }
+      for (SourceLibraryBuilder part in library.parts) {
+        Uri uri = part.uri;
+        if (builders.containsKey(uri)) {
+          yield uri;
+        }
+      }
+    } else if (library is DillLibraryBuilder) {
+      // Imports and exports
+      for (var dependency in library.library.dependencies) {
+        var uriString;
+        if (dependency.importedLibraryReference.node != null) {
+          uriString = '${dependency.targetLibrary.importUri}';
+        } else {
+          uriString =
+              '${dependency.importedLibraryReference.canonicalName.name}';
+        }
+        Uri uri = Uri.parse(uriString);
+        if (builders.containsKey(uri)) {
+          yield uri;
+        }
+      }
+
+      // Parts
+      for (var part in library.library.parts) {
+        Uri uri = part.fileUri;
+        if (builders.containsKey(uri)) {
+          yield uri;
+        }
+      }
+    }
+  }
+}

pkg/front_end/lib/src/fasta/incremental_compiler.dart

@@ -7,23 +7,27 @@ library fasta.incremental_compiler;
 import 'dart:async' show Future;
 
 import 'package:kernel/kernel.dart'
-    show Library, Program, Source, loadProgramFromBytes;
+    show loadProgramFromBytes, Library, Procedure, Program, Source;
 
 import '../api_prototype/incremental_kernel_generator.dart'
     show IncrementalKernelGenerator;
 
+import '../api_prototype/file_system.dart' show FileSystemEntity;
+
 import 'builder/builder.dart' show LibraryBuilder;
 
+import 'builder_graph.dart' show BuilderGraph;
+
+import 'compiler_context.dart' show CompilerContext;
+
+import 'dill/dill_library_builder.dart' show DillLibraryBuilder;
+
 import 'dill/dill_target.dart' show DillTarget;
 
 import 'kernel/kernel_target.dart' show KernelTarget;
 
-import 'source/source_graph.dart' show SourceGraph;
-
 import 'source/source_library_builder.dart' show SourceLibraryBuilder;
 
-import 'compiler_context.dart' show CompilerContext;
-
 import 'ticker.dart' show Ticker;
 
 import 'uri_translator.dart' show UriTranslator;
@@ -35,48 +39,112 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
 
   List<Uri> invalidatedUris = <Uri>[];
 
-  DillTarget platform;
+  DillTarget dillLoadedData;
+  List<LibraryBuilder> platformBuilders;
+  Map<Uri, LibraryBuilder> userBuilders;
+  final Uri bootstrapDill;
+  bool bootstrapSuccess = false;
 
   KernelTarget userCode;
 
-  IncrementalCompiler(this.context) : ticker = context.options.ticker;
+  IncrementalCompiler(this.context, [this.bootstrapDill])
+      : ticker = context.options.ticker;
 
   @override
   Future<Program> computeDelta({Uri entryPoint}) async {
     ticker.reset();
     entryPoint ??= context.options.inputs.single;
     return context.runInContext<Future<Program>>((CompilerContext c) async {
-      if (platform == null) {
+      bool includeUserLoadedLibraries = false;
+      Map<Uri, Source> uriToSource = {};
+      Map<Uri, int> importUriToOrder = {};
+      Procedure userLoadedUriMain;
+      bootstrapSuccess = false;
+      if (dillLoadedData == null) {
         UriTranslator uriTranslator = await c.options.getUriTranslator();
         ticker.logMs("Read packages file");
 
-        platform = new DillTarget(ticker, uriTranslator, c.options.target);
-        List<int> bytes = await c.options.loadSdkSummaryBytes();
-        if (bytes != null) {
+        dillLoadedData =
+            new DillTarget(ticker, uriTranslator, c.options.target);
+        List<int> summaryBytes = await c.options.loadSdkSummaryBytes();
+        int bytesLength = 0;
+        Program program;
+        if (summaryBytes != null) {
           ticker.logMs("Read ${c.options.sdkSummary}");
-          Program program = loadProgramFromBytes(bytes);
+          program = loadProgramFromBytes(summaryBytes);
           ticker.logMs("Deserialized ${c.options.sdkSummary}");
-          platform.loader.appendLibraries(program, byteCount: bytes.length);
-          ticker.logMs("Appended libraries");
+          bytesLength += summaryBytes.length;
+        }
+
+        if (bootstrapDill != null) {
+          FileSystemEntity entity =
+              c.options.fileSystem.entityForUri(bootstrapDill);
+          if (await entity.exists()) {
+            List<int> bootstrapBytes = await entity.readAsBytes();
+            if (bootstrapBytes != null) {
+              Set<Uri> prevLibraryUris = new Set<Uri>.from(
+                  program.libraries.map((Library lib) => lib.importUri));
+              ticker.logMs("Read $bootstrapDill");
+              bool bootstrapFailed = false;
+              try {
+                loadProgramFromBytes(bootstrapBytes, program);
+              } catch (e) {
+                bootstrapFailed = true;
+                program = loadProgramFromBytes(summaryBytes);
+              }
+              if (!bootstrapFailed) {
+                bootstrapSuccess = true;
+                bytesLength += bootstrapBytes.length;
+                for (Library lib in program.libraries) {
+                  if (prevLibraryUris.contains(lib.importUri)) continue;
+                  importUriToOrder[lib.importUri] = importUriToOrder.length;
+                }
+                userLoadedUriMain = program.mainMethod;
+                includeUserLoadedLibraries = true;
+                uriToSource.addAll(program.uriToSource);
+              }
+            }
+          }
+        }
+        summaryBytes = null;
+        if (program != null) {
+          dillLoadedData.loader
+              .appendLibraries(program, byteCount: bytesLength);
         }
-        await platform.buildOutlines();
+        ticker.logMs("Appended libraries");
+        await dillLoadedData.buildOutlines();
+        userBuilders = <Uri, LibraryBuilder>{};
+        platformBuilders = <LibraryBuilder>[];
+        dillLoadedData.loader.builders.forEach((uri, builder) {
+          if (builder.fileUri.scheme == "dart") {
+            platformBuilders.add(builder);
+          } else {
+            userBuilders[uri] = builder;
+          }
+        });
+        if (userBuilders.isEmpty) userBuilders = null;
       }
 
       List<Uri> invalidatedUris = this.invalidatedUris.toList();
       this.invalidatedUris.clear();
 
       List<LibraryBuilder> reusedLibraries =
           computeReusedLibraries(invalidatedUris);
+      Set<Uri> reusedLibraryUris =
+          new Set<Uri>.from(reusedLibraries.map((b) => b.uri));
+      for (Uri uri in new Set<Uri>.from(dillLoadedData.loader.builders.keys)
+        ..removeAll(reusedLibraryUris)) {
+        dillLoadedData.loader.builders.remove(uri);
+      }
+
       if (userCode != null) {
         ticker.logMs("Decided to reuse ${reusedLibraries.length}"
             " of ${userCode.loader.builders.length} libraries");
       }
 
-      platform.loader.builders.forEach((Uri uri, LibraryBuilder builder) {
-        reusedLibraries.add(builder);
-      });
+      reusedLibraries.addAll(platformBuilders);
       userCode = new KernelTarget(
-          c.fileSystem, false, platform, platform.uriTranslator,
+          c.fileSystem, false, dillLoadedData, dillLoadedData.uriTranslator,
           uriToSource: c.uriToSource);
       for (LibraryBuilder library in reusedLibraries) {
         userCode.loader.builders[library.uri] = library;
@@ -94,16 +162,45 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
       Program programWithDill =
           await userCode.buildProgram(verify: c.options.verify);
 
+      List<Library> libraries =
+          new List<Library>.from(userCode.loader.libraries);
+      uriToSource.addAll(userCode.uriToSource);
+      if (includeUserLoadedLibraries) {
+        for (LibraryBuilder library in reusedLibraries) {
+          if (library.fileUri.scheme == "dart") continue;
+          assert(library is DillLibraryBuilder);
+          libraries.add((library as DillLibraryBuilder).library);
+        }
+
+        // For now ensure original order of libraries to produce bit-perfect
+        // output.
+        List<Library> librariesOriginalOrder =
+            new List<Library>.filled(libraries.length, null, growable: true);
+        int lastOpen = libraries.length;
+        for (Library lib in libraries) {
+          int order = importUriToOrder[lib.importUri];
+          if (order != null) {
+            librariesOriginalOrder[order] = lib;
+          } else {
+            librariesOriginalOrder[--lastOpen] = lib;
+          }
+        }
+        libraries = librariesOriginalOrder;
+      }
+
       // This is the incremental program.
-      return new Program(
-          libraries: new List<Library>.from(userCode.loader.libraries),
-          uriToSource: new Map<Uri, Source>.from(userCode.uriToSource))
-        ..mainMethod = programWithDill?.mainMethod;
+      Procedure mainMethod = programWithDill == null
+          ? userLoadedUriMain
+          : programWithDill.mainMethod;
+      return new Program(libraries: libraries, uriToSource: uriToSource)
+        ..mainMethod = mainMethod;
     });
   }
 
   List<LibraryBuilder> computeReusedLibraries(Iterable<Uri> invalidatedUris) {
-    if (userCode == null) return <LibraryBuilder>[];
+    if (userCode == null && userBuilders == null) {
+      return <LibraryBuilder>[];
+    }
 
     // [invalidatedUris] converted to a set.
     Set<Uri> invalidatedFileUris = invalidatedUris.toSet();
@@ -116,25 +213,29 @@ class IncrementalCompiler implements IncrementalKernelGenerator {
     List<Uri> invalidatedImportUris = <Uri>[];
 
     // Compute [builders] and [invalidatedImportUris].
-    addBuilderAndInvalidateUris(Uri uri, LibraryBuilder libraryBuilder) {
-      if (libraryBuilder.loader != platform.loader) {
-        assert(libraryBuilder is SourceLibraryBuilder);
-        SourceLibraryBuilder library = libraryBuilder;
-        builders[uri] = library;
-        if (invalidatedFileUris.contains(uri) ||
-            (uri != library.fileUri &&
-                invalidatedFileUris.contains(library.fileUri))) {
-          invalidatedImportUris.add(uri);
-        }
+    addBuilderAndInvalidateUris(Uri uri, LibraryBuilder library) {
+      builders[uri] = library;
+      if (invalidatedFileUris.contains(uri) ||
+          (uri != library.fileUri &&
+              invalidatedFileUris.contains(library.fileUri)) ||
+          (library is DillLibraryBuilder &&
+              uri != library.library.fileUri &&
+              invalidatedFileUris.contains(library.library.fileUri))) {
+        invalidatedImportUris.add(uri);
+      }
+      if (library is SourceLibraryBuilder) {
         for (var part in library.parts) {
           addBuilderAndInvalidateUris(part.uri, part);
         }
       }
     }
 
-    userCode.loader.builders.forEach(addBuilderAndInvalidateUris);
+    userBuilders?.forEach(addBuilderAndInvalidateUris);
+    if (userCode != null) {
+      userCode.loader.builders.forEach(addBuilderAndInvalidateUris);
+    }
 
-    SourceGraph graph = new SourceGraph(builders);
+    BuilderGraph graph = new BuilderGraph(builders);
 
     // Compute direct dependencies for each import URI (the reverse of the
     // edges returned by `graph.neighborsOf`).