WIP

Author: retronym

src/compiler/scala/tools/nsc/PipelineMain.scala

@@ -33,6 +33,7 @@ class PipelineMainClass(label: String, parallelism: Int, strategy: BuildStrategy
   private var reporter: Reporter = _
 
   implicit val executor = ExecutionContext.fromExecutor(new java.util.concurrent.ForkJoinPool(parallelism))
+  val fileManager = ToolProvider.getSystemJavaCompiler.getStandardFileManager(null, null, null)
 
   private class PickleClassPath[G <: Global](data: mutable.AnyRefMap[G#Symbol, PickleBuffer]) {
     val dir = new VirtualDirectory("fakes", None)
@@ -102,6 +103,34 @@ class PipelineMainClass(label: String, parallelism: Int, strategy: BuildStrategy
     val timer = new Timer
     timer.start()
     strategy match {
+      case OutlineTypeOnly =>
+        val futures = projects.map { p =>
+          val f1 = Future.sequence[Unit, List](dependsOn.getOrElse(p, Nil).map(_.outlineDone.future))
+          p.shouldOutlineType = true
+          f1.map { _ =>
+            p.outlineCompile()
+            p.javaCompile()
+          }
+        }
+
+        val toWait: Future[List[Unit]] = Future.sequence(futures).flatMap(_ => Future.sequence(projects.flatMap(p => p.javaDone.future :: p.outlineDone.future :: Nil) ))
+        Await.result(toWait, Duration.Inf)
+        timer.stop()
+
+        for (p <- projects) {
+          val dependencies = dependsOn(p)
+          def maxByOrZero[A](as: List[A])(f: A => Double): Double = if (as.isEmpty) 0d else as.map(f).max
+          val maxOutlineCriticalPathMs = maxByOrZero(dependencies)(_.outlineCriticalPathMs)
+          p.outlineCriticalPathMs = maxOutlineCriticalPathMs + p.outlineTimer.durationMs
+          p.regularCriticalPathMs = maxOutlineCriticalPathMs + maxByOrZero(p.groups)(_.timer.durationMs)
+          p.fullCriticalPathMs = maxByOrZero(dependencies)(_.fullCriticalPathMs) + p.groups.map(_.timer.durationMs).sum
+        }
+
+        if (parallelism == 1) {
+          val criticalPath = projects.maxBy(_.regularCriticalPathMs)
+          println(f"Critical path: ${criticalPath.regularCriticalPathMs}%.0f ms. Wall Clock: ${timer.durationMs}%.0f ms")
+        } else
+          println(f" Wall Clock: ${timer.durationMs}%.0f ms")
       case OutlineTypePipeline =>
         val futures = projects.map { p =>
           val f1 = Future.sequence[Unit, List](dependsOn.getOrElse(p, Nil).map(_.outlineDone.future))
@@ -202,7 +231,8 @@ class PipelineMainClass(label: String, parallelism: Int, strategy: BuildStrategy
         events += durationEvent(p.label, "outline-type", p.outlineTimer)
       }
       for ((g, ix) <- p.groups.zipWithIndex) {
-        events += durationEvent(p.label, "compile-" + ix, g.timer)
+        if (g.timer.durationMicros > 0d)
+          events += durationEvent(p.label, "compile-" + ix, g.timer)
       }
       events.result()
     }
@@ -229,8 +259,8 @@ class PipelineMainClass(label: String, parallelism: Int, strategy: BuildStrategy
       } else {
         command.settings.classpath.value = command.settings.outputDirs.getSingleOutput.get.toString + File.pathSeparator + command.settings.classpath.value
         val length = files.length
-        val groups = (length.toDouble / 64).toInt.max(1)
-        files.grouped((length.toDouble / groups).ceil.toInt).toList.map(Group(_))
+        val groups = (length.toDouble / 128).toInt.max(1)
+        files.grouped((length.toDouble / groups).ceil.toInt.max(1)).toList.map(Group(_))
       }
     }
     command.settings.outputDirs.getSingleOutput.get.file.mkdirs()
@@ -260,6 +290,7 @@ class PipelineMainClass(label: String, parallelism: Int, strategy: BuildStrategy
       outlineTimer.start()
       command.settings.Youtline.value = true
       command.settings.stopAfter.value = List("pickler")
+      command.settings.Ymacroexpand.value = command.settings.MacroExpand.None
       val run1 = new compiler.Run()
       run1 compile files
       allPickleData.put(command.settings.outputDirs.getSingleOutput.get.file.toPath.toRealPath().normalize(), new PickleClassPath(run1.symData))
@@ -274,6 +305,7 @@ class PipelineMainClass(label: String, parallelism: Int, strategy: BuildStrategy
     def fullCompile(): Unit = {
       command.settings.Youtline.value = false
       command.settings.stopAfter.value = Nil
+      command.settings.Ymacroexpand.value = command.settings.MacroExpand.Normal
 
       for (group <- groups) {
         group.done.completeWith {
@@ -324,7 +356,6 @@ class PipelineMainClass(label: String, parallelism: Int, strategy: BuildStrategy
       val javaSources = files.filter(_.endsWith(".java"))
       if (javaSources.nonEmpty) {
         javaDone.completeWith(Future {
-          val fileManager = ToolProvider.getSystemJavaCompiler.getStandardFileManager(null, null, null)
           val opts = java.util.Arrays.asList("-d", command.settings.outdir.value, "-cp", command.settings.outdir.value + File.pathSeparator + command.settings.classpath.value)
           val compileTask = ToolProvider.getSystemJavaCompiler.getTask(null, null, null, opts, null, fileManager.getJavaFileObjects(javaSources.toArray: _*))
           compileTask.call()
@@ -391,6 +422,7 @@ class PipelineMainClass(label: String, parallelism: Int, strategy: BuildStrategy
 }
 
 sealed abstract class BuildStrategy
+case object OutlineTypeOnly extends BuildStrategy
 /** Outline type check to compute type signatures as pickles as an input to downstream compilation. */
 case object OutlineTypePipeline extends BuildStrategy
 case object Pipeline extends BuildStrategy
@@ -401,7 +433,7 @@ object PipelineMain {
   def main(args: Array[String]): Unit = {
     var i = 0
     //for (_ <- 1 to 10; n <- List(parallel.availableProcessors, 1); strat <- List(Pipeline, OutlineTypePipeline, Traditional)) {
-    for (_ <- 1 to 10; n <- List(parallel.availableProcessors); strat <- List(Pipeline, OutlineTypePipeline, Traditional)) {
+    for (_ <- 1 to 20; n <- List(parallel.availableProcessors); strat <- List(OutlineTypeOnly)) {
         i += 1
         val main = new PipelineMainClass(i.toString, n, strat)
         println(s"====== ITERATION $i=======")