Experimental support for benchmarking compilation under SBT

Works by driving a child SBT process via stdin/stdout scraping.

```
> compilation/jmh:run HotSbtBenchmark -f 1 -psource=scalap
[info] Compiling 1 Scala source to /Users/jz/code/compiler-benchmark/compilation/target/scala-2.11/classes...
Processing 46 classes from /Users/jz/code/compiler-benchmark/compilation/target/scala-2.11/classes with "reflection" generator
Writing out Java source to /Users/jz/code/compiler-benchmark/compilation/target/scala-2.11/src_managed/jmh and resources to /Users/jz/code/compiler-benchmark/compilation/target/scala-2.11/resource_managed/jmh
[info] Compiling 5 Java sources to /Users/jz/code/compiler-benchmark/compilation/target/scala-2.11/classes...
[info] Running scala.bench.ScalacBenchmarkRunner HotSbtBenchmark -f 1 -psource=scalap
[info] # JMH version: 1.19
[info] # VM version: JDK 1.8.0_112, VM 25.112-b16
[info] # VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_112.jdk/Contents/Home/jre/bin/java
[info] # VM options: -Dsbt.launcher=/usr/local/Cellar/sbt/0.13.15/libexec/bin/sbt-launch.jar
[info] # Warmup: 10 iterations, 10 s each
[info] # Measurement: 10 iterations, 10 s each
[info] # Timeout: 10 min per iteration
[info] # Threads: 1 thread, will synchronize iterations
[info] # Benchmark mode: Sampling time
[info] # Benchmark: scala.tools.nsc.HotSbtBenchmark.compile
[info] # Parameters: (corpusVersion = a8c43dc, extraArgs = , source = scalap)
[info]
[info] # Run progress: 0.00% complete, ETA 00:03:20
[info] # Fork: 1 of 1
[info] # Warmup Iteration   1: 6683.623 ms/op
[info] # Warmup Iteration   2: 2724.200 ±(99.9%) 2851.803 ms/op
[info] # Warmup Iteration   3: 1867.863 ±(99.9%) 314.998 ms/op
[info] # Warmup Iteration   4: 2167.197 ±(99.9%) 3228.941 ms/op
[info] # Warmup Iteration   5: 2198.235 ±(99.9%) 713.756 ms/op
[info] # Warmup Iteration   6: 1831.862 ±(99.9%) 452.490 ms/op
[info] # Warmup Iteration   7: 1726.306 ±(99.9%) 115.085 ms/op
[info] # Warmup Iteration   8: 1856.679 ±(99.9%) 1093.828 ms/op
[info] # Warmup Iteration   9: 1615.406 ±(99.9%) 796.722 ms/op
[info] # Warmup Iteration  10: 1492.573 ±(99.9%) 62.763 ms/op
...

⚡ sbt
[info] Loading global plugins from /Users/jz/.sbt/0.13/plugins
[info] Loading project definition from /Users/jz/code/compiler-benchmark/project
[info] Set current project to compiler-benchmark (in build file:/Users/jz/code/compiler-benchmark/)
> compilation/jmh:run HotScalacBenchmark -f 1 -psource=scalap
[info] Running scala.bench.ScalacBenchmarkRunner HotScalacBenchmark -f 1 -psource=scalap
[info] # JMH version: 1.19
[info] # VM version: JDK 1.8.0_112, VM 25.112-b16
[info] # VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_112.jdk/Contents/Home/jre/bin/java
[info] # VM options: -Dsbt.launcher=/usr/local/Cellar/sbt/0.13.15/libexec/bin/sbt-launch.jar
[info] # Warmup: 10 iterations, 10 s each
[info] # Measurement: 10 iterations, 10 s each
[info] # Timeout: 10 min per iteration
[info] # Threads: 1 thread, will synchronize iterations
[info] # Benchmark mode: Sampling time
[info] # Benchmark: scala.tools.nsc.HotScalacBenchmark.compile
[info] # Parameters: (corpusVersion = a8c43dc, extraArgs = , source = scalap)
[info]
[info] # Run progress: 0.00% complete, ETA 00:03:20
[info] # Fork: 1 of 1
[info] # Warmup Iteration   1: 5312.086 ms/op
[info] # Warmup Iteration   2: 2054.790 ±(99.9%) 1420.469 ms/op
[info] # Warmup Iteration   3: 1456.622 ±(99.9%) 141.405 ms/op
[info] # Warmup Iteration   4: 1297.613 ±(99.9%) 100.157 ms/op
[info] # Warmup Iteration   5: 1248.038 ±(99.9%) 38.623 ms/op
[info] # Warmup Iteration   6: 1221.941 ±(99.9%) 31.619 ms/op
[info] # Warmup Iteration   7: 1200.969 ±(99.9%) 33.985 ms/op
[info] # Warmup Iteration   8: 1210.756 ±(99.9%) 44.681 ms/op
```

Author: retronym

compilation/src/main/scala/scala/tools/nsc/BenchmarkUtils.scala

@@ -0,0 +1,48 @@
+package scala.tools.nsc
+
+import java.io.{File, IOException}
+import java.net.URL
+import java.nio.file._
+import java.nio.file.attribute.BasicFileAttributes
+
+import com.typesafe.config.ConfigFactory
+
+import scala.collection.JavaConverters._
+
+object BenchmarkUtils {
+  def deleteRecursive(directory: Path): Unit = {
+    Files.walkFileTree(directory, new SimpleFileVisitor[Path]() {
+      override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
+        Files.delete(file)
+        FileVisitResult.CONTINUE
+      }
+      override def postVisitDirectory(dir: Path, exc: IOException): FileVisitResult = {
+        Files.delete(dir)
+        FileVisitResult.CONTINUE
+      }
+    })
+  }
+  def initDeps(corpusSourcePath: Path): Seq[Path] = {
+    val depsDir = Paths.get(ConfigFactory.load.getString("deps.localdir"))
+    val depsFile = corpusSourcePath.resolve("deps.txt")
+    val depsClasspath = Seq.newBuilder[Path]
+    if (Files.exists(depsFile)) {
+      val res = new StringBuilder()
+      for (depUrlString <- Files.lines(depsFile).iterator().asScala) {
+        val depUrl = new URL(depUrlString)
+        val filename = Paths.get(depUrl.getPath).getFileName.toString
+        val depFile = depsDir.resolve(filename)
+        // TODO: check hash if file exists, or after downloading
+        if (!Files.exists(depFile)) {
+          if (!Files.exists(depsDir)) Files.createDirectories(depsDir)
+          val in = depUrl.openStream
+          Files.copy(in, depFile, StandardCopyOption.REPLACE_EXISTING)
+          in.close()
+        }
+        if (res.nonEmpty) res.append(File.pathSeparator)
+        depsClasspath += depFile
+      }
+      depsClasspath.result()
+    } else Nil
+  }
+}

compilation/src/main/scala/scala/tools/nsc/HotSbtBenchmark.scala

@@ -0,0 +1,141 @@
+package scala.tools.nsc
+
+import java.io._
+import java.net.URL
+import java.nio.file._
+import java.nio.file.attribute.BasicFileAttributes
+import java.util.concurrent.TimeUnit
+
+import com.typesafe.config.ConfigFactory
+import org.openjdk.jmh.annotations.Mode.SampleTime
+import org.openjdk.jmh.annotations._
+
+import scala.collection.JavaConverters._
+
+@State(Scope.Benchmark)
+@BenchmarkMode(Array(SampleTime))
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@Warmup(iterations = 10, time = 10, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 10, time = 10, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 3)
+class HotSbtBenchmark {
+  @Param(value = Array())
+  var source: String = _
+
+  @Param(value = Array(""))
+  var extraArgs: String = _
+
+  // This parameter is set by ScalacBenchmarkRunner / UploadingRunner based on the Scala version.
+  // When running the benchmark directly the "latest" symlink is used.
+  @Param(value = Array("latest"))
+  var corpusVersion: String = _
+
+  var sbtProcess: Process = _
+  var inputRedirect: ProcessBuilder.Redirect = _
+  var outputRedirect: ProcessBuilder.Redirect = _
+  var tempDir: Path = _
+  var scalaHome: Path = _
+  var processOutputReader: BufferedReader = _
+  var processInputReader: BufferedWriter = _
+  var output= new java.lang.StringBuilder()
+
+  def buildDef =
+    s"""
+       |scalaHome := Some(file("${scalaHome.toAbsolutePath.toString}"))
+       |
+       |val cleanClasses = taskKey[Unit]("clean the classes directory")
+       |
+       |cleanClasses := IO.delete((classDirectory in Compile).value)
+       |
+       |scalaSource in Compile := file("${corpusSourcePath.toAbsolutePath.toString}")
+       |
+       |libraryDependencies += "org.scala-lang" % "scala-compiler" % scalaVersion.value
+       |libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value
+       |
+       |// TODO support .java sources
+    """.stripMargin
+
+  @Setup(Level.Trial) def spawn(): Unit = {
+    tempDir = Files.createTempDirectory("sbt-")
+    scalaHome = Files.createTempDirectory("scalaHome-")
+    initDepsClasspath()
+    Files.createDirectory(tempDir.resolve("project"))
+    Files.write(tempDir.resolve("project/build.properties"), java.util.Arrays.asList("sbt.version=0.13.15"))
+    Files.write(tempDir.resolve("build.sbt"), buildDef.getBytes("UTF-8"))
+    val sbtLaucherPath = System.getProperty("sbt.launcher")
+    if (sbtLaucherPath == null) sys.error("System property -Dsbt.launcher absent")
+    val builder = new ProcessBuilder(sys.props("java.home") + "/bin/java", "-Xms2G", "-Xmx2G", "-Dsbt.log.format=false", "-jar", sbtLaucherPath)
+    builder.directory(tempDir.toFile)
+    inputRedirect = builder.redirectInput()
+    outputRedirect = builder.redirectOutput()
+    sbtProcess = builder.start()
+    processOutputReader = new BufferedReader(new InputStreamReader(sbtProcess.getInputStream))
+    processInputReader = new BufferedWriter(new OutputStreamWriter(sbtProcess.getOutputStream))
+    awaitPrompt()
+  }
+
+  @Benchmark
+  def compile(): Unit = {
+    issue(";cleanClasses;compile")
+    awaitPrompt()
+  }
+
+  def issue(str: String) = {
+    processInputReader.write(str + "\n")
+    processInputReader.flush()
+  }
+
+  def awaitPrompt(): Unit = {
+    output.setLength(0)
+    var line = ""
+    val buffer = new Array[Char](128)
+    var read : Int = -1
+    while (true) {
+      read = processOutputReader.read(buffer)
+      if (read == -1) sys.error("EOF")
+      else {
+        output.append(buffer, 0, read)
+        if (output.toString.contains("\n> ")) {
+          if (output.toString.contains("[error")) sys.error(output.toString)
+          return
+        }
+      }
+    }
+
+  }
+
+  private def corpusSourcePath = Paths.get(s"../corpus/$source/$corpusVersion")
+
+  def initDepsClasspath(): Unit = {
+    val libDir = tempDir.resolve("lib")
+    Files.createDirectories(libDir)
+    for (depFile <- BenchmarkUtils.initDeps(corpusSourcePath)) {
+      val libDirFile = libDir.resolve(depFile.getFileName)
+      Files.copy(depFile, libDir)
+    }
+
+    val scalaHomeLibDir = scalaHome.resolve("lib")
+    Files.createDirectories(scalaHomeLibDir)
+    for (elem <- sys.props("java.class.path").split(File.pathSeparatorChar)) {
+      val jarFile = Paths.get(elem)
+      var name = jarFile.getFileName.toString
+      if (name.startsWith("scala") && name.endsWith(".jar")) {
+        if (name.startsWith("scala-library"))
+          name = "scala-library.jar"
+        else if (name.startsWith("scala-reflect"))
+          name = "scala-reflect.jar"
+        else if (name.startsWith("scala-compiler"))
+          name = "scala-compiler.jar"
+        Files.copy(jarFile, scalaHomeLibDir.resolve(name))
+      }
+    }
+
+  }
+
+  @TearDown(Level.Trial) def terminate(): Unit = {
+    processOutputReader.close()
+    sbtProcess.destroyForcibly()
+    BenchmarkUtils.deleteRecursive(tempDir)
+    BenchmarkUtils.deleteRecursive(scalaHome)
+  }
+}

compilation/src/main/scala/scala/tools/nsc/ScalacBenchmark.scala

@@ -82,37 +82,16 @@ class ScalacBenchmark {
     tempDir = tempFile
   }
   @TearDown(Level.Trial) def clearTemp(): Unit = {
-    val directory = tempDir.toPath
-    Files.walkFileTree(directory, new SimpleFileVisitor[Path]() {
-      override def visitFile(file: Path, attrs: BasicFileAttributes): FileVisitResult = {
-        Files.delete(file)
-        FileVisitResult.CONTINUE
-      }
-      override def postVisitDirectory(dir: Path, exc: IOException): FileVisitResult = {
-        Files.delete(dir)
-        FileVisitResult.CONTINUE
-      }
-    })
+    BenchmarkUtils.deleteRecursive(tempDir.toPath)
   }
 
   private def corpusSourcePath = Paths.get(s"../corpus/$source/$corpusVersion")
 
   @Setup(Level.Trial) def initDepsClasspath(): Unit = {
-    val depsDir = Paths.get(ConfigFactory.load.getString("deps.localdir"))
-    val depsFile = corpusSourcePath.resolve("deps.txt")
-    if (Files.exists(depsFile)) {
+    val classPath = BenchmarkUtils.initDeps(corpusSourcePath)
+    if (classPath.nonEmpty) {
       val res = new StringBuilder()
-      for (depUrlString <- Files.lines(depsFile).iterator().asScala) {
-        val depUrl = new URL(depUrlString)
-        val filename = Paths.get(depUrl.getPath).getFileName.toString
-        val depFile = depsDir.resolve(filename)
-        // TODO: check hash if file exists, or after downloading
-        if (!Files.exists(depFile)) {
-          if (!Files.exists(depsDir)) Files.createDirectories(depsDir)
-          val in = depUrl.openStream
-          Files.copy(in, depFile, StandardCopyOption.REPLACE_EXISTING)
-          in.close()
-        }
+      for (depFile <- classPath) {
         if (res.nonEmpty) res.append(File.pathSeparator)
         res.append(depFile.toAbsolutePath.normalize.toString)
       }