WIP classpath plugin

Author: retronym

src/compiler/scala/tools/nsc/backend/JavaPlatform.scala

@@ -20,7 +20,7 @@ trait JavaPlatform extends Platform {
   private[nsc] var currentClassPath: Option[ClassPath] = None
 
   private[nsc] def classPath: ClassPath = {
-    if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings).result)
+    if (currentClassPath.isEmpty) currentClassPath = Some(applyClassPathPlugins(new PathResolver(settings).result))
     currentClassPath.get
   }
 

src/compiler/scala/tools/nsc/backend/Platform.scala

@@ -6,7 +6,10 @@
 package scala.tools.nsc
 package backend
 
+import java.nio.ByteBuffer
+
 import io.AbstractFile
+import scala.tools.nsc.classpath.AggregateClassPath
 import scala.tools.nsc.util.ClassPath
 
 /** The platform dependent pieces of Global.
@@ -37,5 +40,52 @@ trait Platform {
    * a re-compile is triggered. On .NET by contrast classfiles always take precedence.
    */
   def needCompile(bin: AbstractFile, src: AbstractFile): Boolean
-}
 
+  /**
+    * A class path plugin can modify the classpath before it is used by the compiler, and can
+    * customize the way that the compiler reads the contents of class files.
+    *
+    * Applications could include:
+    *
+    *   - Caching the ScalaSignature annotation contents, to avoid the cost of decompressing
+    *     and parsing the classfile, akin to the OpenJDK's .sig format for stripped class files.
+    *   - Starting a downstream compilation job immediately after the upstream job has completed
+    *     the pickler phase ("Build Pipelineing")
+    */
+  abstract class ClassPathPlugin {
+    def info(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo]
+    def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = classPath
+  }
+  sealed abstract class ClassfileInfo {}
+  final case class ClassBytes(data: ByteBuffer) extends ClassfileInfo
+  final case class ScalaRawClass(className: String) extends ClassfileInfo
+  final case class ScalaClass(className: String, pickle: ByteBuffer) extends ClassfileInfo
+
+
+  /** A list of registered classpath plugins */
+  private var classPathPlugins: List[ClassPathPlugin] = Nil
+
+  protected final def applyClassPathPlugins(original: ClassPath): ClassPath = {
+    val entries = original match {
+      case AggregateClassPath(entries) => entries
+      case single => single :: Nil
+    }
+    val entries1 = classPathPlugins.foldLeft(entries) {
+      (entries, plugin) => plugin.modifyClassPath(entries)
+    }
+    AggregateClassPath(entries1)
+  }
+
+
+  /** Registers a new classpath plugin */
+  final def addClassPathPlugin(plugin: ClassPathPlugin): Unit = {
+    if (!classPathPlugins.contains(plugin))
+      classPathPlugins = plugin :: classPathPlugins
+  }
+  final def classFileInfo(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo] = if (classPathPlugins eq Nil) None else {
+    classPathPlugins.foldLeft(Option.empty[ClassfileInfo]) {
+      case (Some(info), _) => Some(info)
+      case (None, plugin) => plugin.info(file, clazz)
+    }
+  }
+}

src/compiler/scala/tools/nsc/symtab/classfile/AbstractFileReader.scala

@@ -19,11 +19,8 @@ import scala.tools.nsc.io.AbstractFile
  * @author Philippe Altherr
  * @version 1.0, 23/03/2004
  */
-class AbstractFileReader(val file: AbstractFile) {
-
-  /** the buffer containing the file
-   */
-  val buf: Array[Byte] = file.toByteArray
+class AbstractFileReader(val file: AbstractFile, val buf: Array[Byte]) {
+  def this(file: AbstractFile) = this(file, file.toByteArray)
 
   /** the current input pointer
    */

src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala

@@ -8,15 +8,15 @@ package tools.nsc
 package symtab
 package classfile
 
-import java.io.{ByteArrayInputStream, DataInputStream, File, IOException}
+import java.io._
 import java.lang.Integer.toHexString
 
 import scala.collection.{immutable, mutable}
 import scala.collection.mutable.{ArrayBuffer, ListBuffer}
 import scala.annotation.switch
 import scala.reflect.internal.JavaAccFlags
 import scala.reflect.internal.pickling.{ByteCodecs, PickleBuffer}
-import scala.reflect.io.NoAbstractFile
+import scala.reflect.io.{NoAbstractFile, VirtualFile}
 import scala.reflect.internal.util.Collections._
 import scala.tools.nsc.util.ClassPath
 import scala.tools.nsc.io.AbstractFile
@@ -145,14 +145,50 @@ abstract class ClassfileParser {
   def parse(file: AbstractFile, clazz: ClassSymbol, module: ModuleSymbol): Unit = {
     this.file = file
     pushBusy(clazz) {
-      this.in           = new AbstractFileReader(file)
       this.clazz        = clazz
       this.staticModule = module
       this.isScala      = false
 
-      parseHeader()
-      this.pool = newConstantPool
-      parseClass()
+      import loaders.platform._
+      classFileInfo(file, clazz) match {
+        case Some(info) =>
+          info match {
+            case ScalaRawClass(className) =>
+              isScalaRaw = true
+              currentClass = TermName(className)
+            case ScalaClass(className, pickle) =>
+              isScala = true
+              currentClass = TermName(className)
+              if (pickle.hasArray) {
+                unpickler.unpickle(pickle.array, pickle.arrayOffset + pickle.position(), clazz, staticModule, file.name)
+              } else {
+                val array = new Array[Byte](pickle.remaining)
+                pickle.get(array)
+                unpickler.unpickle(array, 0, clazz, staticModule, file.name)
+              }
+            case ClassBytes(data) =>
+              val array = new Array[Byte](data.remaining)
+              data.get(array)
+              this.in = new AbstractFileReader(file, array)
+              parseClass()
+          }
+        case None =>
+          this.in = new AbstractFileReader(file)
+          parseHeader()
+          this.pool = new ConstantPool
+          parseClass()
+      }
+      if (isScalaRaw && !isNothingOrNull) {
+        unlinkRaw()
+      }
+    }
+  }
+
+  private def unlinkRaw(): Unit = {
+    val decls = clazz.enclosingPackage.info.decls
+    for (c <- List(clazz, staticModule, staticModule.moduleClass)) {
+      c.setInfo(NoType)
+      decls.unlink(c)
     }
   }
 
@@ -434,7 +470,18 @@ abstract class ClassfileParser {
       lookupClass(name)
   }
 
-  def parseClass() {
+  // TODO: remove after the next 2.13 milestone
+  // A bug in the backend caused classes ending in `$` do get only a Scala marker attribute
+  // instead of a ScalaSig and a Signature annotaiton. This went unnoticed because isScalaRaw
+  // classes were parsed like Java classes. The below covers the cases in the std lib.
+  private def isNothingOrNull = {
+    val n = clazz.fullName.toString
+    n == "scala.runtime.Nothing$" || n == "scala.runtime.Null$"
+  }
+
+  def parseClass(): Unit = {
+    unpickleOrParseInnerClasses()
+
     val jflags   = readClassFlags()
     val sflags   = jflags.toScalaFlags
     val nameIdx  = u2
@@ -466,6 +513,24 @@ abstract class ClassfileParser {
       if (!c.isInstanceOf[StubSymbol] && c != clazz) mismatchError(c)
     }
 
+    if (isScala) {
+      () // We're done
+    } else if (isScalaRaw && !isNothingOrNull) {
+      val decls = clazz.enclosingPackage.info.decls
+      for (c <- List(clazz, staticModule, staticModule.moduleClass)) {
+        c.setInfo(NoType)
+        decls.unlink(c)
+      }
+    } else {
+      val sflags = jflags.toScalaFlags // includes JAVA
+
+      def parseParents(): List[Type] = raiseLoaderLevel {
+        val superType = pool.getSuperClass(u2).tpe_*
+        val ifaceCount = u2
+        val ifaces = for (i <- List.range(0, ifaceCount)) yield pool.getSuperClass(u2).tpe_*
+        superType :: ifaces
+      }
+
     addEnclosingTParams(clazz)
     parseInnerClasses() // also sets the isScala / isScalaRaw flags, see r15956
     // get the class file parser to reuse scopes.

test/junit/scala/tools/nsc/classpath/ClassPluginTest.scala

@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2018 Lightbend. All rights reserved.
+ */
+package scala.tools.nsc.classpath
+
+import java.nio.ByteBuffer
+
+import org.junit.Assert.assertEquals
+import org.junit.Test
+import org.junit.runner.RunWith
+import org.junit.runners.JUnit4
+
+import scala.reflect.io.VirtualDirectory
+import scala.tools.nsc.io.AbstractFile
+import scala.tools.nsc.symtab.SymbolTableForUnitTesting
+import scala.tools.nsc.util.ClassPath
+import scala.tools.testing.BytecodeTesting
+import scala.tools.testing.BytecodeTesting.makeSourceFile
+
+@RunWith(classOf[JUnit4])
+class ClassPluginTest extends BytecodeTesting {
+  // We use this.compiler to generate Scala pickles...
+  override def compilerArgs = "-Ystop-after:pickler"
+
+  // ... and this one to read them with a ClassPathPlugin
+  object symbolTable extends SymbolTableForUnitTesting {
+    val fakeClasses = Map(
+      "fake.C" -> platform.ScalaClass("fake.C", pickleOf("package fake; class C { def foo = 42 }"))
+    )
+    private val fakes = new VirtualDirectory("fakes", None)
+    fakes.subdirectoryNamed("fake").fileNamed("C.class")
+
+    lazy val classpathPlugin = new platform.ClassPathPlugin {
+      override def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = {
+        // Add a classpath entry with the fake/C.class
+        VirtualDirectoryClassPath(fakes) +: classPath
+      }
+
+      override def info(file: AbstractFile, clazz: ClassSymbol): Option[platform.ClassfileInfo] =
+        fakeClasses.get(clazz.fullNameString)
+    }
+    this.platform.addClassPathPlugin(classpathPlugin)
+  }
+
+  @Test def classPathPluginTest(): Unit = {
+    import symbolTable._
+    val CClass = rootMirror.getRequiredClass("fake.C")
+    val C_tpe = CClass.info
+    assertEquals("def foo: Int", definitions.fullyInitializeSymbol(C_tpe.decl(TermName("foo"))).defString)
+  }
+
+  private def pickleOf(code: String): ByteBuffer = {
+    import compiler._
+    val run = newRun
+    run.compileSources(makeSourceFile(code, "unitTestSource.scala") :: Nil)
+    val pickle = run.symData.toList.head._2
+    ByteBuffer.wrap(pickle.bytes, 0, pickle.writeIndex)
+  }
+}

test/junit/scala/tools/nsc/symtab/SymbolTableForUnitTesting.scala

@@ -36,7 +36,7 @@ class SymbolTableForUnitTesting extends SymbolTable {
 
     def platformPhases: List[SubComponent] = Nil
 
-    private[nsc] lazy val classPath: ClassPath = new PathResolver(settings).result
+    private[nsc] lazy val classPath: ClassPath = applyClassPathPlugins(new PathResolver(settings).result)
 
     def isMaybeBoxed(sym: Symbol): Boolean = ???
     def needCompile(bin: AbstractFile, src: AbstractFile): Boolean = ???