WIP use pickles from outline typing on downstream classpath

Author: retronym

src/compiler/scala/tools/nsc/PipelineMain.scala

@@ -4,10 +4,17 @@
  */
 package scala.tools.nsc
 
+import java.net.URL
+import java.nio.ByteBuffer
 import java.nio.file.{Files, Path, Paths}
 
 import scala.collection.mutable
+import scala.reflect.internal.pickling.PickleBuffer
 import scala.reflect.internal.util.FakePos
+import scala.reflect.io.{VirtualDirectory, VirtualFile}
+import scala.tools.nsc.backend.{ClassfileInfo, ScalaClass, ScalaRawClass}
+import scala.tools.nsc.classpath.{DirectoryClassPath, VirtualDirectoryClassPath}
+import scala.tools.nsc.io.AbstractFile
 import scala.tools.nsc.reporters.{ConsoleReporter, Reporter}
 import scala.tools.nsc.util.ClassPath
 
@@ -19,7 +26,53 @@ class PipelineMainClass {
   }
 
   private var reporter: Reporter = _
+
+  private class PickleClassPath[G <: Global](data: mutable.AnyRefMap[G#Symbol, PickleBuffer]) {
+    val dir = new VirtualDirectory("fakes", None)
+    val classpath = VirtualDirectoryClassPath(dir)
+    val dirs = mutable.Map[G#Symbol, AbstractFile]()
+    val classInfo = mutable.Map[AbstractFile, ClassfileInfo]()
+    def packageDir(packSymbol: G#Symbol): AbstractFile = {
+      if (dirs.contains(packSymbol)) dirs(packSymbol)
+      else if (packSymbol.owner.isRoot) {
+        val subDir = dir.subdirectoryNamed(packSymbol.encodedName)
+        dirs.put(packSymbol, subDir)
+        subDir
+      } else {
+        val base = packageDir(packSymbol.owner)
+        val subDir = base.subdirectoryNamed(packSymbol.encodedName)
+        dirs.put(packSymbol, subDir)
+        subDir
+      }
+    }
+    for ((symbol, pickle) <- data) {
+      val base = packageDir(symbol.owner)
+      if (symbol.isClass) {
+        val primary = base.fileNamed(symbol.encodedName + ".class")
+        classInfo(primary) = ScalaClass(symbol.fullNameString, ByteBuffer.wrap(pickle.bytes))
+        if (symbol.companionModule.exists) {
+          val secondary = base.fileNamed(symbol.companionModule.encodedName + "$.class")
+          classInfo(secondary) = ScalaRawClass(symbol.companionModule.fullNameString)
+        }
+      } else if (symbol.isModule) {
+        if (symbol.companionClass.exists) {
+          val primary = base.fileNamed(symbol.encodedName + ".class")
+          classInfo(primary) = ScalaClass(symbol.fullNameString, ByteBuffer.wrap(pickle.bytes))
+          val secondary = base.fileNamed(symbol.companionModule.encodedName + "$.class")
+          classInfo(secondary) = ScalaRawClass(symbol.companionModule.fullNameString)
+        } else {
+          val primary = base.fileNamed(symbol.encodedName + "$.class")
+          classInfo(primary) = ScalaClass(symbol.fullNameString, ByteBuffer.wrap(pickle.bytes))
+        }
+      }
+    }
+  }
+  private val allPickleData =  new java.util.concurrent.ConcurrentHashMap[Path, PickleClassPath[_]]
+
+  var round = 0
+
   def process(args: Array[String]): Boolean = {
+    round = 0
     reporter = new ConsoleReporter(new Settings(scalacError))
 
     def commandFor(argFileArg: String): Task = {
@@ -39,7 +92,6 @@ class PipelineMainClass {
     }
     var toProcess = projects.toList
     val done = mutable.HashSet[Task]()
-    var round = 0
     while (toProcess.nonEmpty) {
       round += 1
       val (nextRound, blocked) = toProcess.partition(x => dependsOn.getOrElse(x, Nil).forall(done))
@@ -53,6 +105,7 @@ class PipelineMainClass {
     }
     true
   }
+
   private case class Task(argsFile: String, command: CompilerCommand) {
     override def toString: String = argsFile
   }
@@ -78,7 +131,42 @@ class PipelineMainClass {
     !reporter.hasErrors
   }
 
-  protected def newCompiler(settings: Settings): Global = Global(settings)
+  protected def newCompiler(settings: Settings): Global = {
+    val g = Global(settings)
+
+    val plugin: g.platform.ClassPathPlugin = new g.platform.ClassPathPlugin {
+      val replacements = mutable.Buffer[PickleClassPath[_]]()
+      override def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = {
+        classPath.flatMap {
+          case dcp: DirectoryClassPath =>
+            val path = dcp.dir.toPath.toRealPath().normalize()
+            allPickleData.get(path) match {
+              case null =>
+                dcp :: Nil
+              case pcp =>
+                replacements += pcp
+                pcp.classpath :: dcp :: Nil // leaving the original classpath for Java compiled files for now
+            }
+          case cp => cp :: Nil
+        }
+      }
+
+      override def info(file: AbstractFile, clazz: g.ClassSymbol): Option[ClassfileInfo] = {
+        file match {
+          case vf: VirtualFile =>
+            if (file.name.contains("ListMap"))
+              getClass
+            val iterator = replacements.iterator.flatMap(_.classInfo.get(vf))
+            if (iterator.hasNext)
+              Some(iterator.next())
+            else None
+          case _ => None
+        }
+      }
+    }
+    g.platform.addClassPathPlugin(plugin)
+    g
+  }
 
   protected def doCompile(command: CompilerCommand, compiler: Global): Unit = {
     if (command.files.isEmpty) {
@@ -95,17 +183,15 @@ class PipelineMainClass {
         println("regular compilation")
         command.settings.Youtline.value = false
         command.settings.stopAfter.value = Nil
-        val pickles = run1.symData
         val compiler2 = newCompiler(command.settings)
+        allPickleData.put(command.settings.outputDirs.getSingleOutput.get.file.toPath.toRealPath().normalize(), new PickleClassPath(run1.symData))
         val run2 = new compiler2.Run()
         run2 compile command.files
         compiler2.reporter.finish()
       }
-
     }
   }
 
-
   def main(args: Array[String]): Unit = {
     for (i <- 1 to 10) {
       val result = process(args)

src/compiler/scala/tools/nsc/backend/Platform.scala

@@ -56,11 +56,6 @@ trait Platform {
     def info(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo]
     def modifyClassPath(classPath: Seq[ClassPath]): Seq[ClassPath] = classPath
   }
-  sealed abstract class ClassfileInfo {}
-  final case class ClassBytes(data: ByteBuffer) extends ClassfileInfo
-  final case class ScalaRawClass(className: String) extends ClassfileInfo
-  final case class ScalaClass(className: String, pickle: ByteBuffer) extends ClassfileInfo
-
 
   /** A list of registered classpath plugins */
   private var classPathPlugins: List[ClassPathPlugin] = Nil
@@ -89,3 +84,9 @@ trait Platform {
     }
   }
 }
+
+sealed abstract class ClassfileInfo {}
+final case class ClassBytes(data: ByteBuffer) extends ClassfileInfo
+final case class ScalaRawClass(className: String) extends ClassfileInfo
+final case class ScalaClass(className: String, pickle: ByteBuffer) extends ClassfileInfo
+

src/compiler/scala/tools/nsc/symtab/SymbolLoaders.scala

@@ -270,6 +270,8 @@ abstract class SymbolLoaders {
 
       val classPathEntries = classPath.list(packageName)
 
+      if (root.name.string_==("immutable"))
+        getClass
       if (!root.isRoot)
         for (entry <- classPathEntries.classesAndSources) initializeFromClassPath(root, entry)
       if (!root.isEmptyPackageClass) {

src/compiler/scala/tools/nsc/symtab/classfile/ClassfileParser.scala

@@ -18,6 +18,7 @@ import scala.reflect.internal.JavaAccFlags
 import scala.reflect.internal.pickling.{ByteCodecs, PickleBuffer}
 import scala.reflect.io.{NoAbstractFile, VirtualFile}
 import scala.reflect.internal.util.Collections._
+import scala.tools.nsc.backend.{ClassBytes, ScalaClass, ScalaRawClass}
 import scala.tools.nsc.util.ClassPath
 import scala.tools.nsc.io.AbstractFile
 import scala.util.control.NonFatal

src/reflect/scala/reflect/internal/Definitions.scala

@@ -282,7 +282,7 @@ trait Definitions extends api.StandardDefinitions {
     }
 
     // top types
-    lazy val AnyClass    = enterNewClass(ScalaPackageClass, tpnme.Any, Nil, ABSTRACT) markAllCompleted
+    lazy val AnyClass    = ScalaPackageClass.newClassSymbol(tpnme.Any, NoPosition, ABSTRACT)
     lazy val AnyRefClass = newAlias(ScalaPackageClass, tpnme.AnyRef, ObjectTpe) markAllCompleted
     lazy val ObjectClass = getRequiredClass(sn.Object.toString)
 
@@ -1451,6 +1451,8 @@ trait Definitions extends api.StandardDefinitions {
     // documented in JavaUniverse.init
     def init() {
       if (isInitialized) return
+      ScalaPackageClass
+      (AnyClass setInfoAndEnter ClassInfoType(Nil, newScope, AnyClass)).markAllCompleted
       ObjectClass.initialize
       ScalaPackageClass.initialize
       symbolsNotPresentInBytecode

src/reflect/scala/reflect/internal/pickling/UnPickler.scala

@@ -242,6 +242,9 @@ abstract class UnPickler {
           else NoSymbol
         }
 
+        if (owner == definitions.ScalaPackageClass && name == tpnme.AnyRef)
+          return definitions.AnyRefClass
+
         // (1) Try name.
         localDummy orElse fromName(name) orElse {
           // (2) Try with expanded name.  Can happen if references to private

test/junit/scala/tools/nsc/classpath/ClassPluginTest.scala

@@ -11,6 +11,7 @@ import org.junit.runner.RunWith
 import org.junit.runners.JUnit4
 
 import scala.reflect.io.VirtualDirectory
+import scala.tools.nsc.backend.{ClassfileInfo, ScalaClass}
 import scala.tools.nsc.io.AbstractFile
 import scala.tools.nsc.symtab.SymbolTableForUnitTesting
 import scala.tools.nsc.util.ClassPath
@@ -25,7 +26,7 @@ class ClassPluginTest extends BytecodeTesting {
   // ... and this one to read them with a ClassPathPlugin
   object symbolTable extends SymbolTableForUnitTesting {
     val fakeClasses = Map(
-      "fake.C" -> platform.ScalaClass("fake.C", pickleOf("package fake; class C { def foo = 42 }"))
+      "fake.C" -> ScalaClass("fake.C", pickleOf("package fake; class C { def foo = 42 }"))
     )
     private val fakes = new VirtualDirectory("fakes", None)
     fakes.subdirectoryNamed("fake").fileNamed("C.class")
@@ -36,7 +37,7 @@ class ClassPluginTest extends BytecodeTesting {
         VirtualDirectoryClassPath(fakes) +: classPath
       }
 
-      override def info(file: AbstractFile, clazz: ClassSymbol): Option[platform.ClassfileInfo] =
+      override def info(file: AbstractFile, clazz: ClassSymbol): Option[ClassfileInfo] =
         fakeClasses.get(clazz.fullNameString)
     }
     this.platform.addClassPathPlugin(classpathPlugin)