Merge pull request #10154 from dotty-staging/test-stdlib-from-tasty-directly-from-jar

Fix #10143 and test stdlib from TASTy directly from Jar and TASTy files

Author: nicolasstucki

compiler/src/dotty/tools/dotc/Driver.scala

@@ -86,6 +86,7 @@ class Driver {
   protected def fromTastySetup(fileNames0: List[String], ctx0: Context): (List[String], Context) =
     given Context = ctx0
     if (ctx0.settings.fromTasty.value) {
+      val fromTastyIgnoreList = ctx0.settings.YfromTastyIgnoreList.value.toSet
       // Resolve classpath and class names of tasty files
       val (classPaths, classNames) = fileNames0.flatMap { name =>
         val path = Paths.get(name)
@@ -96,7 +97,7 @@ class Driver {
           Nil
         else if name.endsWith(".jar") then
           new dotty.tools.io.Jar(File(name)).toList.collect {
-            case e if e.getName.endsWith(".tasty") =>
+            case e if e.getName.endsWith(".tasty") && !fromTastyIgnoreList(e.getName) =>
               (name, e.getName.stripSuffix(".tasty").replace("/", "."))
           }
         else

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -45,7 +45,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val language: Setting[List[String]] = MultiStringSetting("-language", "feature", "Enable one or more language features.") withAbbreviation "--language"
   val rewrite: Setting[Option[Rewrites]] = OptionSetting[Rewrites]("-rewrite", "When used in conjunction with a `...-migration` source version, rewrites sources to migrate to new version.") withAbbreviation "--rewrite"
   val silentWarnings: Setting[Boolean] = BooleanSetting("-nowarn", "Silence all warnings.") withAbbreviation "--no-warnings"
-  val fromTasty: Setting[Boolean] = BooleanSetting("-from-tasty", "Compile classes from tasty in classpath. The arguments are used as class names.") withAbbreviation "--from-tasty"
+  val fromTasty: Setting[Boolean] = BooleanSetting("-from-tasty", "Compile classes from tasty files. The arguments are .tasty or .jar files.") withAbbreviation "--from-tasty"
 
   val newSyntax: Setting[Boolean] = BooleanSetting("-new-syntax", "Require `then` and `do` in control expressions.")
   val oldSyntax: Setting[Boolean] = BooleanSetting("-old-syntax", "Require `(...)` around conditions.")
@@ -158,6 +158,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val YretainTrees: Setting[Boolean] = BooleanSetting("-Yretain-trees", "Retain trees for top-level classes, accessible from ClassSymbol#tree")
   val Ysemanticdb: Setting[Boolean] = BooleanSetting("-Ysemanticdb", "Store information in SemanticDB.")
   val YshowTreeIds: Setting[Boolean] = BooleanSetting("-Yshow-tree-ids", "Uniquely tag all tree nodes in debugging output.")
+  val YfromTastyIgnoreList: Setting[List[String]] = MultiStringSetting("-Yfrom-tasty-ignore-list", "file", "List of `tasty` files in jar files that will not be loaded when using -from-tasty")
 
   val YprofileEnabled: Setting[Boolean] = BooleanSetting("-Yprofile-enabled", "Enable profiling.")
   val YprofileDestination: Setting[String] = StringSetting("-Yprofile-destination", "file", "Where to send profiling output - specify a file, default is to the console.", "")

compiler/src/dotty/tools/dotc/core/tasty/TastyClassName.scala

@@ -25,18 +25,14 @@ class TastyClassName(bytes: Array[Byte]) {
     import dotty.tools.tasty.TastyFormat._
     def unpickle(reader: TastyReader, tastyName: NameTable): (TermName, TermName) = {
       import reader._
-      def readName() = {
-        val idx = readNat()
-        nameAtRef(NameRef(idx))
-      }
       def readNames(packageName: TermName): (TermName, TermName) = {
         val tag = readByte()
         if (tag >= firstLengthTreeTag) {
           val len = readNat()
           val end = currentAddr + len
           tag match {
             case TYPEDEF =>
-              val className = readName()
+              val className = reader.readName()
               goto(end)
               (packageName, className)
             case IMPORT | VALDEF =>
@@ -48,13 +44,26 @@ class TastyClassName(bytes: Array[Byte]) {
         }
         else tag match {
           case TERMREFpkg | TYPEREFpkg =>
-            val subPackageName = readName()
+            val subPackageName = reader.readName()
+            readNames(subPackageName)
+          case SHAREDtype =>
+            val addr = reader.readAddr()
+            val reader2 = reader.subReader(addr, reader.endAddr)
+            val tag2 = reader2.readByte()
+            assert(tag2 == TERMREFpkg || tag2 == TYPEREFpkg)
+            val subPackageName = reader2.readName()
             readNames(subPackageName)
           case _ =>
             readNames(packageName)
         }
       }
       readNames(nme.EMPTY_PACKAGE)
     }
+
+    extension (reader: TastyReader) def readName() = {
+      val idx = reader.readNat()
+      nameAtRef(NameRef(idx))
+    }
   }
+
 }

stdlib-bootstrapped-tasty-tests/test/BootstrappedStdLibTASYyTest.scala

@@ -10,6 +10,7 @@ import dotty.tools.dotc.util.ClasspathFromClassloader
 import scala.quoted._
 
 import java.io.File.pathSeparator
+import java.io.File.separator
 
 class BootstrappedStdLibTASYyTest:
 
@@ -19,17 +20,14 @@ class BootstrappedStdLibTASYyTest:
   @Test def testTastyInspector: Unit =
     loadWithTastyInspector(loadBlacklisted)
 
+  /** Test that we can load and compile trees from TASTy in a Jar */
+  @Test def testFromTastyInJar: Unit =
+    compileFromTastyInJar(loadBlacklisted.union(compileBlacklisted))
+
   /** Test that we can load and compile trees from TASTy */
   @Test def testFromTasty: Unit =
     compileFromTasty(loadBlacklisted.union(compileBlacklisted))
 
-  @Ignore
-  @Test def testWhiteListFromTasty: Unit =
-    val whitelist = Set(
-      "scala.collection.mutable.StringBuilder"
-    )
-    compileFromTasty(x => !whitelist(x))
-
   @Test def blacklistNoDuplicates =
     def testDup(name: String, list: List[String], set: Set[String]) =
       assert(list.size == set.size,
@@ -44,14 +42,14 @@ class BootstrappedStdLibTASYyTest:
         "`compileBlacklist` contains names that are already in `loadBlacklist`: \n  ", "\n  ", "\n\n"))
 
   @Test def blacklistsOnlyContainsClassesThatExist =
-    val scalaLibJarTastyClassNamesSet = scalaLibJarTastyClassNames.toSet
+    val scalaLibTastyPathsSet = scalaLibTastyPaths.toSet
     val intersection = loadBlacklisted & compileBlacklisted
-    assert(loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty,
-      loadBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString(
-        "`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n  ", "\n  ", "\n\n"))
-    assert(compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).isEmpty,
-      compileBlacklisted.diff(scalaLibJarTastyClassNamesSet).mkString(
-        "`loadBlacklisted` contains names that are not in `scalaLibJarTastyClassNames`: \n  ", "\n  ", "\n\n"))
+    assert(loadBlacklisted.diff(scalaLibTastyPathsSet).isEmpty,
+      loadBlacklisted.diff(scalaLibTastyPathsSet).mkString(
+        "`loadBlacklisted` contains names that are not in `scalaLibTastyPaths`: \n  ", "\n  ", "\n\n"))
+    assert(compileBlacklisted.diff(scalaLibTastyPathsSet).isEmpty,
+      compileBlacklisted.diff(scalaLibTastyPathsSet).mkString(
+        "`loadBlacklisted` contains names that are not in `scalaLibTastyPaths`: \n  ", "\n  ", "\n\n"))
 
   @Ignore
   @Test def testLoadBacklistIsMinimal =
@@ -79,7 +77,7 @@ class BootstrappedStdLibTASYyTest:
       val blacklist = blacklist0 - notBlacklisted
       println(s"Trying withouth $notBlacklisted in the blacklist (${i+1}/$size)")
       try {
-        compileFromTasty(blacklist)
+        compileFromTastyInJar(blacklist)
         shouldBeWhitelisted = notBlacklisted :: shouldBeWhitelisted
       }
       catch {
@@ -92,44 +90,59 @@ end BootstrappedStdLibTASYyTest
 
 object BootstrappedStdLibTASYyTest:
 
-  val scalaLibJarPath = System.getProperty("dotty.scala.library")
+  def scalaLibJarPath = System.getProperty("dotty.scala.library")
+  def scalaLibClassesPath =
+    java.nio.file.Paths.get(scalaLibJarPath).getParent.resolve("classes").normalize
 
-  val scalaLibJarTastyClassNames = {
-    val scalaLibJar = Jar(new File(java.nio.file.Paths.get(scalaLibJarPath)))
-    scalaLibJar.toList.map(_.toString).filter(_.endsWith(".tasty"))
-      .map(_.stripSuffix(".tasty").replace("/", "."))
-      .sorted
-  }
+  val scalaLibTastyPaths =
+    new Directory(scalaLibClassesPath).deepFiles
+      .filter(_.`extension` == "tasty")
+      .map(_.normalize.path.stripPrefix(scalaLibClassesPath.toString + "/"))
+      .toList
 
-  def loadWithTastyInspector(blacklisted: String => Boolean): Unit =
+  def loadWithTastyInspector(blacklisted: Set[String]): Unit =
     val inspector = new scala.tasty.inspector.TastyInspector {
       def processCompilationUnit(using QuoteContext)(root: qctx.reflect.Tree): Unit =
         root.showExtractors // Check that we can traverse the full tree
         ()
     }
-    val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted)
-    val hasErrors = inspector.inspectTastyFilesInJar(scalaLibJarPath)
+    val tastyFiles = scalaLibTastyPaths.filterNot(blacklisted)
+    val hasErrors = inspector.inspectTastyFiles(tastyFiles.map(x => scalaLibClassesPath.resolve(x).toString))
     assert(!hasErrors, "Errors reported while loading from TASTy")
 
-  def compileFromTasty(blacklisted: String => Boolean): Unit = {
+  def compileFromTastyInJar(blacklisted: Set[String]): Unit = {
     val driver = new dotty.tools.dotc.Driver
-    val currentClasspath = ClasspathFromClassloader(getClass.getClassLoader)
-    val classNames = scalaLibJarTastyClassNames.filterNot(blacklisted)
+    val yFromTastyBlacklist =
+      blacklisted.mkString("-Yfrom-tasty-ignore-list:", ",", "")
     val args = Array(
-      "-classpath", s"$scalaLibJarPath$pathSeparator$currentClasspath",
+      "-classpath", ClasspathFromClassloader(getClass.getClassLoader),
       "-from-tasty",
-      "-nowarn"
-    ) ++ classNames
+      "-nowarn",
+      yFromTastyBlacklist,
+      scalaLibJarPath,
+    )
     val reporter = driver.process(args)
     assert(reporter.errorCount == 0, "Errors while re-compiling")
   }
 
-  /** List of classes that cannot be loaded from TASTy */
+  def compileFromTasty(blacklisted: Set[String]): Unit = {
+    val driver = new dotty.tools.dotc.Driver
+    val tastyFiles = scalaLibTastyPaths.filterNot(blacklisted)
+    val args = Array(
+      "-classpath", ClasspathFromClassloader(getClass.getClassLoader),
+      "-from-tasty",
+      "-nowarn",
+    ) ++ tastyFiles.map(x => scalaLibClassesPath.resolve(x).toString)
+    val reporter = driver.process(args)
+    assert(reporter.errorCount == 0, "Errors while re-compiling")
+  }
+
+  /** List of tasty files that cannot be loaded from TASTy */
   def loadBlacklist = List[String](
     // No issues :)
   )
 
-  /** List of classes that cannot be recompilied from TASTy */
+  /** List of tasty files that cannot be recompilied from TASTy */
   def compileBlacklist = List[String](
     // See #10048
     // failed: java.lang.AssertionError: assertion failed: class Boolean
@@ -139,22 +152,22 @@ object BootstrappedStdLibTASYyTest:
     //   at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.getClassBTypeAndRegisterInnerClass$(BCodeHelpers.scala:210)
     //   at dotty.tools.backend.jvm.BCodeSkelBuilder$PlainSkelBuilder.getClassBTypeAndRegisterInnerClass(BCodeSkelBuilder.scala:62)
     //   at dotty.tools.backend.jvm.BCodeHelpers$BCInnerClassGen.internalName(BCodeHelpers.scala:237)
-    "scala.Array",
-    "scala.Boolean",
-    "scala.Byte",
-    "scala.Char",
-    "scala.Double",
-    "scala.Float",
-    "scala.Int",
-    "scala.Long",
-    "scala.Short",
-    "scala.Unit",
-  )
-
-  /** Set of classes that cannot be loaded from TASTy */
+    "scala/Array.tasty",
+    "scala/Boolean.tasty",
+    "scala/Byte.tasty",
+    "scala/Char.tasty",
+    "scala/Double.tasty",
+    "scala/Float.tasty",
+    "scala/Int.tasty",
+    "scala/Long.tasty",
+    "scala/Short.tasty",
+    "scala/Unit.tasty",
+  ).map(_.replace("/", separator))
+
+  /** Set of tasty files that cannot be loaded from TASTy */
   def loadBlacklisted = loadBlacklist.toSet
 
-  /** Set of classes that cannot be recompilied from TASTy */
+  /** Set of tasty files that cannot be recompilied from TASTy */
   def compileBlacklisted = compileBlacklist.toSet
 
 end BootstrappedStdLibTASYyTest