Initial steps to generate TASTy for the 2.13 library (#17526)

The aim is to have a project that compiles the 2.13 library using Scala
3 to generate the TASTy. This will require a special compilation mode to
align with Scala 2 semantics. Then in a later step, we can package the
TASTy files in a JAR that can be loaded with the class file JAR of the
Scala 2 standard library.


### `stdlib-bootstrapped` project

This PR changes the purpose of `stdlib-bootstrapped`. Now the project
compiles the Scala 2.13 library (only) sources using `-Yscala2-stdlib`.
With this flag, the compiler will generate code that aligns with the
Scala 2 version of the library. The main purpose is to have TASTy that
contains signatures that align with the Scala 2 library bytecode.

Under `-Yscala2-stdlib` we currently
* change the signature of the case class `unapply` methods,
* do not emit mirrors,
* and inline definitions case class `_N`.

We add MiMa tests to this project to have a better view of the
differences between the Scala 2 generated bytecode and the one generated
in this PR. The bytecode differences are a useful guide of differences
between the TASTy of the library and how applications will link to it.


[skip community_build]

Author: nicolasstucki

.github/workflows/ci.yaml

@@ -253,7 +253,7 @@ jobs:
 
       - name: MiMa
         run: |
-          ./project/scripts/sbt ";scala3-interfaces/mimaReportBinaryIssues ;scala3-library-bootstrapped/mimaReportBinaryIssues ;scala3-library-bootstrappedJS/mimaReportBinaryIssues; tasty-core-bootstrapped/mimaReportBinaryIssues"
+          ./project/scripts/sbt ";scala3-interfaces/mimaReportBinaryIssues ;scala3-library-bootstrapped/mimaReportBinaryIssues ;scala3-library-bootstrappedJS/mimaReportBinaryIssues; tasty-core-bootstrapped/mimaReportBinaryIssues; stdlib-bootstrapped/mimaReportBinaryIssues"
 
   community_build_a:
     runs-on: [self-hosted, Linux]

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -637,7 +637,10 @@ object desugar {
     //       new C[...](p1, ..., pN)(moreParams)
     val (caseClassMeths, enumScaffolding) = {
       def syntheticProperty(name: TermName, tpt: Tree, rhs: Tree) =
-        DefDef(name, Nil, tpt, rhs).withMods(synthetic)
+        val mods =
+          if ctx.settings.Yscala2Stdlib.value then synthetic | Inline
+          else synthetic
+        DefDef(name, Nil, tpt, rhs).withMods(mods)
 
       def productElemMeths =
         val caseParams = derivedVparamss.head.toArray
@@ -735,13 +738,25 @@ object desugar {
               .withMods(appMods) :: Nil
           }
         val unapplyMeth = {
+          def scala2LibCompatUnapplyRhs(unapplyParamName: Name) =
+            assert(arity <= Definitions.MaxTupleArity, "Unexpected case class with tuple larger than 22: "+ cdef.show)
+            if arity == 1 then Apply(scalaDot(nme.Option), Select(Ident(unapplyParamName), nme._1))
+            else
+              val tupleApply = Select(Ident(nme.scala), s"Tuple$arity".toTermName)
+              val members = List.tabulate(arity) { n => Select(Ident(unapplyParamName), s"_${n+1}".toTermName) }
+              Apply(scalaDot(nme.Option), Apply(tupleApply, members))
+
           val hasRepeatedParam = constrVparamss.head.exists {
             case ValDef(_, tpt, _) => isRepeated(tpt)
           }
           val methName = if (hasRepeatedParam) nme.unapplySeq else nme.unapply
           val unapplyParam = makeSyntheticParameter(tpt = classTypeRef)
-          val unapplyRHS = if (arity == 0) Literal(Constant(true)) else Ident(unapplyParam.name)
+          val unapplyRHS =
+            if (arity == 0) Literal(Constant(true))
+            else if ctx.settings.Yscala2Stdlib.value then scala2LibCompatUnapplyRhs(unapplyParam.name)
+            else Ident(unapplyParam.name)
           val unapplyResTp = if (arity == 0) Literal(Constant(true)) else TypeTree()
+
           DefDef(
             methName,
             joinParams(derivedTparams, (unapplyParam :: Nil) :: Nil),

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -367,6 +367,7 @@ private sealed trait YSettings:
   val YfromTastyIgnoreList: Setting[List[String]] = MultiStringSetting("-Yfrom-tasty-ignore-list", "file", "List of `tasty` files in jar files that will not be loaded when using -from-tasty")
   val YnoExperimental: Setting[Boolean] = BooleanSetting("-Yno-experimental", "Disable experimental language features")
   val YlegacyLazyVals: Setting[Boolean] = BooleanSetting("-Ylegacy-lazy-vals", "Use legacy (pre 3.3.0) implementation of lazy vals")
+  val Yscala2Stdlib: Setting[Boolean] = BooleanSetting("-Yscala2-stdlib", "Used when compiling the Scala 2 standard library")
 
   val YprofileEnabled: Setting[Boolean] = BooleanSetting("-Yprofile-enabled", "Enable profiling.")
   val YprofileDestination: Setting[String] = StringSetting("-Yprofile-destination", "file", "Where to send profiling output - specify a file, default is to the console.", "")

compiler/src/dotty/tools/dotc/core/StdNames.scala

@@ -177,6 +177,7 @@ object StdNames {
     final val typeTag: N    = "typeTag"
     final val Expr: N       = "Expr"
     final val String: N     = "String"
+    final val Option: N     = "Option"
     final val Annotation: N = "Annotation"
 
     // fictions we use as both types and terms

compiler/src/dotty/tools/dotc/core/unpickleScala2/Scala2Unpickler.scala

@@ -153,6 +153,7 @@ class Scala2Unpickler(bytes: Array[Byte], classRoot: ClassDenotation, moduleClas
   assert(moduleRoot.isTerm)
 
   checkVersion(using ictx)
+  checkScala2Stdlib(using ictx)
 
   private val loadingMirror = defn(using ictx) // was: mirrorThatLoaded(classRoot)
 
@@ -239,6 +240,9 @@ class Scala2Unpickler(bytes: Array[Byte], classRoot: ClassDenotation, moduleClas
         " in " + source)
   }
 
+  private def checkScala2Stdlib(using Context): Unit =
+    assert(!ctx.settings.Yscala2Stdlib.value, "No Scala 2 libraries should be unpickled under -Yscala2-stdlib")
+
   /** The `decls` scope associated with given symbol */
   protected def symScope(sym: Symbol): Scope = symScopes.getOrElseUpdate(sym, newScope(0))
 

compiler/src/dotty/tools/dotc/transform/SyntheticMembers.scala

@@ -639,8 +639,9 @@ class SyntheticMembers(thisPhase: DenotTransformer) {
     val clazz = ctx.owner.asClass
     val syntheticMembers = serializableObjectMethod(clazz) ::: serializableEnumValueMethod(clazz) ::: caseAndValueMethods(clazz)
     checkInlining(syntheticMembers)
-    addMirrorSupport(
-      cpy.Template(impl)(body = syntheticMembers ::: impl.body))
+    val impl1 = cpy.Template(impl)(body = syntheticMembers ::: impl.body)
+    if ctx.settings.Yscala2Stdlib.value then impl1
+    else addMirrorSupport(impl1)
   }
 
   private def checkInlining(syntheticMembers: List[Tree])(using Context): Unit =

project/Build.scala

@@ -922,25 +922,22 @@ object Build {
       javaOptions := (`scala3-compiler-bootstrapped` / javaOptions).value
     )
 
-  /** Scala library compiled by dotty using the latest published sources of the library */
+  /** Scala 2 library compiled by dotty using the latest published sources of the library.
+   *
+   *  This version of the library is not (yet) TASTy/binary compatible with the Scala 2 compiled library.
+   */
   lazy val `stdlib-bootstrapped` = project.in(file("stdlib-bootstrapped")).
     withCommonSettings(Bootstrapped).
     dependsOn(dottyCompiler(Bootstrapped) % "provided; compile->runtime; test->test").
     settings(commonBootstrappedSettings).
     settings(
       moduleName := "scala-library",
       javaOptions := (`scala3-compiler-bootstrapped` / javaOptions).value,
-      Compile/scalacOptions += "-Yerased-terms",
-      Compile/scalacOptions ++= {
-        Seq(
-          "-sourcepath",
-          Seq(
-            (Compile/sourceManaged).value / "scala-library-src",
-            (Compile/sourceManaged).value / "dotty-library-src",
-          ).mkString(File.pathSeparator),
-        )
+      Compile / scalacOptions ++= {
+        Seq("-sourcepath", ((Compile/sourceManaged).value / "scala-library-src").toString)
       },
       Compile / doc / scalacOptions += "-Ydocument-synthetic-types",
+      scalacOptions += "-Yscala2-stdlib",
       scalacOptions -= "-Xfatal-warnings",
       ivyConfigurations += SourceDeps.hide,
       transitiveClassifiers := Seq("sources"),
@@ -970,36 +967,6 @@ object Build {
           ((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet
         } (Set(scalaLibrarySourcesJar)).toSeq
       }.taskValue,
-      (Compile / sourceGenerators) += Def.task {
-        val s = streams.value
-        val cacheDir = s.cacheDirectory
-        val trgDir = (Compile / sourceManaged).value / "dotty-library-src"
-
-        // NOTE `sourceDirectory` is used for actual copying,
-        // but `sources` are used as cache keys
-        val dottyLibSourceDirs = (`scala3-library-bootstrapped`/Compile/unmanagedSourceDirectories).value
-        def dottyLibSources = dottyLibSourceDirs.foldLeft(PathFinder.empty) { (pf, dir) =>
-          if (!dir.exists) pf else pf +++ (dir ** "*.scala") +++ (dir ** "*.java")
-        }
-
-        val cachedFun = FileFunction.cached(
-          cacheDir / s"copyDottyLibrarySrc",
-          FilesInfo.lastModified,
-          FilesInfo.exists,
-        ) { _ =>
-          if (trgDir.exists) IO.delete(trgDir)
-          dottyLibSourceDirs.foreach { dir =>
-            if (dir.exists) {
-              s.log.info(s"Copying scala3-library sources from $dir to $trgDir...")
-              IO.copyDirectory(dir, trgDir)
-            }
-          }
-
-          ((trgDir ** "*.scala") +++ (trgDir ** "*.java")).get.toSet
-        }
-
-        cachedFun(dottyLibSources.get.toSet).toSeq
-      }.taskValue,
       (Compile / sources) ~= (_.filterNot(file =>
         // sources from https://github.com/scala/scala/tree/2.13.x/src/library-aux
         file.getPath.endsWith("scala-library-src/scala/Any.scala") ||
@@ -1008,9 +975,29 @@ object Build {
         file.getPath.endsWith("scala-library-src/scala/Nothing.scala") ||
         file.getPath.endsWith("scala-library-src/scala/Null.scala") ||
         file.getPath.endsWith("scala-library-src/scala/Singleton.scala"))),
+      (Compile / sources) := {
+        val files = (Compile / sources).value
+        val overwritenSourcesDir = (Compile / scalaSource).value
+        val overwritenSources = files.flatMap(_.relativeTo(overwritenSourcesDir)).toSet
+        val reference = (Compile/sourceManaged).value / "scala-library-src"
+        files.filterNot(_.relativeTo(reference).exists(overwritenSources))
+      },
       (Test / managedClasspath) ~= {
         _.filterNot(file => file.data.getName == s"scala-library-${stdlibVersion(Bootstrapped)}.jar")
       },
+      mimaCheckDirection := "both",
+      mimaBackwardIssueFilters := MiMaFilters.StdlibBootstrappedBackwards,
+      mimaForwardIssueFilters := MiMaFilters.StdlibBootstrappedForward,
+      mimaPreviousArtifacts += "org.scala-lang" % "scala-library" % stdlibVersion(Bootstrapped),
+      mimaExcludeAnnotations ++= Seq(
+        "scala.annotation.experimental",
+        "scala.annotation.specialized",
+        "scala.annotation.unspecialized",
+      ),
+      // TODO package only TASTy files.
+      // We first need to check that a project can depend on a JAR that only contains TASTy files.
+      // Compile / exportJars := true,
+      // Compile / packageBin / mappings ~= { _.filter(_._2.endsWith(".tasty")) },
     )
 
   /** Test the tasty generated by `stdlib-bootstrapped`