add support for pipeline build of Scala and Java files

- rename '-Yjava-tasty-output' to '-Yearly-tasty-output' because
  now Scala TASTy will also be written to this destination.
- add '-Ypickle-java' alias of '-Yjava-tasty', as expected by Zinc
- add '-Ypickle-write' alias of '-Yearly-tasty-output', as expected by Zinc
- move ExtractAPI phase to after Pickler, this way we can do it in
  parallel with generating TASTy bytes. At the end of this phase we
  write the TASTy to the '-Yearly-tasty-output' destination.
  Also ensure that ExtractAPI phase runs with '-Yjava-tasty', even if no
  incremental callback is set (don't extract the API in this case).
- test the pipelining with sbt scripted tests, including for inline
  methods and macros with pipelining
- describe semantics with respect to suspensions,
  introduce -Yno-suspended-units flag for greater control by the user.

Author: bishabosha

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -97,12 +97,15 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
     // when this unit is unsuspended.
     depRecorder.clear()
     if !suspended then
-      if (ctx.settings.XprintSuspension.value)
-        report.echo(i"suspended: $this")
-      suspended = true
-      ctx.run.nn.suspendedUnits += this
-      if ctx.phase == Phases.inliningPhase then
-        suspendedAtInliningPhase = true
+      if ctx.settings.YnoSuspendedUnits.value then
+        report.error(i"Compilation unit suspended $this (-Yno-suspended-units is set)")
+      else
+        if (ctx.settings.XprintSuspension.value)
+          report.echo(i"suspended: $this")
+        suspended = true
+        ctx.run.nn.suspendedUnits += this
+        if ctx.phase == Phases.inliningPhase then
+          suspendedAtInliningPhase = true
     throw CompilationUnit.SuspendException()
 
   private var myAssignmentSpans: Map[Int, List[Span]] | Null = null

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -41,13 +41,13 @@ class Compiler {
     List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files
     List(new PostTyper) ::          // Additional checks and cleanups after type checking
     List(new sjs.PrepJSInterop) ::  // Additional checks and transformations for Scala.js (Scala.js only)
-    List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
     List(new SetRootTree) ::        // Set the `rootTreeOrProvider` on class symbols
     Nil
 
   /** Phases dealing with TASTY tree pickling and unpickling */
   protected def picklerPhases: List[List[Phase]] =
     List(new Pickler) ::            // Generate TASTY info
+    List(new sbt.ExtractAPI) ::     // Sends a representation of the API of classes to sbt via callbacks
     List(new Inlining) ::           // Inline and execute macros
     List(new PostInlining) ::       // Add mirror support for inlined code
     List(new CheckUnused.PostInlining) ::  // Check for unused elements

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -377,6 +377,7 @@ private sealed trait YSettings:
   val YprintPos: Setting[Boolean] = BooleanSetting("-Yprint-pos", "Show tree positions.")
   val YprintPosSyms: Setting[Boolean] = BooleanSetting("-Yprint-pos-syms", "Show symbol definitions positions.")
   val YnoDeepSubtypes: Setting[Boolean] = BooleanSetting("-Yno-deep-subtypes", "Throw an exception on deep subtyping call stacks.")
+  val YnoSuspendedUnits: Setting[Boolean] = BooleanSetting("-Yno-suspended-units", "Do not suspend units, e.g. when calling a macro defined in the same run. This will error instead of suspending.")
   val YnoPatmatOpt: Setting[Boolean] = BooleanSetting("-Yno-patmat-opt", "Disable all pattern matching optimizations.")
   val YplainPrinter: Setting[Boolean] = BooleanSetting("-Yplain-printer", "Pretty-print using a plain printer.")
   val YprintSyms: Setting[Boolean] = BooleanSetting("-Yprint-syms", "When printing trees print info in symbols instead of corresponding info in trees.")
@@ -438,7 +439,7 @@ private sealed trait YSettings:
   val YdebugMacros: Setting[Boolean] = BooleanSetting("-Ydebug-macros", "Show debug info when quote pattern match fails")
 
   // Pipeline compilation options
-  val YjavaTasty: Setting[Boolean] = BooleanSetting("-Yjava-tasty", "Pickler phase should compute pickles for .java defined symbols for use by build tools")
-  val YjavaTastyOutput: Setting[AbstractFile] = OutputSetting("-Yjava-tasty-output", "directory|jar", "(Internal use only!) destination for generated .tasty files containing Java type signatures.", NoAbstractFile)
+  val YjavaTasty: Setting[Boolean] = BooleanSetting("-Yjava-tasty", "Pickler phase should compute pickles for .java defined symbols for use by build tools", aliases = List("-Ypickle-java"))
+  val YearlyTastyOutput: Setting[AbstractFile] = OutputSetting("-Yearly-tasty-output", "directory|jar", "Destination for generated .tasty files containing possibly outline type signatures.", NoAbstractFile, aliases = List("-Ypickle-write"))
   val YallowOutlineFromTasty: Setting[Boolean] = BooleanSetting("-Yallow-outline-from-tasty", "Allow outline TASTy to be loaded with the -from-tasty option.")
 end YSettings

compiler/src/dotty/tools/dotc/config/Settings.scala

@@ -308,8 +308,8 @@ object Settings:
     def MultiStringSetting(name: String, helpArg: String, descr: String, default: List[String] = Nil, aliases: List[String] = Nil): Setting[List[String]] =
       publish(Setting(name, descr, default, helpArg, aliases = aliases))
 
-    def OutputSetting(name: String, helpArg: String, descr: String, default: AbstractFile): Setting[AbstractFile] =
-      publish(Setting(name, descr, default, helpArg))
+    def OutputSetting(name: String, helpArg: String, descr: String, default: AbstractFile, aliases: List[String] = Nil): Setting[AbstractFile] =
+      publish(Setting(name, descr, default, helpArg, aliases = aliases))
 
     def PathSetting(name: String, descr: String, default: String, aliases: List[String] = Nil): Setting[String] =
       publish(Setting(name, descr, default, aliases = aliases))

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -1066,6 +1066,7 @@ class Definitions {
   @tu lazy val PublicInBinaryAnnot: ClassSymbol = requiredClass("scala.annotation.publicInBinary")
 
   @tu lazy val JavaRepeatableAnnot: ClassSymbol = requiredClass("java.lang.annotation.Repeatable")
+  @tu lazy val JavaAnnotationAnnot: ClassSymbol = requiredClass("java.lang.annotation.Annotation")
 
   // Initialization annotations
   @tu lazy val InitModule: Symbol = requiredModule("scala.annotation.init")

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -210,6 +210,7 @@ object Phases {
     private var myTyperPhase: Phase = uninitialized
     private var myPostTyperPhase: Phase = uninitialized
     private var mySbtExtractDependenciesPhase: Phase = uninitialized
+    private var mySbtExtractAPIPhase: Phase = uninitialized
     private var myPicklerPhase: Phase = uninitialized
     private var myInliningPhase: Phase = uninitialized
     private var myStagingPhase: Phase = uninitialized
@@ -235,6 +236,7 @@ object Phases {
     final def typerPhase: Phase = myTyperPhase
     final def postTyperPhase: Phase = myPostTyperPhase
     final def sbtExtractDependenciesPhase: Phase = mySbtExtractDependenciesPhase
+    final def sbtExtractAPIPhase: Phase = mySbtExtractAPIPhase
     final def picklerPhase: Phase = myPicklerPhase
     final def inliningPhase: Phase = myInliningPhase
     final def stagingPhase: Phase = myStagingPhase
@@ -263,6 +265,7 @@ object Phases {
       myTyperPhase = phaseOfClass(classOf[TyperPhase])
       myPostTyperPhase = phaseOfClass(classOf[PostTyper])
       mySbtExtractDependenciesPhase = phaseOfClass(classOf[sbt.ExtractDependencies])
+      mySbtExtractAPIPhase = phaseOfClass(classOf[sbt.ExtractAPI])
       myPicklerPhase = phaseOfClass(classOf[Pickler])
       myInliningPhase = phaseOfClass(classOf[Inlining])
       myStagingPhase = phaseOfClass(classOf[Staging])
@@ -336,19 +339,29 @@ object Phases {
     /** skip the phase for a Java compilation unit, may depend on -Yjava-tasty */
     def skipIfJava(using Context): Boolean = true
 
+    final def isAfterLastJavaPhase(using Context): Boolean =
+      // With `-Yjava-tasty` nominally the final phase is expected be ExtractAPI,
+      // otherwise drop Java sources at the end of TyperPhase.
+      // Checks if the last Java phase is before this phase,
+      // which always fails if the terminal phase is before lastJavaPhase.
+      val lastJavaPhase = if ctx.settings.YjavaTasty.value then sbtExtractAPIPhase else typerPhase
+      lastJavaPhase <= this
+
     /** @pre `isRunnable` returns true */
     def run(using Context): Unit
 
     /** @pre `isRunnable` returns true */
     def runOn(units: List[CompilationUnit])(using runCtx: Context): List[CompilationUnit] =
       val buf = List.newBuilder[CompilationUnit]
-      // factor out typedAsJava check when not needed
-      val doSkipJava = ctx.settings.YjavaTasty.value && this <= picklerPhase && skipIfJava
+
+      // Test that we are in a state where we need to check if the phase should be skipped for a java file,
+      // this prevents checking the expensive `unit.typedAsJava` unnecessarily.
+      val doCheckJava = skipIfJava && !isAfterLastJavaPhase
       for unit <- units do
         given unitCtx: Context = runCtx.fresh.setPhase(this.start).setCompilationUnit(unit).withRootImports
         if ctx.run.enterUnit(unit) then
           try
-            if doSkipJava && unit.typedAsJava then
+            if doCheckJava && unit.typedAsJava then
               ()
             else
               run
@@ -503,6 +516,7 @@ object Phases {
   def typerPhase(using Context): Phase                  = ctx.base.typerPhase
   def postTyperPhase(using Context): Phase              = ctx.base.postTyperPhase
   def sbtExtractDependenciesPhase(using Context): Phase = ctx.base.sbtExtractDependenciesPhase
+  def sbtExtractAPIPhase(using Context): Phase          = ctx.base.sbtExtractAPIPhase
   def picklerPhase(using Context): Phase                = ctx.base.picklerPhase
   def inliningPhase(using Context): Phase               = ctx.base.inliningPhase
   def stagingPhase(using Context): Phase               = ctx.base.stagingPhase

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -456,7 +456,8 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
       val tastyUUID = unpickler.unpickler.header.uuid
       new ClassfileTastyUUIDParser(classfile)(ctx).checkTastyUUID(tastyUUID)
     else
-      // This will be the case in any of our tests that compile with `-Youtput-only-tasty`
+      // This will be the case in any of our tests that compile with `-Youtput-only-tasty`, or when
+      // tasty file compiled by `-Yearly-tasty-output-write` comes from an early output jar.
       report.inform(s"No classfiles found for $tastyFile when checking TASTy UUID")
 
   private def mayLoadTreesFromTasty(using Context): Boolean =

compiler/src/dotty/tools/dotc/inlines/Inliner.scala

@@ -1042,6 +1042,9 @@ class Inliner(val call: tpd.Tree)(using Context):
       for sym <- dependencies do
         if ctx.compilationUnit.source.file == sym.associatedFile then
           report.error(em"Cannot call macro $sym defined in the same source file", call.srcPos)
+        else if ctx.settings.YnoSuspendedUnits.value then
+          val addendum = ", suspension prevented by -Yno-suspended-units"
+          report.error(em"Cannot call macro $sym defined in the same compilation run$addendum", call.srcPos)
         if (suspendable && ctx.settings.XprintSuspension.value)
           report.echo(i"suspension triggered by macro call to ${sym.showLocated} in ${sym.associatedFile}", call.srcPos)
       if suspendable then

compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala

@@ -19,7 +19,8 @@ import StdNames.str
 import NameOps.*
 import inlines.Inlines
 import transform.ValueClasses
-import dotty.tools.io.{File, FileExtension, JarArchive}
+import transform.Pickler
+import dotty.tools.io.{File, FileExtension, JarArchive, ClassfileWriterOps}
 import util.{Property, SourceFile}
 import java.io.PrintWriter
 
@@ -51,7 +52,7 @@ class ExtractAPI extends Phase {
   override def description: String = ExtractAPI.description
 
   override def isRunnable(using Context): Boolean = {
-    super.isRunnable && ctx.runZincPhases
+    super.isRunnable && (ctx.runZincPhases || ctx.settings.YjavaTasty.value)
   }
 
   // Check no needed. Does not transform trees
@@ -65,16 +66,74 @@ class ExtractAPI extends Phase {
   // after `PostTyper` (unlike `ExtractDependencies`, the simplication to trees
   // done by `PostTyper` do not affect this phase because it only cares about
   // definitions, and `PostTyper` does not change definitions).
-  override def runsAfter: Set[String] = Set(transform.PostTyper.name)
+  override def runsAfter: Set[String] = Set(transform.Pickler.name)
 
   override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] =
+    val doZincCallback = ctx.runZincPhases
+    val sigWriter: Option[Pickler.EarlyFileWriter] = ctx.settings.YearlyTastyOutput.value match
+      case earlyOut if earlyOut.isDirectory && earlyOut.exists =>
+        Some(Pickler.EarlyFileWriter(ClassfileWriterOps(earlyOut)))
+      case _ =>
+        None
     val nonLocalClassSymbols = new mutable.HashSet[Symbol]
-    val ctx0 = ctx.withProperty(NonLocalClassSymbolsInCurrentUnits, Some(nonLocalClassSymbols))
-    val units0 = super.runOn(units)(using ctx0)
-    ctx.withIncCallback(recordNonLocalClasses(nonLocalClassSymbols, _))
-    units0
+    val units0 =
+      if doZincCallback then
+        val ctx0 = ctx.withProperty(NonLocalClassSymbolsInCurrentUnits, Some(nonLocalClassSymbols))
+        super.runOn(units)(using ctx0)
+      else
+        units // still run the phase for the side effects (writing TASTy files to -Yearly-tasty-output)
+    sigWriter.foreach(writeSigFiles(units0, _))
+    if doZincCallback then
+      ctx.withIncCallback(recordNonLocalClasses(nonLocalClassSymbols, _))
+    if ctx.settings.YjavaTasty.value then
+      units0.filterNot(_.typedAsJava) // remove java sources, this is the terminal phase when `-Yjava-tasty` is set
+    else
+      units0
   end runOn
 
+  // Why we only write to early output in the first run?
+  // ===================================================
+  // TL;DR the point of pipeline compilation is to start downstream projects early,
+  // so we don't want to wait for suspended units to be compiled.
+  //
+  // But why is it safe to ignore suspended units?
+  // If this project contains a transparent macro that is called in the same project,
+  // the compilation unit of that call will be suspended (if the macro implementation
+  // is also in this project), causing a second run.
+  // However before we do that run, we will have already requested sbt to begin
+  // early downstream compilation. This means that the suspended definitions will not
+  // be visible in *early* downstream compilation.
+  //
+  // However, sbt will by default prevent downstream compilation happening in this scenario,
+  // due to the existence of macro definitions. So we are protected from failure if user tries
+  // to use the suspended definitions.
+  //
+  // Additionally, it is recommended for the user to move macro implementations to another project
+  // if they want to force early output. In this scenario the suspensions will no longer occur, so now
+  // they will become visible in the early-output.
+  //
+  // See `sbt-test/pipelining/pipelining-scala-macro` and `sbt-test/pipelining/pipelining-scala-macro-force`
+  // for examples of this in action.
+  //
+  // Therefore we only need to write to early output in the first run. We also provide the option
+  // to diagnose suspensions with the `-Yno-suspended-units` flag.
+  private def writeSigFiles(units: List[CompilationUnit], writer: Pickler.EarlyFileWriter)(using Context): Unit = {
+    try
+      for
+        unit <- units
+        (cls, pickled) <- unit.pickled
+        if cls.isDefinedInCurrentRun
+      do
+        val binaryName = cls.binaryClassName.replace('.', java.io.File.separatorChar).nn
+        val binaryClassName = if (cls.is(Module)) binaryName.stripSuffix(str.MODULE_SUFFIX).nn else binaryName
+        writer.writeTasty(binaryClassName, pickled())
+    finally
+      writer.close()
+      if ctx.settings.verbose.value then
+        report.echo("[sig files written]")
+    end try
+  }
+
   private def recordNonLocalClasses(nonLocalClassSymbols: mutable.HashSet[Symbol], cb: interfaces.IncrementalCallback)(using Context): Unit =
     for cls <- nonLocalClassSymbols do
       val sourceFile = cls.source

compiler/src/dotty/tools/dotc/transform/Pickler.scala

@@ -46,7 +46,7 @@ class Pickler extends Phase {
 
   // No need to repickle trees coming from TASTY
   override def isRunnable(using Context): Boolean =
-    super.isRunnable && (!ctx.settings.fromTasty.value || ctx.settings.YjavaTasty.value)
+    super.isRunnable && !ctx.settings.fromTasty.value
 
   // when `-Yjava-tasty` is set we actually want to run this phase on Java sources
   override def skipIfJava(using Context): Boolean = false
@@ -83,9 +83,7 @@ class Pickler extends Phase {
 
   private val executor = Executor[Array[Byte]]()
 
-  private def useExecutor(using Context) =
-    Pickler.ParallelPickling && !ctx.settings.YtestPickler.value &&
-    !ctx.settings.YjavaTasty.value // disable parallel pickling when `-Yjava-tasty` is set (internal testing only)
+  private def useExecutor(using Context) = Pickler.ParallelPickling && !ctx.settings.YtestPickler.value
 
   private def printerContext(isOutline: Boolean)(using Context): Context =
     if isOutline then ctx.fresh.setPrinterFn(OutlinePrinter(_))
@@ -182,22 +180,13 @@ class Pickler extends Phase {
   }
 
   override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] = {
-    val sigWriter: Option[Pickler.EarlyFileWriter] = ctx.settings.YjavaTastyOutput.value match
-      case jar: JarArchive if jar.exists =>
-        Some(Pickler.EarlyFileWriter(ClassfileWriterOps(jar)))
-      case _ =>
-        None
-    val units0 =
-      if ctx.settings.fromTasty.value then
-        // we still run the phase for the side effect of writing the pipeline tasty files
-        units
+    val result =
+      if useExecutor then
+        executor.start()
+        try super.runOn(units)
+        finally executor.close()
       else
-        if useExecutor then
-          executor.start()
-          try super.runOn(units)
-          finally executor.close()
-        else
-          super.runOn(units)
+        super.runOn(units)
     if ctx.settings.YtestPickler.value then
       val ctx2 = ctx.fresh
         .setSetting(ctx.settings.YreadComments, true)
@@ -208,34 +197,9 @@ class Pickler extends Phase {
           .setReporter(new ThrowingReporter(ctx.reporter))
           .addMode(Mode.ReadPositions)
       )
-    val result =
-      if ctx.settings.YjavaTasty.value then
-        sigWriter.foreach(writeJavaSigFiles(units0, _))
-        units0.filterNot(_.typedAsJava) // remove java sources, this is the terminal phase when `-Yjava-tasty` is set
-      else
-        units0
     result
   }
 
-  private def writeJavaSigFiles(units: List[CompilationUnit], writer: Pickler.EarlyFileWriter)(using Context): Unit = {
-    var count = 0
-    try
-      for
-        unit <- units if unit.typedAsJava
-        (cls, pickled) <- unit.pickled
-        if cls.isDefinedInCurrentRun
-      do
-        val binaryName = cls.binaryClassName.replace('.', java.io.File.separatorChar).nn
-        val binaryClassName = if (cls.is(Module)) binaryName.stripSuffix(str.MODULE_SUFFIX).nn else binaryName
-        writer.writeTasty(binaryClassName, pickled())
-        count += 1
-    finally
-      writer.close()
-      if ctx.settings.verbose.value then
-        report.echo(s"[$count java sig files written]")
-    end try
-  }
-
   private def testUnpickler(using Context): Unit =
     pickling.println(i"testing unpickler at run ${ctx.runId}")
     ctx.initialize()