Address feedback. Adjust semantics.

Now in backend we block on the composition of the Zinc API callbacks
and tasty being written.

We also now conditionally signal that we are "done" writing tasty,
based on if any units were suspended. This works in line with the
Zinc default, which will ignore the early output anyway under the
presence of macros (user can override this).

TODO: give the user an option to optimise performance by preventing definition
of such problematic macros (which would also avoid suspensions).

Author: bishabosha

compiler/src/dotty/tools/backend/jvm/ClassfileWriters.scala

@@ -20,7 +20,11 @@ import dotty.tools.io.JarArchive
 
 import scala.language.unsafeNulls
 
-
+/** !!! This file is now copied in `dotty.tools.io.FileWriters` in a more general way that does not rely upon
+ * `PostProcessorFrontendAccess`, this should probably be changed to wrap that class instead.
+ *
+ * Until then, any changes to this file should be copied to `dotty.tools.io.FileWriters` as well.
+ */
 class ClassfileWriters(frontendAccess: PostProcessorFrontendAccess) {
   type NullableFile =  AbstractFile | Null
   import frontendAccess.{compilerSettings, backendReporting}

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -12,7 +12,7 @@ import scala.collection.mutable
 import scala.compiletime.uninitialized
 import java.util.concurrent.TimeoutException
 
-import scala.concurrent.duration.given
+import scala.concurrent.duration.Duration
 import scala.concurrent.Await
 
 class GenBCode extends Phase { self =>
@@ -94,20 +94,15 @@ class GenBCode extends Phase { self =>
     try
       val result = super.runOn(units)
       generatedClassHandler.complete()
-      for holder <- ctx.asyncTastyPromise do
-        try
-          val asyncState = Await.result(holder.promise.future, 5.seconds)
-          for reporter <- asyncState.pending do
-            reporter.relayReports(frontendAccess.backendReporting)
-        catch
-          case _: TimeoutException =>
-            report.error(
-              """Timeout (5s) in backend while waiting for async writing of TASTy files to -Yearly-tasty-output,
-                |  this may be a bug in the compiler.
-                |
-                |Alternatively consider turning off pipelining for this project.""".stripMargin
-            )
-      end for
+      try
+        for
+          async <- ctx.asyncTastyHolder
+          reporter <- async.sync
+        do
+          reporter.relayReports(frontendAccess.backendReporting)
+      catch
+        case ex: Exception =>
+          report.error(s"exception from future: $ex, (${Option(ex.getCause())})")
       result
     finally
       // frontendAccess and postProcessor are created lazilly, clean them up only if they were initialized

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -99,9 +99,10 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
     if !suspended then
       suspended = true
       ctx.run.nn.suspendedUnits += this
+      val isInliningPhase = ctx.phase == Phases.inliningPhase
       if ctx.settings.XprintSuspension.value then
-        ctx.run.nn.suspendedHints += (this -> hint)
-      if ctx.phase == Phases.inliningPhase then
+        ctx.run.nn.suspendedHints += (this -> (hint, isInliningPhase))
+      if isInliningPhase then
         suspendedAtInliningPhase = true
     throw CompilationUnit.SuspendException()
 

compiler/src/dotty/tools/dotc/Driver.scala

@@ -35,7 +35,8 @@ class Driver {
         val run = compiler.newRun
         runOrNull = run
         run.compile(files)
-        finish(compiler, run)
+        inContext(ctx.fresh.resetAsyncTasty()):
+          finish(compiler, run)
       catch
         case ex: FatalError =>
           report.error(ex.getMessage.nn) // signals that we should fail compilation.
@@ -54,11 +55,11 @@ class Driver {
       if (ctx.settings.XprintSuspension.value)
         val suspendedHints = run.suspendedHints.toList
         report.echo(i"compiling suspended $suspendedUnits%, %")
-        for (unit, hint) <- suspendedHints do
-          report.echo(s"  $unit: $hint")
+        for (unit, (hint, atInlining)) <- suspendedHints do
+          report.echo(s"  $unit at ${if atInlining then "inlining" else "typer"}: $hint")
       val run1 = compiler.newRun
       run1.compileSuspendedUnits(suspendedUnits)
-      finish(compiler, run1)(using MacroClassLoader.init(ctx.fresh))
+      finish(compiler, run1)(using MacroClassLoader.init(ctx.fresh.resetAsyncTasty()))
 
   protected def initCtx: Context = (new ContextBase).initialCtx
 

compiler/src/dotty/tools/dotc/Run.scala

@@ -130,7 +130,7 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
     myUnits = us
 
   var suspendedUnits: mutable.ListBuffer[CompilationUnit] = mutable.ListBuffer()
-  var suspendedHints: mutable.Map[CompilationUnit, String] = mutable.HashMap()
+  var suspendedHints: mutable.Map[CompilationUnit, (String, Boolean)] = mutable.HashMap()
 
   def checkSuspendedUnits(newUnits: List[CompilationUnit])(using Context): Unit =
     if newUnits.isEmpty && suspendedUnits.nonEmpty && !ctx.reporter.errorsReported then

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -56,7 +56,7 @@ object Contexts {
   private val (importInfoLoc,        store9) = store8.newLocation[ImportInfo | Null]()
   private val (typeAssignerLoc,     store10) = store9.newLocation[TypeAssigner](TypeAssigner)
   private val (progressCallbackLoc, store11) = store10.newLocation[ProgressCallback | Null]()
-  private val (tastyPromiseLoc,     store12) = store11.newLocation[Option[AsyncTastyHolder]](None)
+  private val (asyncTastyLoc,       store12) = store11.newLocation[Option[AsyncTastyHolder]](None)
 
   private val initialStore = store12
 
@@ -200,7 +200,7 @@ object Contexts {
     /** The current settings values */
     def settingsState: SettingsState = store(settingsStateLoc)
 
-    def asyncTastyPromise: Option[AsyncTastyHolder] = store(tastyPromiseLoc)
+    def asyncTastyHolder: Option[AsyncTastyHolder] = store(asyncTastyLoc)
 
     /** The current compilation unit */
     def compilationUnit: CompilationUnit = store(compilationUnitLoc)
@@ -690,9 +690,28 @@ object Contexts {
       updateStore(compilationUnitLoc, compilationUnit)
     }
 
+    private def setAsyncTasty(): this.type =
+      // should we provide a custom ExecutionContext?
+      // currently it is just used to call the `apiPhaseCompleted` and `dependencyPhaseCompleted` callbacks in Zinc
+      import scala.concurrent.ExecutionContext.Implicits.global
+      updateStore(asyncTastyLoc, Some(AsyncTastyHolder.init))
+
     def setInitialAsyncTasty(): this.type =
-      assert(store(tastyPromiseLoc) == None, "trying to set async tasty promise twice!")
-      updateStore(tastyPromiseLoc, Some(AsyncTastyHolder(settings.YearlyTastyOutput.value, Promise())))
+      assert(store(asyncTastyLoc) == None, "trying to set async tasty holder twice!")
+      setAsyncTasty()
+
+    /**replace the async tasty holder if one exists, ready for the next run.
+     */
+    def resetAsyncTasty(): this.type =
+      asyncTastyHolder match
+        case Some(asyncTasty) =>
+          if !reporter.hasErrors && asyncTasty.syncIsDone then
+            updateStore(asyncTastyLoc, None) // we closed so clear the holder.
+          else
+            setAsyncTasty() // reset the holder as we will start a new run and write to it
+        case _ =>
+          this
+
 
     def setCompilerCallback(callback: CompilerCallback): this.type = updateStore(compilerCallbackLoc, callback)
     def setIncCallback(callback: IncrementalCallback): this.type = updateStore(incCallbackLoc, callback)

compiler/src/dotty/tools/dotc/sbt/ExtractAPI.scala

@@ -90,14 +90,7 @@ class ExtractAPI extends Phase {
       val sourceFile = cls.source
       if sourceFile.exists && cls.isDefinedInCurrentRun then
         recordNonLocalClass(cls, sourceFile, cb)
-    for holder <- ctx.asyncTastyPromise do
-      import scala.concurrent.ExecutionContext.Implicits.global
-      // do not expect to be completed with failure
-      holder.promise.future.foreach: state =>
-        if !state.hasErrors then
-          // We also await the promise at GenBCode to emit warnings/errors
-          cb.apiPhaseCompleted()
-          cb.dependencyPhaseCompleted()
+    ctx.asyncTastyHolder.foreach(_.signalAPIComplete())
 
   private def recordNonLocalClass(cls: Symbol, sourceFile: SourceFile, cb: interfaces.IncrementalCallback)(using Context): Unit =
     def registerProductNames(fullClassName: String, binaryClassName: String) =

compiler/src/dotty/tools/dotc/sbt/package.scala

@@ -6,10 +6,29 @@ import dotty.tools.dotc.core.NameOps.stripModuleClassSuffix
 import dotty.tools.dotc.core.Names.Name
 import dotty.tools.dotc.core.Names.termName
 
+import interfaces.IncrementalCallback
+import dotty.tools.io.FileWriters.BufferingReporter
+import dotty.tools.dotc.core.Decorators.em
+
+import scala.util.chaining.given
+import scala.util.control.NonFatal
+
 inline val TermNameHash = 1987 // 300th prime
 inline val TypeNameHash = 1993 // 301st prime
 inline val InlineParamHash = 1997 // 302nd prime
 
+def asyncZincPhasesCompleted(cb: IncrementalCallback, pending: Option[BufferingReporter]): Option[BufferingReporter] =
+  val zincReporter = pending match
+    case Some(buffered) => buffered
+    case None => BufferingReporter()
+  try
+    cb.apiPhaseCompleted()
+    cb.dependencyPhaseCompleted()
+  catch
+    case NonFatal(t) =>
+      zincReporter.exception(em"signaling API and Dependencies phases completion", t)
+  if zincReporter.hasErrors then Some(zincReporter) else None
+
 extension (sym: Symbol)
 
   /** Mangle a JVM symbol name in a format better suited for internal uses by sbt.