Collect macro dependencies

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -56,6 +56,10 @@ class GenBCode extends Phase {
     try super.runOn(units)
     finally myOutput match {
       case jar: JarArchive =>
+        if (ctx.run.suspendedUnits.nonEmpty)
+          // If we close the jar the next run will not be able to write on the jar.
+          // But if we do not close it we cannot use it as part of the macro classpath of the suspended files.
+          ctx.error("Can not suspend and output to a jar at the same time. See suspension with -Xprint-suspension.")
         jar.close()
       case _ =>
     }

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -10,6 +10,7 @@ import dotty.tools.dotc.core.SymDenotations.ClassDenotation
 import dotty.tools.dotc.core.Symbols._
 import dotty.tools.dotc.transform.SymUtils._
 import util.{NoSource, SourceFile}
+import core.Decorators._
 
 class CompilationUnit protected (val source: SourceFile) {
 
@@ -31,10 +32,22 @@ class CompilationUnit protected (val source: SourceFile) {
 
   /** A structure containing a temporary map for generating inline accessors */
   val inlineAccessors: InlineAccessors = new InlineAccessors
+
+  var suspended: Boolean = false
+
+  def suspend()(given ctx: Context): Nothing =
+    if !suspended then
+      if (ctx.settings.XprintSuspension.value)
+        ctx.echo(i"suspended: $this")
+      suspended = true
+      ctx.run.suspendedUnits += this
+    throw CompilationUnit.SuspendException()
 }
 
 object CompilationUnit {
 
+  class SuspendException extends Exception
+
   /** Make a compilation unit for top class `clsd` with the contents of the `unpickled` tree */
   def apply(clsd: ClassDenotation, unpickled: Tree, forceTrees: Boolean)(implicit ctx: Context): CompilationUnit =
     apply(new SourceFile(clsd.symbol.associatedFile, Array.empty[Char]), unpickled, forceTrees)

compiler/src/dotty/tools/dotc/Driver.scala

@@ -10,6 +10,7 @@ import core.{MacroClassLoader, Mode, TypeError}
 import dotty.tools.dotc.ast.Positioned
 import dotty.tools.io.File
 import reporting._
+import core.Decorators._
 
 import scala.util.control.NonFatal
 import fromtasty.{TASTYCompiler, TastyFileUtil}
@@ -30,23 +31,33 @@ class Driver {
 
   protected def doCompile(compiler: Compiler, fileNames: List[String])(implicit ctx: Context): Reporter =
     if (fileNames.nonEmpty)
-      try {
+      try
         val run = compiler.newRun
         run.compile(fileNames)
-        run.printSummary()
-      }
-      catch {
+
+        def finish(run: Run): Unit =
+          run.printSummary()
+          if !ctx.reporter.errorsReported && run.suspendedUnits.nonEmpty then
+            val suspendedUnits = run.suspendedUnits.toList
+            if (ctx.settings.XprintSuspension.value)
+              ctx.echo(i"compiling suspended $suspendedUnits%, %")
+            val run1 = compiler.newRun
+            for unit <- suspendedUnits do unit.suspended = false
+            run1.compileUnits(suspendedUnits)
+            finish(run1)
+
+        finish(run)
+      catch
         case ex: FatalError  =>
           ctx.error(ex.getMessage) // signals that we should fail compilation.
-          ctx.reporter
         case ex: TypeError =>
           println(s"${ex.toMessage} while compiling ${fileNames.mkString(", ")}")
           throw ex
         case ex: Throwable =>
           println(s"$ex while compiling ${fileNames.mkString(", ")}")
           throw ex
-      }
-    else ctx.reporter
+    ctx.reporter
+  end doCompile
 
   protected def initCtx: Context = (new ContextBase).initialCtx
 

compiler/src/dotty/tools/dotc/Run.scala

@@ -78,17 +78,20 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
     */
   def units: List[CompilationUnit] = myUnits
 
+  var suspendedUnits: mutable.ListBuffer[CompilationUnit] = mutable.ListBuffer()
+
   private def units_=(us: List[CompilationUnit]): Unit =
     myUnits = us
 
-  /** The files currently being compiled, this may return different results over time.
-    *  These files do not have to be source files since it's possible to compile
-    *  from TASTY.
-    */
+  /** The files currently being compiled (active or suspended).
+   *  This may return different results over time.
+   *  These files do not have to be source files since it's possible to compile
+   *  from TASTY.
+   */
   def files: Set[AbstractFile] = {
     if (myUnits ne myUnitsCached) {
       myUnitsCached = myUnits
-      myFiles = myUnits.map(_.source.file).toSet
+      myFiles = (myUnits ++ suspendedUnits).map(_.source.file).toSet
     }
     myFiles
   }
@@ -247,11 +250,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
   }
 
   /** Print summary; return # of errors encountered */
-  def printSummary(): Reporter = {
+  def printSummary(): Unit = {
     printMaxConstraint()
     val r = ctx.reporter
     r.printSummary
-    r
   }
 
   override def reset(): Unit = {

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -77,6 +77,7 @@ class ScalaSettings extends Settings.SettingGroup {
   val XprintDiff: Setting[Boolean] = BooleanSetting("-Xprint-diff", "Print changed parts of the tree since last print.")
   val XprintDiffDel: Setting[Boolean] = BooleanSetting("-Xprint-diff-del", "Print changed parts of the tree since last print including deleted parts.")
   val XprintInline: Setting[Boolean] = BooleanSetting("-Xprint-inline", "Show  where inlined code comes from")
+  val XprintSuspension: Setting[Boolean] = BooleanSetting("-Xprint-suspension", "Show when code is suspended until macros are compiled")
   val Xprompt: Setting[Boolean] = BooleanSetting("-Xprompt", "Display a prompt after each error (debugging option).")
   val XnoValueClasses: Setting[Boolean] = BooleanSetting("-Xno-value-classes", "Do not use value classes. Helps debugging.")
   val XreplLineWidth: Setting[Int] = IntSetting("-Xrepl-line-width", "Maximal number of columns per line for REPL output", 390)
@@ -200,7 +201,7 @@ class ScalaSettings extends Settings.SettingGroup {
     "The source repository of your project",
     ""
   )
-  
+
   val projectLogo: Setting[String] = StringSetting(
     "-project-logo",
     "project logo filename",

compiler/src/dotty/tools/dotc/core/MacroClassLoader.scala

@@ -9,7 +9,7 @@ import scala.collection.mutable
 object MacroClassLoader {
 
   /** A key to be used in a context property that caches the class loader used for macro expansion */
-  private val MacroClassLoaderKey = new Property.Key[ClassLoader]
+  private val MacroClassLoaderKey = new Property.Key[MacroClassLoader]
 
   /** Get the macro class loader */
   def fromContext(implicit ctx: Context): ClassLoader =
@@ -19,8 +19,24 @@ object MacroClassLoader {
   def init(ctx: FreshContext): ctx.type =
     ctx.setProperty(MacroClassLoaderKey, makeMacroClassLoader(ctx))
 
-  private def makeMacroClassLoader(implicit ctx: Context): ClassLoader = trace("new macro class loader") {
+  def loadedClasses(implicit ctx: Context): List[String] =
+    ctx.property(MacroClassLoaderKey).get.getLoadedClasses
+
+  private def makeMacroClassLoader(implicit ctx: Context): MacroClassLoader = trace("new macro class loader") {
     val urls = ctx.settings.classpath.value.split(java.io.File.pathSeparatorChar).map(cp => java.nio.file.Paths.get(cp).toUri.toURL)
-    new java.net.URLClassLoader(urls, getClass.getClassLoader)
+    val out = ctx.settings.outputDir.value.jpath.toUri.toURL // to find classes in case of suspended compilation
+    new MacroClassLoader(urls :+ out, getClass.getClassLoader)
+  }
+}
+
+private class MacroClassLoader(urls: Array[java.net.URL], parent: ClassLoader) extends java.net.URLClassLoader(urls, parent) {
+
+  private[this] val loadedClasses: mutable.SortedSet[String] = mutable.SortedSet.empty[String]
+
+  def getLoadedClasses: List[String] = loadedClasses.toList
+
+  override def loadClass(name: String): Class[?] = {
+    loadedClasses.add(name)
+    super.loadClass(name)
   }
 }

compiler/src/dotty/tools/dotc/interactive/InteractiveDriver.scala

@@ -155,7 +155,7 @@ class InteractiveDriver(val settings: List[String]) extends Driver {
 
       run.compileSources(List(source))
       run.printSummary()
-      val unit = ctx.run.units.head
+      val unit = if ctx.run.units.nonEmpty then ctx.run.units.head else ctx.run.suspendedUnits.head
       val t = unit.tpdTree
       cleanup(t)
       myOpenedTrees(uri) = topLevelTrees(t, source)

compiler/src/dotty/tools/dotc/sbt/ExtractDependencies.scala

@@ -361,6 +361,10 @@ private class ExtractDependenciesCollector extends tpd.TreeTraverser { thisTreeT
       case _ =>
     }
 
+    if (tree.isDef && tree.symbol.isTopLevelClass) // FIXME do not add these dependencies if no macro was used in this file
+      for (name <- core.MacroClassLoader.loadedClasses)
+        ctx.sbtCallback.classDependency(name, tree.symbol.showFullName, DependencyContext.LocalDependencyByInheritance)
+
     tree match {
       case Inlined(call, _, _) if !call.isEmpty =>
         // The inlined call is normally ignored by TreeTraverser but we need to

compiler/src/dotty/tools/dotc/transform/Splicer.scala

@@ -47,6 +47,8 @@ object Splicer {
         interpretedExpr.fold(tree)(macroClosure => PickledQuotes.quotedExprToTree(macroClosure(QuoteContext())))
       }
       catch {
+        case ex: CompilationUnit.SuspendException =>
+          throw ex
         case ex: StopInterpretation =>
           ctx.error(ex.msg, ex.pos)
           EmptyTree
@@ -322,20 +324,18 @@ object Splicer {
       try classLoader.loadClass(name)
       catch {
         case _: ClassNotFoundException =>
-          val msg = s"Could not find class $name in classpath$extraMsg"
+          val msg = s"Could not find class $name in classpath"
           throw new StopInterpretation(msg, pos)
       }
 
     private def getMethod(clazz: Class[?], name: Name, paramClasses: List[Class[?]]): Method =
       try clazz.getMethod(name.toString, paramClasses: _*)
       catch {
         case _: NoSuchMethodException =>
-          val msg = em"Could not find method ${clazz.getCanonicalName}.$name with parameters ($paramClasses%, %)$extraMsg"
+          val msg = em"Could not find method ${clazz.getCanonicalName}.$name with parameters ($paramClasses%, %)"
           throw new StopInterpretation(msg, pos)
       }
 
-    private def extraMsg = ". The most common reason for that is that you apply macros in the compilation run that defines them"
-
     private def stopIfRuntimeException[T](thunk: => T, method: Method): T =
       try thunk
       catch {
@@ -348,21 +348,38 @@ object Splicer {
           sw.write("\n")
           throw new StopInterpretation(sw.toString, pos)
         case ex: InvocationTargetException =>
-          val sw = new StringWriter()
-          sw.write("Exception occurred while executing macro expansion.\n")
-          val targetException = ex.getTargetException
-          if (!ctx.settings.Ydebug.value) {
-            val end = targetException.getStackTrace.lastIndexWhere { x =>
-              x.getClassName == method.getDeclaringClass.getCanonicalName && x.getMethodName == method.getName
-            }
-            val shortStackTrace = targetException.getStackTrace.take(end + 1)
-            targetException.setStackTrace(shortStackTrace)
+          ex.getTargetException match {
+            case MissingClassDefinedInCurrentRun(sym) =>
+              if (ctx.settings.XprintSuspension.value)
+                ctx.echo(i"suspension triggered by a dependency on $sym", pos)
+              ctx.compilationUnit.suspend() // this throws a SuspendException
+            case targetException =>
+              val sw = new StringWriter()
+              sw.write("Exception occurred while executing macro expansion.\n")
+              if (!ctx.settings.Ydebug.value) {
+                val end = targetException.getStackTrace.lastIndexWhere { x =>
+                  x.getClassName == method.getDeclaringClass.getCanonicalName && x.getMethodName == method.getName
+                }
+                val shortStackTrace = targetException.getStackTrace.take(end + 1)
+                targetException.setStackTrace(shortStackTrace)
+              }
+              targetException.printStackTrace(new PrintWriter(sw))
+              sw.write("\n")
+              throw new StopInterpretation(sw.toString, pos)
           }
-          targetException.printStackTrace(new PrintWriter(sw))
-          sw.write("\n")
-          throw new StopInterpretation(sw.toString, pos)
       }
 
+    private object MissingClassDefinedInCurrentRun {
+      def unapply(targetException: NoClassDefFoundError)(given ctx: Context): Option[Symbol] = {
+        val className = targetException.getMessage
+        if (className eq null) None
+        else {
+          val sym = ctx.base.staticRef(className.toTypeName).symbol
+          if (sym.isDefinedInCurrentRun) Some(sym) else None
+        }
+      }
+    }
+
     /** List of classes of the parameters of the signature of `sym` */
     private def paramsSig(sym: Symbol): List[Class[?]] = {
       def paramClass(param: Type): Class[?] = {

compiler/src/dotty/tools/dotc/typer/FrontEnd.scala

@@ -6,6 +6,7 @@ import core._
 import Phases._
 import Contexts._
 import Symbols._
+import Decorators._
 import dotty.tools.dotc.parsing.JavaParsers.JavaParser
 import parsing.Parsers.Parser
 import config.Config
@@ -71,11 +72,15 @@ class FrontEnd extends Phase {
   }
 
   def typeCheck(implicit ctx: Context): Unit = monitor("typechecking") {
-    val unit = ctx.compilationUnit
-    unit.tpdTree = ctx.typer.typedExpr(unit.untpdTree)
-    typr.println("typed: " + unit.source)
-    record("retained untyped trees", unit.untpdTree.treeSize)
-    record("retained typed trees after typer", unit.tpdTree.treeSize)
+    try
+      val unit = ctx.compilationUnit
+      if !unit.suspended then
+        unit.tpdTree = ctx.typer.typedExpr(unit.untpdTree)
+        typr.println("typed: " + unit.source)
+        record("retained untyped trees", unit.untpdTree.treeSize)
+        record("retained typed trees after typer", unit.tpdTree.treeSize)
+    catch
+      case ex: CompilationUnit.SuspendException =>
   }
 
   private def firstTopLevelDef(trees: List[tpd.Tree])(implicit ctx: Context): Symbol = trees match {
@@ -86,14 +91,14 @@ class FrontEnd extends Phase {
   }
 
   protected def discardAfterTyper(unit: CompilationUnit)(implicit ctx: Context): Boolean =
-    unit.isJava || firstTopLevelDef(unit.tpdTree :: Nil).isPrimitiveValueClass
+    unit.isJava || unit.suspended || firstTopLevelDef(unit.tpdTree :: Nil).isPrimitiveValueClass
 
   override def runOn(units: List[CompilationUnit])(implicit ctx: Context): List[CompilationUnit] = {
     val unitContexts = for (unit <- units) yield {
       ctx.inform(s"compiling ${unit.source}")
       ctx.fresh.setCompilationUnit(unit)
     }
-    unitContexts foreach (parse(_))
+    unitContexts.foreach(parse(_))
     record("parsedTrees", ast.Trees.ntrees)
     remaining = unitContexts
     while (remaining.nonEmpty) {
@@ -108,7 +113,23 @@ class FrontEnd extends Phase {
 
     unitContexts.foreach(typeCheck(_))
     record("total trees after typer", ast.Trees.ntrees)
-    unitContexts.map(_.compilationUnit).filterNot(discardAfterTyper)
+    val newUnits = unitContexts.map(_.compilationUnit).filterNot(discardAfterTyper)
+    val suspendedUnits = ctx.run.suspendedUnits
+    if newUnits.isEmpty && suspendedUnits.nonEmpty && !ctx.reporter.errorsReported then
+      val where =
+        if suspendedUnits.size == 1 then i"in ${suspendedUnits.head}."
+        else i"""among
+                |
+                |  ${suspendedUnits.toList}%, %
+                |"""
+      val enableXprintSuspensionHint =
+        if (ctx.settings.XprintSuspension.value) ""
+        else "\n\nCompiling with  -Xprint-suspension   gives more information."
+      ctx.error(em"""Cyclic macro dependencies $where
+                    |Compilation stopped since no further progress can be made.
+                    |
+                    |To fix this, place macros in one set of files and their callers in another.$enableXprintSuspensionHint""")
+    newUnits
   }
 
   def run(implicit ctx: Context): Unit = unsupported("run")