blackbox and whitebox macros

This is the first commit in the series. This commit only:
1) Splits Context into BlackboxContext and WhiteboxContext
2) Splits Macro into BlackboxMacro and WhiteboxMacro
3) Introduces the isBundle property in the macro impl binding

Here we just teach the compiler that macros can now be blackbox and whitebox,
without actually imposing any restrictions on blackbox macros. These
restrictions will come in subsequent commits.

For description and documentation of the blackbox/whitebox separation
see the official macro guide at the scaladoc website:
http://docs.scala-lang.org/overviews/macros/blackbox-whitebox.html

Author: xeno-by

src/compiler/scala/reflect/macros/compiler/Errors.scala

@@ -33,7 +33,7 @@ trait Errors extends Traces {
 
   def MacroBundleNonStaticError() = implRefError("macro bundles must be static")
 
-  def MacroBundleWrongShapeError() = implRefError("macro bundles must be monomorphic traits extending scala.reflect.macros.Macro and not implementing its `val c: Context` member")
+  def MacroBundleWrongShapeError() = implRefError("macro bundles must be monomorphic traits extending either BlackboxMacro or WhiteboxMacro and not implementing their `val c: BlackboxContext/WhiteboxContext` member")
 
   // compatibility errors
 

src/compiler/scala/reflect/macros/compiler/Resolvers.scala

@@ -15,10 +15,6 @@ trait Resolvers {
   private val runDefinitions = currentRun.runDefinitions
   import runDefinitions.{Predef_???, _}
 
-  /** Determines the type of context implied by the macro def.
-   */
-  val ctxTpe = MacroContextClass.tpe
-
   /** Resolves a macro impl reference provided in the right-hand side of the given macro definition.
    *
    *  Acceptable shapes of the right-hand side:
@@ -44,22 +40,23 @@ trait Resolvers {
     }
 
     val untypedImplRef = typer.silent(_.typedTypeConstructor(maybeBundleRef)) match {
-      case SilentResultValue(result) if result.tpe.baseClasses.contains(MacroClass) =>
+      case SilentResultValue(result) if mightBeMacroBundleType(result.tpe) =>
         val bundleProto = result.tpe.typeSymbol
         val bundlePkg = bundleProto.enclosingPackageClass
         if (!isMacroBundleProtoType(bundleProto.tpe)) MacroBundleWrongShapeError()
         if (!bundleProto.owner.isStaticOwner) MacroBundleNonStaticError()
 
         // synthesize the bundle, i.e. given a static `trait Foo extends Macro { def expand = ... } `
-        // create a top-level definition `class Foo$Bundle(val c: Context) extends Foo` in a package next to `Foo`
+        // create a top-level definition `class Foo$Bundle(val c: BlackboxContext/WhiteboxContext) extends Foo` in a package next to `Foo`
         val bundlePid = gen.mkUnattributedRef(bundlePkg)
         val bundlePrefix =
           if (bundlePkg == EmptyPackageClass) bundleProto.fullName('$')
           else bundleProto.fullName('$').substring(bundlePkg.fullName('$').length + 1)
         val bundleName = TypeName(bundlePrefix + tpnme.MACRO_BUNDLE_SUFFIX)
         val existingBundle = bundleProto.enclosingPackageClass.info.decl(bundleName)
         if (!currentRun.compiles(existingBundle)) {
-          def mkContextValDef(flags: Long) = ValDef(Modifiers(flags), nme.c, TypeTree(ctxTpe), EmptyTree)
+          val contextType = if (isBlackboxMacroBundleType(bundleProto.tpe)) BlackboxContextClass.tpe else WhiteboxContextClass.tpe
+          def mkContextValDef(flags: Long) = ValDef(Modifiers(flags), nme.c, TypeTree(contextType), EmptyTree)
           val contextField = mkContextValDef(PARAMACCESSOR)
           val contextParam = mkContextValDef(PARAM | PARAMACCESSOR)
           val bundleCtor = DefDef(Modifiers(), nme.CONSTRUCTOR, Nil, List(List(contextParam)), TypeTree(), Block(List(pendingSuperCall), Literal(Constant(()))))
@@ -88,12 +85,13 @@ trait Resolvers {
   // lazy val (isImplBundle, macroImplOwner, macroImpl, macroImplTargs) =
   private lazy val dissectedMacroImplRef =
     macroImplRef match {
-      case MacroImplReference(isBundle, owner, meth, targs) => (isBundle, owner, meth, targs)
+      case MacroImplReference(isBundle, isBlackbox, owner, meth, targs) => (isBundle, isBlackbox, owner, meth, targs)
       case _ => MacroImplReferenceWrongShapeError()
     }
   lazy val isImplBundle = dissectedMacroImplRef._1
   lazy val isImplMethod = !isImplBundle
-  lazy val macroImplOwner = dissectedMacroImplRef._2
-  lazy val macroImpl = dissectedMacroImplRef._3
-  lazy val targs = dissectedMacroImplRef._4
+  lazy val isImplBlackbox = dissectedMacroImplRef._2
+  lazy val macroImplOwner = dissectedMacroImplRef._3
+  lazy val macroImpl = dissectedMacroImplRef._4
+  lazy val targs = dissectedMacroImplRef._5
 }

src/compiler/scala/reflect/macros/compiler/Validators.scala

@@ -49,8 +49,8 @@ trait Validators {
       map2(aparamss.flatten, rparamss.flatten)((aparam, rparam) => {
         if (aparam.name != rparam.name && !rparam.isSynthetic) MacroImplParamNameMismatchError(aparam, rparam)
         if (isRepeated(aparam) ^ isRepeated(rparam)) MacroImplVarargMismatchError(aparam, rparam)
-        val aparamtpe = aparam.tpe.dealias match {
-          case RefinedType(List(tpe), Scope(sym)) if tpe =:= ctxTpe && sym.allOverriddenSymbols.contains(MacroContextPrefixType) => tpe
+        val aparamtpe = aparam.tpe match {
+          case MacroContextType(tpe) => tpe
           case tpe => tpe
         }
         checkMacroImplParamTypeMismatch(atpeToRtpe(aparamtpe), rparam)
@@ -93,20 +93,20 @@ trait Validators {
    *
    *  For the following macro impl:
    *    def fooBar[T: c.WeakTypeTag]
-   *           (c: scala.reflect.macros.Context)
+   *           (c: scala.reflect.macros.BlackboxContext)
    *           (xs: c.Expr[List[T]])
    *           : c.Expr[T] = ...
    *
    *  This function will return:
-   *    (c: scala.reflect.macros.Context)(xs: c.Expr[List[T]])c.Expr[T]
+   *    (c: scala.reflect.macros.BlackboxContext)(xs: c.Expr[List[T]])c.Expr[T]
    *
    *  Note that type tag evidence parameters are not included into the result.
    *  Type tag context bounds for macro impl tparams are optional.
    *  Therefore compatibility checks ignore such parameters, and we don't need to bother about them here.
    *
    *  This method cannot be reduced to just macroImpl.info, because macro implementations might
-   *  come in different shapes. If the implementation is an apply method of a Macro-compatible object,
-   *  then it won't have (c: Context) in its parameters, but will rather refer to Macro.c.
+   *  come in different shapes. If the implementation is an apply method of a BlackboxMacro/WhiteboxMacro-compatible object,
+   *  then it won't have (c: BlackboxContext/WhiteboxContext) in its parameters, but will rather refer to BlackboxMacro/WhiteboxMacro.c.
    *
    *  @param macroImpl The macro implementation symbol
    */
@@ -123,7 +123,8 @@ trait Validators {
    *    def foo[T](xs: List[T]): T = macro fooBar
    *
    *  This function will return:
-   *    (c: scala.reflect.macros.Context)(xs: c.Expr[List[T]])c.Expr[T]
+   *    (c: scala.reflect.macros.BlackboxContext)(xs: c.Expr[List[T]])c.Expr[T] or
+   *    (c: scala.reflect.macros.WhiteboxContext)(xs: c.Expr[List[T]])c.Expr[T]
    *
    *  Note that type tag evidence parameters are not included into the result.
    *  Type tag context bounds for macro impl tparams are optional.
@@ -145,6 +146,7 @@ trait Validators {
     // had to move method's body to an object because of the recursive dependencies between sigma and param
     object SigGenerator {
       val cache = scala.collection.mutable.Map[Symbol, Symbol]()
+      val ctxTpe = if (isImplBlackbox) BlackboxContextClass.tpe else WhiteboxContextClass.tpe
       val ctxPrefix =
         if (isImplMethod) singleType(NoPrefix, makeParam(nme.macroContext, macroDdef.pos, ctxTpe, SYNTHETIC))
         else singleType(ThisType(macroImpl.owner), macroImpl.owner.tpe.member(nme.c))

src/compiler/scala/reflect/macros/contexts/Context.scala

@@ -3,7 +3,8 @@ package contexts
 
 import scala.tools.nsc.Global
 
-abstract class Context extends scala.reflect.macros.Context
+abstract class Context extends scala.reflect.macros.BlackboxContext
+                          with scala.reflect.macros.WhiteboxContext
                           with Aliases
                           with Enclosures
                           with Names

src/compiler/scala/reflect/macros/runtime/MacroRuntimes.scala

@@ -45,7 +45,7 @@ trait MacroRuntimes extends JavaReflectionRuntimes with ScalaReflectionRuntimes
   type MacroRuntime = MacroArgs => Any
   class MacroRuntimeResolver(val macroDef: Symbol) extends JavaReflectionResolvers
                                                       with ScalaReflectionResolvers {
-    val binding = loadMacroImplBinding(macroDef)
+    val binding = loadMacroImplBinding(macroDef).get
     val isBundle = binding.isBundle
     val className = binding.className
     val methName = binding.methName

src/compiler/scala/reflect/macros/util/Helpers.scala

@@ -27,13 +27,13 @@ trait Helpers {
     import runDefinitions._
 
     val MacroContextUniverse = definitions.MacroContextUniverse
-    val treeInfo.MacroImplReference(isBundle, _, macroImpl, _) = macroImplRef
+    val treeInfo.MacroImplReference(isBundle, _, _, macroImpl, _) = macroImplRef
     val paramss = macroImpl.paramss
     val ContextParam = paramss match {
-      case Nil | _ :+ Nil                                             => NoSymbol // no implicit parameters in the signature => nothing to do
-      case _ if isBundle                                              => macroImpl.owner.tpe member nme.c
-      case (cparam :: _) :: _ if cparam.tpe <:< MacroContextClass.tpe => cparam
-      case _                                                          => NoSymbol // no context parameter in the signature => nothing to do
+      case Nil | _ :+ Nil                                       => NoSymbol // no implicit parameters in the signature => nothing to do
+      case _ if isBundle                                        => macroImpl.owner.tpe member nme.c
+      case (cparam :: _) :: _ if isMacroContextType(cparam.tpe) => cparam
+      case _                                                    => NoSymbol // no context parameter in the signature => nothing to do
     }
     def transformTag(param: Symbol): Symbol = param.tpe.dealias match {
       case TypeRef(SingleType(SingleType(_, ContextParam), MacroContextUniverse), WeakTypeTagClass, targ :: Nil) => transform(param, targ.typeSymbol)

src/compiler/scala/tools/nsc/typechecker/ContextErrors.scala

@@ -527,6 +527,9 @@ trait ContextErrors {
       def TooManyArgsPatternError(fun: Tree) =
         NormalTypeError(fun, "too many arguments for unapply pattern, maximum = "+definitions.MaxTupleArity)
 
+      def BlackboxExtractorExpansion(fun: Tree) =
+        NormalTypeError(fun, "extractor macros can only be whitebox")
+
       def WrongShapeExtractorExpansion(fun: Tree) =
         NormalTypeError(fun, "extractor macros can only expand into extractor calls")
 

src/compiler/scala/tools/nsc/typechecker/Implicits.scala

@@ -1147,7 +1147,7 @@ trait Implicits {
             gen.mkAttributedThis(thisSym)
           case _ =>
             // if `pre` is not a PDT, e.g. if someone wrote
-            //   implicitly[scala.reflect.macros.Context#TypeTag[Int]]
+            //   implicitly[scala.reflect.macros.BlackboxContext#TypeTag[Int]]
             // then we need to fail, because we don't know the prefix to use during type reification
             // upd. we also need to fail silently, because this is a very common situation
             // e.g. quite often we're searching for BaseUniverse#TypeTag, e.g. for a type tag in any universe

src/compiler/scala/tools/nsc/typechecker/Macros.scala

@@ -29,7 +29,7 @@ import Fingerprint._
  *  Then fooBar needs to point to a static method of the following form:
  *
  *    def fooBar[T: c.WeakTypeTag] // type tag annotation is optional
- *           (c: scala.reflect.macros.Context)
+ *           (c: scala.reflect.macros.BlackboxContext)
  *           (xs: c.Expr[List[T]])
  *           : c.Expr[T] = {
  *      ...
@@ -67,7 +67,7 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
    *
    *  This solution is very simple, but unfortunately it's also lacking. If we use it, then
    *  signatures of macro defs become transitively dependent on scala-reflect.jar
-   *  (because they refer to macro impls, and macro impls refer to scala.reflect.macros.Context defined in scala-reflect.jar).
+   *  (because they refer to macro impls, and macro impls refer to scala.reflect.macros.BlackboxContext/WhiteboxContext defined in scala-reflect.jar).
    *  More details can be found in comments to https://issues.scala-lang.org/browse/SI-5940.
    *
    *  Therefore we have to avoid putting macro impls into binding pickles and come up with our own serialization format.
@@ -81,54 +81,57 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
    *  and various accounting information necessary when composing an argument list for the reflective invocation.
    */
   case class MacroImplBinding(
-    // Is this macro impl a bundle (a trait extending Macro) or a vanilla def?
-    val isBundle: Boolean,
-    // Java class name of the class that contains the macro implementation
-    // is used to load the corresponding object with Java reflection
-    className: String,
-    // method name of the macro implementation
-    // `className` and `methName` are all we need to reflectively invoke a macro implementation
-    // because macro implementations cannot be overloaded
-    methName: String,
-    // flattens the macro impl's parameter lists having symbols replaced with their fingerprints
-    // currently fingerprints are calculated solely from types of the symbols:
-    //   * c.Expr[T] => LiftedTyped
-    //   * c.Tree => LiftedUntyped
-    //   * c.WeakTypeTag[T] => Tagged(index of the type parameter corresponding to that type tag)
-    //   * everything else (e.g. scala.reflect.macros.Context) => Other
-    // f.ex. for: def impl[T: WeakTypeTag, U, V: WeakTypeTag](c: Context)(x: c.Expr[T], y: c.Tree): (U, V) = ???
-    // `signature` will be equal to List(List(Other), List(LiftedTyped, LiftedUntyped), List(Tagged(0), Tagged(2)))
-    signature: List[List[Fingerprint]],
-    // type arguments part of a macro impl ref (the right-hand side of a macro definition)
-    // these trees don't refer to a macro impl, so we can pickle them as is
-    targs: List[Tree]) {
-
+      // Is this macro impl a bundle (a trait extending BlackboxMacro or WhiteboxMacro) or a vanilla def?
+      val isBundle: Boolean,
+      // Is this macro impl blackbox (i.e. having BlackboxContext in its signature)?
+      val isBlackbox: Boolean,
+      // Java class name of the class that contains the macro implementation
+      // is used to load the corresponding object with Java reflection
+      className: String,
+      // method name of the macro implementation
+      // `className` and `methName` are all we need to reflectively invoke a macro implementation
+      // because macro implementations cannot be overloaded
+      methName: String,
+      // flattens the macro impl's parameter lists having symbols replaced with their fingerprints
+      // currently fingerprints are calculated solely from types of the symbols:
+      //   * c.Expr[T] => LiftedTyped
+      //   * c.Tree => LiftedUntyped
+      //   * c.WeakTypeTag[T] => Tagged(index of the type parameter corresponding to that type tag)
+      //   * everything else (e.g. scala.reflect.macros.BlackboxContext/WhiteboxContext) => Other
+      // f.ex. for: def impl[T: WeakTypeTag, U, V: WeakTypeTag](c: BlackboxContext)(x: c.Expr[T], y: c.Tree): (U, V) = ???
+      // `signature` will be equal to List(List(Other), List(LiftedTyped, LiftedUntyped), List(Tagged(0), Tagged(2)))
+      signature: List[List[Fingerprint]],
+      // type arguments part of a macro impl ref (the right-hand side of a macro definition)
+      // these trees don't refer to a macro impl, so we can pickle them as is
+      targs: List[Tree]) {
     // Was this binding derived from a `def ... = macro ???` definition?
     def is_??? = {
       val Predef_??? = currentRun.runDefinitions.Predef_???
       className == Predef_???.owner.javaClassName && methName == Predef_???.name.encoded
     }
+    def isWhitebox = !isBlackbox
   }
 
   /** Macro def -> macro impl bindings are serialized into a `macroImpl` annotation
    *  with synthetic content that carries the payload described in `MacroImplBinding`.
    *
    *  For example, for a pair of macro definition and macro implementation:
-   *    def impl(c: scala.reflect.macros.Context): c.Expr[Unit] = ???
+   *    def impl(c: scala.reflect.macros.BlackboxContext): c.Expr[Unit] = ???
    *    def foo: Unit = macro impl
    *
    *  We will have the following annotation added on the macro definition `foo`:
    *
    *    @scala.reflect.macros.internal.macroImpl(
    *      `macro`(
    *        "isBundle" = false,
+   *        "isBlackbox" = true,
    *        "signature" = List(Other),
    *        "methodName" = "impl",
    *        "versionFormat" = <current version format>,
    *        "className" = "Macros$"))
    */
   object MacroImplBinding {
-    val versionFormat = 5.0
+    val versionFormat = 6.0
 
     def pickleAtom(obj: Any): Tree =
       obj match {
@@ -151,7 +154,7 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
     def pickle(macroImplRef: Tree): Tree = {
       val runDefinitions = currentRun.runDefinitions
       import runDefinitions._
-      val MacroImplReference(isBundle, owner, macroImpl, targs) = macroImplRef
+      val MacroImplReference(isBundle, isBlackbox, owner, macroImpl, targs) = macroImplRef
 
       // todo. refactor when fixing SI-5498
       def className: String = {
@@ -182,6 +185,7 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
       val payload = List[(String, Any)](
         "versionFormat" -> versionFormat,
         "isBundle"      -> isBundle,
+        "isBlackbox"    -> isBlackbox,
         "className"     -> className,
         "methodName"    -> macroImpl.name.toString,
         "signature"     -> signature
@@ -237,10 +241,11 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
       if (versionFormat != pickleVersionFormat) fail(s"expected version format $versionFormat, actual $pickleVersionFormat")
 
       val isBundle = unpickle("isBundle", classOf[Boolean])
+      val isBlackbox = unpickle("isBlackbox", classOf[Boolean])
       val className = unpickle("className", classOf[String])
       val methodName = unpickle("methodName", classOf[String])
       val signature = unpickle("signature", classOf[List[List[Fingerprint]]])
-      MacroImplBinding(isBundle, className, methodName, signature, targs)
+      MacroImplBinding(isBundle, isBlackbox, className, methodName, signature, targs)
     }
   }
 
@@ -249,14 +254,17 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
     macroDef withAnnotation AnnotationInfo(MacroImplAnnotation.tpe, List(pickle), Nil)
   }
 
-  def loadMacroImplBinding(macroDef: Symbol): MacroImplBinding = {
-    val Some(AnnotationInfo(_, List(pickle), _)) = macroDef.getAnnotation(MacroImplAnnotation)
-    MacroImplBinding.unpickle(pickle)
-  }
+  def loadMacroImplBinding(macroDef: Symbol): Option[MacroImplBinding] =
+    macroDef.getAnnotation(MacroImplAnnotation) collect {
+      case AnnotationInfo(_, List(pickle), _) => MacroImplBinding.unpickle(pickle)
+    }
+
+  def isBlackbox(expandee: Tree): Boolean = isBlackbox(dissectApplied(expandee).core.symbol)
+  def isBlackbox(macroDef: Symbol): Boolean = loadMacroImplBinding(macroDef).map(_.isBlackbox).getOrElse(false)
 
   def computeMacroDefTypeFromMacroImplRef(macroDdef: DefDef, macroImplRef: Tree): Type = {
     macroImplRef match {
-      case MacroImplReference(_, _, macroImpl, targs) =>
+      case MacroImplReference(_, _, _, macroImpl, targs) =>
         // Step I. Transform c.Expr[T] to T and everything else to Any
         var runtimeType = decreaseMetalevel(macroImpl.info.finalResultType)
 
@@ -450,7 +458,7 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
           (trees :+ tags).flatten
         }
 
-        val binding = loadMacroImplBinding(macroDef)
+        val binding = loadMacroImplBinding(macroDef).get
         if (binding.is_???) Nil
         else calculateMacroArgs(binding)
       }
@@ -459,7 +467,7 @@ trait Macros extends FastTrack with MacroRuntimes with Traces with Helpers {
   }
 
   /** Keeps track of macros in-flight.
-   *  See more informations in comments to `openMacros` in `scala.reflect.macros.Context`.
+   *  See more informations in comments to `openMacros` in `scala.reflect.macros.WhiteboxContext`.
    */
   private var _openMacros = List[MacroContext]()
   def openMacros = _openMacros

src/compiler/scala/tools/nsc/typechecker/Typers.scala

@@ -1111,7 +1111,7 @@ trait Typers extends Adaptations with Tags with TypersTracking with PatternTyper
         }
         if (tree.isType)
           adaptType()
-        else if (mode.typingExprNotFun && treeInfo.isMacroApplication(tree))
+        else if (mode.typingExprNotFun && treeInfo.isMacroApplication(tree) && !isMacroExpansionSuppressed(tree))
           macroExpandApply(this, tree, mode, pt)
         else if (mode.typingConstructorPattern)
           typedConstructorPattern(tree, pt)