Add support for lambda serialization

compiler/src/dotty/tools/backend/jvm/DottyBackendInterface.scala

@@ -706,6 +706,7 @@ class DottyBackendInterface(outputDirectory: AbstractFile, val superCallsMap: Ma
     def isJavaEntryPoint: Boolean = CollectEntryPoints.isJavaEntryPoint(sym)
 
     def isClassConstructor: Boolean = toDenot(sym).isClassConstructor
+    def isSerializable: Boolean = toDenot(sym).isSerializable
 
     /**
      * True for module classes of modules that are top-level or owned only by objects. Module classes
@@ -855,6 +856,9 @@ class DottyBackendInterface(outputDirectory: AbstractFile, val superCallsMap: Ma
 
     def samMethod(): Symbol =
       toDenot(sym).info.abstractTermMembers.headOption.getOrElse(toDenot(sym).info.member(nme.apply)).symbol
+
+    def isFunctionClass: Boolean =
+      defn.isFunctionClass(sym)
   }
 
 

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -6,6 +6,7 @@ import dotty.tools.dotc.ast.tpd
 import dotty.tools.dotc.core.Phases.Phase
 
 import scala.collection.mutable
+import scala.collection.JavaConverters._
 import scala.tools.asm.CustomAttr
 import scala.tools.nsc.backend.jvm._
 import dotty.tools.dotc.transform.SymUtils._
@@ -23,6 +24,7 @@ import java.io.DataOutputStream
 
 
 import scala.tools.asm
+import scala.tools.asm.Handle
 import scala.tools.asm.tree._
 import tpd._
 import StdNames._
@@ -304,10 +306,98 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter
     class Worker2 {
       // lazy val localOpt = new LocalOpt(new Settings())
 
-      def localOptimizations(classNode: ClassNode): Unit = {
+      private def localOptimizations(classNode: ClassNode): Unit = {
         // BackendStats.timed(BackendStats.methodOptTimer)(localOpt.methodOptimizations(classNode))
       }
 
+
+      /* Return an array of all serializable lambdas in this class */
+      private def collectSerializableLambdas(classNode: ClassNode): Array[Handle] = {
+        val indyLambdaBodyMethods = new mutable.ArrayBuffer[Handle]
+        for (m <- classNode.methods.asScala) {
+          val iter = m.instructions.iterator
+          while (iter.hasNext) {
+            val insn = iter.next()
+            insn match {
+              case indy: InvokeDynamicInsnNode
+                  // No need to check the exact bsmArgs because we only generate
+                  // altMetafactory indy calls for serializable lambdas.
+                  if indy.bsm == BCodeBodyBuilder.lambdaMetaFactoryAltMetafactoryHandle =>
+                val implMethod = indy.bsmArgs(1).asInstanceOf[Handle]
+                indyLambdaBodyMethods += implMethod
+              case _ =>
+            }
+          }
+        }
+        indyLambdaBodyMethods.toArray
+      }
+
+      /*
+       * Add:
+       *
+       * private static Object $deserializeLambda$(SerializedLambda l) {
+       *   try return indy[scala.runtime.LambdaDeserialize.bootstrap, targetMethodGroup$0](l)
+       *   catch {
+       *     case i: IllegalArgumentException =>
+       *       try return indy[scala.runtime.LambdaDeserialize.bootstrap, targetMethodGroup$1](l)
+       *       catch {
+       *         case i: IllegalArgumentException =>
+       *           ...
+       *             return indy[scala.runtime.LambdaDeserialize.bootstrap, targetMethodGroup${NUM_GROUPS-1}](l)
+       *       }
+       *
+       * We use invokedynamic here to enable caching within the deserializer without needing to
+       * host a static field in the enclosing class. This allows us to add this method to interfaces
+       * that define lambdas in default methods.
+       *
+       * SI-10232 we can't pass arbitrary number of method handles to the final varargs parameter of the bootstrap
+       * method due to a limitation in the JVM. Instead, we emit a separate invokedynamic bytecode for each group of target
+       * methods.
+       */
+      private def addLambdaDeserialize(classNode: ClassNode, implMethodsArray: Array[Handle]): Unit = {
+        import asm.Opcodes._
+        import BCodeBodyBuilder._
+        import bTypes._
+        import coreBTypes._
+
+        val cw = classNode
+
+        // Make sure to reference the ClassBTypes of all types that are used in the code generated
+        // here (e.g. java/util/Map) are initialized. Initializing a ClassBType adds it to
+        // `classBTypeFromInternalNameMap`. When writing the classfile, the asm ClassWriter computes
+        // stack map frames and invokes the `getCommonSuperClass` method. This method expects all
+        // ClassBTypes mentioned in the source code to exist in the map.
+
+        val serlamObjDesc = MethodBType(jliSerializedLambdaRef :: Nil, ObjectReference).descriptor
+
+        val mv = cw.visitMethod(ACC_PRIVATE + ACC_STATIC + ACC_SYNTHETIC, "$deserializeLambda$", serlamObjDesc, null, null)
+        def emitLambdaDeserializeIndy(targetMethods: Seq[Handle]): Unit = {
+          mv.visitVarInsn(ALOAD, 0)
+          mv.visitInvokeDynamicInsn("lambdaDeserialize", serlamObjDesc, lambdaDeserializeBootstrapHandle, targetMethods: _*)
+        }
+
+        val targetMethodGroupLimit = 255 - 1 - 3 // JVM limit. See See MAX_MH_ARITY in CallSite.java
+        val groups: Array[Array[Handle]] = implMethodsArray.grouped(targetMethodGroupLimit).toArray
+        val numGroups = groups.length
+
+        import scala.tools.asm.Label
+        val initialLabels = Array.fill(numGroups - 1)(new Label())
+        val terminalLabel = new Label
+        def nextLabel(i: Int) = if (i == numGroups - 2) terminalLabel else initialLabels(i + 1)
+
+        for ((label, i) <- initialLabels.iterator.zipWithIndex) {
+          mv.visitTryCatchBlock(label, nextLabel(i), nextLabel(i), jlIllegalArgExceptionRef.internalName)
+        }
+        for ((label, i) <- initialLabels.iterator.zipWithIndex) {
+          mv.visitLabel(label)
+          emitLambdaDeserializeIndy(groups(i))
+          mv.visitInsn(ARETURN)
+        }
+        mv.visitLabel(terminalLabel)
+        emitLambdaDeserializeIndy(groups(numGroups - 1))
+        mv.visitInsn(ARETURN)
+      }
+
       def run(): Unit = {
         while (true) {
           val item = q2.poll
@@ -317,7 +407,11 @@ class GenBCodePipeline(val entryPoints: List[Symbol], val int: DottyBackendInter
           }
           else {
             try {
-              localOptimizations(item.plain.classNode)
+              val plainNode = item.plain.classNode
+              localOptimizations(plainNode)
+              val serializableLambdas = collectSerializableLambdas(plainNode)
+              if (serializableLambdas.nonEmpty)
+                addLambdaDeserialize(plainNode, serializableLambdas)
               addToQ3(item)
             } catch {
               case ex: Throwable =>

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -341,8 +341,9 @@ object desugar {
       case _ => false
     }
 
-    val isCaseClass  = mods.is(Case) && !mods.is(Module)
-    val isCaseObject = mods.is(Case) && mods.is(Module)
+    val isObject = mods.is(Module)
+    val isCaseClass  = mods.is(Case) && !isObject
+    val isCaseObject = mods.is(Case) && isObject
     val isImplicit = mods.is(Implicit)
     val isInstance = isImplicit && mods.mods.exists(_.isInstanceOf[Mod.Instance])
     val isEnum = mods.isEnumClass && !mods.is(Module)
@@ -527,13 +528,13 @@ object desugar {
       else Nil
     }
 
-    // Case classes and case objects get Product parents
-    // Enum cases get an inferred parent if no parents are given
     var parents1 = parents
     if (isEnumCase && parents.isEmpty)
       parents1 = enumClassTypeRef :: Nil
     if (isCaseClass | isCaseObject)
       parents1 = parents1 :+ scalaDot(str.Product.toTypeName) :+ scalaDot(nme.Serializable.toTypeName)
+    else if (isObject)
+      parents1 = parents1 :+ scalaDot(nme.Serializable.toTypeName)
     if (isEnum)
       parents1 = parents1 :+ ref(defn.EnumType)
 

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -1001,6 +1001,7 @@ class Definitions {
     tp.derivesFrom(NothingClass) || tp.derivesFrom(NullClass)
 
   /** Is a function class.
+   *   - FunctionXXL
    *   - FunctionN for N >= 0
    *   - ImplicitFunctionN for N >= 0
    *   - ErasedFunctionN for N > 0
@@ -1020,15 +1021,21 @@ class Definitions {
    */
   def isErasedFunctionClass(cls: Symbol): Boolean = scalaClassName(cls).isErasedFunction
 
-  /** Is a class that will be erased to FunctionXXL
+  /** Is either FunctionXXL or  a class that will be erased to FunctionXXL
+   *   - FunctionXXL
    *   - FunctionN for N >= 22
    *   - ImplicitFunctionN for N >= 22
    */
-  def isXXLFunctionClass(cls: Symbol): Boolean = scalaClassName(cls).functionArity > MaxImplementedFunctionArity
+  def isXXLFunctionClass(cls: Symbol): Boolean = {
+    val name = scalaClassName(cls)
+    (name eq tpnme.FunctionXXL) || name.functionArity > MaxImplementedFunctionArity
+  }
 
   /** Is a synthetic function class
    *    - FunctionN for N > 22
-   *    - ImplicitFunctionN for N > 0
+   *    - ImplicitFunctionN for N >= 0
+   *    - ErasedFunctionN for N > 0
+   *    - ErasedImplicitFunctionN for N > 0
    */
   def isSyntheticFunctionClass(cls: Symbol): Boolean = scalaClassName(cls).isSyntheticFunction
 

compiler/src/dotty/tools/dotc/core/NameOps.scala

@@ -183,9 +183,9 @@ object NameOps {
         if (n == 0) -1 else n
       }
 
-    /** Is a function name, i.e one of FunctionN, ImplicitFunctionN for N >= 0 or ErasedFunctionN, ErasedImplicitFunctionN for N > 0
+    /** Is a function name, i.e one of FunctionXXL, FunctionN, ImplicitFunctionN for N >= 0 or ErasedFunctionN, ErasedImplicitFunctionN for N > 0
      */
-    def isFunction: Boolean = functionArity >= 0
+    def isFunction: Boolean = (name eq tpnme.FunctionXXL) || functionArity >= 0
 
     /** Is an implicit function name, i.e one of ImplicitFunctionN for N >= 0 or ErasedImplicitFunctionN for N > 0
      */

compiler/src/dotty/tools/dotc/core/StdNames.scala

@@ -205,6 +205,7 @@ object StdNames {
     final val Singleton: N           = "Singleton"
     final val Throwable: N           = "Throwable"
     final val IOOBException: N       = "IndexOutOfBoundsException"
+    final val FunctionXXL: N         = "FunctionXXL"
 
     final val ClassfileAnnotation: N = "ClassfileAnnotation"
     final val ClassManifest: N       = "ClassManifest"

compiler/src/dotty/tools/dotc/transform/GenericSignatures.scala

@@ -123,6 +123,60 @@ object GenericSignatures {
       builder.append('L').append(name)
     }
 
+    def classSig(sym: Symbol, pre: Type = NoType, args: List[Type] = Nil): Unit = {
+      def argSig(tp: Type): Unit =
+        tp match {
+          case bounds: TypeBounds =>
+            if (!(defn.AnyType <:< bounds.hi)) {
+              builder.append('+')
+              boxedSig(bounds.hi)
+            }
+            else if (!(bounds.lo <:< defn.NothingType)) {
+              builder.append('-')
+              boxedSig(bounds.lo)
+            }
+            else builder.append('*')
+          case PolyType(_, res) =>
+            builder.append('*') // scala/bug#7932
+          case _: HKTypeLambda =>
+            fullNameInSig(tp.typeSymbol)
+            builder.append(';')
+          case _ =>
+            boxedSig(tp)
+        }
+
+      if (pre.exists) {
+        val preRebound = pre.baseType(sym.owner) // #2585
+        if (needsJavaSig(preRebound, Nil)) {
+          val i = builder.length()
+          jsig(preRebound)
+          if (builder.charAt(i) == 'L') {
+            builder.delete(builder.length() - 1, builder.length())// delete ';'
+                                                                  // If the prefix is a module, drop the '$'. Classes (or modules) nested in modules
+                                                                  // are separated by a single '$' in the filename: `object o { object i }` is o$i$.
+            if (preRebound.typeSymbol.is(ModuleClass))
+              builder.delete(builder.length() - 1, builder.length())
+
+            // Ensure every '.' in the generated signature immediately follows
+            // a close angle bracket '>'.  Any which do not are replaced with '$'.
+            // This arises due to multiply nested classes in the face of the
+            // rewriting explained at rebindInnerClass.
+
+            // TODO revisit this. Does it align with javac for code that can be expressed in both languages?
+            val delimiter = if (builder.charAt(builder.length() - 1) == '>') '.' else '$'
+            builder.append(delimiter).append(sanitizeName(sym.name.asSimpleName))
+          } else fullNameInSig(sym)
+        } else fullNameInSig(sym)
+      } else fullNameInSig(sym)
+
+      if (args.nonEmpty) {
+        builder.append('<')
+        args foreach argSig
+        builder.append('>')
+      }
+      builder.append(';')
+    }
+
     @noinline
     def jsig(tp0: Type, toplevel: Boolean = false, primitiveOK: Boolean = true): Unit = {
 
@@ -133,57 +187,6 @@ object GenericSignatures {
           typeParamSig(ref.paramName.lastPart)
 
         case RefOrAppliedType(sym, pre, args) =>
-          def argSig(tp: Type): Unit =
-            tp match {
-              case bounds: TypeBounds =>
-                if (!(defn.AnyType <:< bounds.hi)) {
-                  builder.append('+')
-                  boxedSig(bounds.hi)
-                }
-                else if (!(bounds.lo <:< defn.NothingType)) {
-                  builder.append('-')
-                  boxedSig(bounds.lo)
-                }
-                else builder.append('*')
-              case PolyType(_, res) =>
-                builder.append('*') // scala/bug#7932
-              case _: HKTypeLambda =>
-                fullNameInSig(tp.typeSymbol)
-                builder.append(';')
-              case _ =>
-                boxedSig(tp)
-            }
-          def classSig: Unit = {
-            val preRebound = pre.baseType(sym.owner) // #2585
-            if (needsJavaSig(preRebound, Nil)) {
-              val i = builder.length()
-              jsig(preRebound)
-              if (builder.charAt(i) == 'L') {
-                builder.delete(builder.length() - 1, builder.length())// delete ';'
-                // If the prefix is a module, drop the '$'. Classes (or modules) nested in modules
-                // are separated by a single '$' in the filename: `object o { object i }` is o$i$.
-                if (preRebound.typeSymbol.is(ModuleClass))
-                  builder.delete(builder.length() - 1, builder.length())
-
-                // Ensure every '.' in the generated signature immediately follows
-                // a close angle bracket '>'.  Any which do not are replaced with '$'.
-                // This arises due to multiply nested classes in the face of the
-                // rewriting explained at rebindInnerClass.
-
-                // TODO revisit this. Does it align with javac for code that can be expressed in both languages?
-                val delimiter = if (builder.charAt(builder.length() - 1) == '>') '.' else '$'
-                builder.append(delimiter).append(sanitizeName(sym.name.asSimpleName))
-              } else fullNameInSig(sym)
-            } else fullNameInSig(sym)
-
-            if (args.nonEmpty) {
-              builder.append('<')
-              args foreach argSig
-              builder.append('>')
-            }
-            builder.append(';')
-          }
-
           // If args isEmpty, Array is being used as a type constructor
           if (sym == defn.ArrayClass && args.nonEmpty) {
             if (unboundedGenericArrayLevel(tp) == 1) jsig(defn.ObjectType)
@@ -215,14 +218,14 @@ object GenericSignatures {
             val unboxed     = ValueClasses.valueClassUnbox(sym.asClass).info.finalResultType
             val unboxedSeen = tp.memberInfo(ValueClasses.valueClassUnbox(sym.asClass)).finalResultType
             if (unboxedSeen.isPrimitiveValueType && !primitiveOK)
-              classSig
+              classSig(sym, pre, args)
             else
               jsig(unboxedSeen, toplevel, primitiveOK)
           }
           else if (defn.isXXLFunctionClass(sym))
-            jsig(defn.FunctionXXLType, toplevel, primitiveOK)
+            classSig(defn.FunctionXXLClass)
           else if (sym.isClass)
-            classSig
+            classSig(sym, pre, args)
           else
             jsig(erasure(tp), toplevel, primitiveOK)
 

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -361,6 +361,11 @@ class Typer extends Namer
       if (untpd.isVarPattern(tree) && name.isTermName)
         return typed(desugar.patternVar(tree), pt)
     }
+    // Shortcut for the root package, this is not just a performance
+    // optimization, it also avoids forcing imports thus potentially avoiding
+    // cyclic references.
+    if (name == nme.ROOTPKG)
+      return tree.withType(defn.RootPackage.termRef)
 
     val rawType = {
       val saved1 = unimported
@@ -1606,6 +1611,11 @@ class Typer extends Namer
       var result = if (isTreeType(tree)) typedType(tree)(superCtx) else typedExpr(tree)(superCtx)
       val psym = result.tpe.dealias.typeSymbol
       if (seenParents.contains(psym) && !cls.isRefinementClass) {
+        // Desugaring can adds parents to classes, but we don't want to emit an
+        // error if the same parent was explicitly added in user code.
+        if (!tree.span.isSourceDerived)
+          return EmptyTree
+
         if (!ctx.isAfterTyper) ctx.error(i"$psym is extended twice", tree.sourcePos)
       }
       else seenParents += psym
@@ -1640,7 +1650,7 @@ class Typer extends Namer
 
     completeAnnotations(cdef, cls)
     val constr1 = typed(constr).asInstanceOf[DefDef]
-    val parentsWithClass = ensureFirstTreeIsClass(parents mapconserve typedParent, cdef.nameSpan)
+    val parentsWithClass = ensureFirstTreeIsClass(parents.mapconserve(typedParent).filterConserve(!_.isEmpty), cdef.nameSpan)
     val parents1 = ensureConstrCall(cls, parentsWithClass)(superCtx)
 
     var self1 = typed(self)(ctx.outer).asInstanceOf[ValDef] // outer context where class members are not visible