Merge pull request #7601 from dotty-staging/fix-encoding-2

Revert to the Scala 2 name encoding scheme

Author: smarter

compiler/src/dotty/tools/backend/sjs/JSEncoding.scala

@@ -229,10 +229,7 @@ object JSEncoding {
     fullyMangledString(sym.name)
   }
 
-  /** Work around https://github.com/lampepfl/dotty/issues/5936 by bridging
-   *  most (all?) of the gap in encoding so that Dotty.js artifacts are
-   *  compatible with the restrictions on valid IR identifier names.
-   */
+  /** Convert Dotty mangled names into valid IR identifier names. */
   private def fullyMangledString(name: Name): String = {
     val base = name.mangledString
     val len = base.length
@@ -245,10 +242,8 @@ object JSEncoding {
         val c = base.charAt(i)
         if (c == '_')
           result.append("$und")
-        else if (Character.isJavaIdentifierPart(c) || c == '.')
-          result.append(c)
         else
-          result.append("$u%04x".format(c.toInt))
+          result.append(c)
         i += 1
       }
       result.toString()
@@ -257,7 +252,7 @@ object JSEncoding {
     var i = 0
     while (i != len) {
       val c = base.charAt(i)
-      if (c == '_' || !Character.isJavaIdentifierPart(c))
+      if (c == '_')
         return encodeFurther()
       i += 1
     }

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -9,7 +9,6 @@ import Decorators._, transform.SymUtils._
 import NameKinds.{UniqueName, EvidenceParamName, DefaultGetterName}
 import typer.{FrontEnd, Namer}
 import util.{Property, SourceFile, SourcePosition}
-import util.NameTransformer.avoidIllegalChars
 import collection.mutable.ListBuffer
 import reporting.diagnostic.messages._
 import reporting.trace
@@ -935,7 +934,7 @@ object desugar {
 
   /** Invent a name for an anonympus given of type or template `impl`. */
   def inventGivenName(impl: Tree)(implicit ctx: Context): SimpleName =
-    avoidIllegalChars(s"given_${inventName(impl)}".toTermName.asSimpleName)
+    s"given_${inventName(impl)}".toTermName.asSimpleName
 
   /** The normalized name of `mdef`. This means
    *   1. Check that the name does not redefine a Scala core class.
@@ -1250,7 +1249,7 @@ object desugar {
     else {
       var fileName = ctx.source.file.name
       val sourceName = fileName.take(fileName.lastIndexOf('.'))
-      val groupName = avoidIllegalChars((sourceName ++ str.TOPLEVEL_SUFFIX).toTermName.asSimpleName)
+      val groupName = (sourceName ++ str.TOPLEVEL_SUFFIX).toTermName.asSimpleName
       val grouped = ModuleDef(groupName, Template(emptyConstructor, Nil, Nil, EmptyValDef, nestedStats))
       cpy.PackageDef(pdef)(pdef.pid, topStats :+ grouped)
     }

compiler/src/dotty/tools/dotc/core/Names.scala

@@ -147,8 +147,8 @@ object Names {
     /** Is this name empty? */
     def isEmpty: Boolean
 
-    /** Does (the first part of) this name start with `str`? */
-    def startsWith(str: String): Boolean = firstPart.startsWith(str)
+    /** Does (the first part of) this name starting at index `start` starts with `str`? */
+    def startsWith(str: String, start: Int = 0): Boolean = firstPart.startsWith(str, start)
 
     /** Does (the last part of) this name end with `str`? */
     def endsWith(str: String): Boolean = lastPart.endsWith(str)
@@ -362,9 +362,9 @@ object Names {
 
     override def isEmpty: Boolean = length == 0
 
-    override def startsWith(str: String): Boolean = {
+    override def startsWith(str: String, start: Int): Boolean = {
       var i = 0
-      while (i < str.length && i < length && apply(i) == str(i)) i += 1
+      while (i < str.length && start + i < length && apply(start + i) == str(i)) i += 1
       i == str.length
     }
 

compiler/src/dotty/tools/dotc/core/classfile/ClassfileParser.scala

@@ -686,7 +686,7 @@ class ClassfileParser(
     for (entry <- innerClasses.values) {
       // create a new class member for immediate inner classes
       if (entry.outerName == currentClassName) {
-        val file = ctx.platform.classPath.findClassFile(entry.externalName.mangledString) getOrElse {
+        val file = ctx.platform.classPath.findClassFile(entry.externalName.toString) getOrElse {
           throw new AssertionError(entry.externalName)
         }
         enterClassAndModule(entry, file, entry.jflags)

compiler/src/dotty/tools/dotc/parsing/Scanners.scala

@@ -7,7 +7,6 @@ import core.StdNames._, core.Comments._
 import util.SourceFile
 import java.lang.Character.isDigit
 import scala.internal.Chars._
-import util.NameTransformer.avoidIllegalChars
 import util.Spans.Span
 import config.Config
 import config.Printers.lexical
@@ -929,7 +928,6 @@ object Scanners {
       if (ch == '`') {
         nextChar()
         finishNamed(BACKQUOTED_IDENT)
-        name = avoidIllegalChars(name)
         if (name.length == 0)
           error("empty quoted identifier")
         else if (name == nme.WILDCARD)

compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala

@@ -77,7 +77,7 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
   }
 
   override def nameString(name: Name): String =
-    if (ctx.settings.YdebugNames.value) name.debugString else NameTransformer.decodeIllegalChars(name.toString)
+    if (ctx.settings.YdebugNames.value) name.debugString else name.toString
 
   override protected def simpleNameString(sym: Symbol): String =
     nameString(if (ctx.property(XprintMode).isEmpty) sym.initial.name else sym.name)

compiler/src/dotty/tools/dotc/transform/GenericSignatures.scala

@@ -103,16 +103,10 @@ object GenericSignatures {
         jsig(finalType)
     }
 
-    // This will reject any name that has characters that cannot appear in
-    // names on the JVM. Interop with Java is not guaranteed for those, so we
-    // dont need to generate signatures for them.
-    def sanitizeName(name: Name): String = {
-      val nameString = name.mangledString
-      if (nameString.forall(c => c == '.' || Character.isJavaIdentifierPart(c)))
-        nameString
-      else
-        throw new UnknownSig
-    }
+    // This works as long as mangled names are always valid valid Java identifiers,
+    // if we change our name encoding, we'll have to `throw new UnknownSig` here for
+    // names which are not valid Java identifiers (see git history of this method).
+    def sanitizeName(name: Name): String = name.mangledString
 
     // Anything which could conceivably be a module (i.e. isn't known to be
     // a type parameter or similar) must go through here or the signature is

compiler/src/dotty/tools/dotc/util/NameTransformer.scala

@@ -13,13 +13,16 @@ import scala.annotation.internal.sharable
 object NameTransformer {
 
   private val nops = 128
+  private val ncodes = 26 * 26
 
-  @sharable private val op2code = new Array[String](nops)
-  @sharable private val str2op = new mutable.HashMap[String, Char]
+  private class OpCodes(val op: Char, val code: String, val next: OpCodes)
 
+  @sharable private val op2code = new Array[String](nops)
+  @sharable private val code2op = new Array[OpCodes](ncodes)
   private def enterOp(op: Char, code: String) = {
-    op2code(op) = code
-    str2op(code) = op
+    op2code(op.toInt) = code
+    val c = (code.charAt(1) - 'a') * 26 + code.charAt(2) - 'a'
+    code2op(c.toInt) = new OpCodes(op, code, code2op(c))
   }
 
   /* Note: decoding assumes opcodes are only ever lowercase. */
@@ -42,99 +45,107 @@ object NameTransformer {
   enterOp('?', "$qmark")
   enterOp('@', "$at")
 
-  /** Expand characters that are illegal as JVM method names by `$u`, followed
-   *  by the character's unicode expansion.
-   */
-  def avoidIllegalChars(name: SimpleName): SimpleName = {
-    var i = name.length - 1
-    while (i >= 0 && Chars.isValidJVMMethodChar(name(i))) i -= 1
-    if (i >= 0)
-      termName(
-        name.toString.flatMap(ch =>
-          if (Chars.isValidJVMMethodChar(ch)) ch.toString else "$u%04X".format(ch.toInt)))
-    else name
-  }
-
-  /** Decode expanded characters starting with `$u`, followed by the character's unicode expansion. */
-  def decodeIllegalChars(name: String): String =
-    if (name.contains("$u")) {
-      val sb = new mutable.StringBuilder()
-      var i = 0
-      while (i < name.length)
-        if (i < name.length - 5 && name(i) == '$' && name(i + 1) == 'u') {
-          val numbers = name.substring(i + 2, i + 6)
-          try sb.append(Integer.valueOf(name.substring(i + 2, i + 6), 16).toChar)
-          catch {
-            case _: java.lang.NumberFormatException =>
-              sb.append("$u").append(numbers)
-          }
-          i += 6
-        }
-        else {
-          sb.append(name(i))
-          i += 1
-        }
-      sb.result()
-    }
-    else name
-
-  /** Replace operator symbols by corresponding expansion strings.
-   *
-   *  @param name the string to encode
-   *  @return     the string with all recognized opchars replaced with their encoding
-   *
-   *  Operator symbols are only recognized if they make up the whole name, or
-   *  if they make up the last part of the name which follows a `_`.
+  /** Replace operator symbols by corresponding expansion strings, and replace
+   *  characters that are not valid Java identifiers by "$u" followed by the
+   *  character's unicode expansion.
+   *  Note that no attempt is made to escape the use of '$' in `name`: blindly
+   *  escaping them might make it impossible to call some platform APIs. This
+   *  unfortunately means that `decode(encode(name))` might not be equal to
+   *  `name`, this is considered acceptable since '$' is a reserved character in
+   *  the Scala spec as well as the Java spec.
    */
   def encode(name: SimpleName): SimpleName = {
-    def loop(len: Int, ops: List[String]): SimpleName = {
-      def convert =
-        if (ops.isEmpty) name
-        else {
-          val buf = new java.lang.StringBuilder
-          buf.append(chrs, name.start, len)
-          for (op <- ops) buf.append(op)
-          termName(buf.toString)
+    var buf: StringBuilder = null
+    val len = name.length
+    var i = 0
+    while (i < len) {
+      val c = name(i)
+      if (c < nops && (op2code(c.toInt) ne null)) {
+        if (buf eq null) {
+          buf = new StringBuilder()
+          buf.append(name.sliceToString(0, i))
+        }
+        buf.append(op2code(c.toInt))
+      /* Handle glyphs that are not valid Java/JVM identifiers */
+      }
+      else if (!Character.isJavaIdentifierPart(c)) {
+        if (buf eq null) {
+          buf = new StringBuilder()
+          buf.append(name.sliceToString(0, i))
         }
-      if (len == 0 || name(len - 1) == '_') convert
-      else {
-        val ch = name(len - 1)
-        if (ch <= nops && op2code(ch) != null)
-          loop(len - 1, op2code(ch) :: ops)
-        else if (Chars.isSpecial(ch))
-          loop(len - 1, ch.toString :: ops)
-        else name
+        buf.append("$u%04X".format(c.toInt))
       }
+      else if (buf ne null) {
+        buf.append(c)
+      }
+      i += 1
     }
-    loop(name.length, Nil)
+    if (buf eq null) name else termName(buf.toString)
   }
 
-  /** Replace operator expansions by the operators themselves.
-   *  Operator expansions are only recognized if they make up the whole name, or
-   *  if they make up the last part of the name which follows a `_`.
+  /** Replace operator expansions by the operators themselves,
+   *  and decode `$u....` expansions into unicode characters.
    */
   def decode(name: SimpleName): SimpleName = {
-    def loop(len: Int, ops: List[Char]): SimpleName = {
-      def convert =
-        if (ops.isEmpty) name
-        else {
-          val buf = new java.lang.StringBuilder
-          buf.append(chrs, name.start, len)
-          for (op <- ops) buf.append(op)
-          termName(buf.toString)
-        }
-      if (len == 0 || name(len - 1) == '_') convert
-      else if (Chars.isSpecial(name(len - 1))) loop(len - 1, name(len - 1) :: ops)
-      else {
-        val idx = name.lastIndexOf('$', len - 1)
-        if (idx >= 0 && idx + 2 < len)
-          str2op.get(name.sliceToString(idx, len)) match {
-            case Some(ch) => loop(idx, ch :: ops)
-            case None => name
+    //System.out.println("decode: " + name);//DEBUG
+    var buf: StringBuilder = null
+    val len = name.length
+    var i = 0
+    while (i < len) {
+      var ops: OpCodes = null
+      var unicode = false
+      val c = name(i)
+      if (c == '$' && i + 2 < len) {
+        val ch1 = name(i + 1)
+        if ('a' <= ch1 && ch1 <= 'z') {
+          val ch2 = name(i + 2)
+          if ('a' <= ch2 && ch2 <= 'z') {
+            ops = code2op((ch1 - 'a') * 26 + ch2 - 'a')
+            while ((ops ne null) && !name.startsWith(ops.code, i)) ops = ops.next
+            if (ops ne null) {
+              if (buf eq null) {
+                buf = new StringBuilder()
+                buf.append(name.sliceToString(0, i))
+              }
+              buf.append(ops.op)
+              i += ops.code.length()
+            }
+            /* Handle the decoding of Unicode glyphs that are
+             * not valid Java/JVM identifiers */
+          } else if ((len - i) >= 6 && // Check that there are enough characters left
+                     ch1 == 'u' &&
+                     ((Character.isDigit(ch2)) ||
+                     ('A' <= ch2 && ch2 <= 'F'))) {
+            /* Skip past "$u", next four should be hexadecimal */
+            val hex = name.sliceToString(i+2, i+6)
+            try {
+              val str = Integer.parseInt(hex, 16).toChar
+              if (buf eq null) {
+                buf = new StringBuilder()
+                buf.append(name.sliceToString(0, i))
+              }
+              buf.append(str)
+              /* 2 for "$u", 4 for hexadecimal number */
+              i += 6
+              unicode = true
+            } catch {
+              case _:NumberFormatException =>
+                /* `hex` did not decode to a hexadecimal number, so
+                 * do nothing. */
+            }
           }
-        else name
+        }
+      }
+      /* If we didn't see an opcode or encoded Unicode glyph, and the
+        buffer is non-empty, write the current character and advance
+         one */
+      if ((ops eq null) && !unicode) {
+        if (buf ne null)
+          buf.append(c)
+        i += 1
       }
     }
-    loop(name.length, Nil)
+    //System.out.println("= " + (if (buf == null) name else buf.toString()));//DEBUG
+    if (buf eq null) name else termName(buf.toString)
   }
 }

sbt-dotty/sbt-test/scala2-compat/akka/build.sbt

@@ -0,0 +1,8 @@
+scalaVersion := sys.props("plugin.scalaVersion")
+
+libraryDependencies ++= {
+  Seq(
+    ("com.typesafe.akka" %% "akka-http"   % "10.1.10"),
+    ("com.typesafe.akka" %% "akka-stream" % "2.6.0")
+  ).map(_.withDottyCompat(scalaVersion.value))
+}

sbt-dotty/sbt-test/scala2-compat/akka/i3100.scala

@@ -0,0 +1,12 @@
+import akka.actor.ActorSystem
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model._
+import akka.http.scaladsl.server.Directives._
+import akka.stream.ActorMaterializer
+import scala.io.StdIn
+
+object WebServer {
+  def main(args: Array[String]): Unit = {
+    val x = ContentTypes.`text/html(UTF-8)`
+  }
+}

sbt-dotty/sbt-test/scala2-compat/akka/project/plugins.sbt

@@ -0,0 +1 @@
+addSbtPlugin("ch.epfl.lamp" % "sbt-dotty" % sys.props("plugin.version"))

sbt-dotty/sbt-test/scala2-compat/akka/test

@@ -0,0 +1 @@
+> compile

tests/generic-java-signatures/invalidNames.check

@@ -0,0 +1 @@
+$bang$u005B$u005D$colon$u003B$bang$bang <: java.util.Date