Skip to content

Commit a71e267

Browse files
nicolasstuckinicolasstucki
committed
Add TASTY quote pickling for trees
1 parent 0a603dd commit a71e267

35 files changed

+654
-53
lines changed

‎compiler/src/dotty/tools/dotc/CompilationUnit.scala

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,11 @@ class CompilationUnit(val source: SourceFile) {
2626
* is used in phase ReifyQuotes in order to avoid traversing a quote-less tree.
2727
*/
2828
var containsQuotes: Boolean = false
29+
30+
/** Will be reset to `true` if `untpdTree` contains `Quoted.unary_~` trees. The information
31+
* is used in phase ReifyQuotes in order to avoid traversing a splice-less tree.
32+
*/
33+
var containsSplices: Boolean = false
2934
}
3035

3136
object CompilationUnit {

‎compiler/src/dotty/tools/dotc/core/tasty/TastyString.scala

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package dotty.tools.dotc.core.tasty
2+
3+
/** Utils for String representation of TASTY */
4+
object TastyString {
5+
6+
/** Decode the TASTY String into TASTY bytes */
7+
def stringToTasty(str: String): Array[Byte] = { // TODO factor out this and tastyToString
8+
val bytes = new Array[Byte](str.length)
9+
for (i <- str.indices) bytes(i) = str.charAt(i).toByte
10+
bytes
11+
}
12+
13+
/** Encode TASTY bytes into a TASTY String */
14+
def tastyToString(bytes: Array[Byte]): String = {
15+
val chars = new Array[Char](bytes.length)
16+
for (i <- bytes.indices) chars(i) = (bytes(i) & 0xff).toChar
17+
new String(chars)
18+
}
19+
20+
}

‎compiler/src/dotty/tools/dotc/interpreter/Interpreter.scala

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
package dotty.tools.dotc
2+
package interpreter
3+
4+
import dotty.tools.dotc.ast.tpd
5+
import dotty.tools.dotc.ast.Trees._
6+
import dotty.tools.dotc.core.Constants._
7+
import dotty.tools.dotc.core.Contexts._
8+
import dotty.tools.dotc.core.Decorators._
9+
import dotty.tools.dotc.core.Symbols._
10+
11+
import java.net.URLClassLoader
12+
13+
class Interpreter(implicit ctx: Context) {
14+
import tpd._
15+
16+
private[this] val classLoader = {
17+
// TODO reuse classloader across several splices?
18+
val urls = ctx.settings.classpath.value.split(':').map(cp => java.nio.file.Paths.get(cp).toUri.toURL)
19+
new URLClassLoader(urls, getClass.getClassLoader)
20+
}
21+
22+
def interpretTree(tree: Tree): Object = {
23+
try interpretTreeImpl(tree)
24+
catch {
25+
case ex: NoSuchMethodException =>
26+
ctx.error("Could not find interpreted method in classpath: " + ex.getMessage, tree.pos)
27+
null
28+
case ex: ClassNotFoundException =>
29+
ctx.error("Could not find interpreted class in classpath: " + ex.getMessage, tree.pos)
30+
null
31+
}
32+
}
33+
34+
private def interpretTreeImpl(tree: Tree): Object = {
35+
tree match {
36+
case Apply(_, quote :: Nil) if tree.symbol == defn.quoteMethod =>
37+
new RawExpr(quote)
38+
39+
case TypeApply(_, quote :: Nil) if tree.symbol == defn.typeQuoteMethod =>
40+
new RawType(quote)
41+
42+
case Literal(Constant(c)) =>
43+
c.asInstanceOf[AnyRef]
44+
45+
case Apply(fn, args) if fn.symbol.isConstructor =>
46+
val cls = fn.symbol.owner
47+
val clazz = classLoader.loadClass(cls.symbol.fullName.toString)
48+
val paramClasses = paramsSig(fn.symbol)
49+
val args1: List[Object] = args.map(arg => interpretTree(arg))
50+
clazz.getConstructor(paramClasses: _*).newInstance(args1: _*).asInstanceOf[Object]
51+
52+
case Apply(fun, args) if fun.symbol.isStatic =>
53+
val clazz = classLoader.loadClass(fun.symbol.owner.companionModule.fullName.toString)
54+
val paramClasses = paramsSig(fun.symbol)
55+
val args1: List[Object] = args.map(arg => interpretTree(arg))
56+
val method = clazz.getMethod(fun.symbol.name.toString, paramClasses: _*)
57+
method.invoke(null, args1: _*) // TODO add try catch
58+
59+
case tree: RefTree if tree.symbol.isStatic =>
60+
val clazz = classLoader.loadClass(tree.symbol.owner.companionModule.fullName.toString)
61+
val method = clazz.getMethod(tree.name.toString)
62+
method.invoke(null) // TODO add try catch
63+
64+
case _ =>
65+
val msg =
66+
if (tree.tpe.derivesFrom(defn.QuotedClass)) "Quote needs to be explicit"
67+
else "Value needs to be a explicit"
68+
ctx.error(msg, tree.pos)
69+
null
70+
}
71+
}
72+
73+
private def paramsSig(sym: Symbol): List[Class[_]] = {
74+
sym.signature.paramsSig.map { param =>
75+
val paramString = param.toString
76+
if (paramString == defn.ByteClass.showFullName) classOf[Byte]
77+
else if (paramString == defn.CharClass.showFullName) classOf[Char]
78+
else if (paramString == defn.ShortClass.showFullName) classOf[Short]
79+
else if (paramString == defn.IntClass.showFullName) classOf[Int]
80+
else if (paramString == defn.LongClass.showFullName) classOf[Long]
81+
else if (paramString == defn.DoubleClass.showFullName) classOf[Double]
82+
else if (paramString == defn.DoubleClass.showFullName) classOf[Float]
83+
else classLoader.loadClass(paramString)
84+
}
85+
}
86+
}

‎compiler/src/dotty/tools/dotc/interpreter/RawExpr.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.ast.tpd
4+
5+
class RawExpr(val tree: tpd.Tree) extends quoted.Expr[Any] with RawQuoted {
6+
override def toString: String = s"RawExpr(${tree.toString})"
7+
}

‎compiler/src/dotty/tools/dotc/interpreter/RawQuoted.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.ast.tpd
4+
5+
trait RawQuoted extends quoted.Quoted {
6+
def tree: tpd.Tree
7+
}

‎compiler/src/dotty/tools/dotc/interpreter/RawType.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
package dotty.tools.dotc.interpreter
2+
3+
import dotty.tools.dotc.ast.tpd
4+
5+
class RawType(val tree: tpd.Tree) extends quoted.Type[Any] with RawQuoted {
6+
override def toString: String = s"RawType(${tree.toString})"
7+
}

‎compiler/src/dotty/tools/dotc/transform/PickleQuotes.scala

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package dotty.tools.dotc.transform
2+
3+
import dotty.tools.dotc.ast.tpd
4+
import dotty.tools.dotc.ast.Trees._
5+
import dotty.tools.dotc.config.Printers._
6+
import dotty.tools.dotc.core.Contexts._
7+
import dotty.tools.dotc.core.Decorators._
8+
import dotty.tools.dotc.core.Flags._
9+
import dotty.tools.dotc.core.NameKinds._
10+
import dotty.tools.dotc.core.StdNames._
11+
import dotty.tools.dotc.core.Symbols._
12+
import dotty.tools.dotc.core.Types._
13+
import dotty.tools.dotc.core.tasty._
14+
15+
object PickleQuotes {
16+
import tpd._
17+
18+
/** Serialize `tree`. Embedded splices are represented as nodes of the form
19+
*
20+
* Select(qual, sym)
21+
*
22+
* where `sym` is either `defn.QuotedExpr_~` or `defn.QuotedType_~`. For any splice,
23+
* the `qual` part should not be pickled, since it will be added separately later
24+
* as a splice.
25+
*/
26+
def pickleQuote(tree: Tree, isType: Boolean)(implicit ctx: Context): String = {
27+
if (ctx.reporter.hasErrors) "<error>"
28+
else {
29+
val encapsulated = encapsulateQuote(tree)
30+
val pickled = pickle(encapsulated)
31+
TastyString.tastyToString(pickled)
32+
}
33+
}
34+
35+
/** Encapsulate the tree in a top level def.
36+
* Splices in the tree are replaced with hole represented as the `def`s parameters.
37+
*
38+
* `'{ "foo" + ~x + "bar" + ~y }`
39+
* ===>
40+
* `package _root_ { def '(hole$1: String, hole$2: String): String = "foo" + hole$1 + "bar" + hole$2 }`
41+
*/
42+
private def encapsulateQuote(tree: Tree)(implicit ctx: Context): Tree = {
43+
val quotePackage = defn.RootPackage
44+
class SpliceFinder extends TreeTraverser {
45+
var termSplices = List.newBuilder[Tree]
46+
var typeSplices = List.newBuilder[Tree]
47+
override def traverse(tree: Tree)(implicit ctx: Context): Unit = {
48+
if (tree.symbol == defn.QuotedExpr_~) termSplices += tree
49+
else if (tree.symbol == defn.QuotedType_~) typeSplices += tree
50+
else traverseChildren(tree)
51+
}
52+
}
53+
val spliceFinder = new SpliceFinder()
54+
spliceFinder.traverse(tree)
55+
val typeSplices = spliceFinder.typeSplices.result()
56+
val termSplices = spliceFinder.termSplices.result()
57+
58+
val typeHoleNames = typeSplices.map(_ => UniqueName.fresh("spliceHole".toTermName).toTypeName) // TODO use shorter name
59+
val termHoleNames = termSplices.map(_ => UniqueName.fresh("spliceHole".toTermName)) // TODO use shorter name
60+
61+
def encapsulatedBound(tpe: Type): Type = {
62+
val wtp = tpe.widen
63+
val sym = wtp.typeSymbol
64+
if (sym.is(Param)) sym.info.hiBound
65+
else wtp
66+
}
67+
68+
def methodType = MethodType(termHoleNames)(_ => termSplices.map(t => encapsulatedBound(t.tpe)), _ => defn.AnyType)
69+
def polyType = PolyType(typeHoleNames)(_ => typeSplices.map(t => TypeBounds.empty), _ => methodType)
70+
71+
val sym = ctx.newSymbol(quotePackage, nme.QUOTE, Method, if (typeHoleNames.isEmpty) methodType else polyType).asTerm
72+
73+
val quotedDef = polyDefDef(sym, tpatams => paramss => body(tpatams, paramss.head, tree))(ctx.withOwner(quotePackage))
74+
PackageDef(ref(quotePackage).asInstanceOf[Ident], quotedDef :: Nil)
75+
}
76+
77+
78+
private def body(tpatams: List[Type], params: List[Tree], tree: Tree)(implicit ctx: Context): Tree = {
79+
class placeHoles(typeHoles: List[Type], termHoles: List[Tree]) extends TreeMap { // TODO merge with collectSplices
80+
private[this] var typeHolesLeft = typeHoles
81+
private[this] var termHolesLeft = termHoles
82+
override def transform(tree: Tree)(implicit ctx: Context): Tree = {
83+
if (tree.symbol == defn.QuotedExpr_~) {
84+
val hole :: rest = termHolesLeft
85+
termHolesLeft = rest
86+
hole
87+
}
88+
else if (tree.symbol == defn.QuotedType_~) {
89+
val hole :: rest = typeHolesLeft
90+
typeHolesLeft = rest
91+
TypeTree(hole)
92+
}
93+
else if (typeHolesLeft.isEmpty && termHolesLeft.isEmpty) tree
94+
else super.transform(tree)
95+
96+
}
97+
}
98+
new placeHoles(tpatams, params).transform(tree)
99+
}
100+
101+
private def pickle(tree: Tree)(implicit ctx: Context): Array[Byte] = {
102+
val pickler = new TastyPickler(defn.RootClass)
103+
val treePkl = pickler.treePkl
104+
treePkl.pickle(tree :: Nil)
105+
treePkl.compactify()
106+
pickler.addrOfTree = treePkl.buf.addrOfTree
107+
pickler.addrOfSym = treePkl.addrOfSym
108+
// if (tree.pos.exists)
109+
// new PositionPickler(pickler, treePkl.buf.addrOfTree).picklePositions(tree :: Nil)
110+
111+
// other pickle sections go here.
112+
val pickled = pickler.assembleParts()
113+
114+
if (pickling ne noPrinter) {
115+
println(i"**** pickled quote of \n${tree.show}")
116+
new TastyPrinter(pickled).printContents()
117+
}
118+
119+
pickled
120+
}
121+
}

‎compiler/src/dotty/tools/dotc/transform/PostTyper.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,8 @@ class PostTyper extends MacroTransform with IdentityDenotTransformer { thisPhase
167167
case _ => tree
168168
}
169169
case tree @ Select(qual, name) =>
170+
if (tree.symbol == defn.QuotedExpr_~ || tree.symbol == defn.QuotedType_~)
171+
ctx.compilationUnit.containsSplices = true
170172
if (name.isTypeName) {
171173
Checking.checkRealizable(qual.tpe, qual.pos.focus)
172174
super.transform(tree)

‎compiler/src/dotty/tools/dotc/transform/ReifyQuotes.scala

Lines changed: 9 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,15 @@ package dotty.tools.dotc
22
package transform
33

44
import core._
5-
import Decorators._, Flags._, Types._, Contexts._, Symbols._, Constants._
5+
import Decorators._, Types._, Contexts._, Symbols._, Constants._
66
import Flags._
77
import ast.Trees._
8-
import util.Positions._
9-
import StdNames._
108
import ast.untpd
11-
import MegaPhase.MiniPhase
129
import typer.Implicits._
1310
import NameKinds.OuterSelectName
1411
import scala.collection.mutable
12+
import dotty.tools.dotc.core.StdNames._
13+
1514

1615
/** Translates quoted terms and types to `unpickle` method calls.
1716
* Checks that the phase consistency principle (PCP) holds.
@@ -22,25 +21,14 @@ class ReifyQuotes extends MacroTransformWithImplicits {
2221
override def phaseName: String = "reifyQuotes"
2322

2423
override def run(implicit ctx: Context): Unit =
25-
if (ctx.compilationUnit.containsQuotes) super.run
24+
if (ctx.compilationUnit.containsQuotes || ctx.compilationUnit.containsSplices) super.run
2625

2726
protected def newTransformer(implicit ctx: Context): Transformer = new Reifier
2827

2928
/** Is symbol a splice operation? */
30-
def isSplice(sym: Symbol)(implicit ctx: Context) =
29+
def isSplice(sym: Symbol)(implicit ctx: Context): Boolean =
3130
sym == defn.QuotedExpr_~ || sym == defn.QuotedType_~
3231

33-
/** Serialize `tree`. Embedded splices are represented as nodes of the form
34-
*
35-
* Select(qual, sym)
36-
*
37-
* where `sym` is either `defn.QuotedExpr_~` or `defn.QuotedType_~`. For any splice,
38-
* the `qual` part should not be pickled, since it will be added separately later
39-
* as a splice.
40-
*/
41-
def pickleTree(tree: Tree, isType: Boolean)(implicit ctx: Context): String =
42-
tree.show // TODO: replace with TASTY
43-
4432
private class Reifier extends ImplicitsTransformer {
4533

4634
/** A class for collecting the splices of some quoted expression */
@@ -226,7 +214,7 @@ class ReifyQuotes extends MacroTransformWithImplicits {
226214
ref(if (isType) defn.Unpickler_unpickleType else defn.Unpickler_unpickleExpr)
227215
.appliedToType(if (isType) body1.tpe else body1.tpe.widen)
228216
.appliedTo(
229-
Literal(Constant(pickleTree(body1, isType))),
217+
Literal(Constant(PickleQuotes.pickleQuote(body1, isType))),
230218
SeqLiteral(splices.buf.toList, TypeTree(defn.QuotedType)))
231219
}
232220
finally {
@@ -243,16 +231,16 @@ class ReifyQuotes extends MacroTransformWithImplicits {
243231
case TypeApply(fn, arg :: Nil) if fn.symbol == defn.typeQuoteMethod =>
244232
reify(arg, isType = true)
245233
case tree @ Select(body, name) if isSplice(tree.symbol) =>
246-
currentLevel -= 1
247-
val body1 = try transform(body) finally currentLevel += 1
248234
if (currentLevel > 0) {
235+
currentLevel -= 1
236+
val body1 = try transform(body) finally currentLevel += 1
249237
splicesAtLevel(currentLevel).buf += body1
250238
tree
251239
}
252240
else {
253241
if (currentLevel < 0)
254242
ctx.error(i"splice ~ not allowed under toplevel splice", tree.pos)
255-
cpy.Select(tree)(body1, name)
243+
transform(Splice.splice(body))
256244
}
257245
case Block(stats, _) =>
258246
val last = enteredSyms

0 commit comments

Comments
 (0)