From c36a3e0ea791e22e71a36e95831de238a1b1b162 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Tue, 17 Nov 2020 17:22:36 +0100 Subject: [PATCH 1/3] Fix #6542: Pickle line sizes in TASTy --- .../dotc/core/tasty/DottyUnpickler.scala | 16 ++++--- .../dotc/core/tasty/LineSizesPickler.scala | 35 +++++++++++++++ .../dotc/core/tasty/LineSizesUnpickler.scala | 45 +++++++++++++++++++ .../tools/dotc/core/tasty/TastyPrinter.scala | 21 ++++++++- .../tools/dotc/core/tasty/TreeUnpickler.scala | 15 +++++-- .../dotc/reporting/MessageRendering.scala | 2 +- .../dotty/tools/dotc/transform/Pickler.scala | 2 + .../dotty/tools/dotc/util/SourceFile.scala | 16 +++++-- .../tools/dotc/util/SourcePosition.scala | 12 ++--- tasty/src/dotty/tools/tasty/TastyFormat.scala | 10 ++++- 10 files changed, 153 insertions(+), 21 deletions(-) create mode 100644 compiler/src/dotty/tools/dotc/core/tasty/LineSizesPickler.scala create mode 100644 compiler/src/dotty/tools/dotc/core/tasty/LineSizesUnpickler.scala diff --git a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala index ffcab8dc7a90..cce503c9d868 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala @@ -18,10 +18,10 @@ object DottyUnpickler { /** Exception thrown if classfile is corrupted */ class BadSignature(msg: String) extends RuntimeException(msg) - class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler]) + class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], lineSizesUnpickler: Option[LineSizesUnpickler], commentUnpickler: Option[CommentUnpickler]) extends SectionUnpickler[TreeUnpickler](ASTsSection) { def unpickle(reader: TastyReader, nameAtRef: NameTable): TreeUnpickler = - new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler) + new TreeUnpickler(reader, nameAtRef, posUnpickler, lineSizesUnpickler, commentUnpickler) } class PositionsSectionUnpickler extends SectionUnpickler[PositionUnpickler](PositionsSection) { @@ -29,6 +29,11 @@ object DottyUnpickler { new PositionUnpickler(reader, nameAtRef) } + class LineSizesSectionUnpickler extends SectionUnpickler[LineSizesUnpickler]("LineSizes") { + def unpickle(reader: TastyReader, nameAtRef: NameTable): LineSizesUnpickler = + new LineSizesUnpickler(reader) + } + class CommentsSectionUnpickler extends SectionUnpickler[CommentUnpickler](CommentsSection) { def unpickle(reader: TastyReader, nameAtRef: NameTable): CommentUnpickler = new CommentUnpickler(reader) @@ -45,8 +50,9 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe val unpickler: TastyUnpickler = new TastyUnpickler(bytes) private val posUnpicklerOpt = unpickler.unpickle(new PositionsSectionUnpickler) + private val lineSizesUnpicklerOpt = unpickler.unpickle(new LineSizesSectionUnpickler) private val commentUnpicklerOpt = unpickler.unpickle(new CommentsSectionUnpickler) - private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt)).get + private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, lineSizesUnpicklerOpt, commentUnpicklerOpt)).get /** Enter all toplevel classes and objects into their scopes * @param roots a set of SymDenotations that should be overwritten by unpickling @@ -54,8 +60,8 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe def enter(roots: Set[SymDenotation])(using Context): Unit = treeUnpickler.enter(roots) - protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler], commentUnpicklerOpt: Option[CommentUnpickler]): TreeSectionUnpickler = - new TreeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt) + protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler], lineSizesUnpicklerOpt: Option[LineSizesUnpickler], commentUnpicklerOpt: Option[CommentUnpickler]): TreeSectionUnpickler = + new TreeSectionUnpickler(posUnpicklerOpt, lineSizesUnpicklerOpt, commentUnpicklerOpt) protected def computeRootTrees(using Context): List[Tree] = treeUnpickler.unpickle(mode) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/LineSizesPickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/LineSizesPickler.scala new file mode 100644 index 000000000000..a505ddfcb51a --- /dev/null +++ b/compiler/src/dotty/tools/dotc/core/tasty/LineSizesPickler.scala @@ -0,0 +1,35 @@ +package dotty.tools +package dotc +package core +package tasty + +import dotty.tools.tasty.TastyFormat.SOURCE +import dotty.tools.tasty.TastyBuffer +import TastyBuffer._ + +import ast._ +import ast.Trees._ +import ast.Trees.WithLazyField +import util.{SourceFile, NoSource} +import core._ +import Contexts._, Symbols._, Annotations._, Decorators._ +import collection.mutable +import util.Spans._ + +class LineSizesPickler(pickler: TastyPickler) { + + import ast.tpd._ + val buf: TastyBuffer = new TastyBuffer(5000) + pickler.newSection("LineSizes", buf) + + def pickleLineNumbers(source: SourceFile): Unit = + val content = source.content() + var lastIndex = content.indexOf('\n', 0) + buf.writeInt(lastIndex) // size of first line + while lastIndex != -1 do + val nextIndex = content.indexOf('\n', lastIndex + 1) + val end = if nextIndex != -1 then nextIndex else content.length + buf.writeInt(end - lastIndex - 1) // size of the next line + lastIndex = nextIndex + +} diff --git a/compiler/src/dotty/tools/dotc/core/tasty/LineSizesUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/LineSizesUnpickler.scala new file mode 100644 index 000000000000..7b9d81f0e3f4 --- /dev/null +++ b/compiler/src/dotty/tools/dotc/core/tasty/LineSizesUnpickler.scala @@ -0,0 +1,45 @@ +package dotty.tools +package dotc +package core +package tasty + +import dotty.tools.tasty.{TastyFormat, TastyBuffer, TastyReader} +import TastyFormat.SOURCE +import TastyBuffer.{Addr, NameRef} + +import util.Spans._ +import collection.{mutable, Map} +import Names.TermName + +/** Unpickler for tree positions */ +class LineSizesUnpickler(reader: TastyReader) { + import reader._ + + private var mySizes: Array[Int] = _ + private var isDefined = false + + def ensureDefined(): Unit = + if !isDefined then + val sizeBuf = Array.newBuilder[Int] + // Number of lines if all lines are 127 characters or less + var lowSizeBound = endAddr.index - currentAddr.index + sizeBuf.sizeHint(lowSizeBound) + while !isAtEnd do sizeBuf += readInt() + mySizes = sizeBuf.result() + isDefined = true + + private[tasty] def sizes: Array[Int] = + ensureDefined() + mySizes + + private[tasty] def lineIndices: Array[Int] = + val szs = sizes + val indices = new Array[Int](sizes.length + 1) + var i = 0 + val penultimate = szs.length - 1 + while i < penultimate do + indices(i + 1) = indices(i) + szs(i) + 1 // `+1` for the '\n' at the end of the line + i += 1 + indices(szs.length) = indices(penultimate) + szs(penultimate) // last line does not end with '\n' + indices +} diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala index 074cdd8fca78..9681704f1b7a 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala @@ -51,6 +51,11 @@ class TastyPrinter(bytes: Array[Byte]) { case _ => } sb.append("\n\n") + unpickle(new LineSizesSectionUnpickler) match { + case Some(s) => sb.append(s) + case _ => + } + sb.append("\n\n") unpickle(new CommentSectionUnpickler) match { case Some(s) => sb.append(s) case _ => @@ -141,7 +146,7 @@ class TastyPrinter(bytes: Array[Byte]) { def unpickle(reader: TastyReader, tastyName: NameTable): String = { sb.append(s" ${reader.endAddr.index - reader.currentAddr.index}") val spans = new PositionUnpickler(reader, tastyName).spans - sb.append(s" position bytes:\n") + sb.append(" position bytes:\n") val sorted = spans.toSeq.sortBy(_._1.index) for ((addr, pos) <- sorted) { sb.append(treeStr("%10d".format(addr.index))) @@ -151,6 +156,20 @@ class TastyPrinter(bytes: Array[Byte]) { } } + class LineSizesSectionUnpickler extends SectionUnpickler[String]("LineSizes") { + + private val sb: StringBuilder = new StringBuilder + + def unpickle(reader: TastyReader, tastyName: NameTable): String = { + sb.append(" ").append(reader.endAddr.index - reader.currentAddr.index) + sb.append(" line sizes bytes:\n") + val lineSizes = new LineSizesUnpickler(reader) + sb.append(" sizes: ") + sb.append(lineSizes.sizes.mkString(", ")) + sb.result + } + } + class CommentSectionUnpickler extends SectionUnpickler[String](CommentsSection) { private val sb: StringBuilder = new StringBuilder diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index e3c74e083cba..ee08b721bc4a 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -47,13 +47,15 @@ import scala.annotation.constructorOnly import scala.annotation.internal.sharable /** Unpickler for typed trees - * @param reader the reader from which to unpickle - * @param posUnpicklerOpt the unpickler for positions, if it exists - * @param commentUnpicklerOpt the unpickler for comments, if it exists + * @param reader the reader from which to unpickle + * @param posUnpicklerOpt the unpickler for positions, if it exists + * @param lineSizesUnpicklerOpt the unpickler for line sizes, if it exists + * @param commentUnpicklerOpt the unpickler for comments, if it exists */ class TreeUnpickler(reader: TastyReader, nameAtRef: NameTable, posUnpicklerOpt: Option[PositionUnpickler], + lineSizesUnpicklerOpt: Option[LineSizesUnpickler], commentUnpicklerOpt: Option[CommentUnpickler]) { import TreeUnpickler._ import tpd._ @@ -1363,8 +1365,13 @@ class TreeUnpickler(reader: TastyReader, def sourceChangeContext(addr: Addr = currentAddr)(using Context): Context = { val path = sourcePathAt(addr) if (path.nonEmpty) { + val sourceFile = ctx.getSource(path) + lineSizesUnpicklerOpt match + case Some(lineSizesUnpickler) => + sourceFile.setLineIndices(lineSizesUnpickler.lineIndices) + case _ => pickling.println(i"source change at $addr: $path") - ctx.withSource(ctx.getSource(path)) + ctx.withSource(sourceFile) } else ctx } diff --git a/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala b/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala index 0787ad9cbcba..1c62611210b6 100644 --- a/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala +++ b/compiler/src/dotty/tools/dotc/reporting/MessageRendering.scala @@ -147,7 +147,7 @@ trait MessageRendering { val sb = mutable.StringBuilder() val posString = posStr(pos, diagnosticLevel, msg) if (posString.nonEmpty) sb.append(posString).append(EOL) - if (pos.exists && pos.source.file.exists) { + if (pos.exists) { val pos1 = pos.nonInlined val (srcBefore, srcAfter, offset) = sourceLines(pos1, diagnosticLevel) val marker = columnMarker(pos1, offset, diagnosticLevel) diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index c79796dc804a..312b777b3a27 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -74,6 +74,8 @@ class Pickler extends Phase { if tree.span.exists then new PositionPickler(pickler, treePkl.buf.addrOfTree, treePkl.treeAnnots) .picklePositions(tree :: Nil, positionWarnings) + new LineSizesPickler(pickler) + .pickleLineNumbers(unit.source) if !ctx.settings.YdropComments.value then new CommentPickler(pickler, treePkl.buf.addrOfTree, treePkl.docString) diff --git a/compiler/src/dotty/tools/dotc/util/SourceFile.scala b/compiler/src/dotty/tools/dotc/util/SourceFile.scala index 501d9d78e7e6..702a30f7d3a4 100644 --- a/compiler/src/dotty/tools/dotc/util/SourceFile.scala +++ b/compiler/src/dotty/tools/dotc/util/SourceFile.scala @@ -82,7 +82,9 @@ class SourceFile(val file: AbstractFile, computeContent: => Array[Char]) extends def apply(idx: Int): Char = content().apply(idx) - def length: Int = content().length + def length: Int = + if lineIndicesCache ne null then lineIndicesCache.last + else content().length /** true for all source files except `NoSource` */ def exists: Boolean = true @@ -105,7 +107,8 @@ class SourceFile(val file: AbstractFile, computeContent: => Array[Char]) extends def positionInUltimateSource(position: SourcePosition): SourcePosition = SourcePosition(underlying, position.span shift start) - private def calculateLineIndices(cs: Array[Char]) = { + private def calculateLineIndicesFromContents() = { + val cs = content() val buf = new ArrayBuffer[Int] buf += 0 var i = 0 @@ -120,7 +123,14 @@ class SourceFile(val file: AbstractFile, computeContent: => Array[Char]) extends buf += cs.length // sentinel, so that findLine below works smoother buf.toArray } - private lazy val lineIndices: Array[Int] = calculateLineIndices(content()) + + private var lineIndicesCache: Array[Int] = _ + private def lineIndices: Array[Int] = + if lineIndicesCache eq null then + lineIndicesCache = calculateLineIndicesFromContents() + lineIndicesCache + def setLineIndices(indices: Array[Int]): Unit = + lineIndicesCache = indices /** Map line to offset of first character in line */ def lineToOffset(index: Int): Int = lineIndices(index) diff --git a/compiler/src/dotty/tools/dotc/util/SourcePosition.scala b/compiler/src/dotty/tools/dotc/util/SourcePosition.scala index 35d16cff9f44..160631bc41b0 100644 --- a/compiler/src/dotty/tools/dotc/util/SourcePosition.scala +++ b/compiler/src/dotty/tools/dotc/util/SourcePosition.scala @@ -25,7 +25,7 @@ extends SrcPos, interfaces.SourcePosition, Showable { def point: Int = span.point - def line: Int = if (source.length != 0) source.offsetToLine(point) else -1 + def line: Int = source.offsetToLine(point) /** Extracts the lines from the underlying source file as `Array[Char]`*/ def linesSlice: Array[Char] = @@ -45,16 +45,16 @@ extends SrcPos, interfaces.SourcePosition, Showable { def beforeAndAfterPoint: (List[Int], List[Int]) = lineOffsets.partition(_ <= point) - def column: Int = if (source.content().length != 0) source.column(point) else -1 + def column: Int = source.column(point) def start: Int = span.start - def startLine: Int = if (source.content().length != 0) source.offsetToLine(start) else -1 - def startColumn: Int = if (source.content().length != 0) source.column(start) else -1 + def startLine: Int = source.offsetToLine(start) + def startColumn: Int = source.column(start) def startColumnPadding: String = source.startColumnPadding(start) def end: Int = span.end - def endLine: Int = if (source.content().length != 0) source.offsetToLine(end) else -1 - def endColumn: Int = if (source.content().length != 0) source.column(end) else -1 + def endLine: Int = source.offsetToLine(end) + def endColumn: Int = source.column(end) def withOuter(outer: SourcePosition): SourcePosition = SourcePosition(source, span, outer) def withSpan(range: Span) = SourcePosition(source, range, outer) diff --git a/tasty/src/dotty/tools/tasty/TastyFormat.scala b/tasty/src/dotty/tools/tasty/TastyFormat.scala index b99327393aa7..b370d8531142 100644 --- a/tasty/src/dotty/tools/tasty/TastyFormat.scala +++ b/tasty/src/dotty/tools/tasty/TastyFormat.scala @@ -50,6 +50,7 @@ Macro-format: Note: Unqualified names in the name table are strings. The context decides whether a name is a type-name or a term-name. The same string can represent both. + Standard-Section: "ASTs" TopLevelStat* TopLevelStat = PACKAGE Length Path TopLevelStat* -- package path { topLevelStats } @@ -226,6 +227,7 @@ Note: Tree tags are grouped into 5 categories that determine what follows, and t Category 4 (tags 110-127): tag Nat AST Category 5 (tags 128-255): tag Length + Standard-Section: "Positions" Assoc* Assoc = Header offset_Delta? offset_Delta? point_Delta? @@ -244,7 +246,13 @@ Standard-Section: "Positions" Assoc* All elements of a position section are serialized as Ints -Standard-Section: "Comments" Comment* + +Standard-Section: "LineSizes" LineSize* + + LineSize = Int // Size the i-th line not counting the trailing `\n` + + +Standard Section: "Comments" Comment* Comment = Length Bytes LongInt // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates. From c7106eff5f17b455457ba24bdfa7ad4ad1881f13 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Sat, 21 Nov 2020 11:50:12 +0100 Subject: [PATCH 2/3] Put line sizes in the Positions section --- .../dotc/core/tasty/DottyUnpickler.scala | 16 +++---- .../dotc/core/tasty/LineSizesPickler.scala | 35 --------------- .../dotc/core/tasty/LineSizesUnpickler.scala | 45 ------------------- .../dotc/core/tasty/PositionPickler.scala | 16 ++++++- .../dotc/core/tasty/PositionUnpickler.scala | 13 ++++++ .../tools/dotc/core/tasty/TastyPrinter.scala | 26 +++-------- .../tools/dotc/core/tasty/TreeUnpickler.scala | 14 +++--- .../tools/dotc/quoted/PickledQuotes.scala | 2 +- .../dotty/tools/dotc/transform/Pickler.scala | 4 +- .../dotty/tools/dotc/util/SourceFile.scala | 10 ++++- project/scripts/cmdTests | 1 + tasty/src/dotty/tools/tasty/TastyFormat.scala | 13 +++--- 12 files changed, 62 insertions(+), 133 deletions(-) delete mode 100644 compiler/src/dotty/tools/dotc/core/tasty/LineSizesPickler.scala delete mode 100644 compiler/src/dotty/tools/dotc/core/tasty/LineSizesUnpickler.scala diff --git a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala index cce503c9d868..ffcab8dc7a90 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala @@ -18,10 +18,10 @@ object DottyUnpickler { /** Exception thrown if classfile is corrupted */ class BadSignature(msg: String) extends RuntimeException(msg) - class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], lineSizesUnpickler: Option[LineSizesUnpickler], commentUnpickler: Option[CommentUnpickler]) + class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler]) extends SectionUnpickler[TreeUnpickler](ASTsSection) { def unpickle(reader: TastyReader, nameAtRef: NameTable): TreeUnpickler = - new TreeUnpickler(reader, nameAtRef, posUnpickler, lineSizesUnpickler, commentUnpickler) + new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler) } class PositionsSectionUnpickler extends SectionUnpickler[PositionUnpickler](PositionsSection) { @@ -29,11 +29,6 @@ object DottyUnpickler { new PositionUnpickler(reader, nameAtRef) } - class LineSizesSectionUnpickler extends SectionUnpickler[LineSizesUnpickler]("LineSizes") { - def unpickle(reader: TastyReader, nameAtRef: NameTable): LineSizesUnpickler = - new LineSizesUnpickler(reader) - } - class CommentsSectionUnpickler extends SectionUnpickler[CommentUnpickler](CommentsSection) { def unpickle(reader: TastyReader, nameAtRef: NameTable): CommentUnpickler = new CommentUnpickler(reader) @@ -50,9 +45,8 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe val unpickler: TastyUnpickler = new TastyUnpickler(bytes) private val posUnpicklerOpt = unpickler.unpickle(new PositionsSectionUnpickler) - private val lineSizesUnpicklerOpt = unpickler.unpickle(new LineSizesSectionUnpickler) private val commentUnpicklerOpt = unpickler.unpickle(new CommentsSectionUnpickler) - private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, lineSizesUnpicklerOpt, commentUnpicklerOpt)).get + private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt)).get /** Enter all toplevel classes and objects into their scopes * @param roots a set of SymDenotations that should be overwritten by unpickling @@ -60,8 +54,8 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe def enter(roots: Set[SymDenotation])(using Context): Unit = treeUnpickler.enter(roots) - protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler], lineSizesUnpicklerOpt: Option[LineSizesUnpickler], commentUnpicklerOpt: Option[CommentUnpickler]): TreeSectionUnpickler = - new TreeSectionUnpickler(posUnpicklerOpt, lineSizesUnpicklerOpt, commentUnpicklerOpt) + protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler], commentUnpicklerOpt: Option[CommentUnpickler]): TreeSectionUnpickler = + new TreeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt) protected def computeRootTrees(using Context): List[Tree] = treeUnpickler.unpickle(mode) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/LineSizesPickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/LineSizesPickler.scala deleted file mode 100644 index a505ddfcb51a..000000000000 --- a/compiler/src/dotty/tools/dotc/core/tasty/LineSizesPickler.scala +++ /dev/null @@ -1,35 +0,0 @@ -package dotty.tools -package dotc -package core -package tasty - -import dotty.tools.tasty.TastyFormat.SOURCE -import dotty.tools.tasty.TastyBuffer -import TastyBuffer._ - -import ast._ -import ast.Trees._ -import ast.Trees.WithLazyField -import util.{SourceFile, NoSource} -import core._ -import Contexts._, Symbols._, Annotations._, Decorators._ -import collection.mutable -import util.Spans._ - -class LineSizesPickler(pickler: TastyPickler) { - - import ast.tpd._ - val buf: TastyBuffer = new TastyBuffer(5000) - pickler.newSection("LineSizes", buf) - - def pickleLineNumbers(source: SourceFile): Unit = - val content = source.content() - var lastIndex = content.indexOf('\n', 0) - buf.writeInt(lastIndex) // size of first line - while lastIndex != -1 do - val nextIndex = content.indexOf('\n', lastIndex + 1) - val end = if nextIndex != -1 then nextIndex else content.length - buf.writeInt(end - lastIndex - 1) // size of the next line - lastIndex = nextIndex - -} diff --git a/compiler/src/dotty/tools/dotc/core/tasty/LineSizesUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/LineSizesUnpickler.scala deleted file mode 100644 index 7b9d81f0e3f4..000000000000 --- a/compiler/src/dotty/tools/dotc/core/tasty/LineSizesUnpickler.scala +++ /dev/null @@ -1,45 +0,0 @@ -package dotty.tools -package dotc -package core -package tasty - -import dotty.tools.tasty.{TastyFormat, TastyBuffer, TastyReader} -import TastyFormat.SOURCE -import TastyBuffer.{Addr, NameRef} - -import util.Spans._ -import collection.{mutable, Map} -import Names.TermName - -/** Unpickler for tree positions */ -class LineSizesUnpickler(reader: TastyReader) { - import reader._ - - private var mySizes: Array[Int] = _ - private var isDefined = false - - def ensureDefined(): Unit = - if !isDefined then - val sizeBuf = Array.newBuilder[Int] - // Number of lines if all lines are 127 characters or less - var lowSizeBound = endAddr.index - currentAddr.index - sizeBuf.sizeHint(lowSizeBound) - while !isAtEnd do sizeBuf += readInt() - mySizes = sizeBuf.result() - isDefined = true - - private[tasty] def sizes: Array[Int] = - ensureDefined() - mySizes - - private[tasty] def lineIndices: Array[Int] = - val szs = sizes - val indices = new Array[Int](sizes.length + 1) - var i = 0 - val penultimate = szs.length - 1 - while i < penultimate do - indices(i + 1) = indices(i) + szs(i) + 1 // `+1` for the '\n' at the end of the line - i += 1 - indices(szs.length) = indices(penultimate) + szs(penultimate) // last line does not end with '\n' - indices -} diff --git a/compiler/src/dotty/tools/dotc/core/tasty/PositionPickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/PositionPickler.scala index d7aace27140c..68ee538e89aa 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/PositionPickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/PositionPickler.scala @@ -32,7 +32,21 @@ class PositionPickler( (addrDelta << 3) | (toInt(hasStartDelta) << 2) | (toInt(hasEndDelta) << 1) | toInt(hasPoint) } - def picklePositions(roots: List[Tree], warnings: mutable.ListBuffer[String]): Unit = { + def picklePositions(source: SourceFile, roots: List[Tree], warnings: mutable.ListBuffer[String]): Unit = { + /** Pickle the number of lines followed by the length of each line */ + def pickleLineOffsetts(): Unit = { + val content = source.content() + buf.writeInt(content.count(_ == '\n') + 1) // number of lines + var lastIndex = content.indexOf('\n', 0) + buf.writeInt(lastIndex) // size of first line + while lastIndex != -1 do + val nextIndex = content.indexOf('\n', lastIndex + 1) + val end = if nextIndex != -1 then nextIndex else content.length + buf.writeInt(end - lastIndex - 1) // size of the next line + lastIndex = nextIndex + } + pickleLineOffsetts() + var lastIndex = 0 var lastSpan = Span(0, 0) def pickleDeltas(index: Int, span: Span) = { diff --git a/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala index 872f60837515..011a8b42103c 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala @@ -15,12 +15,20 @@ import Names.TermName class PositionUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName) { import reader._ + private var myLineSizes: Array[Int] = _ private var mySpans: util.HashMap[Addr, Span] = _ private var mySourcePaths: util.HashMap[Addr, String] = _ private var isDefined = false def ensureDefined(): Unit = { if (!isDefined) { + val lines = readInt() + myLineSizes = new Array[Int](lines) + var i = 0 + while i < lines do + myLineSizes(i) += readInt() + i += 1 + mySpans = util.HashMap[Addr, Span]() mySourcePaths = util.HashMap[Addr, String]() var curIndex = 0 @@ -60,6 +68,11 @@ class PositionUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName) { mySourcePaths } + private[tasty] def lineSizes: Array[Int] = { + ensureDefined() + myLineSizes + } + def spanAt(addr: Addr): Span = spans.getOrElse(addr, NoSpan) def sourcePathAt(addr: Addr): String = sourcePaths.getOrElse(addr, "") } diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala index 9681704f1b7a..def4adcfe2a3 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala @@ -51,11 +51,6 @@ class TastyPrinter(bytes: Array[Byte]) { case _ => } sb.append("\n\n") - unpickle(new LineSizesSectionUnpickler) match { - case Some(s) => sb.append(s) - case _ => - } - sb.append("\n\n") unpickle(new CommentSectionUnpickler) match { case Some(s) => sb.append(s) case _ => @@ -144,9 +139,14 @@ class TastyPrinter(bytes: Array[Byte]) { private val sb: StringBuilder = new StringBuilder def unpickle(reader: TastyReader, tastyName: NameTable): String = { + val posUnpickler = new PositionUnpickler(reader, tastyName) sb.append(s" ${reader.endAddr.index - reader.currentAddr.index}") - val spans = new PositionUnpickler(reader, tastyName).spans sb.append(" position bytes:\n") + val lineSizes = posUnpickler.lineSizes + sb.append(s" lines: ${lineSizes.length}\n") + sb.append(posUnpickler.lineSizes.mkString(" line sizes: ", ", ", "\n")) + sb.append(" positions:\n") + val spans = posUnpickler.spans val sorted = spans.toSeq.sortBy(_._1.index) for ((addr, pos) <- sorted) { sb.append(treeStr("%10d".format(addr.index))) @@ -156,20 +156,6 @@ class TastyPrinter(bytes: Array[Byte]) { } } - class LineSizesSectionUnpickler extends SectionUnpickler[String]("LineSizes") { - - private val sb: StringBuilder = new StringBuilder - - def unpickle(reader: TastyReader, tastyName: NameTable): String = { - sb.append(" ").append(reader.endAddr.index - reader.currentAddr.index) - sb.append(" line sizes bytes:\n") - val lineSizes = new LineSizesUnpickler(reader) - sb.append(" sizes: ") - sb.append(lineSizes.sizes.mkString(", ")) - sb.result - } - } - class CommentSectionUnpickler extends SectionUnpickler[String](CommentsSection) { private val sb: StringBuilder = new StringBuilder diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index ee08b721bc4a..de4c145e6632 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -47,15 +47,13 @@ import scala.annotation.constructorOnly import scala.annotation.internal.sharable /** Unpickler for typed trees - * @param reader the reader from which to unpickle - * @param posUnpicklerOpt the unpickler for positions, if it exists - * @param lineSizesUnpicklerOpt the unpickler for line sizes, if it exists - * @param commentUnpicklerOpt the unpickler for comments, if it exists + * @param reader the reader from which to unpickle + * @param posUnpicklerOpt the unpickler for positions, if it exists + * @param commentUnpicklerOpt the unpickler for comments, if it exists */ class TreeUnpickler(reader: TastyReader, nameAtRef: NameTable, posUnpicklerOpt: Option[PositionUnpickler], - lineSizesUnpicklerOpt: Option[LineSizesUnpickler], commentUnpicklerOpt: Option[CommentUnpickler]) { import TreeUnpickler._ import tpd._ @@ -1366,9 +1364,9 @@ class TreeUnpickler(reader: TastyReader, val path = sourcePathAt(addr) if (path.nonEmpty) { val sourceFile = ctx.getSource(path) - lineSizesUnpicklerOpt match - case Some(lineSizesUnpickler) => - sourceFile.setLineIndices(lineSizesUnpickler.lineIndices) + posUnpicklerOpt match + case Some(posUnpickler) => + sourceFile.setLineIndicesFromLineSizes(posUnpickler.lineSizes) case _ => pickling.println(i"source change at $addr: $path") ctx.withSource(sourceFile) diff --git a/compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala b/compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala index d3c91f12bfb6..5e090782a66c 100644 --- a/compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala +++ b/compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala @@ -168,7 +168,7 @@ object PickledQuotes { if tree.span.exists then val positionWarnings = new mutable.ListBuffer[String]() new PositionPickler(pickler, treePkl.buf.addrOfTree, treePkl.treeAnnots) - .picklePositions(tree :: Nil, positionWarnings) + .picklePositions(ctx.compilationUnit.source, tree :: Nil, positionWarnings) positionWarnings.foreach(report.warning(_)) val pickled = pickler.assembleParts() diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala index 312b777b3a27..6cbdf234b676 100644 --- a/compiler/src/dotty/tools/dotc/transform/Pickler.scala +++ b/compiler/src/dotty/tools/dotc/transform/Pickler.scala @@ -73,9 +73,7 @@ class Pickler extends Phase { treePkl.compactify() if tree.span.exists then new PositionPickler(pickler, treePkl.buf.addrOfTree, treePkl.treeAnnots) - .picklePositions(tree :: Nil, positionWarnings) - new LineSizesPickler(pickler) - .pickleLineNumbers(unit.source) + .picklePositions(unit.source, tree :: Nil, positionWarnings) if !ctx.settings.YdropComments.value then new CommentPickler(pickler, treePkl.buf.addrOfTree, treePkl.docString) diff --git a/compiler/src/dotty/tools/dotc/util/SourceFile.scala b/compiler/src/dotty/tools/dotc/util/SourceFile.scala index 702a30f7d3a4..501574c8f989 100644 --- a/compiler/src/dotty/tools/dotc/util/SourceFile.scala +++ b/compiler/src/dotty/tools/dotc/util/SourceFile.scala @@ -129,7 +129,15 @@ class SourceFile(val file: AbstractFile, computeContent: => Array[Char]) extends if lineIndicesCache eq null then lineIndicesCache = calculateLineIndicesFromContents() lineIndicesCache - def setLineIndices(indices: Array[Int]): Unit = + def setLineIndicesFromLineSizes(sizes: Array[Int]): Unit = + val lines = sizes.length + val indices = new Array[Int](lines + 1) + var i = 0 + val penultimate = lines - 1 + while i < penultimate do + indices(i + 1) = indices(i) + sizes(i) + 1 // `+1` for the '\n' at the end of the line + i += 1 + indices(lines) = indices(penultimate) + sizes(penultimate) // last line does not end with '\n' lineIndicesCache = indices /** Map line to offset of first character in line */ diff --git a/project/scripts/cmdTests b/project/scripts/cmdTests index 382c5a76de79..7ab64a4c53af 100755 --- a/project/scripts/cmdTests +++ b/project/scripts/cmdTests @@ -52,6 +52,7 @@ cp tests/neg/i6371/B_2.scala $OUT/B.scala rm $OUT/A.scala "$SBT" "scalac -classpath $OUT1 -d $OUT1 $OUT/B.scala" > "$tmp" 2>&1 || echo "ok" grep -qe "B.scala:2:7" "$tmp" +grep -qe "This location contains code that was inlined from A.scala:3" "$tmp" echo "testing -Ythrough-tasty" clear_out "$OUT" diff --git a/tasty/src/dotty/tools/tasty/TastyFormat.scala b/tasty/src/dotty/tools/tasty/TastyFormat.scala index b370d8531142..06a452101754 100644 --- a/tasty/src/dotty/tools/tasty/TastyFormat.scala +++ b/tasty/src/dotty/tools/tasty/TastyFormat.scala @@ -228,7 +228,9 @@ Note: Tree tags are grouped into 5 categories that determine what follows, and t Category 5 (tags 128-255): tag Length -Standard-Section: "Positions" Assoc* +Standard-Section: "Positions" LinesSizes Assoc* + + LinesSizes = Int Int* // Number of lines followed by the size of each line not counting the trailing `\n` Assoc = Header offset_Delta? offset_Delta? point_Delta? | SOURCE nameref_Int @@ -247,11 +249,6 @@ Standard-Section: "Positions" Assoc* All elements of a position section are serialized as Ints -Standard-Section: "LineSizes" LineSize* - - LineSize = Int // Size the i-th line not counting the trailing `\n` - - Standard Section: "Comments" Comment* Comment = Length Bytes LongInt // Raw comment's bytes encoded as UTF-8, followed by the comment's coordinates. @@ -262,8 +259,8 @@ Standard Section: "Comments" Comment* object TastyFormat { final val header: Array[Int] = Array(0x5C, 0xA1, 0xAB, 0x1F) - val MajorVersion: Int = 25 - val MinorVersion: Int = 1 + val MajorVersion: Int = 26 + val MinorVersion: Int = 0 final val ASTsSection = "ASTs" final val PositionsSection = "Positions" From 361a680279028a2431e2ed85ea4dc9ed85bcb7e3 Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Fri, 27 Nov 2020 10:20:56 +0100 Subject: [PATCH 3/3] Use `Nat`s to pickle line sizes in TASTy --- .../dotty/tools/dotc/core/tasty/PositionPickler.scala | 10 +++++----- .../tools/dotc/core/tasty/PositionUnpickler.scala | 4 ++-- tasty/src/dotty/tools/tasty/TastyFormat.scala | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/PositionPickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/PositionPickler.scala index 68ee538e89aa..36a77fb25c68 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/PositionPickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/PositionPickler.scala @@ -34,18 +34,18 @@ class PositionPickler( def picklePositions(source: SourceFile, roots: List[Tree], warnings: mutable.ListBuffer[String]): Unit = { /** Pickle the number of lines followed by the length of each line */ - def pickleLineOffsetts(): Unit = { + def pickleLineOffsets(): Unit = { val content = source.content() - buf.writeInt(content.count(_ == '\n') + 1) // number of lines + buf.writeNat(content.count(_ == '\n') + 1) // number of lines var lastIndex = content.indexOf('\n', 0) - buf.writeInt(lastIndex) // size of first line + buf.writeNat(lastIndex) // size of first line while lastIndex != -1 do val nextIndex = content.indexOf('\n', lastIndex + 1) val end = if nextIndex != -1 then nextIndex else content.length - buf.writeInt(end - lastIndex - 1) // size of the next line + buf.writeNat(end - lastIndex - 1) // size of the next line lastIndex = nextIndex } - pickleLineOffsetts() + pickleLineOffsets() var lastIndex = 0 var lastSpan = Span(0, 0) diff --git a/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala index 011a8b42103c..782a6ac3a598 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/PositionUnpickler.scala @@ -22,11 +22,11 @@ class PositionUnpickler(reader: TastyReader, nameAtRef: NameRef => TermName) { def ensureDefined(): Unit = { if (!isDefined) { - val lines = readInt() + val lines = readNat() myLineSizes = new Array[Int](lines) var i = 0 while i < lines do - myLineSizes(i) += readInt() + myLineSizes(i) += readNat() i += 1 mySpans = util.HashMap[Addr, Span]() diff --git a/tasty/src/dotty/tools/tasty/TastyFormat.scala b/tasty/src/dotty/tools/tasty/TastyFormat.scala index 06a452101754..23bbb6391cdf 100644 --- a/tasty/src/dotty/tools/tasty/TastyFormat.scala +++ b/tasty/src/dotty/tools/tasty/TastyFormat.scala @@ -230,7 +230,7 @@ Note: Tree tags are grouped into 5 categories that determine what follows, and t Standard-Section: "Positions" LinesSizes Assoc* - LinesSizes = Int Int* // Number of lines followed by the size of each line not counting the trailing `\n` + LinesSizes = Nat Nat* // Number of lines followed by the size of each line not counting the trailing `\n` Assoc = Header offset_Delta? offset_Delta? point_Delta? | SOURCE nameref_Int