Merge pull request swiftlang#74 from akyrtzi/opt3-rawsyntax

Optimize creation of RawSyntax objects for parsing

Author: akyrtzi

Sources/SwiftSyntax/RawSyntax.swift

@@ -191,14 +191,14 @@ extension _SyntaxBase {
   /// the first token syntax contained by this node. Without such token, this
   /// property will return nil.
   var leadingTrivia: Trivia? {
-    return raw.leadingTrivia
+    return raw.formLeadingTrivia()
   }
 
   /// The trailing trivia of this syntax node. Trailing trivia is attached to
   /// the last token syntax contained by this node. Without such token, this
   /// property will return nil.
   var trailingTrivia: Trivia? {
-    return raw.trailingTrivia
+    return raw.formTrailingTrivia()
   }
 
   /// The length this node's leading trivia takes up spelled out in source.
@@ -497,7 +497,7 @@ public struct TokenSyntax: _SyntaxBase, Hashable {
       fatalError("TokenSyntax must have token as its raw")
     }
     let newRaw = RawSyntax.createAndCalcLength(kind: tokenKind,
-      leadingTrivia: raw.leadingTrivia!, trailingTrivia: raw.trailingTrivia!,
+      leadingTrivia: raw.formLeadingTrivia()!, trailingTrivia: raw.formTrailingTrivia()!,
       presence: raw.presence)
     let newData = data.replacingSelf(newRaw)
     return TokenSyntax(newData)
@@ -509,8 +509,8 @@ public struct TokenSyntax: _SyntaxBase, Hashable {
     guard raw.kind == .token else {
       fatalError("TokenSyntax must have token as its raw")
     }
-    let newRaw = RawSyntax.createAndCalcLength(kind: raw.tokenKind!,
-      leadingTrivia: leadingTrivia, trailingTrivia: raw.trailingTrivia!,
+    let newRaw = RawSyntax.createAndCalcLength(kind: raw.formTokenKind()!,
+      leadingTrivia: leadingTrivia, trailingTrivia: raw.formTrailingTrivia()!,
       presence: raw.presence)
     let newData = data.replacingSelf(newRaw)
     return TokenSyntax(newData)
@@ -522,8 +522,8 @@ public struct TokenSyntax: _SyntaxBase, Hashable {
     guard raw.kind == .token else {
       fatalError("TokenSyntax must have token as its raw")
     }
-    let newRaw = RawSyntax.createAndCalcLength(kind: raw.tokenKind!,
-                           leadingTrivia: raw.leadingTrivia!,
+    let newRaw = RawSyntax.createAndCalcLength(kind: raw.formTokenKind()!,
+                           leadingTrivia: raw.formLeadingTrivia()!,
                            trailingTrivia: trailingTrivia,
                            presence: raw.presence)
     let newData = data.replacingSelf(newRaw)
@@ -547,26 +547,17 @@ public struct TokenSyntax: _SyntaxBase, Hashable {
 
   /// The leading trivia (spaces, newlines, etc.) associated with this token.
   public var leadingTrivia: Trivia {
-    guard raw.kind == .token else {
-      fatalError("TokenSyntax must have token as its raw")
-    }
-    return raw.leadingTrivia!
+    return raw.formLeadingTrivia()!
   }
 
   /// The trailing trivia (spaces, newlines, etc.) associated with this token.
   public var trailingTrivia: Trivia {
-    guard raw.kind == .token else {
-      fatalError("TokenSyntax must have token as its raw")
-    }
-    return raw.trailingTrivia!
+    return raw.formTrailingTrivia()!
   }
 
   /// The kind of token this node represents.
   public var tokenKind: TokenKind {
-    guard raw.kind == .token else {
-      fatalError("TokenSyntax must have token as its raw")
-    }
-    return raw.tokenKind!
+    return raw.formTokenKind()!
   }
 
   /// The length this node takes up spelled out in the source, excluding its

Sources/SwiftSyntax/Syntax.swift

@@ -45,9 +45,9 @@ public struct ${Builder} {
     if let list = layout[idx] {
       layout[idx] = list.appending(elt.raw)
     } else {
-      layout[idx] = RawSyntax(kind: SyntaxKind.${child.swift_syntax_kind},
-                              layout: [elt.raw], length: elt.raw.totalLength,
-                              presence: SourcePresence.present)
+      layout[idx] = RawSyntax.create(kind: SyntaxKind.${child.swift_syntax_kind},
+        layout: [elt.raw], length: elt.raw.totalLength,
+        presence: SourcePresence.present)
     }
   }
 %       else:

Sources/SwiftSyntax/SyntaxBuilders.swift.gyb

@@ -29,10 +29,10 @@ struct RawSyntaxChildren: Sequence {
 
     mutating func next() -> (RawSyntax?, AbsoluteSyntaxInfo)? {
       let idx = Int(nextChildInfo.indexInParent)
-      guard idx < parent.layout.endIndex else {
+      guard idx < parent.numberOfChildren else {
         return nil
       }
-      let child = parent.layout[idx]
+      let child = parent.child(at: idx)
       let thisItem = (child, nextChildInfo)
       nextChildInfo = nextChildInfo.advancedBySibling(child)
       return thisItem
@@ -115,10 +115,10 @@ struct ReversedPresentRawSyntaxChildren: Sequence {
     mutating func next() -> AbsoluteRawSyntax? {
       while true {
         let prevIdx = Int(previousChildInfo.indexInParent)
-        guard prevIdx > parent.layout.startIndex else {
+        guard prevIdx > 0 else {
           return nil
         }
-        let child = parent.layout[prevIdx-1]
+        let child = parent.child(at: prevIdx-1)
         previousChildInfo = previousChildInfo.reversedBySibling(child)
         if let n = child, n.isPresent {
           return AbsoluteRawSyntax(raw: n, info: previousChildInfo)

Sources/SwiftSyntax/SyntaxChildren.swift

@@ -62,7 +62,7 @@ public struct ${node.name}: _SyntaxBase, Hashable, SyntaxCollection {
   /// - Returns: A new `${node.name}` with that element appended to the end.
   public func appending(
     _ syntax: ${node.collection_element_type}) -> ${node.name} {
-    var newLayout = data.raw.layout
+    var newLayout = data.raw.formLayoutArray()
     newLayout.append(syntax.raw)
     return replacingLayout(newLayout)
   }
@@ -88,7 +88,7 @@ public struct ${node.name}: _SyntaxBase, Hashable, SyntaxCollection {
   /// - Returns: A new `${node.name}` with that element appended to the end.
   public func inserting(_ syntax: ${node.collection_element_type},
                         at index: Int) -> ${node.name} {
-    var newLayout = data.raw.layout
+    var newLayout = data.raw.formLayoutArray()
     /// Make sure the index is a valid insertion index (0 to 1 past the end)
     precondition((newLayout.startIndex...newLayout.endIndex).contains(index),
                  "inserting node at invalid index \(index)")
@@ -106,7 +106,7 @@ public struct ${node.name}: _SyntaxBase, Hashable, SyntaxCollection {
   /// - Returns: A new `${node.name}` with the new element at the provided index.
   public func replacing(childAt index: Int,
                         with syntax: ${node.collection_element_type}) -> ${node.name} {
-    var newLayout = data.raw.layout
+    var newLayout = data.raw.formLayoutArray()
     /// Make sure the index is a valid index for replacing
     precondition((newLayout.startIndex..<newLayout.endIndex).contains(index),
                  "replacing node at invalid index \(index)")
@@ -121,7 +121,7 @@ public struct ${node.name}: _SyntaxBase, Hashable, SyntaxCollection {
   /// - Returns: A new `${node.name}` with the element at the provided index
   ///            removed.
   public func removing(childAt index: Int) -> ${node.name} {
-    var newLayout = data.raw.layout
+    var newLayout = data.raw.formLayoutArray()
     newLayout.remove(at: index)
     return replacingLayout(newLayout)
   }
@@ -130,7 +130,7 @@ public struct ${node.name}: _SyntaxBase, Hashable, SyntaxCollection {
   ///
   /// - Returns: A new `${node.name}` with the first element removed.
   public func removingFirst() -> ${node.name} {
-    var newLayout = data.raw.layout
+    var newLayout = data.raw.formLayoutArray()
     newLayout.removeFirst()
     return replacingLayout(newLayout)
   }
@@ -139,7 +139,7 @@ public struct ${node.name}: _SyntaxBase, Hashable, SyntaxCollection {
   ///
   /// - Returns: A new `${node.name}` with the last element removed.
   public func removingLast() -> ${node.name} {
-    var newLayout = data.raw.layout
+    var newLayout = data.raw.formLayoutArray()
     newLayout.removeLast()
     return replacingLayout(newLayout)
   }

Sources/SwiftSyntax/SyntaxCollections.swift.gyb

@@ -152,7 +152,7 @@ struct SyntaxData {
   ///             normally the Syntax node that this `SyntaxData` belongs to.
   /// - Returns: The child's data at the provided index.
   func child(at index: Int, parent: _SyntaxBase) -> SyntaxData? {
-    if raw.layout[index] == nil { return nil }
+    if !raw.hasChild(at: index) { return nil }
     var iter = RawSyntaxChildren(absoluteRaw).makeIterator()
     for _ in 0..<index { _ = iter.next() }
     let (raw, info) = iter.next()!

Sources/SwiftSyntax/SyntaxData.swift

@@ -84,7 +84,7 @@ public enum SyntaxFactory {
 
 %   if not node.is_base():
   public static func makeBlank${node.syntax_kind}() -> ${node.name} {
-    let data = SyntaxData.forRoot(RawSyntax(kind: .${node.swift_syntax_kind},
+    let data = SyntaxData.forRoot(RawSyntax.create(kind: .${node.swift_syntax_kind},
       layout: [
 %     for child in node.children:
 %       if child.is_optional:

Sources/SwiftSyntax/SyntaxFactory.swift.gyb

@@ -130,10 +130,8 @@ public struct ${node.name}: ${base_type}, _SyntaxBase, Hashable {
     if let col = raw[Cursor.${child.swift_name}] {
       collection = col.appending(element.raw)
     } else {
-      collection = RawSyntax(kind: SyntaxKind.${child_node.swift_syntax_kind},
-                             layout: [element.raw],
-                             length: element.raw.totalLength,
-                             presence: .present)
+      collection = RawSyntax.create(kind: SyntaxKind.${child_node.swift_syntax_kind},
+        layout: [element.raw], length: element.raw.totalLength, presence: .present)
     }
     let newData = data.replacingChild(collection,
                                       at: Cursor.${child.swift_name})

Sources/SwiftSyntax/SyntaxNodes.swift.gyb

@@ -24,7 +24,10 @@ typealias CTriviaPiecePtr = UnsafePointer<CTriviaPiece>
 typealias CSyntaxKind = swiftparse_syntax_kind_t
 typealias CTokenKind = swiftparse_token_kind_t
 typealias CTriviaKind = swiftparse_trivia_kind_t
+typealias CTokenData = swiftparse_token_data_t
+typealias CLayoutData = swiftparse_layout_data_t
 typealias CParseLookupResult = swiftparse_lookup_result_t
+typealias CClientNode = swiftparse_client_node_t
 
 /// A list of possible errors that could be encountered while parsing a
 /// Syntax tree.
@@ -128,7 +131,7 @@ public struct SyntaxParser {
     }
 
     let nodeHandler = { (cnode: CSyntaxNodePtr!) -> UnsafeMutableRawPointer in
-      let node = makeRawNode(cnode, source: source)
+      let node = RawSyntax.create(from: cnode, source: source)
       // Transfer ownership of the object to the C parser. We get ownership back
       // via `moveFromCRawNode()`.
       let bits = Unmanaged.passRetained(node)
@@ -152,98 +155,8 @@ public struct SyntaxParser {
     }
 
     let c_top = swiftparse_parse_string(c_parser, source)
-    return moveFromCRawNode(c_top)!
-  }
-}
-
-fileprivate
-func makeRawToken(_ c_node: CSyntaxNode, source: String) -> RawSyntax {
-  let tokdat = c_node.token_data
-  let kind = tokdat.kind
-  var leadingTriviaLen = 0
-  for i in 0..<Int(tokdat.leading_trivia_count) {
-    leadingTriviaLen += Int(tokdat.leading_trivia![i].length)
-  }
-  var trailingTriviaLen = 0
-  for i in 0..<Int(tokdat.trailing_trivia_count) {
-    trailingTriviaLen += Int(tokdat.trailing_trivia![i].length)
-  }
-
-  let offset = Int(c_node.range.offset)
-  let totalLen = Int(c_node.range.length)
-  let tokOffset = offset + leadingTriviaLen
-  let tokLen = totalLen - (leadingTriviaLen + trailingTriviaLen)
-
-  let text = source.utf8Slice(offset: tokOffset, length: tokLen)
-  let tokKind = TokenKind.fromRawValue(kind: kind, text: text)
-  let leadingTrivia = toTrivia(tokdat.leading_trivia,
-                               count: Int(tokdat.leading_trivia_count),
-                               offset: offset, source: source)
-  let trailingTrivia = toTrivia(tokdat.trailing_trivia,
-                                count: Int(tokdat.trailing_trivia_count),
-                                offset: tokOffset+tokLen, source: source)
-  let presence: SourcePresence = c_node.present ? .present : .missing
-  return RawSyntax(kind: tokKind, leadingTrivia: leadingTrivia,
-                   trailingTrivia: trailingTrivia,
-                   length: SourceLength(utf8Length: totalLen),
-                   presence: presence)
-}
-
-fileprivate
-func makeRawNode(_ cnodeptr: CSyntaxNodePtr, source: String) -> RawSyntax {
-  let cnode = cnodeptr.pointee
-  let kind = SyntaxKind.fromRawValue(cnode.kind)
-  if kind == .token {
-    return makeRawToken(cnode, source: source)
-  } else {
-    var layout = [RawSyntax?]()
-    layout.reserveCapacity(Int(cnode.layout_data.nodes_count))
-    for i in 0..<Int(cnode.layout_data.nodes_count) {
-      // The parser guarantees that the `RawSyntax` pointers we passed via the
-      // node handler, we'll get them back in a depth-first fashion.
-      // From the memory management perspective we gave ownership of the
-      // `RawSyntax` object to the C parser via the node handler and now we get
-      // ownership back.
-      let subnode = moveFromCRawNode(cnode.layout_data.nodes![i])
-      layout.append(subnode)
-    }
-    let totalLen = Int(cnode.range.length)
-    let presence: SourcePresence = cnode.present ? .present : .missing
-    return RawSyntax(kind: kind, layout: layout,
-      length: SourceLength(utf8Length: totalLen), presence: presence)
-  }
-}
 
-fileprivate
-func moveFromCRawNode(_ ptr: UnsafeMutableRawPointer?) -> RawSyntax? {
-  guard let ptr = ptr else {
-    return nil
-  }
   // Get ownership back from the C parser.
-  let node: RawSyntax = Unmanaged.fromOpaque(ptr).takeRetainedValue()
-  return node
-}
-
-fileprivate
-func toTrivia(_ cptr: CTriviaPiecePtr?, count: Int, offset: Int,
-              source: String) -> Trivia {
-  var pieces = [TriviaPiece]()
-  pieces.reserveCapacity(count)
-  var offs = offset
-  for i in 0..<count {
-    let c_piece = cptr![i]
-    let piece = toTriviaPiece(c_piece, offset: offs, source: source)
-    pieces.append(piece)
-    offs += Int(c_piece.length)
+    return RawSyntax.moveFromOpaque(c_top)!
   }
-  return Trivia(pieces: pieces)
-}
-
-fileprivate
-func toTriviaPiece(_ c_piece: CTriviaPiece, offset: Int,
-                   source: String) -> TriviaPiece {
-  let kind = c_piece.kind
-  let text = source.utf8Slice(offset: offset, length: Int(c_piece.length))
-  return TriviaPiece.fromRawValue(kind: kind, length: Int(c_piece.length),
-                                  text: text)
 }

Sources/SwiftSyntax/SyntaxParser.swift

@@ -92,7 +92,7 @@ open class SyntaxRewriter {
     }
 
     // Sanity check, ensure the new children are the same length.
-    assert(newLayout.count == node.raw.layout.count)
+    assert(newLayout.count == node.raw.numberOfChildren)
 
     let newRaw = node.raw.replacingLayout(newLayout)
     let newData = node.base.data.replacingSelf(newRaw)

Sources/SwiftSyntax/SyntaxRewriter.swift.gyb

@@ -105,27 +105,43 @@ extension TokenKind: Equatable {
 
 extension TokenKind {
   static func fromRawValue(kind: CTokenKind,
-                           text: Substring) -> TokenKind {
+                           textBuffer: UnsafeBufferPointer<UInt8>) -> TokenKind {
     switch kind {
     case 0: return .eof
 % for token in SYNTAX_TOKENS:
     case ${token.serialization_code}:
 %   if token.text: # The token does not have text associated with it
       return .${token.swift_kind()}
 %   else:
-      return .${token.swift_kind()}(String(text))
+      return .${token.swift_kind()}(.fromBuffer(textBuffer))
 %   end
 % end
     default:
-      if !text.isEmpty {
+      if !textBuffer.isEmpty {
         // Default to an unknown token with the passed text if we don't know
         // its kind.
-        return .unknown(String(text))
+        return .unknown(.fromBuffer(textBuffer))
       } else {
         // If we were not passed the token's text, we cannot recover since we
         // would lose roundtripness.
         fatalError("unexpected token kind \(kind)")
       }
     }
   }
+
+  static func hasText(kind: CTokenKind) -> Bool {
+    switch kind {
+    case 0: return false
+% for token in SYNTAX_TOKENS:
+    case ${token.serialization_code}:
+%   if token.text: # The token does not have text associated with it
+      return false
+%   else:
+      return true
+%   end
+% end
+    default:
+      fatalError("unexpected token kind \(kind)")
+    }
+  }
 }