From d0eaf60d5e4bd0d5be63a41ed9ff861dca95c932 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <nnethercote@mozilla.com>
Date: Wed, 16 Oct 2019 09:30:28 +1100
Subject: [PATCH 1/4] Remove two no-op `into()` calls.
---
src/libsyntax/parse/parser.rs | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs
index 9cb410a8ae318..6bbd8be0cb982 100644
--- a/src/libsyntax/parse/parser.rs
+++ b/src/libsyntax/parse/parser.rs
@@ -285,10 +285,10 @@ impl TokenCursor {
token::NoDelim,
&if doc_comment_style(&name.as_str()) == AttrStyle::Inner {
[TokenTree::token(token::Pound, sp), TokenTree::token(token::Not, sp), body]
- .iter().cloned().collect::<TokenStream>().into()
+ .iter().cloned().collect::<TokenStream>()
} else {
[TokenTree::token(token::Pound, sp), body]
- .iter().cloned().collect::<TokenStream>().into()
+ .iter().cloned().collect::<TokenStream>()
},
)));
From a6eef299d3b0ca24f8ffc0c3dc03283c09ec7945 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <nnethercote@mozilla.com>
Date: Mon, 14 Oct 2019 10:37:21 +1100
Subject: [PATCH 2/4] Make `TokenStream::from_iter` less general and more
efficient.
The current code has this impl:
```
impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream
```
If given an `IntoIterator<Item = TokenTree>`, it will convert each individual
`TokenTree` to a `TokenStream` (at the cost of two allocations: a `Vec`
and an `Lrc`). It will then merge those `TokenStream`s into a single
`TokenStream`. This is inefficient.
This commit changes the impl to this less general one:
```
impl iter::FromIterator<TokenTree> for TokenStream
```
It collects the `TokenTree`s into a single `Vec` first and then converts that
to a `TokenStream` by wrapping it in a single `Lrc`. The previous generality
was unnecessary; no other code needs changing.
This change speeds up several benchmarks by up to 4%.
---
src/libsyntax/tokenstream.rs | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index db6832d642322..3d89e73d729f3 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -202,9 +202,9 @@ impl From<TokenTree> for TreeAndJoint {
}
}
-impl<T: Into<TokenStream>> iter::FromIterator<T> for TokenStream {
- fn from_iter<I: IntoIterator<Item = T>>(iter: I) -> Self {
- TokenStream::from_streams(iter.into_iter().map(Into::into).collect::<SmallVec<_>>())
+impl iter::FromIterator<TokenTree> for TokenStream {
+ fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
+ TokenStream::new(iter.into_iter().map(Into::into).collect::<Vec<TreeAndJoint>>())
}
}
From 212ae58f36e4c9f2e0f46153f1c50a657e6c25db Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <nnethercote@mozilla.com>
Date: Mon, 14 Oct 2019 11:24:46 +1100
Subject: [PATCH 3/4] Change `Lit::tokens()` to `Lit::token_tree()`.
Because most of the call sites have an easier time working with a
`TokenTree` instead of a `TokenStream`.
---
src/libsyntax/attr/mod.rs | 9 +++++----
src/libsyntax/parse/literal.rs | 8 ++++----
src/libsyntax/parse/parser/attr.rs | 9 ++++-----
3 files changed, 13 insertions(+), 13 deletions(-)
diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs
index 402c2cad72f99..8786c850bfdd2 100644
--- a/src/libsyntax/attr/mod.rs
+++ b/src/libsyntax/attr/mod.rs
@@ -541,9 +541,10 @@ impl MetaItemKind {
match *self {
MetaItemKind::Word => TokenStream::default(),
MetaItemKind::NameValue(ref lit) => {
- let mut vec = vec![TokenTree::token(token::Eq, span).into()];
- lit.tokens().append_to_tree_and_joint_vec(&mut vec);
- TokenStream::new(vec)
+ TokenStream::new(vec![
+ TokenTree::token(token::Eq, span).into(),
+ lit.token_tree().into(),
+ ])
}
MetaItemKind::List(ref list) => {
let mut tokens = Vec::new();
@@ -606,7 +607,7 @@ impl NestedMetaItem {
fn tokens(&self) -> TokenStream {
match *self {
NestedMetaItem::MetaItem(ref item) => item.tokens(),
- NestedMetaItem::Literal(ref lit) => lit.tokens(),
+ NestedMetaItem::Literal(ref lit) => lit.token_tree().into(),
}
}
diff --git a/src/libsyntax/parse/literal.rs b/src/libsyntax/parse/literal.rs
index 14e1696610a4d..7952e293a532d 100644
--- a/src/libsyntax/parse/literal.rs
+++ b/src/libsyntax/parse/literal.rs
@@ -3,7 +3,7 @@
use crate::ast::{self, Lit, LitKind};
use crate::parse::token::{self, Token};
use crate::symbol::{kw, sym, Symbol};
-use crate::tokenstream::{TokenStream, TokenTree};
+use crate::tokenstream::TokenTree;
use log::debug;
use rustc_data_structures::sync::Lrc;
@@ -216,13 +216,13 @@ impl Lit {
Lit { token: kind.to_lit_token(), kind, span }
}
- /// Losslessly convert an AST literal into a token stream.
- crate fn tokens(&self) -> TokenStream {
+ /// Losslessly convert an AST literal into a token tree.
+ crate fn token_tree(&self) -> TokenTree {
let token = match self.token.kind {
token::Bool => token::Ident(self.token.symbol, false),
_ => token::Literal(self.token),
};
- TokenTree::token(token, self.span).into()
+ TokenTree::token(token, self.span)
}
}
diff --git a/src/libsyntax/parse/parser/attr.rs b/src/libsyntax/parse/parser/attr.rs
index 6f7d1ead4c17c..188a144cac9de 100644
--- a/src/libsyntax/parse/parser/attr.rs
+++ b/src/libsyntax/parse/parser/attr.rs
@@ -6,7 +6,6 @@ use crate::tokenstream::{TokenStream, TokenTree};
use crate::source_map::Span;
use log::debug;
-use smallvec::smallvec;
#[derive(Debug)]
enum InnerAttributeParsePolicy<'a> {
@@ -193,15 +192,15 @@ impl<'a> Parser<'a> {
is_interpolated_expr = true;
}
}
- let tokens = if is_interpolated_expr {
+ let token_tree = if is_interpolated_expr {
// We need to accept arbitrary interpolated expressions to continue
// supporting things like `doc = $expr` that work on stable.
// Non-literal interpolated expressions are rejected after expansion.
- self.parse_token_tree().into()
+ self.parse_token_tree()
} else {
- self.parse_unsuffixed_lit()?.tokens()
+ self.parse_unsuffixed_lit()?.token_tree()
};
- TokenStream::from_streams(smallvec![eq.into(), tokens])
+ TokenStream::new(vec![eq.into(), token_tree.into()])
} else {
TokenStream::default()
};
From e4ec4a6da350ae2564971ae826a1bc3ec9a41988 Mon Sep 17 00:00:00 2001
From: Nicholas Nethercote <nnethercote@mozilla.com>
Date: Mon, 14 Oct 2019 14:06:00 +1100
Subject: [PATCH 4/4] Change `MetaItem::tokens()` to
`MetaItem::token_trees_and_joints()`.
Likewise for `NestedMetaItem::tokens()`. Also, add
`MetaItemKind::token_trees_and_joints()`, which `MetaItemKind::tokens()`
now calls.
This avoids some unnecessary `TokenTree` to `TokenStream` conversions,
and removes the need for the clumsy
`TokenStream::append_to_tree_and_joint_vec()`.
---
src/libsyntax/attr/mod.rs | 43 ++++++++++++++++++++++--------------
src/libsyntax/tokenstream.rs | 4 ----
2 files changed, 26 insertions(+), 21 deletions(-)
diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs
index 8786c850bfdd2..4aec50408812f 100644
--- a/src/libsyntax/attr/mod.rs
+++ b/src/libsyntax/attr/mod.rs
@@ -22,7 +22,7 @@ use crate::ptr::P;
use crate::sess::ParseSess;
use crate::symbol::{sym, Symbol};
use crate::ThinVec;
-use crate::tokenstream::{TokenStream, TokenTree, DelimSpan};
+use crate::tokenstream::{DelimSpan, TokenStream, TokenTree, TreeAndJoint};
use crate::GLOBALS;
use log::debug;
@@ -463,7 +463,7 @@ pub fn first_attr_value_str_by_name(attrs: &[Attribute], name: Symbol) -> Option
}
impl MetaItem {
- fn tokens(&self) -> TokenStream {
+ fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> {
let mut idents = vec![];
let mut last_pos = BytePos(0 as u32);
for (i, segment) in self.path.segments.iter().enumerate() {
@@ -477,8 +477,8 @@ impl MetaItem {
idents.push(TokenTree::Token(Token::from_ast_ident(segment.ident)).into());
last_pos = segment.ident.span.hi();
}
- self.kind.tokens(self.span).append_to_tree_and_joint_vec(&mut idents);
- TokenStream::new(idents)
+ idents.extend(self.kind.token_trees_and_joints(self.span));
+ idents
}
fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItem>
@@ -537,14 +537,14 @@ impl MetaItem {
}
impl MetaItemKind {
- pub fn tokens(&self, span: Span) -> TokenStream {
+ pub fn token_trees_and_joints(&self, span: Span) -> Vec<TreeAndJoint> {
match *self {
- MetaItemKind::Word => TokenStream::default(),
+ MetaItemKind::Word => vec![],
MetaItemKind::NameValue(ref lit) => {
- TokenStream::new(vec![
+ vec![
TokenTree::token(token::Eq, span).into(),
lit.token_tree().into(),
- ])
+ ]
}
MetaItemKind::List(ref list) => {
let mut tokens = Vec::new();
@@ -552,17 +552,26 @@ impl MetaItemKind {
if i > 0 {
tokens.push(TokenTree::token(token::Comma, span).into());
}
- item.tokens().append_to_tree_and_joint_vec(&mut tokens);
+ tokens.extend(item.token_trees_and_joints())
}
- TokenTree::Delimited(
- DelimSpan::from_single(span),
- token::Paren,
- TokenStream::new(tokens).into(),
- ).into()
+ vec![
+ TokenTree::Delimited(
+ DelimSpan::from_single(span),
+ token::Paren,
+ TokenStream::new(tokens).into(),
+ ).into()
+ ]
}
}
}
+ // Premature conversions of `TokenTree`s to `TokenStream`s can hurt
+ // performance. Do not use this function if `token_trees_and_joints()` can
+ // be used instead.
+ pub fn tokens(&self, span: Span) -> TokenStream {
+ TokenStream::new(self.token_trees_and_joints(span))
+ }
+
fn from_tokens<I>(tokens: &mut iter::Peekable<I>) -> Option<MetaItemKind>
where I: Iterator<Item = TokenTree>,
{
@@ -604,10 +613,10 @@ impl NestedMetaItem {
}
}
- fn tokens(&self) -> TokenStream {
+ fn token_trees_and_joints(&self) -> Vec<TreeAndJoint> {
match *self {
- NestedMetaItem::MetaItem(ref item) => item.tokens(),
- NestedMetaItem::Literal(ref lit) => lit.token_tree().into(),
+ NestedMetaItem::MetaItem(ref item) => item.token_trees_and_joints(),
+ NestedMetaItem::Literal(ref lit) => vec![lit.token_tree().into()],
}
}
diff --git a/src/libsyntax/tokenstream.rs b/src/libsyntax/tokenstream.rs
index 3d89e73d729f3..ac155556cdae2 100644
--- a/src/libsyntax/tokenstream.rs
+++ b/src/libsyntax/tokenstream.rs
@@ -271,10 +271,6 @@ impl TokenStream {
}
}
- pub fn append_to_tree_and_joint_vec(self, vec: &mut Vec<TreeAndJoint>) {
- vec.extend(self.0.iter().cloned());
- }
-
pub fn trees(&self) -> Cursor {
self.clone().into_trees()
}