Skip to content

Commit cd04959

Browse files
brendanzabbrendanzab
committed
Use an enum rather than a bool in token::Ident
1 parent fcb78d6 commit cd04959

File tree

6 files changed

+96
-57
lines changed

6 files changed

+96
-57
lines changed

‎src/grammar/verify.rs

+3-2
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ use syntax::parse::lexer::TokenAndSpan;
3535

3636
fn parse_token_list(file: &str) -> HashMap<String, Token> {
3737
fn id() -> Token {
38-
token::Ident(ast::Ident { name: Name(0), ctxt: 0, }, false)
38+
token::Ident(ast::Ident { name: Name(0), ctxt: 0, }, token::Plain)
3939
}
4040

4141
let mut res = HashMap::new();
@@ -198,7 +198,8 @@ fn parse_antlr_token(s: &str, tokens: &HashMap<String, Token>) -> TokenAndSpan {
198198
token::LitFloat(..) => token::LitFloat(nm),
199199
token::LitBinary(..) => token::LitBinary(nm),
200200
token::LitBinaryRaw(..) => token::LitBinaryRaw(fix(content), count(content)),
201-
token::Ident(..) => token::Ident(ast::Ident { name: nm, ctxt: 0 }, true),
201+
token::Ident(..) => token::Ident(ast::Ident { name: nm, ctxt: 0 },
202+
token::ModName),
202203
token::Lifetime(..) => token::Lifetime(ast::Ident { name: nm, ctxt: 0 }),
203204
ref t => t.clone()
204205
};

‎src/libsyntax/ext/quote.rs

+7-2
Original file line numberDiff line numberDiff line change
@@ -531,6 +531,7 @@ fn mk_binop(cx: &ExtCtxt, sp: Span, bop: token::BinOpToken) -> P<ast::Expr> {
531531
mk_token_path(cx, sp, name)
532532
}
533533

534+
#[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot
534535
fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
535536
match *tok {
536537
token::BinOp(binop) => {
@@ -575,10 +576,14 @@ fn mk_token(cx: &ExtCtxt, sp: Span, tok: &token::Token) -> P<ast::Expr> {
575576
vec!(mk_name(cx, sp, ident.ident()), cx.expr_uint(sp, n)));
576577
}
577578

578-
token::Ident(ident, b) => {
579+
token::Ident(ident, style) => {
579580
return cx.expr_call(sp,
580581
mk_token_path(cx, sp, "Ident"),
581-
vec!(mk_ident(cx, sp, ident), cx.expr_bool(sp, b)));
582+
vec![mk_ident(cx, sp, ident),
583+
match style {
584+
ModName => mk_token_path(cx, sp, "ModName"),
585+
Plain => mk_token_path(cx, sp, "Plain"),
586+
}]);
582587
}
583588

584589
token::Lifetime(ident) => {

‎src/libsyntax/parse/lexer/mod.rs

+36-25
Original file line numberDiff line numberDiff line change
@@ -921,12 +921,14 @@ impl<'a> StringReader<'a> {
921921
if string == "_" {
922922
token::Underscore
923923
} else {
924-
let is_mod_name = self.curr_is(':') && self.nextch_is(':');
925-
926924
// FIXME: perform NFKC normalization here. (Issue #2253)
927-
token::Ident(str_to_ident(string), is_mod_name)
925+
if self.curr_is(':') && self.nextch_is(':') {
926+
token::Ident(str_to_ident(string), token::ModName)
927+
} else {
928+
token::Ident(str_to_ident(string), token::Plain)
929+
}
928930
}
929-
})
931+
});
930932
}
931933

932934
if is_dec_digit(c) {
@@ -937,8 +939,11 @@ impl<'a> StringReader<'a> {
937939
match (c.unwrap(), self.nextch(), self.nextnextch()) {
938940
('\x00', Some('n'), Some('a')) => {
939941
let ast_ident = self.scan_embedded_hygienic_ident();
940-
let is_mod_name = self.curr_is(':') && self.nextch_is(':');
941-
return token::Ident(ast_ident, is_mod_name);
942+
return if self.curr_is(':') && self.nextch_is(':') {
943+
token::Ident(ast_ident, token::ModName)
944+
} else {
945+
token::Ident(ast_ident, token::Plain)
946+
};
942947
}
943948
_ => {}
944949
}
@@ -1056,7 +1061,7 @@ impl<'a> StringReader<'a> {
10561061
str_to_ident(lifetime_name)
10571062
});
10581063
let keyword_checking_token =
1059-
&token::Ident(keyword_checking_ident, false);
1064+
&token::Ident(keyword_checking_ident, token::Plain);
10601065
let last_bpos = self.last_pos;
10611066
if keyword_checking_token.is_keyword(token::keywords::Self) {
10621067
self.err_span_(start,
@@ -1434,7 +1439,7 @@ mod test {
14341439
assert_eq!(string_reader.next_token().tok, token::Whitespace);
14351440
let tok1 = string_reader.next_token();
14361441
let tok2 = TokenAndSpan{
1437-
tok:token::Ident(id, false),
1442+
tok:token::Ident(id, token::Plain),
14381443
sp:Span {lo:BytePos(21),hi:BytePos(23),expn_id: NO_EXPANSION}};
14391444
assert_eq!(tok1,tok2);
14401445
assert_eq!(string_reader.next_token().tok, token::Whitespace);
@@ -1443,7 +1448,7 @@ mod test {
14431448
// read another token:
14441449
let tok3 = string_reader.next_token();
14451450
let tok4 = TokenAndSpan{
1446-
tok:token::Ident(str_to_ident("main"), false),
1451+
tok:token::Ident(str_to_ident("main"), token::Plain),
14471452
sp:Span {lo:BytePos(24),hi:BytePos(28),expn_id: NO_EXPANSION}};
14481453
assert_eq!(tok3,tok4);
14491454
// the lparen is already read:
@@ -1458,39 +1463,45 @@ mod test {
14581463
}
14591464
}
14601465

1461-
// make the identifier by looking up the string in the interner
1466+
#[cfg(stage0)]
14621467
fn mk_ident (id: &str, is_mod_name: bool) -> token::Token {
1463-
token::Ident (str_to_ident(id),is_mod_name)
1468+
token::Ident(str_to_ident(id), is_mod_name)
1469+
}
1470+
1471+
// make the identifier by looking up the string in the interner
1472+
#[cfg(not(stage0))]
1473+
fn mk_ident(id: &str, style: token::IdentStyle) -> token::Token {
1474+
token::Ident(str_to_ident(id), style)
14641475
}
14651476

14661477
#[test] fn doublecolonparsing () {
14671478
check_tokenization(setup(&mk_sh(), "a b".to_string()),
1468-
vec!(mk_ident("a",false),
1469-
token::Whitespace,
1470-
mk_ident("b",false)));
1479+
vec![mk_ident("a", token::Plain),
1480+
token::Whitespace,
1481+
mk_ident("b", token::Plain)]);
14711482
}
14721483

14731484
#[test] fn dcparsing_2 () {
14741485
check_tokenization(setup(&mk_sh(), "a::b".to_string()),
1475-
vec!(mk_ident("a",true),
1476-
token::ModSep,
1477-
mk_ident("b",false)));
1486+
vec![mk_ident("a",token::ModName),
1487+
token::ModSep,
1488+
mk_ident("b", token::Plain)]);
14781489
}
14791490

14801491
#[test] fn dcparsing_3 () {
14811492
check_tokenization(setup(&mk_sh(), "a ::b".to_string()),
1482-
vec!(mk_ident("a",false),
1483-
token::Whitespace,
1484-
token::ModSep,
1485-
mk_ident("b",false)));
1493+
vec![mk_ident("a", token::Plain),
1494+
token::Whitespace,
1495+
token::ModSep,
1496+
mk_ident("b", token::Plain)]);
14861497
}
14871498

14881499
#[test] fn dcparsing_4 () {
14891500
check_tokenization(setup(&mk_sh(), "a:: b".to_string()),
1490-
vec!(mk_ident("a",true),
1491-
token::ModSep,
1492-
token::Whitespace,
1493-
mk_ident("b",false)));
1501+
vec![mk_ident("a",token::ModName),
1502+
token::ModSep,
1503+
token::Whitespace,
1504+
mk_ident("b", token::Plain)]);
14941505
}
14951506

14961507
#[test] fn character_a() {

‎src/libsyntax/parse/mod.rs

+9-9
Original file line numberDiff line numberDiff line change
@@ -793,9 +793,9 @@ mod test {
793793
let tts = string_to_tts("macro_rules! zip (($a)=>($a))".to_string());
794794
let tts: &[ast::TokenTree] = tts.as_slice();
795795
match tts {
796-
[ast::TtToken(_, token::Ident(name_macro_rules, false)),
796+
[ast::TtToken(_, token::Ident(name_macro_rules, token::Plain)),
797797
ast::TtToken(_, token::Not),
798-
ast::TtToken(_, token::Ident(name_zip, false)),
798+
ast::TtToken(_, token::Ident(name_zip, token::Plain)),
799799
ast::TtDelimited(_, ref macro_delimed)]
800800
if name_macro_rules.as_str() == "macro_rules"
801801
&& name_zip.as_str() == "zip" => {
@@ -810,7 +810,7 @@ mod test {
810810
match (first_open, first_tts.as_slice(), first_close) {
811811
(&ast::Delimiter { token: token::LParen, .. },
812812
[ast::TtToken(_, token::Dollar),
813-
ast::TtToken(_, token::Ident(name, false))],
813+
ast::TtToken(_, token::Ident(name, token::Plain))],
814814
&ast::Delimiter { token: token::RParen, .. })
815815
if name.as_str() == "a" => {},
816816
_ => fail!("value 3: {}", **first_delimed),
@@ -819,7 +819,7 @@ mod test {
819819
match (second_open, second_tts.as_slice(), second_close) {
820820
(&ast::Delimiter { token: token::LParen, .. },
821821
[ast::TtToken(_, token::Dollar),
822-
ast::TtToken(_, token::Ident(name, false))],
822+
ast::TtToken(_, token::Ident(name, token::Plain))],
823823
&ast::Delimiter { token: token::RParen, .. })
824824
if name.as_str() == "a" => {},
825825
_ => fail!("value 4: {}", **second_delimed),
@@ -845,7 +845,7 @@ mod test {
845845
\"variant\":\"Ident\",\
846846
\"fields\":[\
847847
\"fn\",\
848-
false\
848+
\"Plain\"\
849849
]\
850850
}\
851851
]\
@@ -858,7 +858,7 @@ mod test {
858858
\"variant\":\"Ident\",\
859859
\"fields\":[\
860860
\"a\",\
861-
false\
861+
\"Plain\"\
862862
]\
863863
}\
864864
]\
@@ -881,7 +881,7 @@ mod test {
881881
\"variant\":\"Ident\",\
882882
\"fields\":[\
883883
\"b\",\
884-
false\
884+
\"Plain\"\
885885
]\
886886
}\
887887
]\
@@ -901,7 +901,7 @@ mod test {
901901
\"variant\":\"Ident\",\
902902
\"fields\":[\
903903
\"int\",\
904-
false\
904+
\"Plain\"\
905905
]\
906906
}\
907907
]\
@@ -932,7 +932,7 @@ mod test {
932932
\"variant\":\"Ident\",\
933933
\"fields\":[\
934934
\"b\",\
935-
false\
935+
\"Plain\"\
936936
]\
937937
}\
938938
]\

‎src/libsyntax/parse/parser.rs

+6-6
Original file line numberDiff line numberDiff line change
@@ -2067,10 +2067,10 @@ impl<'a> Parser<'a> {
20672067
},
20682068
// FIXME #13626: Should be able to stick in
20692069
// token::SELF_KEYWORD_NAME
2070-
token::Ident(id @ ast::Ident{
2071-
name: ast::Name(token::SELF_KEYWORD_NAME_NUM),
2072-
ctxt: _
2073-
} ,false) => {
2070+
token::Ident(id @ ast::Ident {
2071+
name: ast::Name(token::SELF_KEYWORD_NAME_NUM),
2072+
ctxt: _
2073+
}, token::Plain) => {
20742074
self.bump();
20752075
let path = ast_util::ident_to_path(mk_sp(lo, hi), id);
20762076
ex = ExprPath(path);
@@ -4094,14 +4094,14 @@ impl<'a> Parser<'a> {
40944094

40954095
fn is_self_ident(&mut self) -> bool {
40964096
match self.token {
4097-
token::Ident(id, false) => id.name == special_idents::self_.name,
4097+
token::Ident(id, token::Plain) => id.name == special_idents::self_.name,
40984098
_ => false
40994099
}
41004100
}
41014101

41024102
fn expect_self_ident(&mut self) -> ast::Ident {
41034103
match self.token {
4104-
token::Ident(id, false) if id.name == special_idents::self_.name => {
4104+
token::Ident(id, token::Plain) if id.name == special_idents::self_.name => {
41054105
self.bump();
41064106
id
41074107
},

‎src/libsyntax/parse/token.rs

+35-13
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,21 @@ pub enum BinOpToken {
9898
Shr,
9999
}
100100

101+
#[cfg(stage0)]
102+
#[allow(non_uppercase_statics)]
103+
pub const ModName: bool = true;
104+
#[cfg(stage0)]
105+
#[allow(non_uppercase_statics)]
106+
pub const Plain: bool = false;
107+
108+
#[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)]
109+
#[cfg(not(stage0))]
110+
pub enum IdentStyle {
111+
/// `::` follows the identifier with no whitespace in-between.
112+
ModName,
113+
Plain,
114+
}
115+
101116
#[allow(non_camel_case_types)]
102117
#[deriving(Clone, Encodable, Decodable, PartialEq, Eq, Hash, Show)]
103118
pub enum Token {
@@ -149,10 +164,10 @@ pub enum Token {
149164
LitBinaryRaw(ast::Name, uint), /* raw binary str delimited by n hash symbols */
150165

151166
/* Name components */
152-
/// An identifier contains an "is_mod_name" boolean,
153-
/// indicating whether :: follows this token with no
154-
/// whitespace in between.
167+
#[cfg(stage0)]
155168
Ident(ast::Ident, bool),
169+
#[cfg(not(stage0))]
170+
Ident(ast::Ident, IdentStyle),
156171
Underscore,
157172
Lifetime(ast::Ident),
158173

@@ -252,10 +267,11 @@ impl Token {
252267

253268
/// Returns `true` if the token is a path that is not followed by a `::`
254269
/// token.
270+
#[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot
255271
pub fn is_plain_ident(&self) -> bool {
256272
match *self {
257-
Ident(_, false) => true,
258-
_ => false,
273+
Ident(_, Plain) => true,
274+
_ => false,
259275
}
260276
}
261277

@@ -299,18 +315,20 @@ impl Token {
299315
}
300316

301317
/// Returns `true` if the token is a given keyword, `kw`.
318+
#[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot
302319
pub fn is_keyword(&self, kw: keywords::Keyword) -> bool {
303320
match *self {
304-
Ident(sid, false) => kw.to_name() == sid.name,
305-
_ => false,
321+
Ident(sid, Plain) => kw.to_name() == sid.name,
322+
_ => false,
306323
}
307324
}
308325

309326
/// Returns `true` if the token is either a special identifier, or a strict
310327
/// or reserved keyword.
328+
#[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot
311329
pub fn is_any_keyword(&self) -> bool {
312330
match *self {
313-
Ident(sid, false) => {
331+
Ident(sid, Plain) => {
314332
let n = sid.name;
315333

316334
n == SELF_KEYWORD_NAME
@@ -324,9 +342,10 @@ impl Token {
324342
}
325343

326344
/// Returns `true` if the token may not appear as an identifier.
345+
#[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot
327346
pub fn is_strict_keyword(&self) -> bool {
328347
match *self {
329-
Ident(sid, false) => {
348+
Ident(sid, Plain) => {
330349
let n = sid.name;
331350

332351
n == SELF_KEYWORD_NAME
@@ -335,7 +354,7 @@ impl Token {
335354
|| STRICT_KEYWORD_START <= n
336355
&& n <= STRICT_KEYWORD_FINAL
337356
},
338-
Ident(sid, true) => {
357+
Ident(sid, ModName) => {
339358
let n = sid.name;
340359

341360
n != SELF_KEYWORD_NAME
@@ -349,9 +368,10 @@ impl Token {
349368

350369
/// Returns `true` if the token is a keyword that has been reserved for
351370
/// possible future use.
371+
#[allow(non_uppercase_statics)] // NOTE(stage0): remove this attribute after the next snapshot
352372
pub fn is_reserved_keyword(&self) -> bool {
353373
match *self {
354-
Ident(sid, false) => {
374+
Ident(sid, Plain) => {
355375
let n = sid.name;
356376

357377
RESERVED_KEYWORD_START <= n
@@ -382,8 +402,10 @@ pub enum Nonterminal {
382402
NtPat( P<ast::Pat>),
383403
NtExpr( P<ast::Expr>),
384404
NtTy( P<ast::Ty>),
385-
/// See IDENT, above, for meaning of bool in NtIdent:
405+
#[cfg(stage0)]
386406
NtIdent(Box<ast::Ident>, bool),
407+
#[cfg(not(stage0))]
408+
NtIdent(Box<ast::Ident>, IdentStyle),
387409
/// Stuff inside brackets for attributes
388410
NtMeta( P<ast::MetaItem>),
389411
NtPath(Box<ast::Path>),
@@ -857,6 +879,6 @@ mod test {
857879
assert!(Gt.mtwt_eq(&Gt));
858880
let a = str_to_ident("bac");
859881
let a1 = mark_ident(a,92);
860-
assert!(Ident(a,true).mtwt_eq(&Ident(a1,false)));
882+
assert!(Ident(a, ModName).mtwt_eq(&Ident(a1, Plain)));
861883
}
862884
}

0 commit comments

Comments
 (0)