Update grammar to allow e-suffixes

This is aiming to match the behaviour of #131656

 - Integer_literal: use SUFFIX rather than SUFFIX_NO_E
 - FLOAT_BODY_WITHOUT_EXPONENT: remove eE not-predicate before suffix
 - reserved_float_empty_exponent:
   - no longer apply if sign is absent
   - move before float_body_without_exponent in priority order
 - reserved_float_based:
   - no longer apply if both sign and exponent are absent

Author: mattheww

src/lex_via_peg/pretokenisation/pest_pretokeniser.rs

@@ -173,7 +173,7 @@ fn interpret_pest_pair(pair: Pair<Rule>) -> Result<PretokenData, &'static str> {
         }
         Rule::Unterminated_literal_2015 | Rule::Reserved_literal_2021 => Ok(PretokenData::Reserved),
         Rule::Reserved_guard_2024 => Ok(PretokenData::Reserved),
-        Rule::Float_literal => {
+        Rule::Float_literal_1 | Rule::Float_literal_2 => {
             let mut body = None;
             let mut suffix = None;
             for sub in pair.into_inner().flatten() {
@@ -192,7 +192,9 @@ fn interpret_pest_pair(pair: Pair<Rule>) -> Result<PretokenData, &'static str> {
                 suffix: suffix.map(Into::into),
             })
         }
-        Rule::Reserved_float => Ok(PretokenData::Reserved),
+        Rule::Reserved_float_empty_exponent | Rule::Reserved_float_based => {
+            Ok(PretokenData::Reserved)
+        }
         Rule::Integer_literal => {
             let mut base = None;
             let mut digits = None;

src/lex_via_peg/pretokenisation/pretokenise.pest

@@ -8,8 +8,10 @@ PRETOKEN_2015 = {
     Double_quoted_literal_2015 |
     Raw_double_quoted_literal_2015 |
     Unterminated_literal_2015 |
-    Float_literal |
-    Reserved_float |
+    Float_literal_1 |
+    Reserved_float_empty_exponent |
+    Float_literal_2 |
+    Reserved_float_based |
     Integer_literal |
     Lifetime_or_label |
     Raw_identifier |
@@ -27,8 +29,10 @@ PRETOKEN_2021 = {
     Double_quoted_literal_2021 |
     Raw_double_quoted_literal_2021 |
     Reserved_literal_2021 |
-    Float_literal |
-    Reserved_float |
+    Float_literal_1 |
+    Reserved_float_empty_exponent |
+    Float_literal_2 |
+    Reserved_float_based |
     Integer_literal |
     Raw_lifetime_or_label_2021 |
     Reserved_lifetime_or_label_prefix_2021 |
@@ -49,8 +53,10 @@ PRETOKEN_2024 = {
     Raw_double_quoted_literal_2021 |
     Reserved_literal_2021 |
     Reserved_guard_2024 |
-    Float_literal |
-    Reserved_float |
+    Float_literal_1 |
+    Reserved_float_empty_exponent |
+    Float_literal_2 |
+    Reserved_float_based |
     Integer_literal |
     Raw_lifetime_or_label_2021 |
     Reserved_lifetime_or_label_prefix_2021 |
@@ -166,9 +172,11 @@ DECIMAL_PART = { '0'..'9' ~ DECIMAL_DIGITS }
 
 
 // ANCHOR: float_literal
-Float_literal = {
-    FLOAT_BODY_WITH_EXPONENT ~ SUFFIX ? |
-    FLOAT_BODY_WITHOUT_EXPONENT ~ !("e"|"E") ~ SUFFIX ? |
+Float_literal_1 = {
+    FLOAT_BODY_WITH_EXPONENT ~ SUFFIX ?
+}
+Float_literal_2 = {
+    FLOAT_BODY_WITHOUT_EXPONENT ~ SUFFIX ? |
     FLOAT_BODY_WITH_FINAL_DOT ~ !"." ~ !IDENT_START
 }
 
@@ -188,40 +196,34 @@ FLOAT_BODY_WITH_FINAL_DOT = {
 // ANCHOR_END: float_literal
 
 // ANCHOR: reserved_float
-Reserved_float = {
-    RESERVED_FLOAT_EMPTY_EXPONENT | RESERVED_FLOAT_BASED
-}
-RESERVED_FLOAT_EMPTY_EXPONENT = {
+Reserved_float_empty_exponent = {
     DECIMAL_PART ~ ("." ~ DECIMAL_PART ) ? ~
-    ("e"|"E") ~ ("+"|"-") ?
+    ("e"|"E") ~ ("+"|"-")
 }
-RESERVED_FLOAT_BASED = {
+Reserved_float_based = {
     (
         ("0b" | "0o") ~ LOW_BASE_PRETOKEN_DIGITS |
         "0x" ~ HEXADECIMAL_DIGITS
     )  ~  (
-        ("e"|"E") |
+        ("e"|"E") ~ ("+"|"-" | EXPONENT_DIGITS) |
         "." ~ !"." ~ !IDENT_START
     )
 }
 // ANCHOR_END: reserved_float
 
-
 // ANCHOR: integer_literals
 Integer_literal = {
     ( INTEGER_BINARY_LITERAL |
       INTEGER_OCTAL_LITERAL |
       INTEGER_HEXADECIMAL_LITERAL |
       INTEGER_DECIMAL_LITERAL ) ~
-    SUFFIX_NO_E ?
+    SUFFIX ?
 }
 
 INTEGER_BINARY_LITERAL = { "0b" ~ LOW_BASE_PRETOKEN_DIGITS }
 INTEGER_OCTAL_LITERAL = { "0o" ~ LOW_BASE_PRETOKEN_DIGITS }
 INTEGER_HEXADECIMAL_LITERAL = { "0x" ~ HEXADECIMAL_DIGITS }
 INTEGER_DECIMAL_LITERAL = { DECIMAL_PART }
-
-SUFFIX_NO_E = { !("e"|"E") ~ SUFFIX }
 // ANCHOR_END: integer_literals
 
 

writeup/numeric_literal_pretokens.md

@@ -27,6 +27,9 @@ The following nonterminals are common to the definitions below:
 > Note: The `! "."` subexpression makes sure that forms like `1..2` aren't treated as starting with a float.
 > The `! IDENT_START` subexpression makes sure that forms like `1.some_method()` aren't treated as starting with a float.
 
+> Note: The `Reserved_float_empty_exponent` pretoken nonterminal is placed between `Float_literal_1` and `Float_literal_2` in priority order
+> (which is why there are two pretoken nonterminals producing `FloatLiteral`).
+
 
 #### Reserved float { .rule }
 
@@ -41,6 +44,8 @@ The following nonterminals are common to the definitions below:
 ##### Attributes
 (none)
 
+> Note: The `Reserved_float_empty_exponent` pretoken nonterminal is placed between `Float_literal_1` and `Float_literal_2` in priority order.
+> This ordering makes sure that forms like `123.4e+` are reserved, rather than being accepted by `FLOAT_BODY_WITHOUT_EXPONENT`).
 
 #### Integer literal { .rule }
 

writeup/pretokenising.md

@@ -27,6 +27,7 @@ It's also available on a [single page](complete_pretoken_grammar.md).
 
 The pretoken nonterminals are presented in an order consistent with their appearance in the edition nonterminals.
 That means they appear in priority order (highest priority first).
+There is one exception, for floating-point literals and their related reserved forms (see [Float literal]).
 
 
 ### Extracting pretokens
@@ -51,6 +52,7 @@ Each pretoken nonterminal produces a single kind of pretoken.
 In most cases a given kind of pretoken is produced only by a single pretoken nonterminal.
 The exceptions are:
 - Several pretoken nonterminals produce `Reserved` pretokens.
+- There are two pretoken nonterminals producing `FloatLiteral` pretokens.
 - In some cases there are variant pretoken nonterminals for different editions.
 
 Each pretoken nonterminal (or group of edition variants) has a subsection on the following pages,
@@ -79,3 +81,5 @@ it's a bug in this specification.
 In other cases the attributes table entry defines the attribute value explicitly,
 depending on the characters consumed by the pretoken nonterminal or on which subexpression of the pretoken nonterminal matched.
 
+
+[Float literal]: numeric_literal_pretokens.md#float-literal