[Parser] Do not eagerly lex IDs (#6542)

Lex them on demand instead to avoid wasted work.

Author: tlively

src/parser/lexer.cpp

@@ -1015,21 +1015,6 @@ std::optional<std::string_view> Token::getString() const {
   return {};
 }
 
-std::optional<std::string_view> Token::getID() const {
-  if (auto* tok = std::get_if<IdTok>(&data)) {
-    if (tok->str) {
-      return std::string_view(*tok->str);
-    }
-    if (tok->isStr) {
-      // Remove '$' and quotes.
-      return span.substr(2, span.size() - 3);
-    }
-    // Remove '$'.
-    return span.substr(1);
-  }
-  return {};
-}
-
 void Lexer::skipSpace() {
   while (true) {
     if (auto ctx = annotation(next())) {
@@ -1069,6 +1054,26 @@ bool Lexer::takeRParen() {
   return false;
 }
 
+std::optional<Name> Lexer::takeID() {
+  if (curr) {
+    return std::nullopt;
+  }
+  if (auto result = ident(next())) {
+    index += result->span.size();
+    advance();
+    if (result->str) {
+      return Name(*result->str);
+    }
+    if (result->isStr) {
+      // Remove '$' and quotes.
+      return Name(result->span.substr(2, result->span.size() - 3));
+    }
+    // Remove '$'.
+    return Name(result->span.substr(1));
+  }
+  return std::nullopt;
+}
+
 std::optional<std::string_view> Lexer::takeKeyword() {
   if (curr) {
     return std::nullopt;
@@ -1123,9 +1128,7 @@ std::optional<uint32_t> Lexer::takeAlign() {
 void Lexer::lexToken() {
   // TODO: Ensure we're getting the longest possible match.
   Token tok;
-  if (auto t = ident(next())) {
-    tok = Token{t->span, IdTok{t->isStr, t->str}};
-  } else if (auto t = integer(next())) {
+  if (auto t = integer(next())) {
     tok = Token{t->span, IntTok{t->n, t->sign}};
   } else if (auto t = float_(next())) {
     tok = Token{t->span, FloatTok{t->nanPayload, t->d}};
@@ -1186,8 +1189,6 @@ std::ostream& operator<<(std::ostream& os, const TextPos& pos) {
   return os << pos.line << ":" << pos.col;
 }
 
-std::ostream& operator<<(std::ostream& os, const IdTok&) { return os << "id"; }
-
 std::ostream& operator<<(std::ostream& os, const IntTok& tok) {
   return os << (tok.sign == Pos ? "+" : tok.sign == Neg ? "-" : "") << tok.n;
 }

src/parser/lexer.h

@@ -45,17 +45,6 @@ struct TextPos {
 // Tokens
 // ======
 
-struct IdTok {
-  // Whether this ID has `$"..."` format
-  bool isStr;
-
-  // If the ID is a string ID and contains escapes, this is its contents.
-  std::optional<std::string> str;
-
-  bool operator==(const IdTok&) const { return true; }
-  friend std::ostream& operator<<(std::ostream&, const IdTok&);
-};
-
 enum Sign { NoSign, Pos, Neg };
 
 struct IntTok {
@@ -88,7 +77,7 @@ struct StringTok {
 };
 
 struct Token {
-  using Data = std::variant<IdTok, IntTok, FloatTok, StringTok>;
+  using Data = std::variant<IntTok, FloatTok, StringTok>;
   std::string_view span;
   Data data;
 
@@ -102,7 +91,6 @@ struct Token {
   std::optional<double> getF64() const;
   std::optional<float> getF32() const;
   std::optional<std::string_view> getString() const;
-  std::optional<std::string_view> getID() const;
 
   bool operator==(const Token&) const;
   friend std::ostream& operator<<(std::ostream& os, const Token&);
@@ -164,16 +152,7 @@ struct Lexer {
     }
   }
 
-  std::optional<Name> takeID() {
-    if (curr) {
-      if (auto id = curr->getID()) {
-        advance();
-        // See comment on takeName.
-        return Name(std::string(*id));
-      }
-    }
-    return {};
-  }
+  std::optional<Name> takeID();
 
   std::optional<std::string_view> takeKeyword();
   bool takeKeyword(std::string_view expected);