lexer: convert Token to be an ES6 class (#2234)

Author: IvanGoncharov

src/language/ast.d.ts

@@ -38,7 +38,7 @@ export class Location {
  * Represents a range of characters represented by a lexical token
  * within a Source.
  */
-export interface Token {
+export class Token {
   /**
    * The kind of Token.
    */
@@ -76,6 +76,16 @@ export interface Token {
    */
   readonly prev: Token | null;
   readonly next: Token | null;
+
+  constructor(
+    kind: TokenKindEnum,
+    start: number,
+    end: number,
+    line: number,
+    column: number,
+    prev: Token | null,
+    value?: string,
+  );
 }
 
 /**

src/language/ast.js

@@ -53,47 +53,74 @@ defineToJSON(Location, function() {
  * Represents a range of characters represented by a lexical token
  * within a Source.
  */
-export type Token = {
+export class Token {
   /**
    * The kind of Token.
    */
-  +kind: TokenKindEnum,
+  +kind: TokenKindEnum;
 
   /**
    * The character offset at which this Node begins.
    */
-  +start: number,
+  +start: number;
 
   /**
    * The character offset at which this Node ends.
    */
-  +end: number,
+  +end: number;
 
   /**
    * The 1-indexed line number on which this Token appears.
    */
-  +line: number,
+  +line: number;
 
   /**
    * The 1-indexed column number at which this Token begins.
    */
-  +column: number,
+  +column: number;
 
   /**
    * For non-punctuation tokens, represents the interpreted value of the token.
    */
-  +value: string | void,
+  +value: string | void;
 
   /**
    * Tokens exist as nodes in a double-linked-list amongst all tokens
    * including ignored tokens. <SOF> is always the first node and <EOF>
    * the last.
    */
-  +prev: Token | null,
-  +next: Token | null,
+  +prev: Token | null;
+  +next: Token | null;
+
+  constructor(
+    kind: TokenKindEnum,
+    start: number,
+    end: number,
+    line: number,
+    column: number,
+    prev: Token | null,
+    value?: string,
+  ) {
+    this.kind = kind;
+    this.start = start;
+    this.end = end;
+    this.line = line;
+    this.column = column;
+    this.value = value;
+    this.prev = prev;
+    this.next = null;
+  }
+}
 
-  ...
-};
+// Print a simplified form when appearing in JSON/util.inspect.
+defineToJSON(Token, function() {
+  return {
+    kind: this.kind,
+    value: this.value,
+    line: this.line,
+    column: this.column,
+  };
+});
 
 /**
  * The list of all possible AST node types.

src/language/lexer.js

@@ -1,10 +1,8 @@
 // @flow strict
 
-import defineToJSON from '../jsutils/defineToJSON';
-
 import { syntaxError } from '../error/syntaxError';
 
-import { type Token } from './ast';
+import { Token } from './ast';
 import { type Source } from './source';
 import { dedentBlockStringValue } from './blockString';
 import { type TokenKindEnum, TokenKind } from './tokenKind';
@@ -41,7 +39,7 @@ export class Lexer {
   lineStart: number;
 
   constructor(source: Source) {
-    const startOfFileToken = new Tok(TokenKind.SOF, 0, 0, 0, 0, null);
+    const startOfFileToken = new Token(TokenKind.SOF, 0, 0, 0, 0, null);
 
     this.source = source;
     this.lastToken = startOfFileToken;
@@ -97,38 +95,6 @@ export function isPunctuatorTokenKind(kind: TokenKindEnum) {
   );
 }
 
-/**
- * Helper function for constructing the Token object.
- */
-function Tok(
-  kind: TokenKindEnum,
-  start: number,
-  end: number,
-  line: number,
-  column: number,
-  prev: Token | null,
-  value?: string,
-) {
-  this.kind = kind;
-  this.start = start;
-  this.end = end;
-  this.line = line;
-  this.column = column;
-  this.value = value;
-  this.prev = prev;
-  this.next = null;
-}
-
-// Print a simplified form when appearing in JSON/util.inspect.
-defineToJSON(Tok, function() {
-  return {
-    kind: this.kind,
-    value: this.value,
-    line: this.line,
-    column: this.column,
-  };
-});
-
 function printCharCode(code) {
   return (
     // NaN/undefined represents access beyond the end of the file.
@@ -159,7 +125,7 @@ function readToken(lexer: Lexer, prev: Token): Token {
   const col = 1 + pos - lexer.lineStart;
 
   if (pos >= bodyLength) {
-    return new Tok(TokenKind.EOF, bodyLength, bodyLength, line, col, prev);
+    return new Token(TokenKind.EOF, bodyLength, bodyLength, line, col, prev);
   }
 
   const code = body.charCodeAt(pos);
@@ -168,52 +134,52 @@ function readToken(lexer: Lexer, prev: Token): Token {
   switch (code) {
     // !
     case 33:
-      return new Tok(TokenKind.BANG, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.BANG, pos, pos + 1, line, col, prev);
     // #
     case 35:
       return readComment(source, pos, line, col, prev);
     // $
     case 36:
-      return new Tok(TokenKind.DOLLAR, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.DOLLAR, pos, pos + 1, line, col, prev);
     // &
     case 38:
-      return new Tok(TokenKind.AMP, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.AMP, pos, pos + 1, line, col, prev);
     // (
     case 40:
-      return new Tok(TokenKind.PAREN_L, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.PAREN_L, pos, pos + 1, line, col, prev);
     // )
     case 41:
-      return new Tok(TokenKind.PAREN_R, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.PAREN_R, pos, pos + 1, line, col, prev);
     // .
     case 46:
       if (body.charCodeAt(pos + 1) === 46 && body.charCodeAt(pos + 2) === 46) {
-        return new Tok(TokenKind.SPREAD, pos, pos + 3, line, col, prev);
+        return new Token(TokenKind.SPREAD, pos, pos + 3, line, col, prev);
       }
       break;
     // :
     case 58:
-      return new Tok(TokenKind.COLON, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.COLON, pos, pos + 1, line, col, prev);
     // =
     case 61:
-      return new Tok(TokenKind.EQUALS, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.EQUALS, pos, pos + 1, line, col, prev);
     // @
     case 64:
-      return new Tok(TokenKind.AT, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.AT, pos, pos + 1, line, col, prev);
     // [
     case 91:
-      return new Tok(TokenKind.BRACKET_L, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.BRACKET_L, pos, pos + 1, line, col, prev);
     // ]
     case 93:
-      return new Tok(TokenKind.BRACKET_R, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.BRACKET_R, pos, pos + 1, line, col, prev);
     // {
     case 123:
-      return new Tok(TokenKind.BRACE_L, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.BRACE_L, pos, pos + 1, line, col, prev);
     // |
     case 124:
-      return new Tok(TokenKind.PIPE, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.PIPE, pos, pos + 1, line, col, prev);
     // }
     case 125:
-      return new Tok(TokenKind.BRACE_R, pos, pos + 1, line, col, prev);
+      return new Token(TokenKind.BRACE_R, pos, pos + 1, line, col, prev);
     // A-Z _ a-z
     case 65:
     case 66:
@@ -364,7 +330,7 @@ function readComment(source, start, line, col, prev): Token {
     (code > 0x001f || code === 0x0009)
   );
 
-  return new Tok(
+  return new Token(
     TokenKind.COMMENT,
     start,
     position,
@@ -439,7 +405,7 @@ function readNumber(source, start, firstCode, line, col, prev): Token {
     );
   }
 
-  return new Tok(
+  return new Token(
     isFloat ? TokenKind.FLOAT : TokenKind.INT,
     start,
     position,
@@ -493,7 +459,7 @@ function readString(source, start, line, col, prev): Token {
     // Closing Quote (")
     if (code === 34) {
       value += body.slice(chunkStart, position);
-      return new Tok(
+      return new Token(
         TokenKind.STRING,
         start,
         position + 1,
@@ -600,7 +566,7 @@ function readBlockString(source, start, line, col, prev, lexer): Token {
       body.charCodeAt(position + 2) === 34
     ) {
       rawValue += body.slice(chunkStart, position);
-      return new Tok(
+      return new Token(
         TokenKind.BLOCK_STRING,
         start,
         position + 3,
@@ -711,7 +677,7 @@ function readName(source, start, line, col, prev): Token {
   ) {
     ++position;
   }
-  return new Tok(
+  return new Token(
     TokenKind.NAME,
     start,
     position,