Parse bounds expression in function parameter list scope.

This change addresses issue checkedc#7 (process bounds expressions for parameters
with all the parameters available).  Currently, bounds expressions for
parameters are being parsed and type checked in the scope of the parameters
seen so far in a parameter list.  They need to be typechecked in a scope with
all the parameters for the parameter list.

To do this, parsing and type checking of bounds expressios for parameters
is deferred until the entire parameter list has been seen.  This is done
in two steps.  During the initial parse, the tokens for a superset of bounds
expressions are parsed and stored in a list.  They are then re-parsed using
ParseBoundsExpressions after all parameters have been seen.

This change also improve serror messages when a bounds expression is parsed
with misspelled contextual keyword.  For example, int x : boounds(x, x + 5).
Before, the code would simply give up after parsing the misspelled keyword
and the '('.  This led to additional error messages when the code tried to
 match the closing ')'.  This fix is to skip all the tokens up to but not
including the closing ')'.

Testing:
* Passes existing clang test suite.
* Added additional tests for parsing of function parameter lists with bounds
  expressions.  This will be commited separtely to the checked repo.

Author: dtarditi

include/clang/Parse/Parser.h

@@ -1633,6 +1633,8 @@ class Parser : public CodeCompletionHandler {
   // Checked C Expressions
 
   ExprResult ParseBoundsExpression();
+  bool ConsumeAndStoreBoundsExpression(CachedTokens &Toks);
+  ExprResult DeferredParseBoundsExpression(CachedTokens &Toks);
 
   //===--------------------------------------------------------------------===//
   // clang Expressions

lib/Parse/ParseDecl.cpp

@@ -5859,21 +5859,29 @@ void Parser::ParseFunctionDeclaratorIdentifierList(
 ///         parameter-list ',' parameter-declaration
 ///
 ///       parameter-declaration: [C99 6.7.5]
-///         declaration-specifiers declarator
-/// [C++]   declaration-specifiers declarator '=' assignment-expression
-/// [C++11]                                       initializer-clause
-/// [GNU]   declaration-specifiers declarator attributes
-///         declaration-specifiers abstract-declarator[opt]
-/// [C++]   declaration-specifiers abstract-declarator[opt]
-///           '=' assignment-expression
-/// [GNU]   declaration-specifiers abstract-declarator[opt] attributes
-/// [C++11] attribute-specifier-seq parameter-declaration
+///             declaration-specifiers declarator
+/// [Checked C] declaration-specifiers declarator ':' bounds-expression
+/// [C++]       declaration-specifiers declarator '=' assignment-expression
+/// [C++11]                                           initializer-clause
+/// [GNU]       declaration-specifiers declarator attributes
+///             declaration-specifiers abstract-declarator[opt]
+/// [C++]       declaration-specifiers abstract-declarator[opt]
+///               '=' assignment-expression
+/// [GNU]       declaration-specifiers abstract-declarator[opt] attributes
+/// [C++11]     attribute-specifier-seq parameter-declaration
 ///
 void Parser::ParseParameterDeclarationClause(
        Declarator &D,
        ParsedAttributes &FirstArgAttrs,
        SmallVectorImpl<DeclaratorChunk::ParamInfo> &ParamInfo,
        SourceLocation &EllipsisLoc) {
+  // Delay parsing/semantic checking of bounds expressions until after the
+  // parameter list is parsed. For each variable with a bounds declaration,
+  // keep a list of tokens for the bounds expression.   Parsing/checking
+  // bounds expressions for parameters is done in a scope that includes all
+  // the parameters in the parameter list.
+  typedef std::pair<ParmVarDecl *, CachedTokens *> BoundsExprInfo;
+  SmallVector<BoundsExprInfo, 4> deferredBoundsExpressions;
   do {
     // FIXME: Issue a diagnostic if we parsed an attribute-specifier-seq
     // before deciding this was a parameter-declaration-clause.
@@ -5944,14 +5952,21 @@ void Parser::ParseParameterDeclarationClause(
       // added to the current scope.
       ParmVarDecl *Param = Actions.ActOnParamDeclarator(getCurScope(), ParmDeclarator);
 
-      // Parse the bounds expression, if any.
+      // Parse and store the tokens for the bounds expression, if there is one.
       if (getLangOpts().CheckedC && Tok.is(tok::colon)) {
         ConsumeToken();
-        ExprResult Bounds = ParseBoundsExpression();
-        if (Bounds.isInvalid()) {
+        // Consume and store tokens until a bounds-like expression has been
+        // read or a parsing error has happened.  Store the tokens even if a
+        // parsing error occurs so that ParseBoundsExpression can generate the
+        // error message.  This way the error messages from parsing of bounds
+        // expressions will be the same or very similar regardless of whether
+        // parsing is deferred or not.
+        CachedTokens *BoundsExprTokens = new CachedTokens;
+        bool ParsingError = !ConsumeAndStoreBoundsExpression(*BoundsExprTokens);
+        deferredBoundsExpressions.push_back(BoundsExprInfo(Param, BoundsExprTokens));
+        if (ParsingError) {
           SkipUntil(tok::comma, tok::r_paren, StopAtSemi | StopBeforeMatch);
-          Actions.ActOnInvalidBoundsExpr(Param);
-        } else Actions.ActOnBoundsExpr(Param, cast<BoundsExpr>(Bounds.get()));
+        }
       }
 
       // Parse the default argument, if any. We parse the default
@@ -6044,6 +6059,18 @@ void Parser::ParseParameterDeclarationClause(
 
     // If the next token is a comma, consume it and keep reading arguments.
   } while (TryConsumeToken(tok::comma));
+
+  // Now parse the deferred bounds expressions
+  for (auto Pair : deferredBoundsExpressions) {
+    ParmVarDecl *Param = Pair.first;
+    CachedTokens *Tokens = Pair.second;
+    ExprResult Bounds = DeferredParseBoundsExpression(*Tokens);
+    delete Tokens;
+    if (Bounds.isInvalid())
+      Actions.ActOnInvalidBoundsExpr(Param);
+    else
+      Actions.ActOnBoundsExpr(Param, cast<BoundsExpr>(Bounds.get()));
+  }
 }
 
 /// [C90]   direct-declarator '[' constant-expression[opt] ']'

lib/Parse/ParseExpr.cpp

@@ -2813,13 +2813,71 @@ ExprResult Parser::ParseBoundsExpression() {
     // The identifier is not a valid contextual keyword for the start of a
     // a bounds expression.
     Diag(Tok, diag::err_expected_bounds_expr);
+    SkipUntil(tok::r_paren, StopAtSemi | StopBeforeMatch);
     Result = ExprError();
   }
 
   PT.consumeClose();
   return Result;
 }
 
+/// Consume and store tokens for an expression shaped like a bounds expression
+/// in the passed token container. Returns \c true if it reached the end of
+/// something bounds-expression shaped, \c false if a parsing error occurred,
+///
+/// Stop when the end of the expression is reached or a parsing error occurs
+/// for which ParseBoundsExpression doesn't know how to make forward progress.
+/// An expression is shaped like a bounds expression if it consists of
+///   identifier '(' sequence of tokens ')'
+/// where parentheses balance within the sequence of tokens.
+bool Parser::ConsumeAndStoreBoundsExpression(CachedTokens &Toks) {
+  Toks.push_back(Tok);
+  if (Tok.getKind() != tok::identifier) {
+    return false;
+  }
+  ConsumeToken();
+
+  Toks.push_back(Tok);
+  if (Tok.getKind() != tok::l_paren) {
+    return false;
+  }
+  ConsumeParen();
+
+  return ConsumeAndStoreUntil(tok::r_paren, Toks, /*StopAtSemi=*/true);
+}
+
+/// Given a list of tokens that have the same shape as a bounds
+/// expression, parse them to create a bounds expression.
+ExprResult Parser::DeferredParseBoundsExpression(CachedTokens &Toks) {
+  Token LastBoundsExprToken = Toks.back();
+  Token BoundsExprEnd;
+  BoundsExprEnd.startToken();
+  BoundsExprEnd.setKind(tok::eof);
+  SourceLocation OrigLoc = LastBoundsExprToken.getEndLoc();
+  BoundsExprEnd.setLocation(OrigLoc);
+  Toks.push_back(BoundsExprEnd);
+
+  Toks.push_back(Tok); // Save the current token at the end of the new tokens
+                       // so it isn't lost.
+  PP.EnterTokenStream(Toks, true);
+  ConsumeAnyToken();   // Skip past the current token to the new tokens.
+  ExprResult Result = ParseBoundsExpression();
+
+  // There could be leftover tokens because of an error.
+  // Skip through them until we reach the eof token.
+  if (Tok.isNot(tok::eof)) {
+    assert(Result.isInvalid());
+    while (Tok.isNot(tok::eof))
+      ConsumeAnyToken();
+  }
+
+  // Clean up the remaining EOF token.
+  if (Tok.is(tok::eof) && Tok.getLocation() == OrigLoc)
+    ConsumeAnyToken();
+
+  return Result;
+}
+
 /// ParseBlockLiteralExpression - Parse a block literal, which roughly looks
 /// like ^(int x){ return x+1; }
 ///