Revert "[flang] Recognize compiler directives after expansion in comm… (llvm#180982)

klausler · web-flow · commit 8e93ac7f1a0d · 2026-02-11T10:32:08.000-08:00
…ent (llvm#180062)" This reverts commit 0d64801. Lines like "!MACRO ... &" in which MACRO expands to a compiler directive are now failing because they are not being recognized as having line continuations. Will fix and try again.
diff --git a/flang/include/flang/Parser/preprocessor.h b/flang/include/flang/Parser/preprocessor.h
@@ -49,8 +49,8 @@ class Definition {
 
   bool set_isDisabled(bool disable);
 
-  TokenSequence Apply(const std::vector<TokenSequence> &args,
-      const Prescanner &, bool inIfExpression = false) const;
+  TokenSequence Apply(const std::vector<TokenSequence> &args, Prescanner &,
+      bool inIfExpression = false);
 
   void Print(llvm::raw_ostream &out, const char *macroName = "") const;
 
@@ -95,7 +95,7 @@ class Preprocessor {
   // that result and try again.  All other Fortran preprocessors share this
   // behavior.
   std::optional<TokenSequence> MacroReplacement(const TokenSequence &,
-      const Prescanner &,
+      Prescanner &,
       std::optional<std::size_t> *partialFunctionLikeMacro = nullptr,
       bool inIfExpression = false);
 
@@ -109,7 +109,7 @@ class Preprocessor {
   enum class CanDeadElseAppear { No, Yes };
 
   CharBlock SaveTokenAsName(const CharBlock &);
-  TokenSequence ReplaceMacros(const TokenSequence &, const Prescanner &,
+  TokenSequence ReplaceMacros(const TokenSequence &, Prescanner &,
       std::optional<std::size_t> *partialFunctionLikeMacro = nullptr,
       bool inIfExpression = false);
   void SkipDisabledConditionalCode(
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
@@ -214,7 +214,7 @@ constexpr bool IsDefinedKeyword(CharBlock token) {
 }
 
 TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
-    const Prescanner &prescanner, bool inIfExpression) const {
+    Prescanner &prescanner, bool inIfExpression) {
   TokenSequence result;
   bool skipping{false};
   int parenthesesNesting{0};
@@ -254,9 +254,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
         CHECK(resultSize > 0 &&
             result.TokenAt(resultSize - 1) == replacement_.TokenAt(prev - 1));
         result.pop_back();
-        AllSources &allSources{
-            *const_cast<AllSources *>(&prescanner.allSources())};
-        result.CopyAll(Stringify(args[index], allSources));
+        result.CopyAll(Stringify(args[index], prescanner.allSources()));
       } else {
         const TokenSequence *arg{&args[index]};
         std::optional<TokenSequence> replaced;
@@ -269,9 +267,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
           auto next{replacement_.SkipBlanks(j + 1)};
           if (next >= tokens || !IsTokenPasting(replacement_.TokenAt(next))) {
             // Apply macro replacement to the actual argument
-            Preprocessor &preprocessor{
-                *const_cast<Preprocessor *>(&prescanner.preprocessor())};
-            replaced = preprocessor.MacroReplacement(
+            replaced = prescanner.preprocessor().MacroReplacement(
                 *arg, prescanner, nullptr, inIfExpression);
             if (replaced) {
               arg = &*replaced;
@@ -282,9 +278,9 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
       }
     } else if (bytes == 11 && isVariadic_ &&
         token.ToString() == "__VA_ARGS__") {
-      AllSources &allSources{
-          *const_cast<AllSources *>(&prescanner.allSources())};
-      Provenance commaProvenance{allSources.CompilerInsertionProvenance(',')};
+      Provenance commaProvenance{
+          prescanner.preprocessor().allSources().CompilerInsertionProvenance(
+              ',')};
       for (std::size_t k{argumentCount()}; k < args.size(); ++k) {
         if (k > argumentCount()) {
           result.Put(","s, commaProvenance);
@@ -444,7 +440,7 @@ void Preprocessor::Define(const std::string &macro, const std::string &value) {
 void Preprocessor::Undefine(std::string macro) { definitions_.erase(macro); }
 
 std::optional<TokenSequence> Preprocessor::MacroReplacement(
-    const TokenSequence &input, const Prescanner &prescanner,
+    const TokenSequence &input, Prescanner &prescanner,
     std::optional<std::size_t> *partialFunctionLikeMacro, bool inIfExpression) {
   // Do quick scan for any use of a defined name.
   if (!inIfExpression && definitions_.empty()) {
@@ -667,7 +663,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
 }
 
 TokenSequence Preprocessor::ReplaceMacros(const TokenSequence &tokens,
-    const Prescanner &prescanner,
+    Prescanner &prescanner,
     std::optional<std::size_t> *partialFunctionLikeMacro, bool inIfExpression) {
   if (std::optional<TokenSequence> repl{MacroReplacement(
           tokens, prescanner, partialFunctionLikeMacro, inIfExpression)}) {
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
@@ -212,14 +212,6 @@ void Prescanner::Statement() {
     }
     break;
   }
-  case LineClassification::Kind::CompilerDirectiveAfterMacroExpansion:
-    BeginStatementAndAdvance();
-    SkipSpaces();
-    if (*at_ == '!') {
-      tokens.Put(at_++, 1, GetCurrentProvenance());
-      ++column_;
-    }
-    break;
   case LineClassification::Kind::Source: {
     BeginStatementAndAdvance();
     bool checkLabelField{false};
@@ -246,17 +238,30 @@ void Prescanner::Statement() {
     // a comment marker or directive sentinel.  If so, disable line
     // continuation, so that NextToken() won't consume anything from
     // following lines.
-    if (auto kwName{GetKeywordMacroName(at_)}) {
-      checkLabelField = false;
-      Provenance here{GetCurrentProvenance()};
-      TokenSequence replacement{ExpandKeywordMacro(*kwName, here)};
-      auto newLineClass{ClassifyLine(replacement, here)};
-      if (newLineClass.kind == LineClassification::Kind::CompilerDirective) {
-        directiveSentinel_ = newLineClass.sentinel;
-        disableSourceContinuation_ = false;
-      } else {
-        disableSourceContinuation_ = !replacement.empty() &&
-            newLineClass.kind != LineClassification::Kind::Source;
+    if (IsLegalIdentifierStart(*at_)) {
+      // TODO: Only bother with these cases when any keyword macro has
+      // been defined with replacement text that could begin a comment
+      // or directive sentinel.
+      const char *p{at_};
+      while (IsLegalInIdentifier(*++p)) {
+      }
+      CharBlock id{at_, static_cast<std::size_t>(p - at_)};
+      if (preprocessor_.IsNameDefined(id) &&
+          !preprocessor_.IsFunctionLikeDefinition(id)) {
+        checkLabelField = false;
+        TokenSequence toks;
+        toks.Put(id, GetProvenance(at_));
+        if (auto replaced{preprocessor_.MacroReplacement(toks, *this)}) {
+          auto newLineClass{ClassifyLine(*replaced, GetCurrentProvenance())};
+          if (newLineClass.kind ==
+              LineClassification::Kind::CompilerDirective) {
+            directiveSentinel_ = newLineClass.sentinel;
+            disableSourceContinuation_ = false;
+          } else {
+            disableSourceContinuation_ = !replaced->empty() &&
+                newLineClass.kind != LineClassification::Kind::Source;
+          }
+        }
       }
     }
     if (checkLabelField) {
@@ -299,7 +304,6 @@ void Prescanner::Statement() {
       CheckAndEmitLine(preprocessed->ToLowerCase(), newlineProvenance);
       break;
     case LineClassification::Kind::CompilerDirective:
-    case LineClassification::Kind::CompilerDirectiveAfterMacroExpansion:
       if (preprocessed->HasRedundantBlanks()) {
         preprocessed->RemoveRedundantBlanks();
       }
@@ -331,39 +335,40 @@ void Prescanner::Statement() {
           preprocessed->ToLowerCase().ClipComment(*this), newlineProvenance);
       break;
     }
-  } else if (line.kind == LineClassification::Kind::CompilerDirective) {
-    while (CompilerDirectiveContinuation(tokens, line.sentinel)) {
-      newlineProvenance = GetCurrentProvenance();
-    }
-    if (preprocessingOnly_ && inFixedForm_ && InConditionalLine() &&
-        nextLine_ < limit_) {
-      // In -E mode, when the line after !$ conditional compilation is a
-      // regular fixed form continuation line, append a '&' to the line.
-      const char *p{nextLine_};
-      int col{1};
-      while (int n{IsSpace(p)}) {
-        if (*p == '\t') {
-          break;
+  } else { // no macro replacement
+    if (line.kind == LineClassification::Kind::CompilerDirective) {
+      while (CompilerDirectiveContinuation(tokens, line.sentinel)) {
+        newlineProvenance = GetCurrentProvenance();
+      }
+      if (preprocessingOnly_ && inFixedForm_ && InConditionalLine() &&
+          nextLine_ < limit_) {
+        // In -E mode, when the line after !$ conditional compilation is a
+        // regular fixed form continuation line, append a '&' to the line.
+        const char *p{nextLine_};
+        int col{1};
+        while (int n{IsSpace(p)}) {
+          if (*p == '\t') {
+            break;
+          }
+          p += n;
+          ++col;
+        }
+        if (col == 6 && *p != '0' && *p != '\t' && *p != '\n') {
+          EmitChar(tokens, '&');
+          tokens.CloseToken();
         }
-        p += n;
-        ++col;
       }
-      if (col == 6 && *p != '0' && *p != '\t' && *p != '\n') {
-        EmitChar(tokens, '&');
-        tokens.CloseToken();
+      tokens.ToLowerCase();
+      if (!SourceFormChange(tokens.ToString())) {
+        CheckAndEmitLine(tokens, newlineProvenance);
+      }
+    } else { // Kind::Source
+      tokens.ToLowerCase();
+      if (inFixedForm_) {
+        EnforceStupidEndStatementRules(tokens);
       }
-    }
-    tokens.ToLowerCase();
-    if (!SourceFormChange(tokens.ToString())) {
       CheckAndEmitLine(tokens, newlineProvenance);
     }
-  } else {
-    CHECK(line.kind == LineClassification::Kind::Source);
-    tokens.ToLowerCase();
-    if (inFixedForm_) {
-      EnforceStupidEndStatementRules(tokens);
-    }
-    CheckAndEmitLine(tokens, newlineProvenance);
   }
   directiveSentinel_ = nullptr;
 }
@@ -562,8 +567,7 @@ bool Prescanner::MustSkipToEndOfLine() const {
     return true; // skip over ignored columns in right margin (73:80)
   } else if (*at_ == '!' && !inCharLiteral_ &&
       (!inFixedForm_ || tabInCurrentLine_ || column_ != 6)) {
-    return InCompilerDirective() ||
-        !IsCompilerDirectiveSentinelAfterKeywordMacro(at_ + 1);
+    return InCompilerDirective() || !IsCompilerDirectiveSentinel(at_ + 1);
   } else {
     return false;
   }
@@ -1612,9 +1616,6 @@ Prescanner::IsFixedFormCompilerDirectiveLine(const char *start) const {
   if (!IsFixedFormCommentChar(col1)) {
     return std::nullopt;
   }
-  // TODO: Handle keyword macros that expand to directives in fixed form.
-  // The comment character can't be 'c' or 'C'.  Need to figure out whether
-  // fixed form continuation should apply to the expansions.
   char sentinel[5], *sp{sentinel};
   int column{2};
   for (; column < 6; ++column) {
@@ -1677,28 +1678,20 @@ Prescanner::IsFixedFormCompilerDirectiveLine(const char *start) const {
 std::optional<Prescanner::LineClassification>
 Prescanner::IsFreeFormCompilerDirectiveLine(const char *start) const {
   if (const char *p{SkipWhiteSpaceIncludingEmptyMacros(start)};
-      p && *p == '!') {
-    if (auto lnClass{IsCompilerDirectiveSentinelAfterKeywordMacro(p + 1)}) {
-      if (lnClass->kind == LineClassification::Kind::CompilerDirective) {
-        const char *sentinel{lnClass->sentinel};
-        CHECK(sentinel != nullptr);
-        const char *payload{nullptr};
-        if (sentinel[0] == '$' && sentinel[1] == '\0') {
-          payload = p + 2; // !$
-        } else if (sentinel[1] == '@') {
-          payload = p + 5; // !@acc or !@cuf
-        }
-        if (payload) {
-          if (const char *comment{IsFreeFormComment(payload)}) {
-            if (*comment == '!') { // !$ !blah or !@acc !blah
-              // Conditional line comment - treat as comment
-              return std::nullopt;
-            }
+      p && *p++ == '!') {
+    if (auto maybePair{IsCompilerDirectiveSentinel(p)}) {
+      auto offset{static_cast<std::size_t>(p - start - 1)};
+      const char *sentinel{maybePair->first};
+      if ((sentinel[0] == '$' && sentinel[1] == '\0') || sentinel[1] == '@') {
+        if (const char *comment{IsFreeFormComment(maybePair->second)}) {
+          if (*comment == '!') {
+            // Conditional line comment - treat as comment
+            return std::nullopt;
           }
         }
-        lnClass->payloadOffset = static_cast<std::size_t>(p - start);
       }
-      return lnClass;
+      return {LineClassification{
+          LineClassification::Kind::CompilerDirective, offset, sentinel}};
     }
   }
   return std::nullopt;
@@ -1715,31 +1708,6 @@ Prescanner &Prescanner::AddCompilerDirectiveSentinel(const std::string &dir) {
   return *this;
 }
 
-std::optional<CharBlock> Prescanner::GetKeywordMacroName(
-    const char *start) const {
-  if (IsLegalIdentifierStart(*start)) {
-    // TODO: Only bother with these cases when any keyword macro has
-    // been defined with replacement text that could begin a comment
-    // or directive sentinel.
-    const char *p{start};
-    while (IsLegalInIdentifier(*++p)) {
-    }
-    CharBlock name{start, static_cast<std::size_t>(p - start)};
-    if (preprocessor_.IsNameDefined(name) &&
-        !preprocessor_.IsFunctionLikeDefinition(name)) {
-      return name;
-    }
-  }
-  return std::nullopt;
-}
-
-TokenSequence Prescanner::ExpandKeywordMacro(
-    CharBlock name, Provenance provenance) const {
-  TokenSequence toks;
-  toks.Put(name, provenance);
-  return preprocessor_.MacroReplacement(toks, *this).value();
-}
-
 const char *Prescanner::IsCompilerDirectiveSentinel(
     const char *sentinel, std::size_t len) const {
   std::uint64_t packed{0};
@@ -1797,26 +1765,8 @@ Prescanner::IsCompilerDirectiveSentinel(const char *p) const {
   return std::nullopt;
 }
 
-auto Prescanner::IsCompilerDirectiveSentinelAfterKeywordMacro(
-    const char *p) const -> std::optional<LineClassification> {
-  if (auto name{GetKeywordMacroName(p)}) {
-    Provenance provenance{GetProvenance(p)};
-    TokenSequence expansion{ExpandKeywordMacro(*name, provenance)};
-    expansion.Put("\n", 1, provenance); // termination
-    CharBlock block{expansion.ToLowerCase().ToCharBlock()};
-    if (auto maybePair{IsCompilerDirectiveSentinel(block.begin())}) {
-      return LineClassification{
-          LineClassification::Kind::CompilerDirectiveAfterMacroExpansion,
-          name->size(), maybePair->first};
-    }
-  } else if (auto maybePair{IsCompilerDirectiveSentinel(p)}) {
-    return LineClassification{LineClassification::Kind::CompilerDirective,
-        static_cast<std::size_t>(maybePair->second - p), maybePair->first};
-  }
-  return std::nullopt;
-}
-
-auto Prescanner::ClassifyLine(const char *start) const -> LineClassification {
+Prescanner::LineClassification Prescanner::ClassifyLine(
+    const char *start) const {
   if (inFixedForm_) {
     if (std::optional<LineClassification> lc{
             IsFixedFormCompilerDirectiveLine(start)}) {
diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h
@@ -81,9 +81,6 @@ class Prescanner {
   TokenSequence TokenizePreprocessorDirective();
   Provenance GetCurrentProvenance() const { return GetProvenance(at_); }
 
-  std::optional<CharBlock> GetKeywordMacroName(const char *) const;
-  TokenSequence ExpandKeywordMacro(CharBlock, Provenance) const;
-
   const char *IsCompilerDirectiveSentinel(const char *, std::size_t) const;
   const char *IsCompilerDirectiveSentinel(CharBlock) const;
   // 'first' is the sentinel, 'second' is beginning of payload
@@ -112,7 +109,6 @@ class Prescanner {
       PreprocessorDirective,
       IncludeLine, // Fortran INCLUDE
       CompilerDirective,
-      CompilerDirectiveAfterMacroExpansion, // !MACRO -> !$OMP ...
       Source
     };
     LineClassification(Kind k, std::size_t po = 0, const char *s = nullptr)
@@ -160,7 +156,7 @@ class Prescanner {
   }
 
   void EmitInsertedChar(TokenSequence &tokens, char ch) {
-    Provenance provenance{allSources().CompilerInsertionProvenance(ch)};
+    Provenance provenance{allSources_.CompilerInsertionProvenance(ch)};
     tokens.PutNextTokenChar(ch, provenance);
   }
 
@@ -244,8 +240,6 @@ class Prescanner {
   bool SourceFormChange(std::string &&);
   bool CompilerDirectiveContinuation(TokenSequence &, const char *sentinel);
   bool SourceLineContinuation(TokenSequence &);
-  std::optional<LineClassification>
-  IsCompilerDirectiveSentinelAfterKeywordMacro(const char *p) const;
 
   Messages &messages_;
   CookedSource &cooked_;
@@ -304,9 +298,9 @@ class Prescanner {
   const std::size_t firstCookedCharacterOffset_{cooked_.BufferedBytes()};
 
   const Provenance spaceProvenance_{
-      allSources().CompilerInsertionProvenance(' ')};
+      allSources_.CompilerInsertionProvenance(' ')};
   const Provenance backslashProvenance_{
-      allSources().CompilerInsertionProvenance('\\')};
+      allSources_.CompilerInsertionProvenance('\\')};
 
   // To avoid probing the set of active compiler directive sentinel strings
   // on every comment line, they're checked first with a cheap Bloom filter.
diff --git a/flang/test/Preprocessing/bug178481.F90 b/flang/test/Preprocessing/bug178481.F90
