[flang] Handle "defined" in macro expansions (#114844)

klausler · web-flow · commit 850d42fb145c · 2024-11-05T13:18:27.000-08:00
The preprocessor implements "defined(X)" and "defined X" in if/elif directive expressions in such a way that they only work at the top level, not when they appear in macro expansions. Fix that, which is a little tricky due to the need to detect the "defined" keyword before applying any macro expansion to its argument, and add a bunch of tests. Fixes #114064.
diff --git a/flang/include/flang/Parser/preprocessor.h b/flang/include/flang/Parser/preprocessor.h
@@ -48,7 +48,8 @@ class Definition {
 
   bool set_isDisabled(bool disable);
 
-  TokenSequence Apply(const std::vector<TokenSequence> &args, Prescanner &);
+  TokenSequence Apply(const std::vector<TokenSequence> &args, Prescanner &,
+      bool inIfExpression = false);
 
   void Print(llvm::raw_ostream &out, const char *macroName = "") const;
 
@@ -93,7 +94,8 @@ class Preprocessor {
   // behavior.
   std::optional<TokenSequence> MacroReplacement(const TokenSequence &,
       Prescanner &,
-      std::optional<std::size_t> *partialFunctionLikeMacro = nullptr);
+      std::optional<std::size_t> *partialFunctionLikeMacro = nullptr,
+      bool inIfExpression = false);
 
   // Implements a preprocessor directive.
   void Directive(const TokenSequence &, Prescanner &);
@@ -106,7 +108,8 @@ class Preprocessor {
 
   CharBlock SaveTokenAsName(const CharBlock &);
   TokenSequence ReplaceMacros(const TokenSequence &, Prescanner &,
-      std::optional<std::size_t> *partialFunctionLikeMacro = nullptr);
+      std::optional<std::size_t> *partialFunctionLikeMacro = nullptr,
+      bool inIfExpression = false);
   void SkipDisabledConditionalCode(
       const std::string &, IsElseActive, Prescanner &, ProvenanceRange);
   bool IsIfPredicateTrue(const TokenSequence &expr, std::size_t first,
diff --git a/flang/include/flang/Parser/token-sequence.h b/flang/include/flang/Parser/token-sequence.h
@@ -79,6 +79,7 @@ class TokenSequence {
   }
 
   std::size_t SkipBlanks(std::size_t) const;
+  std::optional<std::size_t> SkipBlanksBackwards(std::size_t) const;
 
   // True if anything remains in the sequence at & after the given offset
   // except blanks and line-ending C++ and Fortran free-form comments.
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
@@ -177,8 +177,13 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
   return result;
 }
 
-TokenSequence Definition::Apply(
-    const std::vector<TokenSequence> &args, Prescanner &prescanner) {
+constexpr bool IsDefinedKeyword(CharBlock token) {
+  return token.size() == 7 && (token[0] == 'd' || token[0] == 'D') &&
+      ToLowerCaseLetters(token.ToString()) == "defined";
+}
+
+TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
+    Prescanner &prescanner, bool inIfExpression) {
   TokenSequence result;
   bool skipping{false};
   int parenthesesNesting{0};
@@ -223,13 +228,16 @@ TokenSequence Definition::Apply(
         const TokenSequence *arg{&args[index]};
         std::optional<TokenSequence> replaced;
         // Don't replace macros in the actual argument if it is preceded or
-        // followed by the token-pasting operator ## in the replacement text.
-        if (prev == 0 || !IsTokenPasting(replacement_.TokenAt(prev - 1))) {
+        // followed by the token-pasting operator ## in the replacement text,
+        // or if we have to worry about "defined(X)"/"defined X" in an
+        // #if/#elif expression.
+        if (!inIfExpression &&
+            (prev == 0 || !IsTokenPasting(replacement_.TokenAt(prev - 1)))) {
           auto next{replacement_.SkipBlanks(j + 1)};
           if (next >= tokens || !IsTokenPasting(replacement_.TokenAt(next))) {
             // Apply macro replacement to the actual argument
-            replaced =
-                prescanner.preprocessor().MacroReplacement(*arg, prescanner);
+            replaced = prescanner.preprocessor().MacroReplacement(
+                *arg, prescanner, nullptr, inIfExpression);
             if (replaced) {
               arg = &*replaced;
             }
@@ -301,7 +309,7 @@ void Preprocessor::Undefine(std::string macro) { definitions_.erase(macro); }
 
 std::optional<TokenSequence> Preprocessor::MacroReplacement(
     const TokenSequence &input, Prescanner &prescanner,
-    std::optional<std::size_t> *partialFunctionLikeMacro) {
+    std::optional<std::size_t> *partialFunctionLikeMacro, bool inIfExpression) {
   // Do quick scan for any use of a defined name.
   if (definitions_.empty()) {
     return std::nullopt;
@@ -311,7 +319,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
   for (; j < tokens; ++j) {
     CharBlock token{input.TokenAt(j)};
     if (!token.empty() && IsLegalIdentifierStart(token[0]) &&
-        IsNameDefined(token)) {
+        (IsNameDefined(token) || (inIfExpression && IsDefinedKeyword(token)))) {
       break;
     }
   }
@@ -326,17 +334,17 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
   // replacement text and attempt to proceed.  Otherwise, return, so that
   // the caller may try again with remaining tokens in its input.
   auto CompleteFunctionLikeMacro{
-      [this, &input, &prescanner, &result, &partialFunctionLikeMacro](
-          std::size_t after, const TokenSequence &replacement,
+      [this, &input, &prescanner, &result, &partialFunctionLikeMacro,
+          inIfExpression](std::size_t after, const TokenSequence &replacement,
           std::size_t pFLMOffset) {
         if (after < input.SizeInTokens()) {
           result.Put(replacement, 0, pFLMOffset);
           TokenSequence suffix;
           suffix.Put(
               replacement, pFLMOffset, replacement.SizeInTokens() - pFLMOffset);
           suffix.Put(input, after, input.SizeInTokens() - after);
-          auto further{
-              ReplaceMacros(suffix, prescanner, partialFunctionLikeMacro)};
+          auto further{ReplaceMacros(
+              suffix, prescanner, partialFunctionLikeMacro, inIfExpression)};
           if (partialFunctionLikeMacro && *partialFunctionLikeMacro) {
             // still not closed
             **partialFunctionLikeMacro += result.SizeInTokens();
@@ -357,7 +365,28 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
       result.Put(input, j);
       continue;
     }
+    // Process identifier in replacement text.
     auto it{definitions_.find(token)};
+    // Is in the X in "defined(X)" or "defined X" in an #if/#elif expression?
+    if (inIfExpression) {
+      if (auto prev{result.SkipBlanksBackwards(result.SizeInTokens())}) {
+        bool ok{true};
+        std::optional<std::size_t> rightParenthesis;
+        if (result.TokenAt(*prev).OnlyNonBlank() == '(') {
+          prev = result.SkipBlanksBackwards(*prev);
+          rightParenthesis = input.SkipBlanks(j + 1);
+          ok = *rightParenthesis < tokens &&
+              input.TokenAt(*rightParenthesis).OnlyNonBlank() == ')';
+        }
+        if (ok && prev && IsDefinedKeyword(result.TokenAt(*prev))) {
+          result = TokenSequence{result, 0, *prev}; // trims off "defined ("
+          char truth{it != definitions_.end() ? '1' : '0'};
+          result.Put(&truth, 1, allSources_.CompilerInsertionProvenance(truth));
+          j = rightParenthesis.value_or(j);
+          continue;
+        }
+      }
+    }
     if (it == definitions_.end()) {
       result.Put(input, j);
       continue;
@@ -403,8 +432,8 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
       }
       std::optional<std::size_t> partialFLM;
       def->set_isDisabled(true);
-      TokenSequence replaced{TokenPasting(
-          ReplaceMacros(def->replacement(), prescanner, &partialFLM))};
+      TokenSequence replaced{TokenPasting(ReplaceMacros(
+          def->replacement(), prescanner, &partialFLM, inIfExpression))};
       def->set_isDisabled(false);
       if (partialFLM &&
           CompleteFunctionLikeMacro(j + 1, replaced, *partialFLM)) {
@@ -476,11 +505,11 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
             (n + 1 == argStart.size() ? k : argStart[n + 1] - 1) - at};
         args.emplace_back(TokenSequence(input, at, count));
       }
-      TokenSequence applied{def->Apply(args, prescanner)};
+      TokenSequence applied{def->Apply(args, prescanner, inIfExpression)};
       std::optional<std::size_t> partialFLM;
       def->set_isDisabled(true);
-      TokenSequence replaced{
-          ReplaceMacros(std::move(applied), prescanner, &partialFLM)};
+      TokenSequence replaced{ReplaceMacros(
+          std::move(applied), prescanner, &partialFLM, inIfExpression)};
       def->set_isDisabled(false);
       if (partialFLM &&
           CompleteFunctionLikeMacro(k + 1, replaced, *partialFLM)) {
@@ -501,9 +530,9 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
 
 TokenSequence Preprocessor::ReplaceMacros(const TokenSequence &tokens,
     Prescanner &prescanner,
-    std::optional<std::size_t> *partialFunctionLikeMacro) {
-  if (std::optional<TokenSequence> repl{
-          MacroReplacement(tokens, prescanner, partialFunctionLikeMacro)}) {
+    std::optional<std::size_t> *partialFunctionLikeMacro, bool inIfExpression) {
+  if (std::optional<TokenSequence> repl{MacroReplacement(
+          tokens, prescanner, partialFunctionLikeMacro, inIfExpression)}) {
     return std::move(*repl);
   }
   return tokens;
@@ -1215,50 +1244,27 @@ static std::int64_t ExpressionValue(const TokenSequence &token,
   return left;
 }
 
-bool Preprocessor::IsIfPredicateTrue(const TokenSequence &expr,
+bool Preprocessor::IsIfPredicateTrue(const TokenSequence &directive,
     std::size_t first, std::size_t exprTokens, Prescanner &prescanner) {
-  TokenSequence expr1{expr, first, exprTokens};
-  if (expr1.HasBlanks()) {
-    expr1.RemoveBlanks();
-  }
-  TokenSequence expr2;
-  for (std::size_t j{0}; j < expr1.SizeInTokens(); ++j) {
-    if (ToLowerCaseLetters(expr1.TokenAt(j).ToString()) == "defined") {
-      CharBlock name;
-      if (j + 3 < expr1.SizeInTokens() &&
-          expr1.TokenAt(j + 1).OnlyNonBlank() == '(' &&
-          expr1.TokenAt(j + 3).OnlyNonBlank() == ')') {
-        name = expr1.TokenAt(j + 2);
-        j += 3;
-      } else if (j + 1 < expr1.SizeInTokens() &&
-          IsLegalIdentifierStart(expr1.TokenAt(j + 1))) {
-        name = expr1.TokenAt(++j);
-      }
-      if (!name.empty()) {
-        char truth{IsNameDefined(name) ? '1' : '0'};
-        expr2.Put(&truth, 1, allSources_.CompilerInsertionProvenance(truth));
-        continue;
-      }
-    }
-    expr2.Put(expr1, j);
-  }
-  TokenSequence expr3{ReplaceMacros(expr2, prescanner)};
-  if (expr3.HasBlanks()) {
-    expr3.RemoveBlanks();
+  TokenSequence expr{directive, first, exprTokens};
+  TokenSequence replaced{
+      ReplaceMacros(expr, prescanner, nullptr, /*inIfExpression=*/true)};
+  if (replaced.HasBlanks()) {
+    replaced.RemoveBlanks();
   }
-  if (expr3.empty()) {
+  if (replaced.empty()) {
     prescanner.Say(expr.GetProvenanceRange(), "empty expression"_err_en_US);
     return false;
   }
   std::size_t atToken{0};
   std::optional<Message> error;
-  bool result{ExpressionValue(expr3, 0, &atToken, &error) != 0};
+  bool result{ExpressionValue(replaced, 0, &atToken, &error) != 0};
   if (error) {
     prescanner.Say(std::move(*error));
-  } else if (atToken < expr3.SizeInTokens() &&
-      expr3.TokenAt(atToken).ToString() != "!") {
-    prescanner.Say(expr3.GetIntervalProvenanceRange(
-                       atToken, expr3.SizeInTokens() - atToken),
+  } else if (atToken < replaced.SizeInTokens() &&
+      replaced.TokenAt(atToken).ToString() != "!") {
+    prescanner.Say(replaced.GetIntervalProvenanceRange(
+                       atToken, replaced.SizeInTokens() - atToken),
         atToken == 0 ? "could not parse any expression"_err_en_US
                      : "excess characters after expression"_err_en_US);
   }
diff --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
@@ -61,6 +61,16 @@ std::size_t TokenSequence::SkipBlanks(std::size_t at) const {
   return tokens; // even if at > tokens
 }
 
+std::optional<std::size_t> TokenSequence::SkipBlanksBackwards(
+    std::size_t at) const {
+  while (at-- > 0) {
+    if (!TokenAt(at).IsBlank()) {
+      return at;
+    }
+  }
+  return std::nullopt;
+}
+
 // C-style /*comments*/ are removed from preprocessing directive
 // token sequences by the prescanner, but not C++ or Fortran
 // free-form line-ending comments (//...  and !...) because
diff --git a/flang/test/Preprocessing/defined-in-macro.F90 b/flang/test/Preprocessing/defined-in-macro.F90
@@ -0,0 +1,117 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+
+! CHECK: print *, 'pass 1'
+#define IS_DEFINED
+#define M1 defined(IS_DEFINED)
+#if M1
+print *, 'pass 1'
+#else
+print *, 'fail 1'
+#endif
+
+! CHECK: print *, 'pass 2'
+#define M2 defined IS_DEFINED
+#if M2
+print *, 'pass 2'
+#else
+print *, 'fail 2'
+#endif
+
+! CHECK: print *, 'pass 3'
+#define M3 defined(IS_UNDEFINED)
+#if M3
+print *, 'fail 3'
+#else
+print *, 'pass 3'
+#endif
+
+! CHECK: print *, 'pass 4'
+#define M4 defined IS_UNDEFINED
+#if M4
+print *, 'fail 4'
+#else
+print *, 'pass 4'
+#endif
+
+! CHECK: print *, 'pass 5'
+#define DEFINED_KEYWORD defined
+#define M5(x) DEFINED_KEYWORD(x)
+#define KWM1 1
+#if M5(KWM1)
+print *, 'pass 5'
+#else
+print *, 'fail 5'
+#endif
+
+! CHECK: print *, 'pass 6'
+#define KWM2 KWM1
+#if M5(KWM2)
+print *, 'pass 6'
+#else
+print *, 'fail 6'
+#endif
+
+! CHECK: print *, 'pass 7'
+#if M5(IS_UNDEFINED)
+print *, 'fail 7'
+#else
+print *, 'pass 7'
+#endif
+
+! CHECK: print *, 'pass 8'
+#define KWM3 IS_UNDEFINED
+#if M5(KWM3)
+print *, 'pass 8'
+#else
+print *, 'fail 8'
+#endif
+
+! CHECK: print *, 'pass 9'
+#define M6(x) defined(x)
+#if M6(KWM1)
+print *, 'pass 9'
+#else
+print *, 'fail 9'
+#endif
+
+! CHECK: print *, 'pass 10'
+#if M6(KWM2)
+print *, 'pass 10'
+#else
+print *, 'fail 10'
+#endif
+
+! CHECK: print *, 'pass 11'
+#if M6(IS_UNDEFINED)
+print *, 'fail 11'
+#else
+print *, 'pass 11'
+#endif
+
+! CHECK: print *, 'pass 12'
+#if M6(KWM3)
+print *, 'pass 12'
+#else
+print *, 'fail 12'
+#endif
+
+! CHECK: print *, 'pass 13'
+#define M7(A, B) ((A) * 10000 + (B) * 100)
+#define M8(A, B, C, AA, BB) ( \
+  (defined(AA) && defined(BB)) && \
+  (M7(A, B) C M7(AA, BB)))
+#if M8(9, 5, >, BAZ, FUX)
+print *, 'fail 13'
+#else
+print *, 'pass 13'
+#endif
+
+! CHECK: print *, 'pass 14'
+#define M9() (defined(IS_UNDEFINED))
+#if M9()
+print *, 'fail 14'
+#else
+print *, 'pass 14'
+#endif
+
+end

Original file line number	Diff line number	Diff line change
`@@ -79,6 +79,7 @@ class TokenSequence {`
`79`	`79`	`}`
`80`	`80`
`81`	`81`	`std::size_t SkipBlanks(std::size_t) const;`
	`82`	`+ std::optional<std::size_t> SkipBlanksBackwards(std::size_t) const;`
`82`	`83`
`83`	`84`	`// True if anything remains in the sequence at & after the given offset`
`84`	`85`	`// except blanks and line-ending C++ and Fortran free-form comments.`