When calling IsCompilerDirectiveSentinel,the prefix comment character need to be skipped.

Fixes #117693 and also handles continuation lines beginning with Macros

Author: shivaramaarao

flang/include/flang/Parser/token-sequence.h

@@ -123,6 +123,7 @@ class TokenSequence {
   bool HasRedundantBlanks(std::size_t firstChar = 0) const;
   TokenSequence &RemoveBlanks(std::size_t firstChar = 0);
   TokenSequence &RemoveRedundantBlanks(std::size_t firstChar = 0);
+  TokenSequence &RemoveRedundantCompilerDirectives(const Prescanner &);
   TokenSequence &ClipComment(const Prescanner &, bool skipFirst = false);
   const TokenSequence &CheckBadFortranCharacters(
       Messages &, const Prescanner &, bool allowAmpersand) const;

flang/lib/Parser/prescan.cpp

@@ -175,7 +175,7 @@ void Prescanner::Statement() {
       EmitChar(tokens, '!');
       ++at_, ++column_;
       for (const char *sp{directiveSentinel_}; *sp != '\0';
-           ++sp, ++at_, ++column_) {
+          ++sp, ++at_, ++column_) {
         EmitChar(tokens, *sp);
       }
       if (IsSpaceOrTab(at_)) {
@@ -346,6 +346,7 @@ void Prescanner::CheckAndEmitLine(
       tokens.CheckBadParentheses(messages_);
     }
   }
+  tokens.RemoveRedundantCompilerDirectives(*this);
   tokens.Emit(cooked_);
   if (omitNewline_) {
     omitNewline_ = false;
@@ -511,7 +512,7 @@ bool Prescanner::MustSkipToEndOfLine() const {
   if (inFixedForm_ && column_ > fixedFormColumnLimit_ && !tabInCurrentLine_) {
     return true; // skip over ignored columns in right margin (73:80)
   } else if (*at_ == '!' && !inCharLiteral_) {
-    return !IsCompilerDirectiveSentinel(at_);
+    return !IsCompilerDirectiveSentinel(at_ + 1);
   } else {
     return false;
   }
@@ -1073,7 +1074,7 @@ std::optional<std::size_t> Prescanner::IsIncludeLine(const char *start) const {
   }
   if (IsDecimalDigit(*p)) { // accept & ignore a numeric kind prefix
     for (p = SkipWhiteSpace(p + 1); IsDecimalDigit(*p);
-         p = SkipWhiteSpace(p + 1)) {
+        p = SkipWhiteSpace(p + 1)) {
     }
     if (*p != '_') {
       return std::nullopt;
@@ -1121,7 +1122,7 @@ void Prescanner::FortranInclude(const char *firstQuote) {
   llvm::raw_string_ostream error{buf};
   Provenance provenance{GetProvenance(nextLine_)};
   std::optional<std::string> prependPath;
-  if (const SourceFile * currentFile{allSources_.GetSourceFile(provenance)}) {
+  if (const SourceFile *currentFile{allSources_.GetSourceFile(provenance)}) {
     prependPath = DirectoryName(currentFile->path());
   }
   const SourceFile *included{

flang/lib/Parser/token-sequence.cpp

@@ -304,6 +304,46 @@ TokenSequence &TokenSequence::ClipComment(
   return *this;
 }
 
+TokenSequence &TokenSequence::RemoveRedundantCompilerDirectives(
+    const Prescanner &prescanner) {
+  // When the toekn sqeuence is "<compiler directive> <clause1>  <compiler
+  // directive> <clause2>" convert it to "<compiler directive> <clause1>
+  // <clause2>"
+  std::size_t tokens{SizeInTokens()};
+  TokenSequence result;
+  bool firstDirective{true};
+  for (std::size_t j{0}; j < tokens; ++j) {
+    CharBlock tok{TokenAt(j)};
+    bool isSentinel{false};
+    std::size_t blanks{tok.CountLeadingBlanks()};
+    if (blanks < tok.size() && tok[blanks] == '!') {
+      // Retain active compiler directive sentinels (e.g. "!dir$", "!$omp")
+      for (std::size_t k{j + 1}; k < tokens && tok.size() <= blanks + 5; ++k) {
+        if (tok.begin() + tok.size() == TokenAt(k).begin()) {
+          tok.ExtendToCover(TokenAt(k));
+        } else {
+          break;
+        }
+      }
+    }
+    if (tok.size() > blanks + 5) {
+      isSentinel =
+          prescanner.IsCompilerDirectiveSentinel(&tok[blanks + 1]).has_value();
+    }
+    if (isSentinel &&
+        !firstDirective) { // skip the directives if not the first one
+      j++;
+    } else {
+      result.Put(*this, j);
+    }
+    if (isSentinel && firstDirective) {
+      firstDirective = false;
+    }
+  }
+  swap(result);
+  return *this;
+}
+
 void TokenSequence::Emit(CookedSource &cooked) const {
   if (auto n{char_.size()}) {
     cooked.Put(&char_[0], n);

flang/test/Preprocessing/bug117693.f90

@@ -0,0 +1,14 @@
+! RUN: %flang -fopenmp -E %s 2>&1 | FileCheck %s
+!CHECK: !$OMP DO SCHEDULE(STATIC)
+program main
+IMPLICIT NONE
+INTEGER:: I
+#define OMPSUPPORT
+!$ INTEGER :: omp_id
+!$OMP PARALLEL DO
+OMPSUPPORT !$OMP DO SCHEDULE(STATIC)
+DO I=1,100
+print *, omp_id
+ENDDO
+!$OMP END PARALLEL DO
+end program

flang/test/Preprocessing/bug117693_2.f90

@@ -0,0 +1,15 @@
+! RUN: %flang -fopenmp -E %s 2>&1 | FileCheck %s
+!CHECK: !$OMP DO SCHEDULE(STATIC) DEFAULT(NONE)
+!CHECK-NOT: !$OMP DEFAULT(NONE)
+program main
+IMPLICIT NONE
+INTEGER:: I
+#define OMPSUPPORT
+!$ INTEGER :: omp_id
+!$OMP PARALLEL DO
+OMPSUPPORT !$OMP DO SCHEDULE(STATIC) !$OMP DEFAULT(NONE)
+DO I=1,100
+print *, omp_id
+ENDDO
+!$OMP END PARALLEL DO
+end program

flang/test/Preprocessing/bug117693_3.f90

@@ -0,0 +1,7 @@
+! RUN: %flang -fopenmp -E %s 2>&1 | FileCheck %s
+!CHECK-NOT: DO I=1,100 !$OMP
+program main
+INTEGER::n
+DO I=1,100 !!$OMP 
+ENDDO
+END PROGRAM