[flang] Accomodate historic preprocessing usage (#78868)

Some Fortran codes use line continuation as a form of token pasting; see
#78797. This works in
compilers that run a C-like preprocessor and then apply line
continuation to its output; f18 implements line continuation during
tokenization and preprocessing, but can still handle this case.

In the rare case when an identifier is split across two or more
continuation lines, this patch allows its parts to be distinct
preprocessing tokens for the purpose of macro replacemnt. They (or their
replacement texts) can be effectively rejoined later as a single
identifier when the cooked character stream is tokenized in parsing.

Fixes #78797.

Author: klausler

flang/lib/Parser/prescan.cpp

@@ -438,7 +438,8 @@ void Prescanner::NextChar() {
 // character is reached; handles C-style comments in preprocessing
 // directives, Fortran ! comments, stuff after the right margin in
 // fixed form, and all forms of line continuation.
-void Prescanner::SkipToNextSignificantCharacter() {
+bool Prescanner::SkipToNextSignificantCharacter() {
+  auto anyContinuationLine{false};
   if (inPreprocessorDirective_) {
     SkipCComments();
   } else {
@@ -449,6 +450,7 @@ void Prescanner::SkipToNextSignificantCharacter() {
       mightNeedSpace = *at_ == '\n';
     }
     for (; Continuation(mightNeedSpace); mightNeedSpace = false) {
+      anyContinuationLine = true;
       ++continuationLines_;
       if (MustSkipToEndOfLine()) {
         SkipToEndOfLine();
@@ -458,6 +460,7 @@ void Prescanner::SkipToNextSignificantCharacter() {
       tabInCurrentLine_ = true;
     }
   }
+  return anyContinuationLine;
 }
 
 void Prescanner::SkipCComments() {
@@ -625,7 +628,23 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
     }
     preventHollerith_ = false;
   } else if (IsLegalInIdentifier(*at_)) {
-    while (IsLegalInIdentifier(EmitCharAndAdvance(tokens, *at_))) {
+    int parts{1};
+    do {
+      EmitChar(tokens, *at_);
+      ++at_, ++column_;
+      if (SkipToNextSignificantCharacter() && IsLegalIdentifierStart(*at_)) {
+        tokens.CloseToken();
+        ++parts;
+      }
+    } while (IsLegalInIdentifier(*at_));
+    if (parts >= 3) {
+      // Subtlety: When an identifier is split across three or more continuation
+      // lines, its parts are kept as distinct pp-tokens so that macro
+      // operates on them independently.  This trick accommodates the historic
+      // practice of using line continuation for token pasting after
+      // replacement.
+    } else if (parts == 2) {
+      tokens.ReopenLastToken();
     }
     if (InFixedFormSource()) {
       SkipSpaces();

flang/lib/Parser/prescan.h

@@ -159,7 +159,8 @@ class Prescanner {
   void SkipToEndOfLine();
   bool MustSkipToEndOfLine() const;
   void NextChar();
-  void SkipToNextSignificantCharacter();
+  // True when input flowed to a continuation line
+  bool SkipToNextSignificantCharacter();
   void SkipCComments();
   void SkipSpaces();
   static const char *SkipWhiteSpace(const char *);

flang/test/Preprocessing/pp133.F90

@@ -0,0 +1,9 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+! CHECK: print *, ADC
+#define B D
+implicit none
+real ADC
+print *, A&
+  &B&
+  &C
+end