[flang] Produce proper "preprocessor output" for -E option

Rename the current -E option to "-E -Xflang -fno-reformat".

Add a new Parsing::EmitPreprocessedSource() routine to convert the
cooked character stream output of the prescanner back to something
more closely resembling output from a traditional preprocessor;
call this new routine when -E appears.

The new -E output is suitable for use as fixed form Fortran source to
compilation by (one hopes) any Fortran compiler.  If the original
top-level source file had been free form source, the output will be
suitable for use as free form source as well; otherwise there may be
diagnostics about missing spaces if they were indeed absent in the
original fixed form source.

Unless the -P option appears, #line directives are interspersed
with the output (but be advised, f18 will ignore these if presented
with them in a later compilation).

An effort has been made to preserve original alphabetic character case
and source indentation.

Add -P and -fno-reformat to the new drivers.

Tweak test options to avoid confusion with prior -E output; use
-fno-reformat where needed, but prefer to keep -E, sometimes
in concert with -P, on most, updating expected results accordingly.

Differential Revision: https://reviews.llvm.org/D106727

Author: klausler

clang/include/clang/Driver/Options.td

@@ -701,7 +701,7 @@ def ObjC : Flag<["-"], "ObjC">, Flags<[NoXarchOption]>,
 def O : Joined<["-"], "O">, Group<O_Group>, Flags<[CC1Option]>;
 def O_flag : Flag<["-"], "O">, Flags<[CC1Option]>, Alias<O>, AliasArgs<["1"]>;
 def Ofast : Joined<["-"], "Ofast">, Group<O_Group>, Flags<[CC1Option]>;
-def P : Flag<["-"], "P">, Flags<[CC1Option]>, Group<Preprocessor_Group>,
+def P : Flag<["-"], "P">, Flags<[CC1Option,FlangOption,FC1Option]>, Group<Preprocessor_Group>,
   HelpText<"Disable linemarker output in -E mode">,
   MarshallingInfoNegativeFlag<PreprocessorOutputOpts<"ShowLineMarkers">>;
 def Qy : Flag<["-"], "Qy">, Flags<[CC1Option]>,
@@ -4514,7 +4514,7 @@ def fdebug_measure_parse_tree : Flag<["-"], "fdebug-measure-parse-tree">, Group<
   HelpText<"Measure the parse tree">;
 def fdebug_pre_fir_tree : Flag<["-"], "fdebug-pre-fir-tree">, Group<Action_Group>,
   HelpText<"Dump the pre-FIR tree">;
-def fdebug_module_writer : Flag<["-"],"fdebug-module-writer">, 
+def fdebug_module_writer : Flag<["-"],"fdebug-module-writer">,
   HelpText<"Enable debug messages while writing module files">;
 def fget_symbols_sources : Flag<["-"], "fget-symbols-sources">, Group<Action_Group>,
   HelpText<"Dump symbols and their source code locations">;
@@ -4526,7 +4526,8 @@ def fanalyzed_objects_for_unparse : Flag<["-"],
 def fno_analyzed_objects_for_unparse : Flag<["-"],
   "fno-analyzed-objects-for-unparse">,  Group<f_Group>,
   HelpText<"Do not use the analyzed objects when unparsing">;
-
+def fno_reformat : Flag<["-"], "fno-reformat">, Group<Preprocessor_Group>,
+  HelpText<"Dump the cooked character stream in -E mode">;
 }
 
 //===----------------------------------------------------------------------===//

clang/lib/Driver/ToolChains/Flang.cpp

@@ -37,8 +37,9 @@ void Flang::AddFortranDialectOptions(const ArgList &Args,
 
 void Flang::AddPreprocessingOptions(const ArgList &Args,
                                     ArgStringList &CmdArgs) const {
-  Args.AddAllArgs(CmdArgs, {options::OPT_D, options::OPT_U, options::OPT_I,
-                            options::OPT_cpp, options::OPT_nocpp});
+  Args.AddAllArgs(CmdArgs,
+                  {options::OPT_P, options::OPT_D, options::OPT_U,
+                   options::OPT_I, options::OPT_cpp, options::OPT_nocpp});
 }
 
 void Flang::AddOtherOptions(const ArgList &Args, ArgStringList &CmdArgs) const {

flang/include/flang/Frontend/FrontendOptions.h

@@ -25,7 +25,7 @@ enum ActionKind {
   /// -test-io mode
   InputOutputTest,
 
-  /// -E mode.
+  /// -E mode
   PrintPreprocessedInput,
 
   /// -fsyntax-only

flang/include/flang/Frontend/PreprocessorOptions.h

@@ -44,6 +44,12 @@ class PreprocessorOptions {
 
   PPMacrosFlag macrosFlag_ = PPMacrosFlag::Unknown;
 
+  // -P: Suppress #line directives in -E output
+  bool noLineDirectives{false};
+
+  // -fno-reformat: Emit cooked character stream as -E output
+  bool noReformat{false};
+
 public:
   PreprocessorOptions() {}
 

flang/include/flang/Parser/parsing.h

@@ -37,6 +37,7 @@ struct Options {
   bool isModuleFile{false};
   bool needProvenanceRangeToCharBlockMappings{false};
   Fortran::parser::Encoding encoding{Fortran::parser::Encoding::UTF_8};
+  bool prescanAndReformat{false}; // -E
 };
 
 class Parsing {
@@ -47,12 +48,15 @@ class Parsing {
   bool consumedWholeFile() const { return consumedWholeFile_; }
   const char *finalRestingPlace() const { return finalRestingPlace_; }
   AllCookedSources &allCooked() { return allCooked_; }
+  const AllCookedSources &allCooked() const { return allCooked_; }
   Messages &messages() { return messages_; }
   std::optional<Program> &parseTree() { return parseTree_; }
 
   const CookedSource &cooked() const { return DEREF(currentCooked_); }
 
   const SourceFile *Prescan(const std::string &path, Options);
+  void EmitPreprocessedSource(
+      llvm::raw_ostream &, bool lineDirectives = true) const;
   void DumpCookedChars(llvm::raw_ostream &) const;
   void DumpProvenance(llvm::raw_ostream &) const;
   void DumpParsingLog(llvm::raw_ostream &) const;

flang/lib/Frontend/CompilerInvocation.cpp

@@ -371,6 +371,9 @@ static void parsePreprocessorArgs(
         (currentArg->getOption().matches(clang::driver::options::OPT_cpp))
         ? PPMacrosFlag::Include
         : PPMacrosFlag::Exclude;
+
+  opts.noReformat = args.hasArg(clang::driver::options::OPT_fno_reformat);
+  opts.noLineDirectives = args.hasArg(clang::driver::options::OPT_P);
 }
 
 /// Parses all semantic related arguments and populates the variables

flang/lib/Frontend/FrontendActions.cpp

@@ -208,9 +208,14 @@ void PrintPreprocessedAction::ExecuteAction() {
   std::string buf;
   llvm::raw_string_ostream outForPP{buf};
 
-  // Run the preprocessor
+  // Format or dump the prescanner's output
   CompilerInstance &ci = this->instance();
-  ci.parsing().DumpCookedChars(outForPP);
+  if (ci.invocation().preprocessorOpts().noReformat) {
+    ci.parsing().DumpCookedChars(outForPP);
+  } else {
+    ci.parsing().EmitPreprocessedSource(
+        outForPP, !ci.invocation().preprocessorOpts().noLineDirectives);
+  }
 
   // If a pre-defined output stream exists, dump the preprocessed content there
   if (!ci.IsOutputStreamNull()) {
@@ -219,7 +224,7 @@ void PrintPreprocessedAction::ExecuteAction() {
     return;
   }
 
-  // Print diagnostics from the preprocessor
+  // Print diagnostics from the prescanner
   ci.parsing().messages().Emit(llvm::errs(), ci.allCookedSources());
 
   // Create a file and save the preprocessed output there
@@ -228,7 +233,6 @@ void PrintPreprocessedAction::ExecuteAction() {
     (*os) << outForPP.str();
   } else {
     llvm::errs() << "Unable to create the output file\n";
-    return;
   }
 }
 

flang/lib/Frontend/FrontendOptions.cpp

@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "flang/Frontend/FrontendOptions.h"
-#include "flang/Evaluate/expression.h"
 
 using namespace Fortran::frontend;
 

flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp

@@ -28,61 +28,43 @@ static std::unique_ptr<FrontendAction> CreateFrontendBaseAction(
   switch (ak) {
   case InputOutputTest:
     return std::make_unique<InputOutputTestAction>();
-    break;
   case PrintPreprocessedInput:
     return std::make_unique<PrintPreprocessedAction>();
-    break;
   case ParseSyntaxOnly:
     return std::make_unique<ParseSyntaxOnlyAction>();
   case EmitObj:
     return std::make_unique<EmitObjAction>();
-    break;
   case DebugUnparse:
     return std::make_unique<DebugUnparseAction>();
-    break;
   case DebugUnparseNoSema:
     return std::make_unique<DebugUnparseNoSemaAction>();
-    break;
   case DebugUnparseWithSymbols:
     return std::make_unique<DebugUnparseWithSymbolsAction>();
-    break;
   case DebugDumpSymbols:
     return std::make_unique<DebugDumpSymbolsAction>();
-    break;
   case DebugDumpParseTree:
     return std::make_unique<DebugDumpParseTreeAction>();
-    break;
   case DebugDumpParseTreeNoSema:
     return std::make_unique<DebugDumpParseTreeNoSemaAction>();
-    break;
   case DebugDumpAll:
     return std::make_unique<DebugDumpAllAction>();
-    break;
   case DebugDumpProvenance:
     return std::make_unique<DebugDumpProvenanceAction>();
-    break;
   case DebugDumpParsingLog:
     return std::make_unique<DebugDumpParsingLogAction>();
-    break;
   case DebugMeasureParseTree:
     return std::make_unique<DebugMeasureParseTreeAction>();
-    break;
   case DebugPreFIRTree:
     return std::make_unique<DebugPreFIRTreeAction>();
-    break;
   case GetDefinition:
     return std::make_unique<GetDefinitionAction>();
-    break;
   case GetSymbolsSources:
     return std::make_unique<GetSymbolsSourcesAction>();
-    break;
   case InitOnly:
     return std::make_unique<InitOnlyAction>();
-    break;
   default:
     break;
     // TODO:
-    // case RunPreprocessor:
     // case ParserSyntaxOnly:
     // case EmitLLVM:
     // case EmitLLVMOnly:

flang/lib/Parser/parsing.cpp

@@ -95,6 +95,94 @@ const SourceFile *Parsing::Prescan(const std::string &path, Options options) {
   return sourceFile;
 }
 
+void Parsing::EmitPreprocessedSource(
+    llvm::raw_ostream &out, bool lineDirectives) const {
+  const SourceFile *sourceFile{nullptr};
+  int sourceLine{0};
+  int column{1};
+  bool inDirective{false};
+  bool inContinuation{false};
+  const AllSources &allSources{allCooked().allSources()};
+  for (const char &atChar : cooked().AsCharBlock()) {
+    char ch{atChar};
+    if (ch == '\n') {
+      out << '\n'; // TODO: DOS CR-LF line ending if necessary
+      column = 1;
+      inDirective = false;
+      inContinuation = false;
+      ++sourceLine;
+    } else {
+      if (ch == '!') {
+        // Other comment markers (C, *, D) in original fixed form source
+        // input card column 1 will have been deleted or normalized to !,
+        // which signifies a comment (directive) in both source forms.
+        inDirective = true;
+      }
+      auto provenance{cooked().GetProvenanceRange(CharBlock{&atChar, 1})};
+      std::optional<SourcePosition> position{provenance
+              ? allSources.GetSourcePosition(provenance->start())
+              : std::nullopt};
+      if (lineDirectives && column == 1 && position) {
+        if (&position->file != sourceFile) {
+          out << "#line \"" << position->file.path() << "\" " << position->line
+              << '\n';
+        } else if (position->line != sourceLine) {
+          if (sourceLine < position->line &&
+              sourceLine + 10 >= position->line) {
+            // Emit a few newlines to catch up when they'll likely
+            // require fewer bytes than a #line directive would have
+            // occupied.
+            while (sourceLine++ < position->line) {
+              out << '\n';
+            }
+          } else {
+            out << "#line " << position->line << '\n';
+          }
+        }
+        sourceFile = &position->file;
+        sourceLine = position->line;
+      }
+      if (column > 72) {
+        // Wrap long lines in a portable fashion that works in both
+        // of the Fortran source forms.  The first free-form continuation
+        // marker ("&") lands in column 73, which begins the card commentary
+        // field of fixed form, and the second one is put in column 6,
+        // where it signifies fixed form line continuation.
+        // The standard Fortran fixed form column limit (72) is used
+        // for output, even if the input was parsed with a nonstandard
+        // column limit override option.
+        out << "&\n     &";
+        column = 7; // start of fixed form source field
+        ++sourceLine;
+        inContinuation = true;
+      } else if (!inDirective && ch != ' ' && (ch < '0' || ch > '9')) {
+        // Put anything other than a label or directive into the
+        // Fortran fixed form source field (columns [7:72]).
+        for (; column < 7; ++column) {
+          out << ' ';
+        }
+      }
+      if (!inContinuation && position && position->column <= 72 && ch != ' ') {
+        // Preserve original indentation
+        for (; column < position->column; ++column) {
+          out << ' ';
+        }
+      }
+      if (ch >= 'a' && ch <= 'z' && provenance && provenance->size() == 1) {
+        // Preserve original case
+        if (const char *orig{allSources.GetSource(*provenance)}) {
+          auto upper{static_cast<char>(ch + 'A' - 'a')};
+          if (*orig == upper) {
+            ch = upper;
+          }
+        }
+      }
+      out << ch;
+      ++column;
+    }
+  }
+}
+
 void Parsing::DumpCookedChars(llvm::raw_ostream &out) const {
   UserState userState{allCooked_, common::LanguageFeatureControl{}};
   ParseState parseState{cooked()};