chore: Preprocess with the streaming API

Signed-off-by: Roberto Raggi <[email protected]>

Author: robertoraggi

src/frontend/cxx/cxx_document.cc

@@ -181,7 +181,6 @@ void CxxDocument::parse(std::string source, std::string fileName) {
   unit.setSource(std::move(source), fileName);
 
   auto preprocessor = unit.preprocessor();
-  preprocessor->squeeze();
 
   unit.parse(ParserConfiguration{
       .checkTypes = cli.opt_fcheck,

src/frontend/cxx/frontend.cc

@@ -266,8 +266,6 @@ auto runOnFile(const CLI& cli, const std::string& fileName) -> bool {
   }
 
   if (!shouldExit) {
-    preprocessor->squeeze();
-
     unit.parse(ParserConfiguration{
         .checkTypes = cli.opt_fcheck,
         .fuzzyTemplateResolution = true,

src/parser/cxx/preprocessor.cc

@@ -1135,7 +1135,7 @@ struct Preprocessor::Private {
   [[nodiscard]] auto parseMacroDefinition(TokList *ts) -> Macro;
 
   [[nodiscard]] auto expand(const std::function<void(const Tok *)> &emitToken)
-      -> Status;
+      -> PreprocessingState;
 
   [[nodiscard]] auto expandTokens(TokIterator it, TokIterator last,
                                   bool inConditionalExpression) -> TokIterator;
@@ -1650,7 +1650,7 @@ auto Preprocessor::Private::expandTokens(TokIterator it, TokIterator last,
 }
 
 auto Preprocessor::Private::expand(
-    const std::function<void(const Tok *)> &emitToken) -> Status {
+    const std::function<void(const Tok *)> &emitToken) -> PreprocessingState {
   if (buffers_.empty()) return ProcessingComplete{};
 
   auto buffer = buffers_.back();
@@ -2934,8 +2934,7 @@ void Preprocessor::preprocess(std::string source, std::string fileName,
   endPreprocessing(tokens);
 }
 
-void Preprocessor::PendingInclude::resolveWith(
-    std::optional<std::string> fileName) const {
+void PendingInclude::resolveWith(std::optional<std::string> fileName) const {
   auto d = preprocessor.d.get();
 
   if (!fileName.has_value()) {
@@ -2952,7 +2951,7 @@ void Preprocessor::PendingInclude::resolveWith(
   fs::path dirpath = fs::path(continuation->fileName);
   dirpath.remove_filename();
 
-  d->buffers_.push_back(Private::Buffer{
+  d->buffers_.push_back(Preprocessor::Private::Buffer{
       .source = continuation,
       .currentPath = dirpath,
       .ts = continuation->tokens,
@@ -3003,7 +3002,8 @@ void Preprocessor::endPreprocessing(std::vector<Token> &tokens) {
   tk.setFileId(mainSourceFileId);
 }
 
-auto Preprocessor::continuePreprocessing(std::vector<Token> &tokens) -> Status {
+auto Preprocessor::continuePreprocessing(std::vector<Token> &tokens)
+    -> PreprocessingState {
   // consume the continuation if there is one
   std::function<void()> continuation;
   std::swap(continuation, d->continuation_);
@@ -3208,24 +3208,20 @@ auto Preprocessor::resolve(const Include &include, bool isIncludeNext) const
   return d->resolve(include, isIncludeNext);
 }
 
-void DefaultPreprocessorState::operator()(
-    const Preprocessor::ProcessingComplete &) {
+void DefaultPreprocessorState::operator()(const ProcessingComplete &) {
   done = true;
 }
 
-void DefaultPreprocessorState::operator()(
-    const Preprocessor::CanContinuePreprocessing &) {}
+void DefaultPreprocessorState::operator()(const CanContinuePreprocessing &) {}
 
-void DefaultPreprocessorState::operator()(
-    const Preprocessor::PendingInclude &status) {
+void DefaultPreprocessorState::operator()(const PendingInclude &status) {
   auto resolvedInclude = self.resolve(status.include, status.isIncludeNext);
 
   status.resolveWith(resolvedInclude);
 }
 
-void DefaultPreprocessorState::operator()(
-    const Preprocessor::PendingHasIncludes &status) {
-  using Request = Preprocessor::PendingHasIncludes::Request;
+void DefaultPreprocessorState::operator()(const PendingHasIncludes &status) {
+  using Request = PendingHasIncludes::Request;
 
   std::ranges::for_each(status.requests, [&](const Request &dep) {
     auto resolved = self.resolve(dep.include, dep.isIncludeNext);

src/parser/cxx/preprocessor.h

@@ -20,15 +20,14 @@
 
 #pragma once
 
-#include <cxx/cxx_fwd.h>
+#include <cxx/preprocessor_fwd.h>
 
 #include <functional>
 #include <iosfwd>
 #include <memory>
 #include <optional>
 #include <string>
 #include <string_view>
-#include <variant>
 #include <vector>
 
 namespace cxx {
@@ -40,25 +39,6 @@ class Preprocessor;
 class PreprocessorDelegate;
 class SourcePosition;
 
-struct SystemInclude {
-  std::string fileName;
-
-  SystemInclude() = default;
-
-  explicit SystemInclude(std::string fileName)
-      : fileName(std::move(fileName)) {}
-};
-
-struct QuoteInclude {
-  std::string fileName;
-
-  QuoteInclude() = default;
-
-  explicit QuoteInclude(std::string fileName) : fileName(std::move(fileName)) {}
-};
-
-using Include = std::variant<SystemInclude, QuoteInclude>;
-
 class CommentHandler {
  public:
   virtual ~CommentHandler() = default;
@@ -108,37 +88,8 @@ class Preprocessor {
 
   void endPreprocessing(std::vector<Token> &outputTokens);
 
-  struct PendingInclude {
-    Preprocessor &preprocessor;
-    Include include;
-    bool isIncludeNext = false;
-    void *loc = nullptr;
-
-    void resolveWith(std::optional<std::string> fileName) const;
-  };
-
-  struct PendingHasIncludes {
-    struct Request {
-      Include include;
-      bool isIncludeNext = false;
-      bool &exists;
-
-      void setExists(bool value) const { exists = value; }
-    };
-
-    Preprocessor &preprocessor;
-    std::vector<Request> requests;
-  };
-
-  struct CanContinuePreprocessing {};
-
-  struct ProcessingComplete {};
-
-  using Status = std::variant<PendingInclude, PendingHasIncludes,
-                              CanContinuePreprocessing, ProcessingComplete>;
-
   [[nodiscard]] auto continuePreprocessing(std::vector<Token> &outputTokens)
-      -> Status;
+      -> PreprocessingState;
 
   [[nodiscard]] auto sourceFileName(uint32_t sourceFileId) const
       -> const std::string &;
@@ -180,6 +131,7 @@ class Preprocessor {
  private:
   struct Private;
   struct ParseArguments;
+  friend struct PendingInclude;
   std::unique_ptr<Private> d;
 };
 
@@ -192,10 +144,10 @@ class DefaultPreprocessorState {
 
   explicit operator bool() const { return !done; }
 
-  void operator()(const Preprocessor::ProcessingComplete &);
-  void operator()(const Preprocessor::CanContinuePreprocessing &);
-  void operator()(const Preprocessor::PendingInclude &status);
-  void operator()(const Preprocessor::PendingHasIncludes &status);
+  void operator()(const ProcessingComplete &);
+  void operator()(const CanContinuePreprocessing &);
+  void operator()(const PendingInclude &status);
+  void operator()(const PendingHasIncludes &status);
 };
 
 }  // namespace cxx

src/parser/cxx/preprocessor_fwd.h

@@ -0,0 +1,84 @@
+// Copyright (c) 2024 Roberto Raggi <[email protected]>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include <cxx/cxx_fwd.h>
+
+#include <optional>
+#include <string>
+#include <variant>
+#include <vector>
+
+namespace cxx {
+
+class Preprocessor;
+class CommentHandler;
+
+struct SystemInclude {
+  std::string fileName;
+
+  SystemInclude() = default;
+
+  explicit SystemInclude(std::string fileName)
+      : fileName(std::move(fileName)) {}
+};
+
+struct QuoteInclude {
+  std::string fileName;
+
+  QuoteInclude() = default;
+
+  explicit QuoteInclude(std::string fileName) : fileName(std::move(fileName)) {}
+};
+
+using Include = std::variant<SystemInclude, QuoteInclude>;
+
+struct PendingInclude {
+  Preprocessor &preprocessor;
+  Include include;
+  bool isIncludeNext = false;
+  void *loc = nullptr;
+
+  void resolveWith(std::optional<std::string> fileName) const;
+};
+
+struct PendingHasIncludes {
+  struct Request {
+    Include include;
+    bool isIncludeNext = false;
+    bool &exists;
+
+    void setExists(bool value) const { exists = value; }
+  };
+
+  Preprocessor &preprocessor;
+  std::vector<Request> requests;
+};
+
+struct CanContinuePreprocessing {};
+
+struct ProcessingComplete {};
+
+using PreprocessingState =
+    std::variant<PendingInclude, PendingHasIncludes, CanContinuePreprocessing,
+                 ProcessingComplete>;
+
+}  // namespace cxx

src/parser/cxx/translation_unit.cc

@@ -73,8 +73,27 @@ auto TranslationUnit::changeDiagnosticsClient(
 }
 
 void TranslationUnit::setSource(std::string source, std::string fileName) {
+  beginPreprocessing(std::move(source), std::move(fileName));
+  DefaultPreprocessorState state{*preprocessor_};
+  while (state) {
+    std::visit(state, continuePreprocessing());
+  }
+  endPreprocessing();
+}
+
+void TranslationUnit::beginPreprocessing(std::string source,
+                                         std::string fileName) {
   fileName_ = std::move(fileName);
-  preprocessor_->preprocess(std::move(source), fileName_, tokens_);
+  preprocessor_->beginPreprocessing(std::move(source), fileName_, tokens_);
+}
+
+auto TranslationUnit::continuePreprocessing() -> PreprocessingState {
+  return preprocessor_->continuePreprocessing(tokens_);
+}
+
+void TranslationUnit::endPreprocessing() {
+  preprocessor_->endPreprocessing(tokens_);
+  preprocessor_->squeeze();
 }
 
 auto TranslationUnit::tokenLength(SourceLocation loc) const -> int {
@@ -125,6 +144,10 @@ auto TranslationUnit::tokenEndPosition(SourceLocation loc) const
 }
 
 void TranslationUnit::parse(const ParserConfiguration& config) {
+  if (ast_) {
+    cxx_runtime_error("translation unit already parsed");
+  }
+
   Parser parse(this);
   parse.setConfig(config);
   parse(ast_);

src/parser/cxx/translation_unit.h

@@ -26,6 +26,7 @@
 #include <cxx/literals_fwd.h>
 #include <cxx/names_fwd.h>
 #include <cxx/parser_fwd.h>
+#include <cxx/preprocessor_fwd.h>
 #include <cxx/source_location.h>
 #include <cxx/symbols_fwd.h>
 #include <cxx/token.h>
@@ -67,8 +68,15 @@ class TranslationUnit {
     return preprocessor_.get();
   }
 
+  // set source and preprocess, deprecated.
   void setSource(std::string source, std::string fileName);
 
+  void beginPreprocessing(std::string source, std::string fileName);
+
+  [[nodiscard]] auto continuePreprocessing() -> PreprocessingState;
+
+  void endPreprocessing();
+
   [[nodiscard]] auto fatalErrors() const -> bool {
     return diagnosticsClient_->fatalErrors();
   }