chore: Add APIs for reading files from JS

Dockerfile.emsdk

@@ -1,4 +1,4 @@
-FROM emscripten/emsdk:3.1.70 as em
+FROM emscripten/emsdk:3.1.71 as em
 
 RUN apt-get update && apt-get install -y \
     ninja-build \

packages/cxx-frontend/src/Parser.ts

@@ -33,11 +33,19 @@ interface ParserParams {
    */
   source: string;
 
+  /**
+   * Function to resolve include directives.
+   */
   resolve?: (
     name: string,
     kind: "quoted" | "angled",
     next: boolean,
   ) => Promise<string | undefined>;
+
+  /**
+   * Function to read files.
+   */
+  readFile?: (path: string) => Promise<string | undefined>;
 }
 
 export class Parser {
@@ -76,7 +84,7 @@ export class Parser {
   }
 
   constructor(options: ParserParams) {
-    const { path, source, resolve } = options;
+    const { path, source, resolve, readFile } = options;
 
     if (typeof path !== "string") {
       throw new TypeError("expected parameter 'path' of type 'string'");
@@ -86,7 +94,7 @@ export class Parser {
       throw new TypeError("expected parameter 'source' of type 'string'");
     }
 
-    this.#unit = cxx.createUnit(source, path, { resolve });
+    this.#unit = cxx.createUnit(source, path, { resolve, readFile });
   }
 
   async parse() {

packages/cxx-frontend/src/cxx-js.d.ts

@@ -89,6 +89,8 @@ interface Api {
     kind: "quoted" | "angled",
     isIncludeNext: boolean,
   ) => Promise<string | undefined>;
+
+  readFile?: (path: string) => Promise<string | undefined>;
 }
 
 export type CXX = {

src/js/cxx/api.cc

@@ -31,8 +31,10 @@
 #include <emscripten/bind.h>
 #include <emscripten/val.h>
 
+#include <cstdlib>
 #include <format>
 #include <iostream>
+#include <optional>
 #include <sstream>
 
 using namespace emscripten;
@@ -87,9 +89,11 @@ struct WrappedUnit {
 
   auto parse() -> val {
     val resolve = val::undefined();
+    val readFile = val::undefined();
 
     if (!api.isUndefined()) {
       resolve = api["resolve"];
+      readFile = api["readFile"];
     }
 
     struct {
@@ -152,6 +156,20 @@ struct WrappedUnit {
 
           request.setExists(resolved.isString());
         }
+      } else if (auto pendingFileContent =
+                     std::get_if<cxx::PendingFileContent>(&state)) {
+        if (readFile.isUndefined()) {
+          pendingFileContent->setContent(std::nullopt);
+          continue;
+        }
+
+        val content = co_await readFile(pendingFileContent->fileName);
+
+        if (content.isString()) {
+          pendingFileContent->setContent(content.as<std::string>());
+        } else {
+          pendingFileContent->setContent(std::nullopt);
+        }
       }
     }
 
@@ -258,8 +276,9 @@ auto getASTSlotCount(std::intptr_t handle, int slot) -> int {
   return static_cast<int>(slotCount);
 }
 
-auto createUnit(std::string source, std::string filename) -> WrappedUnit* {
-  auto wrapped = new WrappedUnit(std::move(source), std::move(filename));
+auto createUnit(std::string source, std::string filename, val api)
+    -> WrappedUnit* {
+  auto wrapped = new WrappedUnit(std::move(source), std::move(filename), api);
 
   return wrapped;
 }

src/parser/cxx/preprocessor.cc

@@ -45,6 +45,7 @@
 #include <unordered_set>
 #include <variant>
 
+#include "cxx/preprocessor_fwd.h"
 #include "pp_keywords-priv.h"
 
 namespace {
@@ -1004,13 +1005,6 @@ struct Preprocessor::Private {
     return fs::exists(file);
   }
 
-  [[nodiscard]] auto readFile(const fs::path &file) const -> std::string {
-    std::ifstream in(file);
-    std::ostringstream out;
-    out << in.rdbuf();
-    return out.str();
-  }
-
   [[nodiscard]] auto checkHeaderProtection(TokList *ts) const -> TokList *;
 
   [[nodiscard]] auto checkPragmaOnceProtected(TokList *ts) const -> bool;
@@ -1193,9 +1187,6 @@ struct Preprocessor::Private {
   [[nodiscard]] auto parseIncludeDirective(TokList *directive, TokList *ts)
       -> std::optional<ParsedIncludeDirective>;
 
-  [[nodiscard]] auto findOrCreateSourceFile(const std::string &fileName)
-      -> SourceFile *;
-
   [[nodiscard]] auto parseHeaderName(TokList *ts)
       -> std::tuple<TokList *, std::optional<Include>>;
 
@@ -1313,6 +1304,110 @@ struct Preprocessor::ParseArguments {
   }
 };
 
+void PendingInclude::resolveWith(
+    std::optional<std::string> resolvedFileName) const {
+  auto d = preprocessor.d.get();
+
+  if (!resolvedFileName.has_value()) {
+    const auto &header = getHeaderName(include);
+    d->error(static_cast<TokList *>(loc),
+             std::format("file '{}' not found", header));
+    return;
+  }
+
+  auto fileName = resolvedFileName.value();
+
+  auto resume = [=, this]() -> std::optional<PreprocessingState> {
+    auto sourceFile = d->findSourceFile(fileName);
+    if (!sourceFile) {
+      PendingFileContent request{
+          .preprocessor = preprocessor,
+          .fileName = fileName,
+      };
+      return request;
+    }
+
+    if (sourceFile->pragmaOnceProtected) {
+      // nothing to do
+      return std::nullopt;
+    }
+
+    if (auto it = d->ifndefProtectedFiles_.find(fileName);
+        it != d->ifndefProtectedFiles_.end() &&
+        d->macros_.contains(it->second)) {
+      return std::nullopt;
+    }
+
+    auto continuation = sourceFile;
+    if (!continuation) return std::nullopt;
+
+    // make the continuation the current file
+    auto dirpath = fs::path(continuation->fileName);
+    dirpath.remove_filename();
+
+    d->buffers_.push_back(Preprocessor::Private::Buffer{
+        .source = continuation,
+        .currentPath = dirpath,
+        .ts = continuation->tokens,
+        .includeDepth = d->includeDepth_ + 1,
+    });
+
+    if (d->willIncludeHeader_) {
+      d->willIncludeHeader_(fileName, d->includeDepth_ + 1);
+    }
+
+    return std::nullopt;
+  };
+
+  auto sourceFile = d->findSourceFile(fileName);
+  if (sourceFile) {
+    resume();
+    return;
+  }
+
+  d->continuation_ = std::move(resume);
+}
+
+void PendingFileContent::setContent(std::optional<std::string> content) const {
+  auto d = preprocessor.d.get();
+
+  if (!content.has_value()) {
+    // report error
+    return;
+  }
+
+  auto sourceFile = d->createSourceFile(fileName, std::move(*content));
+
+  sourceFile->pragmaOnceProtected =
+      d->checkPragmaOnceProtected(sourceFile->tokens);
+
+  sourceFile->headerProtection = d->checkHeaderProtection(sourceFile->tokens);
+
+  if (sourceFile->headerProtection) {
+    sourceFile->headerProtectionLevel = d->evaluating_.size();
+
+    d->ifndefProtectedFiles_.insert_or_assign(
+        sourceFile->fileName, sourceFile->headerProtection->tok->text);
+  }
+
+  auto continuation = sourceFile;
+
+  // make the continuation the current file
+  auto dirpath = fs::path(continuation->fileName);
+  dirpath.remove_filename();
+
+  d->buffers_.push_back(Preprocessor::Private::Buffer{
+      .source = continuation,
+      .currentPath = dirpath,
+      .ts = continuation->tokens,
+      .includeDepth = d->includeDepth_ + 1,
+  });
+
+  if (d->willIncludeHeader_) {
+    d->willIncludeHeader_(fileName, d->includeDepth_ + 1);
+  }
+}
+
 Preprocessor::Private::Private() {
   skipping_.push_back(false);
   evaluating_.push_back(true);
@@ -2021,43 +2116,6 @@ auto Preprocessor::Private::parseIncludeDirective(TokList *directive,
   return std::nullopt;
 }
 
-auto Preprocessor::Private::findOrCreateSourceFile(const std::string &fileName)
-    -> SourceFile * {
-  if (auto it = ifndefProtectedFiles_.find(fileName);
-      it != ifndefProtectedFiles_.end() && macros_.contains(it->second)) {
-    return nullptr;
-  }
-
-  auto sourceFile = findSourceFile(fileName);
-
-  if (sourceFile && sourceFile->pragmaOnceProtected) {
-    // nothing to do
-    return nullptr;
-  }
-
-  if (!sourceFile) {
-    sourceFile = createSourceFile(fileName, readFile(fileName));
-
-    sourceFile->pragmaOnceProtected =
-        checkPragmaOnceProtected(sourceFile->tokens);
-
-    sourceFile->headerProtection = checkHeaderProtection(sourceFile->tokens);
-
-    if (sourceFile->headerProtection) {
-      sourceFile->headerProtectionLevel = evaluating_.size();
-
-      ifndefProtectedFiles_.insert_or_assign(
-          sourceFile->fileName, sourceFile->headerProtection->tok->text);
-    }
-  }
-
-  if (willIncludeHeader_) {
-    willIncludeHeader_(fileName, includeDepth_ + 1);
-  }
-
-  return sourceFile;
-}
-
 auto Preprocessor::Private::parseHeaderName(TokList *ts)
     -> std::tuple<TokList *, std::optional<Include>> {
   if (lookat(ts, TokenKind::T_STRING_LITERAL)) {
@@ -2932,43 +2990,6 @@ void Preprocessor::preprocess(std::string source, std::string fileName,
   endPreprocessing(tokens);
 }
 
-void PendingInclude::resolveWith(std::optional<std::string> fileName) const {
-  auto d = preprocessor.d.get();
-
-  if (!fileName.has_value()) {
-    const auto &header = getHeaderName(include);
-    d->error(static_cast<TokList *>(loc),
-             std::format("file '{}' not found", header));
-    return;
-  }
-
-  auto resume = [=]() -> std::optional<PreprocessingState> {
-    auto continuation = d->findOrCreateSourceFile(*fileName);
-    if (!continuation) return std::nullopt;
-
-    // make the continuation the current file
-    auto dirpath = fs::path(continuation->fileName);
-    dirpath.remove_filename();
-
-    d->buffers_.push_back(Preprocessor::Private::Buffer{
-        .source = continuation,
-        .currentPath = dirpath,
-        .ts = continuation->tokens,
-        .includeDepth = d->includeDepth_ + 1,
-    });
-
-    return std::nullopt;
-  };
-
-  auto sourceFile = d->findSourceFile(*fileName);
-  if (sourceFile) {
-    resume();
-    return;
-  }
-
-  d->continuation_ = std::move(resume);
-}
-
 void Preprocessor::beginPreprocessing(std::string source, std::string fileName,
                                       std::vector<Token> &tokens) {
   assert(!d->findSourceFile(fileName));
@@ -3242,4 +3263,12 @@ void DefaultPreprocessorState::operator()(const PendingHasIncludes &status) {
   });
 }
 
+void DefaultPreprocessorState::operator()(const PendingFileContent &request) {
+  std::ifstream in(request.fileName);
+  std::ostringstream out;
+  out << in.rdbuf();
+
+  request.setContent(out.str());
+}
+
 }  // namespace cxx

src/parser/cxx/preprocessor.h

@@ -121,6 +121,7 @@ class Preprocessor {
   struct Private;
   struct ParseArguments;
   friend struct PendingInclude;
+  friend struct PendingFileContent;
   std::unique_ptr<Private> d;
 };
 
@@ -137,6 +138,7 @@ class DefaultPreprocessorState {
   void operator()(const CanContinuePreprocessing &);
   void operator()(const PendingInclude &status);
   void operator()(const PendingHasIncludes &status);
+  void operator()(const PendingFileContent &status);
 };
 
 }  // namespace cxx

src/parser/cxx/preprocessor_fwd.h

@@ -75,12 +75,19 @@ struct PendingHasIncludes {
   std::vector<Request> requests;
 };
 
+struct PendingFileContent {
+  Preprocessor &preprocessor;
+  std::string fileName;
+
+  void setContent(std::optional<std::string> content) const;
+};
+
 struct CanContinuePreprocessing {};
 
 struct ProcessingComplete {};
 
 using PreprocessingState =
-    std::variant<PendingInclude, PendingHasIncludes, CanContinuePreprocessing,
-                 ProcessingComplete>;
+    std::variant<PendingInclude, PendingHasIncludes, PendingFileContent,
+                 CanContinuePreprocessing, ProcessingComplete>;
 
 }  // namespace cxx