diff --git a/Dockerfile.emsdk b/Dockerfile.emsdk index a0e60b3f..e31d6b06 100644 --- a/Dockerfile.emsdk +++ b/Dockerfile.emsdk @@ -1,4 +1,4 @@ -FROM emscripten/emsdk:3.1.70 as em +FROM emscripten/emsdk:3.1.71 as em RUN apt-get update && apt-get install -y \ ninja-build \ diff --git a/packages/cxx-frontend/src/Parser.ts b/packages/cxx-frontend/src/Parser.ts index 33bacd24..24fca212 100644 --- a/packages/cxx-frontend/src/Parser.ts +++ b/packages/cxx-frontend/src/Parser.ts @@ -33,11 +33,19 @@ interface ParserParams { */ source: string; + /** + * Function to resolve include directives. + */ resolve?: ( name: string, kind: "quoted" | "angled", next: boolean, ) => Promise; + + /** + * Function to read files. + */ + readFile?: (path: string) => Promise; } export class Parser { @@ -76,7 +84,7 @@ export class Parser { } constructor(options: ParserParams) { - const { path, source, resolve } = options; + const { path, source, resolve, readFile } = options; if (typeof path !== "string") { throw new TypeError("expected parameter 'path' of type 'string'"); @@ -86,7 +94,7 @@ export class Parser { throw new TypeError("expected parameter 'source' of type 'string'"); } - this.#unit = cxx.createUnit(source, path, { resolve }); + this.#unit = cxx.createUnit(source, path, { resolve, readFile }); } async parse() { diff --git a/packages/cxx-frontend/src/cxx-js.d.ts b/packages/cxx-frontend/src/cxx-js.d.ts index b1f0a6d2..9d278631 100644 --- a/packages/cxx-frontend/src/cxx-js.d.ts +++ b/packages/cxx-frontend/src/cxx-js.d.ts @@ -89,6 +89,8 @@ interface Api { kind: "quoted" | "angled", isIncludeNext: boolean, ) => Promise; + + readFile?: (path: string) => Promise; } export type CXX = { diff --git a/src/js/cxx/api.cc b/src/js/cxx/api.cc index a1cf6474..674ae49f 100644 --- a/src/js/cxx/api.cc +++ b/src/js/cxx/api.cc @@ -31,8 +31,10 @@ #include #include +#include #include #include +#include #include using namespace emscripten; @@ -87,9 +89,11 @@ struct WrappedUnit { auto parse() -> val { val resolve = val::undefined(); + val readFile = val::undefined(); if (!api.isUndefined()) { resolve = api["resolve"]; + readFile = api["readFile"]; } struct { @@ -152,6 +156,20 @@ struct WrappedUnit { request.setExists(resolved.isString()); } + } else if (auto pendingFileContent = + std::get_if(&state)) { + if (readFile.isUndefined()) { + pendingFileContent->setContent(std::nullopt); + continue; + } + + val content = co_await readFile(pendingFileContent->fileName); + + if (content.isString()) { + pendingFileContent->setContent(content.as()); + } else { + pendingFileContent->setContent(std::nullopt); + } } } @@ -258,8 +276,9 @@ auto getASTSlotCount(std::intptr_t handle, int slot) -> int { return static_cast(slotCount); } -auto createUnit(std::string source, std::string filename) -> WrappedUnit* { - auto wrapped = new WrappedUnit(std::move(source), std::move(filename)); +auto createUnit(std::string source, std::string filename, val api) + -> WrappedUnit* { + auto wrapped = new WrappedUnit(std::move(source), std::move(filename), api); return wrapped; } diff --git a/src/parser/cxx/preprocessor.cc b/src/parser/cxx/preprocessor.cc index 2730eb4e..80d3c7a4 100644 --- a/src/parser/cxx/preprocessor.cc +++ b/src/parser/cxx/preprocessor.cc @@ -45,6 +45,7 @@ #include #include +#include "cxx/preprocessor_fwd.h" #include "pp_keywords-priv.h" namespace { @@ -1004,13 +1005,6 @@ struct Preprocessor::Private { return fs::exists(file); } - [[nodiscard]] auto readFile(const fs::path &file) const -> std::string { - std::ifstream in(file); - std::ostringstream out; - out << in.rdbuf(); - return out.str(); - } - [[nodiscard]] auto checkHeaderProtection(TokList *ts) const -> TokList *; [[nodiscard]] auto checkPragmaOnceProtected(TokList *ts) const -> bool; @@ -1193,9 +1187,6 @@ struct Preprocessor::Private { [[nodiscard]] auto parseIncludeDirective(TokList *directive, TokList *ts) -> std::optional; - [[nodiscard]] auto findOrCreateSourceFile(const std::string &fileName) - -> SourceFile *; - [[nodiscard]] auto parseHeaderName(TokList *ts) -> std::tuple>; @@ -1313,6 +1304,110 @@ struct Preprocessor::ParseArguments { } }; +void PendingInclude::resolveWith( + std::optional resolvedFileName) const { + auto d = preprocessor.d.get(); + + if (!resolvedFileName.has_value()) { + const auto &header = getHeaderName(include); + d->error(static_cast(loc), + std::format("file '{}' not found", header)); + return; + } + + auto fileName = resolvedFileName.value(); + + auto resume = [=, this]() -> std::optional { + auto sourceFile = d->findSourceFile(fileName); + if (!sourceFile) { + PendingFileContent request{ + .preprocessor = preprocessor, + .fileName = fileName, + }; + return request; + } + + if (sourceFile->pragmaOnceProtected) { + // nothing to do + return std::nullopt; + } + + if (auto it = d->ifndefProtectedFiles_.find(fileName); + it != d->ifndefProtectedFiles_.end() && + d->macros_.contains(it->second)) { + return std::nullopt; + } + + auto continuation = sourceFile; + if (!continuation) return std::nullopt; + + // make the continuation the current file + auto dirpath = fs::path(continuation->fileName); + dirpath.remove_filename(); + + d->buffers_.push_back(Preprocessor::Private::Buffer{ + .source = continuation, + .currentPath = dirpath, + .ts = continuation->tokens, + .includeDepth = d->includeDepth_ + 1, + }); + + if (d->willIncludeHeader_) { + d->willIncludeHeader_(fileName, d->includeDepth_ + 1); + } + + return std::nullopt; + }; + + auto sourceFile = d->findSourceFile(fileName); + if (sourceFile) { + resume(); + return; + } + + d->continuation_ = std::move(resume); +} + +void PendingFileContent::setContent(std::optional content) const { + auto d = preprocessor.d.get(); + + if (!content.has_value()) { + // report error + return; + } + + auto sourceFile = d->createSourceFile(fileName, std::move(*content)); + + sourceFile->pragmaOnceProtected = + d->checkPragmaOnceProtected(sourceFile->tokens); + + sourceFile->headerProtection = d->checkHeaderProtection(sourceFile->tokens); + + if (sourceFile->headerProtection) { + sourceFile->headerProtectionLevel = d->evaluating_.size(); + + d->ifndefProtectedFiles_.insert_or_assign( + sourceFile->fileName, sourceFile->headerProtection->tok->text); + } + + auto continuation = sourceFile; + + // make the continuation the current file + auto dirpath = fs::path(continuation->fileName); + dirpath.remove_filename(); + + d->buffers_.push_back(Preprocessor::Private::Buffer{ + .source = continuation, + .currentPath = dirpath, + .ts = continuation->tokens, + .includeDepth = d->includeDepth_ + 1, + }); + + if (d->willIncludeHeader_) { + d->willIncludeHeader_(fileName, d->includeDepth_ + 1); + } +} + Preprocessor::Private::Private() { skipping_.push_back(false); evaluating_.push_back(true); @@ -2021,43 +2116,6 @@ auto Preprocessor::Private::parseIncludeDirective(TokList *directive, return std::nullopt; } -auto Preprocessor::Private::findOrCreateSourceFile(const std::string &fileName) - -> SourceFile * { - if (auto it = ifndefProtectedFiles_.find(fileName); - it != ifndefProtectedFiles_.end() && macros_.contains(it->second)) { - return nullptr; - } - - auto sourceFile = findSourceFile(fileName); - - if (sourceFile && sourceFile->pragmaOnceProtected) { - // nothing to do - return nullptr; - } - - if (!sourceFile) { - sourceFile = createSourceFile(fileName, readFile(fileName)); - - sourceFile->pragmaOnceProtected = - checkPragmaOnceProtected(sourceFile->tokens); - - sourceFile->headerProtection = checkHeaderProtection(sourceFile->tokens); - - if (sourceFile->headerProtection) { - sourceFile->headerProtectionLevel = evaluating_.size(); - - ifndefProtectedFiles_.insert_or_assign( - sourceFile->fileName, sourceFile->headerProtection->tok->text); - } - } - - if (willIncludeHeader_) { - willIncludeHeader_(fileName, includeDepth_ + 1); - } - - return sourceFile; -} - auto Preprocessor::Private::parseHeaderName(TokList *ts) -> std::tuple> { if (lookat(ts, TokenKind::T_STRING_LITERAL)) { @@ -2932,43 +2990,6 @@ void Preprocessor::preprocess(std::string source, std::string fileName, endPreprocessing(tokens); } -void PendingInclude::resolveWith(std::optional fileName) const { - auto d = preprocessor.d.get(); - - if (!fileName.has_value()) { - const auto &header = getHeaderName(include); - d->error(static_cast(loc), - std::format("file '{}' not found", header)); - return; - } - - auto resume = [=]() -> std::optional { - auto continuation = d->findOrCreateSourceFile(*fileName); - if (!continuation) return std::nullopt; - - // make the continuation the current file - auto dirpath = fs::path(continuation->fileName); - dirpath.remove_filename(); - - d->buffers_.push_back(Preprocessor::Private::Buffer{ - .source = continuation, - .currentPath = dirpath, - .ts = continuation->tokens, - .includeDepth = d->includeDepth_ + 1, - }); - - return std::nullopt; - }; - - auto sourceFile = d->findSourceFile(*fileName); - if (sourceFile) { - resume(); - return; - } - - d->continuation_ = std::move(resume); -} - void Preprocessor::beginPreprocessing(std::string source, std::string fileName, std::vector &tokens) { assert(!d->findSourceFile(fileName)); @@ -3242,4 +3263,12 @@ void DefaultPreprocessorState::operator()(const PendingHasIncludes &status) { }); } +void DefaultPreprocessorState::operator()(const PendingFileContent &request) { + std::ifstream in(request.fileName); + std::ostringstream out; + out << in.rdbuf(); + + request.setContent(out.str()); +} + } // namespace cxx diff --git a/src/parser/cxx/preprocessor.h b/src/parser/cxx/preprocessor.h index c888e5b3..166cc0ab 100644 --- a/src/parser/cxx/preprocessor.h +++ b/src/parser/cxx/preprocessor.h @@ -121,6 +121,7 @@ class Preprocessor { struct Private; struct ParseArguments; friend struct PendingInclude; + friend struct PendingFileContent; std::unique_ptr d; }; @@ -137,6 +138,7 @@ class DefaultPreprocessorState { void operator()(const CanContinuePreprocessing &); void operator()(const PendingInclude &status); void operator()(const PendingHasIncludes &status); + void operator()(const PendingFileContent &status); }; } // namespace cxx diff --git a/src/parser/cxx/preprocessor_fwd.h b/src/parser/cxx/preprocessor_fwd.h index dadd4747..c63a1e3f 100644 --- a/src/parser/cxx/preprocessor_fwd.h +++ b/src/parser/cxx/preprocessor_fwd.h @@ -75,12 +75,19 @@ struct PendingHasIncludes { std::vector requests; }; +struct PendingFileContent { + Preprocessor &preprocessor; + std::string fileName; + + void setContent(std::optional content) const; +}; + struct CanContinuePreprocessing {}; struct ProcessingComplete {}; using PreprocessingState = - std::variant; + std::variant; } // namespace cxx diff --git a/templates/cxx-parse/index.js b/templates/cxx-parse/index.js index fb83b0a4..265fab2a 100644 --- a/templates/cxx-parse/index.js +++ b/templates/cxx-parse/index.js @@ -19,11 +19,9 @@ // SOFTWARE. const { Parser, AST, ASTKind, ASTSlot } = require("cxx-frontend"); -const { readFileSync } = require("fs"); +const { readFile } = require("fs/promises"); const source = ` -#include - template concept CanAdd = requires(T n) { n + n; @@ -42,7 +40,7 @@ int main() { async function main() { const wasmBinaryFile = require.resolve("cxx-frontend/dist/wasm/cxx-js.wasm"); - const wasm = readFileSync(wasmBinaryFile); + const wasm = await readFile(wasmBinaryFile); await Parser.init({ wasm }); const parser = new Parser({