Skip to content

Commit 331bed1

Browse files
robertoraggirobertoraggi
committed
chore: Add APIs for reading files from JS
1 parent 5ee8198 commit 331bed1

File tree

8 files changed

+163
-95
lines changed

8 files changed

+163
-95
lines changed

‎Dockerfile.emsdk‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM emscripten/emsdk:3.1.70 as em
1+
FROM emscripten/emsdk:3.1.71 as em
22

33
RUN apt-get update && apt-get install -y \
44
ninja-build \

‎packages/cxx-frontend/src/Parser.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,19 @@ interface ParserParams {
3333
*/
3434
source: string;
3535

36+
/**
37+
* Function to resolve include directives.
38+
*/
3639
resolve?: (
3740
name: string,
3841
kind: "quoted" | "angled",
3942
next: boolean,
4043
) => Promise<string | undefined>;
44+
45+
/**
46+
* Function to read files.
47+
*/
48+
readFile?: (path: string) => Promise<string | undefined>;
4149
}
4250

4351
export class Parser {
@@ -76,7 +84,7 @@ export class Parser {
7684
}
7785

7886
constructor(options: ParserParams) {
79-
const { path, source, resolve } = options;
87+
const { path, source, resolve, readFile } = options;
8088

8189
if (typeof path !== "string") {
8290
throw new TypeError("expected parameter 'path' of type 'string'");
@@ -86,7 +94,7 @@ export class Parser {
8694
throw new TypeError("expected parameter 'source' of type 'string'");
8795
}
8896

89-
this.#unit = cxx.createUnit(source, path, { resolve });
97+
this.#unit = cxx.createUnit(source, path, { resolve, readFile });
9098
}
9199

92100
async parse() {

‎packages/cxx-frontend/src/cxx-js.d.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@ interface Api {
8989
kind: "quoted" | "angled",
9090
isIncludeNext: boolean,
9191
) => Promise<string | undefined>;
92+
93+
readFile?: (path: string) => Promise<string | undefined>;
9294
}
9395

9496
export type CXX = {

‎src/js/cxx/api.cc‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,10 @@
3131
#include <emscripten/bind.h>
3232
#include <emscripten/val.h>
3333

34+
#include <cstdlib>
3435
#include <format>
3536
#include <iostream>
37+
#include <optional>
3638
#include <sstream>
3739

3840
using namespace emscripten;
@@ -87,9 +89,14 @@ struct WrappedUnit {
8789

8890
auto parse() -> val {
8991
val resolve = val::undefined();
92+
val readFile = val::undefined();
9093

9194
if (!api.isUndefined()) {
9295
resolve = api["resolve"];
96+
readFile = api["readFile"];
97+
std::cerr << "set resolve and read file\n";
98+
} else {
99+
std::cerr << "no resolve and read file\n";
93100
}
94101

95102
struct {
@@ -152,6 +159,20 @@ struct WrappedUnit {
152159

153160
request.setExists(resolved.isString());
154161
}
162+
} else if (auto pendingFileContent =
163+
std::get_if<cxx::PendingFileContent>(&state)) {
164+
if (readFile.isUndefined()) {
165+
pendingFileContent->setContent(std::nullopt);
166+
continue;
167+
}
168+
169+
val content = co_await readFile(pendingFileContent->fileName);
170+
171+
if (content.isString()) {
172+
pendingFileContent->setContent(content.as<std::string>());
173+
} else {
174+
pendingFileContent->setContent(std::nullopt);
175+
}
155176
}
156177
}
157178

@@ -258,8 +279,9 @@ auto getASTSlotCount(std::intptr_t handle, int slot) -> int {
258279
return static_cast<int>(slotCount);
259280
}
260281

261-
auto createUnit(std::string source, std::string filename) -> WrappedUnit* {
262-
auto wrapped = new WrappedUnit(std::move(source), std::move(filename));
282+
auto createUnit(std::string source, std::string filename, val api)
283+
-> WrappedUnit* {
284+
auto wrapped = new WrappedUnit(std::move(source), std::move(filename), api);
263285

264286
return wrapped;
265287
}

‎src/parser/cxx/preprocessor.cc‎

Lines changed: 113 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
#include <unordered_set>
4646
#include <variant>
4747

48+
#include "cxx/preprocessor_fwd.h"
4849
#include "pp_keywords-priv.h"
4950

5051
namespace {
@@ -1004,13 +1005,6 @@ struct Preprocessor::Private {
10041005
return fs::exists(file);
10051006
}
10061007

1007-
[[nodiscard]] auto readFile(const fs::path &file) const -> std::string {
1008-
std::ifstream in(file);
1009-
std::ostringstream out;
1010-
out << in.rdbuf();
1011-
return out.str();
1012-
}
1013-
10141008
[[nodiscard]] auto checkHeaderProtection(TokList *ts) const -> TokList *;
10151009

10161010
[[nodiscard]] auto checkPragmaOnceProtected(TokList *ts) const -> bool;
@@ -1193,9 +1187,6 @@ struct Preprocessor::Private {
11931187
[[nodiscard]] auto parseIncludeDirective(TokList *directive, TokList *ts)
11941188
-> std::optional<ParsedIncludeDirective>;
11951189

1196-
[[nodiscard]] auto findOrCreateSourceFile(const std::string &fileName)
1197-
-> SourceFile *;
1198-
11991190
[[nodiscard]] auto parseHeaderName(TokList *ts)
12001191
-> std::tuple<TokList *, std::optional<Include>>;
12011192

@@ -1313,6 +1304,110 @@ struct Preprocessor::ParseArguments {
13131304
}
13141305
};
13151306

1307+
void PendingInclude::resolveWith(
1308+
std::optional<std::string> resolvedFileName) const {
1309+
auto d = preprocessor.d.get();
1310+
1311+
if (!resolvedFileName.has_value()) {
1312+
const auto &header = getHeaderName(include);
1313+
d->error(static_cast<TokList *>(loc),
1314+
std::format("file '{}' not found", header));
1315+
return;
1316+
}
1317+
1318+
auto fileName = resolvedFileName.value();
1319+
1320+
auto resume = [=, this]() -> std::optional<PreprocessingState> {
1321+
auto sourceFile = d->findSourceFile(fileName);
1322+
if (!sourceFile) {
1323+
PendingFileContent request{
1324+
.preprocessor = preprocessor,
1325+
.fileName = fileName,
1326+
};
1327+
return request;
1328+
}
1329+
1330+
if (sourceFile->pragmaOnceProtected) {
1331+
// nothing to do
1332+
return std::nullopt;
1333+
}
1334+
1335+
if (auto it = d->ifndefProtectedFiles_.find(fileName);
1336+
it != d->ifndefProtectedFiles_.end() &&
1337+
d->macros_.contains(it->second)) {
1338+
return std::nullopt;
1339+
}
1340+
1341+
auto continuation = sourceFile;
1342+
if (!continuation) return std::nullopt;
1343+
1344+
// make the continuation the current file
1345+
auto dirpath = fs::path(continuation->fileName);
1346+
dirpath.remove_filename();
1347+
1348+
d->buffers_.push_back(Preprocessor::Private::Buffer{
1349+
.source = continuation,
1350+
.currentPath = dirpath,
1351+
.ts = continuation->tokens,
1352+
.includeDepth = d->includeDepth_ + 1,
1353+
});
1354+
1355+
if (d->willIncludeHeader_) {
1356+
d->willIncludeHeader_(fileName, d->includeDepth_ + 1);
1357+
}
1358+
1359+
return std::nullopt;
1360+
};
1361+
1362+
auto sourceFile = d->findSourceFile(fileName);
1363+
if (sourceFile) {
1364+
resume();
1365+
return;
1366+
}
1367+
1368+
d->continuation_ = std::move(resume);
1369+
}
1370+
1371+
void PendingFileContent::setContent(std::optional<std::string> content) const {
1372+
auto d = preprocessor.d.get();
1373+
1374+
if (!content.has_value()) {
1375+
// report error
1376+
return;
1377+
}
1378+
1379+
auto sourceFile = d->createSourceFile(fileName, std::move(*content));
1380+
1381+
sourceFile->pragmaOnceProtected =
1382+
d->checkPragmaOnceProtected(sourceFile->tokens);
1383+
1384+
sourceFile->headerProtection = d->checkHeaderProtection(sourceFile->tokens);
1385+
1386+
if (sourceFile->headerProtection) {
1387+
sourceFile->headerProtectionLevel = d->evaluating_.size();
1388+
1389+
d->ifndefProtectedFiles_.insert_or_assign(
1390+
sourceFile->fileName, sourceFile->headerProtection->tok->text);
1391+
}
1392+
1393+
auto continuation = sourceFile;
1394+
1395+
// make the continuation the current file
1396+
auto dirpath = fs::path(continuation->fileName);
1397+
dirpath.remove_filename();
1398+
1399+
d->buffers_.push_back(Preprocessor::Private::Buffer{
1400+
.source = continuation,
1401+
.currentPath = dirpath,
1402+
.ts = continuation->tokens,
1403+
.includeDepth = d->includeDepth_ + 1,
1404+
});
1405+
1406+
if (d->willIncludeHeader_) {
1407+
d->willIncludeHeader_(fileName, d->includeDepth_ + 1);
1408+
}
1409+
}
1410+
13161411
Preprocessor::Private::Private() {
13171412
skipping_.push_back(false);
13181413
evaluating_.push_back(true);
@@ -2021,43 +2116,6 @@ auto Preprocessor::Private::parseIncludeDirective(TokList *directive,
20212116
return std::nullopt;
20222117
}
20232118

2024-
auto Preprocessor::Private::findOrCreateSourceFile(const std::string &fileName)
2025-
-> SourceFile * {
2026-
if (auto it = ifndefProtectedFiles_.find(fileName);
2027-
it != ifndefProtectedFiles_.end() && macros_.contains(it->second)) {
2028-
return nullptr;
2029-
}
2030-
2031-
auto sourceFile = findSourceFile(fileName);
2032-
2033-
if (sourceFile && sourceFile->pragmaOnceProtected) {
2034-
// nothing to do
2035-
return nullptr;
2036-
}
2037-
2038-
if (!sourceFile) {
2039-
sourceFile = createSourceFile(fileName, readFile(fileName));
2040-
2041-
sourceFile->pragmaOnceProtected =
2042-
checkPragmaOnceProtected(sourceFile->tokens);
2043-
2044-
sourceFile->headerProtection = checkHeaderProtection(sourceFile->tokens);
2045-
2046-
if (sourceFile->headerProtection) {
2047-
sourceFile->headerProtectionLevel = evaluating_.size();
2048-
2049-
ifndefProtectedFiles_.insert_or_assign(
2050-
sourceFile->fileName, sourceFile->headerProtection->tok->text);
2051-
}
2052-
}
2053-
2054-
if (willIncludeHeader_) {
2055-
willIncludeHeader_(fileName, includeDepth_ + 1);
2056-
}
2057-
2058-
return sourceFile;
2059-
}
2060-
20612119
auto Preprocessor::Private::parseHeaderName(TokList *ts)
20622120
-> std::tuple<TokList *, std::optional<Include>> {
20632121
if (lookat(ts, TokenKind::T_STRING_LITERAL)) {
@@ -2932,43 +2990,6 @@ void Preprocessor::preprocess(std::string source, std::string fileName,
29322990
endPreprocessing(tokens);
29332991
}
29342992

2935-
void PendingInclude::resolveWith(std::optional<std::string> fileName) const {
2936-
auto d = preprocessor.d.get();
2937-
2938-
if (!fileName.has_value()) {
2939-
const auto &header = getHeaderName(include);
2940-
d->error(static_cast<TokList *>(loc),
2941-
std::format("file '{}' not found", header));
2942-
return;
2943-
}
2944-
2945-
auto resume = [=]() -> std::optional<PreprocessingState> {
2946-
auto continuation = d->findOrCreateSourceFile(*fileName);
2947-
if (!continuation) return std::nullopt;
2948-
2949-
// make the continuation the current file
2950-
auto dirpath = fs::path(continuation->fileName);
2951-
dirpath.remove_filename();
2952-
2953-
d->buffers_.push_back(Preprocessor::Private::Buffer{
2954-
.source = continuation,
2955-
.currentPath = dirpath,
2956-
.ts = continuation->tokens,
2957-
.includeDepth = d->includeDepth_ + 1,
2958-
});
2959-
2960-
return std::nullopt;
2961-
};
2962-
2963-
auto sourceFile = d->findSourceFile(*fileName);
2964-
if (sourceFile) {
2965-
resume();
2966-
return;
2967-
}
2968-
2969-
d->continuation_ = std::move(resume);
2970-
}
2971-
29722993
void Preprocessor::beginPreprocessing(std::string source, std::string fileName,
29732994
std::vector<Token> &tokens) {
29742995
assert(!d->findSourceFile(fileName));
@@ -3242,4 +3263,12 @@ void DefaultPreprocessorState::operator()(const PendingHasIncludes &status) {
32423263
});
32433264
}
32443265

3266+
void DefaultPreprocessorState::operator()(const PendingFileContent &request) {
3267+
std::ifstream in(request.fileName);
3268+
std::ostringstream out;
3269+
out << in.rdbuf();
3270+
3271+
request.setContent(out.str());
3272+
}
3273+
32453274
} // namespace cxx

‎src/parser/cxx/preprocessor.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ class Preprocessor {
121121
struct Private;
122122
struct ParseArguments;
123123
friend struct PendingInclude;
124+
friend struct PendingFileContent;
124125
std::unique_ptr<Private> d;
125126
};
126127

@@ -137,6 +138,7 @@ class DefaultPreprocessorState {
137138
void operator()(const CanContinuePreprocessing &);
138139
void operator()(const PendingInclude &status);
139140
void operator()(const PendingHasIncludes &status);
141+
void operator()(const PendingFileContent &status);
140142
};
141143

142144
} // namespace cxx

‎src/parser/cxx/preprocessor_fwd.h‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,12 +75,19 @@ struct PendingHasIncludes {
7575
std::vector<Request> requests;
7676
};
7777

78+
struct PendingFileContent {
79+
Preprocessor &preprocessor;
80+
std::string fileName;
81+
82+
void setContent(std::optional<std::string> content) const;
83+
};
84+
7885
struct CanContinuePreprocessing {};
7986

8087
struct ProcessingComplete {};
8188

8289
using PreprocessingState =
83-
std::variant<PendingInclude, PendingHasIncludes, CanContinuePreprocessing,
84-
ProcessingComplete>;
90+
std::variant<PendingInclude, PendingHasIncludes, PendingFileContent,
91+
CanContinuePreprocessing, ProcessingComplete>;
8592

8693
} // namespace cxx

0 commit comments

Comments
 (0)