From 9f995bd687099a4768b3865b1918035bd55344f8 Mon Sep 17 00:00:00 2001 From: Roberto Raggi Date: Thu, 14 Nov 2024 20:16:40 +0100 Subject: [PATCH 1/3] chore: Add code completion token Signed-off-by: Roberto Raggi --- packages/cxx-frontend/src/TokenKind.ts | 1 + packages/cxx-gen-ast/src/tokens.ts | 1 + src/parser/cxx/token_fwd.h | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/cxx-frontend/src/TokenKind.ts b/packages/cxx-frontend/src/TokenKind.ts index 02e8ec15..09ca7269 100644 --- a/packages/cxx-frontend/src/TokenKind.ts +++ b/packages/cxx-frontend/src/TokenKind.ts @@ -33,6 +33,7 @@ export enum TokenKind { UTF8_STRING_LITERAL, WIDE_STRING_LITERAL, PP_INTERNAL_VARIABLE, + CODE_COMPLETION, AMP_AMP, AMP_EQUAL, AMP, diff --git a/packages/cxx-gen-ast/src/tokens.ts b/packages/cxx-gen-ast/src/tokens.ts index 51e3c1b0..611dce29 100644 --- a/packages/cxx-gen-ast/src/tokens.ts +++ b/packages/cxx-gen-ast/src/tokens.ts @@ -33,6 +33,7 @@ export const BASE_TOKENS: string[] = [ "UTF8_STRING_LITERAL", "WIDE_STRING_LITERAL", "PP_INTERNAL_VARIABLE", + "CODE_COMPLETION", ]; export const OPERATORS: Array<[kind: string, spelling: string]> = [ diff --git a/src/parser/cxx/token_fwd.h b/src/parser/cxx/token_fwd.h index e06bad54..7276ed77 100644 --- a/src/parser/cxx/token_fwd.h +++ b/src/parser/cxx/token_fwd.h @@ -43,7 +43,8 @@ class Token; V(UTF32_STRING_LITERAL, "") \ V(UTF8_STRING_LITERAL, "") \ V(WIDE_STRING_LITERAL, "") \ - V(PP_INTERNAL_VARIABLE, "") + V(PP_INTERNAL_VARIABLE, "") \ + V(CODE_COMPLETION, "") #define FOR_EACH_OPERATOR(V) \ V(AMP_AMP, "&&") \ From 9efb8d710e13a392909ed350d927069931031d63 Mon Sep 17 00:00:00 2001 From: Roberto Raggi Date: Thu, 14 Nov 2024 20:17:15 +0100 Subject: [PATCH 2/3] chore: Remove duplicate linking of the parser library Signed-off-by: Roberto Raggi --- src/frontend/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/frontend/CMakeLists.txt b/src/frontend/CMakeLists.txt index 8ff7068e..5fb33bfa 100644 --- a/src/frontend/CMakeLists.txt +++ b/src/frontend/CMakeLists.txt @@ -21,7 +21,7 @@ aux_source_directory(cxx SOURCES) add_executable(cxx ${SOURCES}) -target_link_libraries(cxx PRIVATE cxx-parser cxx-lsp) +target_link_libraries(cxx PRIVATE cxx-lsp) if(EMSCRIPTEN) target_link_options(cxx PRIVATE From b2f761c600a3afe0476a08e3dfe614b4c5a99ca3 Mon Sep 17 00:00:00 2001 From: Roberto Raggi Date: Thu, 14 Nov 2024 20:28:00 +0100 Subject: [PATCH 3/3] chore: Add APIs to request code completion at a specific location Signed-off-by: Roberto Raggi --- src/parser/cxx/preprocessor.cc | 74 ++++++++++++++++++++++++++-------- src/parser/cxx/preprocessor.h | 2 + 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/parser/cxx/preprocessor.cc b/src/parser/cxx/preprocessor.cc index 0f622405..83b87f3f 100644 --- a/src/parser/cxx/preprocessor.cc +++ b/src/parser/cxx/preprocessor.cc @@ -637,24 +637,37 @@ struct SourceFile { initLineMap(); } - void getTokenStartPosition(unsigned offset, unsigned *line, unsigned *column, - std::string_view *fileName) const { + [[nodiscard]] auto getTokenStartPosition(unsigned offset) const + -> SourcePosition { auto it = std::lower_bound(lines.cbegin(), lines.cend(), static_cast(offset)); if (*it != static_cast(offset)) --it; assert(*it <= int(offset)); - if (line) *line = int(std::distance(cbegin(lines), it) + 1); + auto line = std::uint32_t(std::distance(cbegin(lines), it) + 1); - if (column) { - const auto start = cbegin(source) + *it; - const auto end = cbegin(source) + offset; + const auto start = cbegin(source) + *it; + const auto end = cbegin(source) + offset; - *column = utf8::unchecked::distance(start, end) + 1; - } + const auto column = + std::uint32_t(utf8::unchecked::distance(start, end) + 1); - if (fileName) *fileName = this->fileName; + return SourcePosition{fileName, line, column}; + } + + [[nodiscard]] auto offsetAt(std::uint32_t line, std::uint32_t column) const + -> std::uint32_t { + if (line == 0 && column == 0) return 0; + if (line > lines.size()) return static_cast(source.size()); + const auto start = source.data(); + const auto end = start + source.size(); + const auto offsetOfTheLine = lines[line - 1]; + auto it = start + offsetOfTheLine; + for (std::uint32_t i = 1; i < column; ++i) { + utf8::unchecked::next(it); + } + return static_cast(it - start); } private: @@ -713,6 +726,8 @@ struct Preprocessor::Private { }; std::vector dependencies_; std::functionstd::optional> continuation_; + std::optional codeCompletionLocation_; + std::uint32_t codeCompletionOffset_ = 0; int localCount_ = 0; int counter_ = 0; @@ -1593,6 +1608,18 @@ void Preprocessor::Private::finalizeToken(std::vector &tokens, const auto fileId = tk->sourceFile; TokenValue value{}; + if (tk->sourceFile == 1 && codeCompletionLocation_.has_value()) { + if (codeCompletionOffset_ < tk->offset || + (codeCompletionOffset_ >= tk->offset && + codeCompletionOffset_ < tk->offset + tk->length)) { + auto &completionToken = + tokens.emplace_back(TokenKind::T_CODE_COMPLETION, tk->offset, 0); + completionToken.setFileId(fileId); + + codeCompletionLocation_ = std::nullopt; + } + } + switch (tk->kind) { case TokenKind::T_IDENTIFIER: { kind = Lexer::classifyKeyword(tk->text); @@ -2987,6 +3014,10 @@ void Preprocessor::beginPreprocessing(std::string source, std::string fileName, if (tokens.empty()) { tokens.emplace_back(TokenKind::T_ERROR); } + + if (auto loc = d->codeCompletionLocation_) { + d->codeCompletionOffset_ = sourceFile->offsetAt(loc->line, loc->column); + } } void Preprocessor::endPreprocessing(std::vector &tokens) { @@ -3002,6 +3033,16 @@ void Preprocessor::endPreprocessing(std::vector &tokens) { // place the EOF token at the end of the main source file const auto offset = d->sourceFiles_[mainSourceFileId - 1]->source.size(); + + if (d->codeCompletionLocation_.has_value()) { + auto sourceFile = d->sourceFiles_[0].get(); + + auto &tk = tokens.emplace_back(TokenKind::T_CODE_COMPLETION, offset, 0); + tk.setFileId(mainSourceFileId); + + d->codeCompletionLocation_ = std::nullopt; + } + auto &tk = tokens.emplace_back(TokenKind::T_EOF_SYMBOL, offset); tk.setFileId(mainSourceFileId); } @@ -3165,10 +3206,7 @@ auto Preprocessor::tokenStartPosition(const Token &token) const } auto &sourceFile = *d->sourceFiles_[token.fileId() - 1]; - SourcePosition pos; - sourceFile.getTokenStartPosition(token.offset(), &pos.line, &pos.column, - &pos.fileName); - return pos; + return sourceFile.getTokenStartPosition(token.offset()); } auto Preprocessor::tokenEndPosition(const Token &token) const @@ -3179,10 +3217,7 @@ auto Preprocessor::tokenEndPosition(const Token &token) const auto &sourceFile = *d->sourceFiles_[token.fileId() - 1]; - SourcePosition pos; - sourceFile.getTokenStartPosition(token.offset() + token.length(), &pos.line, - &pos.column, &pos.fileName); - return pos; + return sourceFile.getTokenStartPosition(token.offset() + token.length()); } auto Preprocessor::getTextLine(const Token &token) const -> std::string_view { @@ -3215,6 +3250,11 @@ auto Preprocessor::resolve(const Include &include, bool isIncludeNext) const return d->resolve(include, isIncludeNext); } +void Preprocessor::requestCodeCompletionAt(std::uint32_t line, + std::uint32_t column) { + d->codeCompletionLocation_ = SourcePosition{{}, line, column}; +} + void DefaultPreprocessorState::operator()(const ProcessingComplete &) { done = true; } diff --git a/src/parser/cxx/preprocessor.h b/src/parser/cxx/preprocessor.h index 166cc0ab..f49d0a77 100644 --- a/src/parser/cxx/preprocessor.h +++ b/src/parser/cxx/preprocessor.h @@ -115,6 +115,8 @@ class Preprocessor { [[nodiscard]] auto resolve(const Include &include, bool isIncludeNext) const -> std::optional; + void requestCodeCompletionAt(std::uint32_t line, std::uint32_t column); + void squeeze(); private: