Skip to content

Commit 93d7000

Browse files
bitjammerbitjammer
authored
Merge pull request swiftlang#29874 from bitjammer/acgarland/rdar-58339492-sg-source-locations
SymbolGraph: Serialize source locations and doc comment ranges
2 parents 6b1af59 + edd7fdb commit 93d7000

File tree

15 files changed

+362
-59
lines changed

15 files changed

+362
-59
lines changed

‎include/swift/AST/Decl.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -665,9 +665,10 @@ class alignas(1 << DeclAlignInBits) Decl {
665665
SourceLoc Loc;
666666
SourceLoc StartLoc;
667667
SourceLoc EndLoc;
668+
SmallVector<CharSourceRange, 4> DocRanges;
668669
};
669-
mutable CachedExternalSourceLocs const *CachedLocs = nullptr;
670-
const CachedExternalSourceLocs *calculateSerializedLocs() const;
670+
mutable CachedExternalSourceLocs const *CachedSerializedLocs = nullptr;
671+
const CachedExternalSourceLocs *getSerializedLocs() const;
671672
protected:
672673

673674
Decl(DeclKind kind, llvm::PointerUnion<DeclContext *, ASTContext *> context)
@@ -828,7 +829,7 @@ class alignas(1 << DeclAlignInBits) Decl {
828829
}
829830

830831
/// \returns the unparsed comment attached to this declaration.
831-
RawComment getRawComment() const;
832+
RawComment getRawComment(bool SerializedOK = false) const;
832833

833834
Optional<StringRef> getGroupName() const;
834835

‎include/swift/AST/RawComment.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ struct LineColumn {
8686

8787
struct BasicDeclLocs {
8888
StringRef SourceFilePath;
89+
SmallVector<std::pair<LineColumn, uint32_t>, 4> DocRanges;
8990
LineColumn Loc;
9091
LineColumn StartLoc;
9192
LineColumn EndLoc;

‎lib/AST/Decl.cpp

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -569,7 +569,10 @@ case DeclKind::ID: return cast<ID##Decl>(this)->getLocFromSource();
569569
llvm_unreachable("Unknown decl kind");
570570
}
571571

572-
const Decl::CachedExternalSourceLocs *Decl::calculateSerializedLocs() const {
572+
const Decl::CachedExternalSourceLocs *Decl::getSerializedLocs() const {
573+
if (CachedSerializedLocs) {
574+
return CachedSerializedLocs;
575+
}
573576
auto *File = cast<FileUnit>(getDeclContext()->getModuleScopeContext());
574577
auto Locs = File->getBasicLocsForDecl(this);
575578
if (!Locs.hasValue()) {
@@ -585,6 +588,14 @@ Result->X = SM.getLocFromExternalSource(Locs->SourceFilePath, Locs->X.Line, \
585588
CASE(StartLoc)
586589
CASE(EndLoc)
587590
#undef CASE
591+
592+
for (const auto &LineColumnAndLength : Locs->DocRanges) {
593+
auto Start = SM.getLocFromExternalSource(Locs->SourceFilePath,
594+
LineColumnAndLength.first.Line,
595+
LineColumnAndLength.first.Column);
596+
Result->DocRanges.push_back({ Start, LineColumnAndLength.second });
597+
}
598+
588599
return Result;
589600
}
590601

@@ -625,10 +636,7 @@ static_assert(sizeof(checkSourceLocType(&ID##Decl::getLoc)) == 2, \
625636
case FileUnitKind::SerializedAST: {
626637
if (!SerializedOK)
627638
return SourceLoc();
628-
if (!CachedLocs) {
629-
CachedLocs = calculateSerializedLocs();
630-
}
631-
return CachedLocs->Loc;
639+
return getSerializedLocs()->Loc;
632640
}
633641
case FileUnitKind::Builtin:
634642
case FileUnitKind::ClangModule:

‎lib/AST/DocComment.cpp

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
#include "swift/AST/ASTContext.h"
2020
#include "swift/AST/Comment.h"
2121
#include "swift/AST/Decl.h"
22+
#include "swift/AST/FileUnit.h"
2223
#include "swift/AST/Types.h"
2324
#include "swift/AST/PrettyStackTrace.h"
2425
#include "swift/AST/RawComment.h"
@@ -502,14 +503,23 @@ StringRef Decl::getBriefComment() const {
502503
if (!this->canHaveComment())
503504
return StringRef();
504505

505-
// Ensure the serialized doc is populated to ASTContext.
506-
auto RC = getRawComment();
507-
508506
// Check the cache in ASTContext.
509507
auto &Context = getASTContext();
510508
if (Optional<StringRef> Comment = Context.getBriefComment(this))
511509
return Comment.getValue();
512510

511+
// Check if the serialized module may have the brief comment available.
512+
if (auto *Unit =
513+
dyn_cast<FileUnit>(this->getDeclContext()->getModuleScopeContext())) {
514+
if (Optional<CommentInfo> C = Unit->getCommentForDecl(this)) {
515+
Context.setBriefComment(this, C->Brief);
516+
return C->Brief;
517+
}
518+
}
519+
520+
// Otherwise, parse the brief from the raw comment itself.
521+
auto RC = getRawComment();
522+
513523
StringRef Result;
514524
if (RC.isEmpty())
515525
if (auto *docD = getDocCommentProvidingDecl(this))

‎lib/AST/Module.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -726,6 +726,14 @@ SourceFile::getBasicLocsForDecl(const Decl *D) const {
726726
SourceManager &SM = getASTContext().SourceMgr;
727727
BasicDeclLocs Result;
728728
Result.SourceFilePath = SM.getDisplayNameForLoc(D->getLoc());
729+
730+
for (const auto &SRC : D->getRawComment().Comments) {
731+
Result.DocRanges.push_back(std::make_pair(
732+
LineColumn { SRC.StartLine, SRC.StartColumn },
733+
SRC.Range.getByteLength())
734+
);
735+
}
736+
729737
auto setLineColumn = [&SM](LineColumn &Home, SourceLoc Loc) {
730738
if (Loc.isValid()) {
731739
std::tie(Home.Line, Home.Column) = SM.getLineAndColumn(Loc);

‎lib/AST/RawComment.cpp

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ static RawComment toRawComment(ASTContext &Context, CharSourceRange Range) {
128128
return Result;
129129
}
130130

131-
RawComment Decl::getRawComment() const {
131+
RawComment Decl::getRawComment(bool SerializedOK) const {
132132
if (!this->canHaveComment())
133133
return RawComment();
134134

@@ -147,9 +147,24 @@ RawComment Decl::getRawComment() const {
147147
// Ask the parent module.
148148
if (auto *Unit =
149149
dyn_cast<FileUnit>(this->getDeclContext()->getModuleScopeContext())) {
150+
if (SerializedOK) {
151+
if (const auto *CachedLocs = getSerializedLocs()) {
152+
if (!CachedLocs->DocRanges.empty()) {
153+
SmallVector<SingleRawComment, 4> SRCs;
154+
for (const auto &Range : CachedLocs->DocRanges) {
155+
SRCs.push_back({ Range, Context.SourceMgr });
156+
}
157+
auto RC = RawComment(Context.AllocateCopy(llvm::makeArrayRef(SRCs)));
158+
159+
if (!RC.isEmpty()) {
160+
Context.setRawComment(this, RC);
161+
return RC;
162+
}
163+
}
164+
}
165+
}
166+
150167
if (Optional<CommentInfo> C = Unit->getCommentForDecl(this)) {
151-
swift::markup::MarkupContext MC;
152-
Context.setBriefComment(this, C->Brief);
153168
Context.setRawComment(this, C->Raw);
154169
return C->Raw;
155170
}

‎lib/Markup/LineList.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ LineList MarkupContext::getLineList(swift::RawComment RC) {
9797
auto CleanedStartLoc =
9898
C.Range.getStart().getAdvancedLocOrInvalid(CommentMarkerBytes);
9999
auto CleanedEndLoc =
100-
C.Range.getStart().getAdvancedLocOrInvalid(Cleaned.size());
100+
CleanedStartLoc.getAdvancedLocOrInvalid(Cleaned.size());
101101
Builder.addLine(Cleaned, { CleanedStartLoc, CleanedEndLoc });
102102
} else {
103103
// Skip comment markers at the beginning and at the end.
@@ -139,13 +139,13 @@ LineList MarkupContext::getLineList(swift::RawComment RC) {
139139
StringRef Line = Cleaned.substr(0, Pos);
140140
auto CleanedEndLoc = CleanedStartLoc.getAdvancedLocOrInvalid(Pos);
141141

142+
Builder.addLine(Line, { CleanedStartLoc, CleanedEndLoc });
143+
142144
Cleaned = Cleaned.drop_front(Pos);
143145
unsigned NewlineBytes = swift::measureNewline(Cleaned);
144146
Cleaned = Cleaned.drop_front(NewlineBytes);
145147
Pos += NewlineBytes;
146148
CleanedStartLoc = CleanedStartLoc.getAdvancedLocOrInvalid(Pos);
147-
148-
Builder.addLine(Line, { CleanedStartLoc, CleanedEndLoc });
149149
}
150150
}
151151
}

‎lib/Serialization/ModuleFile.cpp

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1460,6 +1460,9 @@ bool ModuleFile::readDeclLocsBlock(llvm::BitstreamCursor &cursor) {
14601460
case decl_locs_block::DECL_USRS:
14611461
DeclUSRsTable = readDeclUSRsTable(scratch, blobData);
14621462
break;
1463+
case decl_locs_block::DOC_RANGES:
1464+
DocRangesData = blobData;
1465+
break;
14631466
default:
14641467
// Unknown index kind, which this version of the compiler won't use.
14651468
break;
@@ -2697,7 +2700,10 @@ ModuleFile::getBasicDeclLocsForDecl(const Decl *D) const {
26972700
// Size of BasicDeclLocs in the buffer.
26982701
// FilePathOffset + LocNum * LineColumn
26992702
uint32_t LineColumnCount = 3;
2700-
uint32_t RecordSize = NumSize + NumSize * 2 * LineColumnCount;
2703+
uint32_t RecordSize =
2704+
NumSize + // Offset into source filename blob
2705+
NumSize + // Offset into doc ranges blob
2706+
NumSize * 2 * LineColumnCount; // Line/column of: Loc, StartLoc, EndLoc
27012707
uint32_t RecordOffset = RecordSize * UsrId;
27022708
assert(RecordOffset < BasicDeclLocsData.size());
27032709
assert(BasicDeclLocsData.size() % RecordSize == 0);
@@ -2711,6 +2717,23 @@ ModuleFile::getBasicDeclLocsForDecl(const Decl *D) const {
27112717
size_t TerminatorOffset = FilePath.find('\0');
27122718
assert(TerminatorOffset != StringRef::npos && "unterminated string data");
27132719
Result.SourceFilePath = FilePath.slice(0, TerminatorOffset);
2720+
2721+
const auto DocRangesOffset = ReadNext();
2722+
if (DocRangesOffset) {
2723+
assert(!DocRangesData.empty());
2724+
const auto *Data = DocRangesData.data() + DocRangesOffset;
2725+
const auto NumLocs = endian::readNext<uint32_t, little, unaligned>(Data);
2726+
assert(NumLocs);
2727+
2728+
for (uint32_t i = 0; i < NumLocs; ++i) {
2729+
LineColumn LC;
2730+
LC.Line = endian::readNext<uint32_t, little, unaligned>(Data);
2731+
LC.Column = endian::readNext<uint32_t, little, unaligned>(Data);
2732+
auto Length = endian::readNext<uint32_t, little, unaligned>(Data);
2733+
Result.DocRanges.push_back(std::make_pair(LC, Length));
2734+
}
2735+
}
2736+
27142737
#define READ_FIELD(X) \
27152738
Result.X.Line = ReadNext(); \
27162739
Result.X.Column = ReadNext();

‎lib/Serialization/ModuleFile.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,6 +424,10 @@ class ModuleFile
424424
/// \c DeclUSRsTable.
425425
StringRef BasicDeclLocsData;
426426

427+
/// An array of fixed-size location data for each `SingleRawComment` piece
428+
/// of declaration's documentation `RawComment`s.
429+
StringRef DocRangesData;
430+
427431
struct ModuleBits {
428432
/// The decl ID of the main class in this module file, if it has one.
429433
unsigned EntryPointDeclID : 31;

‎lib/Serialization/SerializeDoc.cpp

Lines changed: 73 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -522,6 +522,7 @@ void serialization::writeDocToStream(raw_ostream &os, ModuleOrSourceFile DC,
522522
namespace {
523523
struct DeclLocationsTableData {
524524
uint32_t SourceFileOffset;
525+
uint32_t DocRangesOffset;
525526
LineColumn Loc;
526527
LineColumn StartLoc;
527528
LineColumn EndLoc;
@@ -614,13 +615,72 @@ class StringWriter {
614615
}
615616
};
616617

618+
619+
/**
620+
Records the locations of `SingleRawComment` pieces for a declaration
621+
and emits them into a blob for the DOC_RANGES record.
622+
623+
See: \c decl_locs_block::DocRangesLayout
624+
*/
625+
class DocRangeWriter {
626+
llvm::DenseMap<const Decl *, uint32_t> DeclOffsetMap;
627+
llvm::SmallString<1024> Buffer;
628+
public:
629+
DocRangeWriter() {
630+
/**
631+
Offset 0 is reserved to mean "no offset", meaning that a declaration
632+
didn't have a doc comment.
633+
*/
634+
Buffer.push_back(0);
635+
}
636+
637+
/**
638+
\returns the offset into the doc ranges buffer for a declaration. Calling this
639+
twice on the same declaration will not duplicate data but return the
640+
original offset.
641+
*/
642+
uint32_t getDocRangesOffset(const Decl *D,
643+
ArrayRef<std::pair<LineColumn, uint32_t>> DocRanges) {
644+
if (DocRanges.empty()) {
645+
return 0;
646+
}
647+
const auto EntryAndAlreadyFound = DeclOffsetMap.insert({ D, Buffer.size() });
648+
const auto StartOffset = EntryAndAlreadyFound.first->getSecond();
649+
const auto AlreadyInMap = !EntryAndAlreadyFound.second;
650+
if (AlreadyInMap) {
651+
return StartOffset;
652+
}
653+
654+
llvm::raw_svector_ostream OS(Buffer);
655+
656+
endian::write<uint32_t>(OS, DocRanges.size(), little);
657+
658+
for (const auto &LineColumnAndLength : DocRanges) {
659+
endian::write<uint32_t>(OS, LineColumnAndLength.first.Line, little);
660+
endian::write<uint32_t>(OS, LineColumnAndLength.first.Column, little);
661+
endian::write<uint32_t>(OS, LineColumnAndLength.second, little);
662+
}
663+
664+
return StartOffset;
665+
}
666+
667+
void emitDocRangesRecord(llvm::BitstreamWriter &Out) {
668+
decl_locs_block::DocRangesLayout DocRangesBlob(Out);
669+
SmallVector<uint64_t, 8> Scratch;
670+
DocRangesBlob.emit(Scratch, Buffer);
671+
}
672+
};
673+
617674
struct BasicDeclLocsTableWriter : public ASTWalker {
618675
llvm::SmallString<1024> Buffer;
619676
DeclUSRsTableWriter &USRWriter;
620677
StringWriter &FWriter;
678+
DocRangeWriter &DocWriter;
621679
BasicDeclLocsTableWriter(DeclUSRsTableWriter &USRWriter,
622-
StringWriter &FWriter): USRWriter(USRWriter),
623-
FWriter(FWriter) {}
680+
StringWriter &FWriter,
681+
DocRangeWriter &DocWriter): USRWriter(USRWriter),
682+
FWriter(FWriter),
683+
DocWriter(DocWriter) {}
624684

625685
std::pair<bool, Stmt *> walkToStmtPre(Stmt *S) override { return { false, S };}
626686
std::pair<bool, Expr *> walkToExprPre(Expr *E) override { return { false, E };}
@@ -632,6 +692,7 @@ struct BasicDeclLocsTableWriter : public ASTWalker {
632692
llvm::raw_svector_ostream out(Buffer);
633693
endian::Writer writer(out, little);
634694
writer.write<uint32_t>(data.SourceFileOffset);
695+
writer.write<uint32_t>(data.DocRangesOffset);
635696
#define WRITE_LINE_COLUMN(X) \
636697
writer.write<uint32_t>(data.X.Line); \
637698
writer.write<uint32_t>(data.X.Column);
@@ -668,6 +729,8 @@ writer.write<uint32_t>(data.X.Column);
668729
llvm::SmallString<128> AbsolutePath = Locs->SourceFilePath;
669730
llvm::sys::fs::make_absolute(AbsolutePath);
670731
Result.SourceFileOffset = FWriter.getTextOffset(AbsolutePath.str());
732+
Result.DocRangesOffset = DocWriter.getDocRangesOffset(D,
733+
llvm::makeArrayRef(Locs->DocRanges));
671734
#define COPY_LINE_COLUMN(X) \
672735
Result.X.Line = Locs->X.Line; \
673736
Result.X.Column = Locs->X.Column;
@@ -713,10 +776,11 @@ Result.X.Column = Locs->X.Column;
713776
static void emitBasicLocsRecord(llvm::BitstreamWriter &Out,
714777
ModuleOrSourceFile MSF,
715778
DeclUSRsTableWriter &USRWriter,
716-
StringWriter &FWriter) {
779+
StringWriter &FWriter,
780+
DocRangeWriter &DocWriter) {
717781
assert(MSF);
718782
const decl_locs_block::BasicDeclLocsLayout DeclLocsList(Out);
719-
BasicDeclLocsTableWriter Writer(USRWriter, FWriter);
783+
BasicDeclLocsTableWriter Writer(USRWriter, FWriter, DocWriter);
720784
if (auto *SF = MSF.dyn_cast<SourceFile*>()) {
721785
SF->walk(Writer);
722786
} else {
@@ -754,6 +818,7 @@ class SourceInfoSerializer : public SerializerBase {
754818
BLOCK_RECORD(decl_locs_block, BASIC_DECL_LOCS);
755819
BLOCK_RECORD(decl_locs_block, DECL_USRS);
756820
BLOCK_RECORD(decl_locs_block, TEXT_DATA);
821+
BLOCK_RECORD(decl_locs_block, DOC_RANGES);
757822

758823
#undef BLOCK
759824
#undef BLOCK_RECORD
@@ -791,14 +856,17 @@ void serialization::writeSourceInfoToStream(raw_ostream &os,
791856
BCBlockRAII restoreBlock(S.Out, DECL_LOCS_BLOCK_ID, 4);
792857
DeclUSRsTableWriter USRWriter;
793858
StringWriter FPWriter;
794-
emitBasicLocsRecord(S.Out, DC, USRWriter, FPWriter);
859+
DocRangeWriter DocWriter;
860+
emitBasicLocsRecord(S.Out, DC, USRWriter, FPWriter, DocWriter);
795861
// Emit USR table mapping from a USR to USR Id.
796862
// The basic locs record uses USR Id instead of actual USR, so that we
797863
// don't need to repeat USR texts for newly added records.
798864
USRWriter.emitUSRsRecord(S.Out);
799865
// A blob of 0 terminated strings referenced by the location records,
800866
// e.g. file paths.
801867
FPWriter.emitSourceFilesRecord(S.Out);
868+
// A blob of fixed-size location records of `SingleRawComment` pieces.
869+
DocWriter.emitDocRangesRecord(S.Out);
802870
}
803871
}
804872

0 commit comments

Comments
 (0)