Skip to content

Commit a57f4e0

Browse files
sheetalkamatsheetalkamat
authored
Compute the snapshot changes in parallel (#1475)
1 parent f1a4f69 commit a57f4e0

13 files changed

+520
-465
lines changed

‎internal/collections/manytomanyset.go

Lines changed: 0 additions & 49 deletions
This file was deleted.

‎internal/collections/syncmanytomanyset.go

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package collections
2+
3+
import "fmt"
4+
5+
type SyncManyToManySet[K comparable, V comparable] struct {
6+
keyToValueSet SyncMap[K, *Set[V]]
7+
valueToKeySet SyncMap[V, *SyncSet[K]]
8+
}
9+
10+
func (m *SyncManyToManySet[K, V]) GetKeys(value V) (*SyncSet[K], bool) {
11+
keys, present := m.valueToKeySet.Load(value)
12+
return keys, present
13+
}
14+
15+
func (m *SyncManyToManySet[K, V]) GetValues(key K) (*Set[V], bool) {
16+
values, present := m.keyToValueSet.Load(key)
17+
return values, present
18+
}
19+
20+
func (m *SyncManyToManySet[K, V]) Keys() *SyncMap[K, *Set[V]] {
21+
return &m.keyToValueSet
22+
}
23+
24+
func (m *SyncManyToManySet[K, V]) Store(key K, valueSet *Set[V]) {
25+
_, hasExisting := m.keyToValueSet.LoadOrStore(key, valueSet)
26+
if hasExisting {
27+
panic("ManyToManySet.Set: key already exists: " + fmt.Sprintf("%v", key))
28+
}
29+
for value := range valueSet.Keys() {
30+
// Add to valueToKeySet
31+
keySetForValue, _ := m.valueToKeySet.LoadOrStore(value, &SyncSet[K]{})
32+
keySetForValue.Add(key)
33+
}
34+
}

‎internal/collections/syncmap.go

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package collections
22

3-
import "sync"
3+
import (
4+
"iter"
5+
"sync"
6+
)
47

58
type SyncMap[K comparable, V any] struct {
69
m sync.Map
@@ -57,3 +60,14 @@ func (s *SyncMap[K, V]) ToMap() map[K]V {
5760
})
5861
return m
5962
}
63+
64+
func (s *SyncMap[K, V]) Keys() iter.Seq[K] {
65+
return func(yield func(K) bool) {
66+
s.m.Range(func(key, value any) bool {
67+
if !yield(key.(K)) {
68+
return false
69+
}
70+
return true
71+
})
72+
}
73+
}

‎internal/collections/syncset.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package collections
22

3+
import "iter"
4+
35
type SyncSet[T comparable] struct {
46
m SyncMap[T, struct{}]
57
}
@@ -23,6 +25,18 @@ func (s *SyncSet[T]) Range(fn func(key T) bool) {
2325
})
2426
}
2527

28+
// Size returns the approximate number of items in the map.
29+
// Note that this is not a precise count, as the map may be modified
30+
// concurrently while this method is running.
31+
func (s *SyncSet[T]) Size() int {
32+
count := 0
33+
s.m.Range(func(_ T, _ struct{}) bool {
34+
count++
35+
return true
36+
})
37+
return count
38+
}
39+
2640
func (s *SyncSet[T]) ToSlice() []T {
2741
var arr []T
2842
arr = make([]T, 0, s.m.Size())
@@ -32,3 +46,14 @@ func (s *SyncSet[T]) ToSlice() []T {
3246
})
3347
return arr
3448
}
49+
50+
func (s *SyncSet[T]) Keys() iter.Seq[T] {
51+
return func(yield func(T) bool) {
52+
s.m.Range(func(key T, value struct{}) bool {
53+
if !yield(key) {
54+
return false
55+
}
56+
return true
57+
})
58+
}
59+
}

‎internal/execute/testsys_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,8 +183,8 @@ func (s *testSys) baselineProgram(baseline *strings.Builder, program *incrementa
183183
baseline.WriteString("SemanticDiagnostics::\n")
184184
testingData := program.GetTestingData(program.GetProgram())
185185
for _, file := range program.GetProgram().GetSourceFiles() {
186-
if diagnostics, ok := testingData.SemanticDiagnosticsPerFile[file.Path()]; ok {
187-
if oldDiagnostics, ok := testingData.OldProgramSemanticDiagnosticsPerFile[file.Path()]; !ok || oldDiagnostics != diagnostics {
186+
if diagnostics, ok := testingData.SemanticDiagnosticsPerFile.Load(file.Path()); ok {
187+
if oldDiagnostics, ok := testingData.OldProgramSemanticDiagnosticsPerFile.Load(file.Path()); !ok || oldDiagnostics != diagnostics {
188188
baseline.WriteString("*refresh* " + file.FileName() + "\n")
189189
}
190190
} else {

‎internal/incremental/affectedfileshandler.go

Lines changed: 19 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package incremental
22

33
import (
44
"context"
5+
"iter"
56
"maps"
67
"slices"
78
"sync"
@@ -41,8 +42,8 @@ func (h *affectedFilesHandler) getDtsMayChange(affectedFilePath tspath.Path, aff
4142

4243
func (h *affectedFilesHandler) isChangedSignature(path tspath.Path) bool {
4344
newSignature, _ := h.updatedSignatures.Load(path)
44-
oldSignature := h.program.snapshot.fileInfos[path].signature
45-
return newSignature != oldSignature
45+
oldInfo, _ := h.program.snapshot.fileInfos.Load(path)
46+
return newSignature != oldInfo.signature
4647
}
4748

4849
func (h *affectedFilesHandler) removeSemanticDiagnosticsOf(path tspath.Path) {
@@ -81,7 +82,7 @@ func (h *affectedFilesHandler) updateShapeSignature(file *ast.SourceFile, useFil
8182
return false
8283
}
8384

84-
info := h.program.snapshot.fileInfos[file.Path()]
85+
info, _ := h.program.snapshot.fileInfos.Load(file.Path())
8586
prevSignature := info.signature
8687
var latestSignature string
8788
var updateKind SignatureUpdateKind
@@ -110,7 +111,7 @@ func (h *affectedFilesHandler) getFilesAffectedBy(path tspath.Path) []*ast.Sourc
110111
return []*ast.SourceFile{file}
111112
}
112113

113-
if info := h.program.snapshot.fileInfos[file.Path()]; info.affectsGlobalScope {
114+
if info, _ := h.program.snapshot.fileInfos.Load(file.Path()); info.affectsGlobalScope {
114115
h.hasAllFilesExcludingDefaultLibraryFile.Store(true)
115116
h.program.snapshot.getAllFilesExcludingDefaultLibraryFile(h.program.program, file)
116117
}
@@ -139,10 +140,10 @@ func (h *affectedFilesHandler) getFilesAffectedBy(path tspath.Path) []*ast.Sourc
139140
}
140141

141142
// Gets the files referenced by the the file path
142-
func (h *affectedFilesHandler) getReferencedByPaths(file tspath.Path) map[tspath.Path]struct{} {
143+
func (h *affectedFilesHandler) getReferencedByPaths(file tspath.Path) iter.Seq[tspath.Path] {
143144
keys, ok := h.program.snapshot.referencedMap.GetKeys(file)
144145
if !ok {
145-
return nil
146+
return func(yield func(tspath.Path) bool) {}
146147
}
147148
return keys.Keys()
148149
}
@@ -154,8 +155,7 @@ func (h *affectedFilesHandler) forEachFileReferencedBy(file *ast.SourceFile, fn
154155
seenFileNamesMap := map[tspath.Path]*ast.SourceFile{}
155156
// Start with the paths this file was referenced by
156157
seenFileNamesMap[file.Path()] = file
157-
references := h.getReferencedByPaths(file.Path())
158-
queue := slices.Collect(maps.Keys(references))
158+
queue := slices.Collect(h.getReferencedByPaths(file.Path()))
159159
for len(queue) > 0 {
160160
currentPath := queue[len(queue)-1]
161161
queue = queue[:len(queue)-1]
@@ -290,7 +290,7 @@ func (h *affectedFilesHandler) handleDtsMayChangeOfFileAndExportsOfFile(dtsMayCh
290290
}
291291

292292
func (h *affectedFilesHandler) handleDtsMayChangeOfGlobalScope(dtsMayChange dtsMayChange, filePath tspath.Path, invalidateJsFiles bool) bool {
293-
if info, ok := h.program.snapshot.fileInfos[filePath]; !ok || !info.affectsGlobalScope {
293+
if info, ok := h.program.snapshot.fileInfos.Load(filePath); !ok || !info.affectsGlobalScope {
294294
return false
295295
}
296296
// Every file needs to be handled
@@ -331,7 +331,9 @@ func (h *affectedFilesHandler) updateSnapshot() {
331331
return
332332
}
333333
h.updatedSignatures.Range(func(filePath tspath.Path, signature string) bool {
334-
h.program.snapshot.fileInfos[filePath].signature = signature
334+
if info, ok := h.program.snapshot.fileInfos.Load(filePath); ok {
335+
info.signature = signature
336+
}
335337
return true
336338
})
337339
if h.updatedSignatureKinds != nil {
@@ -341,33 +343,34 @@ func (h *affectedFilesHandler) updateSnapshot() {
341343
})
342344
}
343345
h.filesToRemoveDiagnostics.Range(func(file tspath.Path) bool {
344-
delete(h.program.snapshot.semanticDiagnosticsPerFile, file)
346+
h.program.snapshot.semanticDiagnosticsPerFile.Delete(file)
345347
return true
346348
})
347349
for _, change := range h.dtsMayChange {
348350
for filePath, emitKind := range change {
349351
h.program.snapshot.addFileToAffectedFilesPendingEmit(filePath, emitKind)
350352
}
351353
}
352-
h.program.snapshot.changedFilesSet = &collections.Set[tspath.Path]{}
353-
h.program.snapshot.buildInfoEmitPending = true
354+
h.program.snapshot.changedFilesSet = collections.SyncSet[tspath.Path]{}
355+
h.program.snapshot.buildInfoEmitPending.Store(true)
354356
}
355357

356358
func collectAllAffectedFiles(ctx context.Context, program *Program) {
357-
if program.snapshot.changedFilesSet.Len() == 0 {
359+
if program.snapshot.changedFilesSet.Size() == 0 {
358360
return
359361
}
360362

361363
handler := affectedFilesHandler{ctx: ctx, program: program, updatedSignatureKinds: core.IfElse(program.updatedSignatureKinds == nil, nil, &collections.SyncMap[tspath.Path, SignatureUpdateKind]{})}
362364
wg := core.NewWorkGroup(handler.program.program.SingleThreaded())
363365
var result collections.SyncSet[*ast.SourceFile]
364-
for file := range program.snapshot.changedFilesSet.Keys() {
366+
program.snapshot.changedFilesSet.Range(func(file tspath.Path) bool {
365367
wg.Queue(func() {
366368
for _, affectedFile := range handler.getFilesAffectedBy(file) {
367369
result.Add(affectedFile)
368370
}
369371
})
370-
}
372+
return true
373+
})
371374
wg.RunAndWait()
372375

373376
if ctx.Err() != nil {

‎internal/incremental/buildInfo.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -328,12 +328,13 @@ func (b *BuildInfoEmitSignature) noEmitSignature() bool {
328328
return b.Signature == "" && !b.DiffersOnlyInDtsMap && !b.DiffersInOptions
329329
}
330330

331-
func (b *BuildInfoEmitSignature) toEmitSignature(path tspath.Path, emitSignatures map[tspath.Path]*emitSignature) *emitSignature {
331+
func (b *BuildInfoEmitSignature) toEmitSignature(path tspath.Path, emitSignatures *collections.SyncMap[tspath.Path, *emitSignature]) *emitSignature {
332332
var signature string
333333
var signatureWithDifferentOptions []string
334334
if b.DiffersOnlyInDtsMap {
335335
signatureWithDifferentOptions = make([]string, 0, 1)
336-
signatureWithDifferentOptions = append(signatureWithDifferentOptions, emitSignatures[path].signature)
336+
info, _ := emitSignatures.Load(path)
337+
signatureWithDifferentOptions = append(signatureWithDifferentOptions, info.signature)
337338
} else if b.DiffersInOptions {
338339
signatureWithDifferentOptions = make([]string, 0, 1)
339340
signatureWithDifferentOptions = append(signatureWithDifferentOptions, b.Signature)

‎internal/incremental/tosnapshot.go renamed to ‎internal/incremental/buildinfotosnapshot.go

Lines changed: 15 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -93,66 +93,63 @@ func (t *toSnapshot) setCompilerOptions() {
9393
}
9494

9595
func (t *toSnapshot) setFileInfoAndEmitSignatures() {
96-
t.snapshot.fileInfos = make(map[tspath.Path]*fileInfo, len(t.buildInfo.FileInfos))
97-
t.snapshot.createEmitSignaturesMap()
96+
isComposite := t.snapshot.options.Composite.IsTrue()
9897
for index, buildInfoFileInfo := range t.buildInfo.FileInfos {
9998
path := t.toFilePath(BuildInfoFileId(index + 1))
10099
info := buildInfoFileInfo.GetFileInfo()
101-
t.snapshot.fileInfos[path] = info
100+
t.snapshot.fileInfos.Store(path, info)
102101
// Add default emit signature as file's signature
103-
if info.signature != "" && t.snapshot.emitSignatures != nil {
104-
t.snapshot.emitSignatures[path] = &emitSignature{signature: info.signature}
102+
if info.signature != "" && isComposite {
103+
t.snapshot.emitSignatures.Store(path, &emitSignature{signature: info.signature})
105104
}
106105
}
107106
// Fix up emit signatures
108107
for _, value := range t.buildInfo.EmitSignatures {
109108
if value.noEmitSignature() {
110-
delete(t.snapshot.emitSignatures, t.toFilePath(value.FileId))
109+
t.snapshot.emitSignatures.Delete(t.toFilePath(value.FileId))
111110
} else {
112111
path := t.toFilePath(value.FileId)
113-
t.snapshot.emitSignatures[path] = value.toEmitSignature(path, t.snapshot.emitSignatures)
112+
t.snapshot.emitSignatures.Store(path, value.toEmitSignature(path, &t.snapshot.emitSignatures))
114113
}
115114
}
116115
}
117116

118117
func (t *toSnapshot) setReferencedMap() {
119118
for _, entry := range t.buildInfo.ReferencedMap {
120-
t.snapshot.referencedMap.Set(t.toFilePath(entry.FileId), t.toFilePathSet(entry.FileIdListId))
119+
t.snapshot.referencedMap.Store(t.toFilePath(entry.FileId), t.toFilePathSet(entry.FileIdListId))
121120
}
122121
}
123122

124123
func (t *toSnapshot) setChangeFileSet() {
125-
t.snapshot.changedFilesSet = collections.NewSetWithSizeHint[tspath.Path](len(t.buildInfo.ChangeFileSet))
126124
for _, fileId := range t.buildInfo.ChangeFileSet {
127125
filePath := t.toFilePath(fileId)
128126
t.snapshot.changedFilesSet.Add(filePath)
129127
}
130128
}
131129

132130
func (t *toSnapshot) setSemanticDiagnostics() {
133-
t.snapshot.semanticDiagnosticsPerFile = make(map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName, len(t.snapshot.fileInfos))
134-
for path := range t.snapshot.fileInfos {
131+
t.snapshot.fileInfos.Range(func(path tspath.Path, info *fileInfo) bool {
135132
// Initialize to have no diagnostics if its not changed file
136133
if !t.snapshot.changedFilesSet.Has(path) {
137-
t.snapshot.semanticDiagnosticsPerFile[path] = &diagnosticsOrBuildInfoDiagnosticsWithFileName{}
134+
t.snapshot.semanticDiagnosticsPerFile.Store(path, &diagnosticsOrBuildInfoDiagnosticsWithFileName{})
138135
}
139-
}
136+
return true
137+
})
140138
for _, diagnostic := range t.buildInfo.SemanticDiagnosticsPerFile {
141139
if diagnostic.FileId != 0 {
142140
filePath := t.toFilePath(diagnostic.FileId)
143-
delete(t.snapshot.semanticDiagnosticsPerFile, filePath) // does not have cached diagnostics
141+
t.snapshot.semanticDiagnosticsPerFile.Delete(filePath) // does not have cached diagnostics
144142
} else {
145143
filePath := t.toFilePath(diagnostic.Diagnostics.FileId)
146-
t.snapshot.semanticDiagnosticsPerFile[filePath] = t.toDiagnosticsOrBuildInfoDiagnosticsWithFileName(diagnostic.Diagnostics)
144+
t.snapshot.semanticDiagnosticsPerFile.Store(filePath, t.toDiagnosticsOrBuildInfoDiagnosticsWithFileName(diagnostic.Diagnostics))
147145
}
148146
}
149147
}
150148

151149
func (t *toSnapshot) setEmitDiagnostics() {
152-
t.snapshot.emitDiagnosticsPerFile = make(map[tspath.Path]*diagnosticsOrBuildInfoDiagnosticsWithFileName, len(t.snapshot.fileInfos))
153150
for _, diagnostic := range t.buildInfo.EmitDiagnosticsPerFile {
154151
filePath := t.toFilePath(diagnostic.FileId)
155-
t.snapshot.emitDiagnosticsPerFile[filePath] = t.toDiagnosticsOrBuildInfoDiagnosticsWithFileName(diagnostic)
152+
t.snapshot.emitDiagnosticsPerFile.Store(filePath, t.toDiagnosticsOrBuildInfoDiagnosticsWithFileName(diagnostic))
156153
}
157154
}
158155

@@ -161,8 +158,7 @@ func (t *toSnapshot) setAffectedFilesPendingEmit() {
161158
return
162159
}
163160
ownOptionsEmitKind := GetFileEmitKind(t.snapshot.options)
164-
t.snapshot.affectedFilesPendingEmit = make(map[tspath.Path]FileEmitKind, len(t.buildInfo.AffectedFilesPendingEmit))
165161
for _, pendingEmit := range t.buildInfo.AffectedFilesPendingEmit {
166-
t.snapshot.affectedFilesPendingEmit[t.toFilePath(pendingEmit.FileId)] = core.IfElse(pendingEmit.EmitKind == 0, ownOptionsEmitKind, pendingEmit.EmitKind)
162+
t.snapshot.affectedFilesPendingEmit.Store(t.toFilePath(pendingEmit.FileId), core.IfElse(pendingEmit.EmitKind == 0, ownOptionsEmitKind, pendingEmit.EmitKind))
167163
}
168164
}

0 commit comments

Comments
 (0)