Skip to content

Commit 25d8ceb

Browse files
jakebaileyjakebailey
authored
Reorganize fileLoader, make it lock-free via SyncMaps (#779)
1 parent 09b9592 commit 25d8ceb

File tree

3 files changed

+145
-82
lines changed

3 files changed

+145
-82
lines changed

‎internal/compiler/fileloader.go‎

Lines changed: 87 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -5,35 +5,37 @@ import (
55
"iter"
66
"slices"
77
"strings"
8-
"sync"
8+
"sync/atomic"
99

1010
"github.com/microsoft/typescript-go/internal/ast"
11+
"github.com/microsoft/typescript-go/internal/collections"
1112
"github.com/microsoft/typescript-go/internal/compiler/module"
1213
"github.com/microsoft/typescript-go/internal/core"
1314
"github.com/microsoft/typescript-go/internal/tsoptions"
1415
"github.com/microsoft/typescript-go/internal/tspath"
1516
)
1617

1718
type fileLoader struct {
18-
host CompilerHost
19-
programOptions ProgramOptions
20-
compilerOptions *core.CompilerOptions
21-
22-
resolver *module.Resolver
23-
resolvedModulesMutex sync.Mutex
24-
resolvedModules map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule]
25-
26-
sourceFileMetaDatasMutex sync.RWMutex
27-
sourceFileMetaDatas map[tspath.Path]*ast.SourceFileMetaData
28-
29-
mu sync.Mutex
30-
wg core.WorkGroup
31-
tasksByFileName map[string]*parseTask
32-
currentNodeModulesDepth int
33-
defaultLibraryPath string
34-
comparePathsOptions tspath.ComparePathsOptions
35-
rootTasks []*parseTask
36-
supportedExtensions []string
19+
host CompilerHost
20+
programOptions ProgramOptions
21+
compilerOptions *core.CompilerOptions
22+
resolver *module.Resolver
23+
defaultLibraryPath string
24+
comparePathsOptions tspath.ComparePathsOptions
25+
wg core.WorkGroup
26+
supportedExtensions []string
27+
28+
tasksByFileName collections.SyncMap[string, *parseTask]
29+
rootTasks []*parseTask
30+
31+
totalFileCount atomic.Int32
32+
libFileCount atomic.Int32
33+
}
34+
35+
type processedFiles struct {
36+
files []*ast.SourceFile
37+
resolvedModules map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule]
38+
sourceFileMetaDatas map[tspath.Path]*ast.SourceFileMetaData
3739
}
3840

3941
func processAllProgramFiles(
@@ -43,14 +45,13 @@ func processAllProgramFiles(
4345
resolver *module.Resolver,
4446
rootFiles []string,
4547
libs []string,
46-
) (files []*ast.SourceFile, resolvedModules map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule], sourceFileMetaDatas map[tspath.Path]*ast.SourceFileMetaData) {
48+
) processedFiles {
4749
supportedExtensions := tsoptions.GetSupportedExtensions(compilerOptions, nil /*extraFileExtensions*/)
4850
loader := fileLoader{
4951
host: host,
5052
programOptions: programOptions,
5153
compilerOptions: compilerOptions,
5254
resolver: resolver,
53-
tasksByFileName: make(map[string]*parseTask),
5455
defaultLibraryPath: tspath.GetNormalizedAbsolutePath(host.DefaultLibraryPath(), host.GetCurrentDirectory()),
5556
comparePathsOptions: tspath.ComparePathsOptions{
5657
UseCaseSensitiveFileNames: host.FS().UseCaseSensitiveFileNames(),
@@ -69,17 +70,35 @@ func processAllProgramFiles(
6970

7071
loader.wg.RunAndWait()
7172

72-
files, libFiles := []*ast.SourceFile{}, []*ast.SourceFile{}
73+
totalFileCount := int(loader.totalFileCount.Load())
74+
libFileCount := int(loader.libFileCount.Load())
75+
76+
files := make([]*ast.SourceFile, 0, totalFileCount-libFileCount)
77+
libFiles := make([]*ast.SourceFile, 0, totalFileCount) // totalFileCount here since we append files to it later to construct the final list
78+
79+
resolvedModules := make(map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule], totalFileCount)
80+
sourceFileMetaDatas := make(map[tspath.Path]*ast.SourceFileMetaData, totalFileCount)
81+
7382
for task := range loader.collectTasks(loader.rootTasks) {
83+
file := task.file
7484
if task.isLib {
75-
libFiles = append(libFiles, task.file)
85+
libFiles = append(libFiles, file)
7686
} else {
77-
files = append(files, task.file)
87+
files = append(files, file)
7888
}
89+
path := file.Path()
90+
resolvedModules[path] = task.resolutionsInFile
91+
sourceFileMetaDatas[path] = task.metadata
7992
}
8093
loader.sortLibs(libFiles)
8194

82-
return append(libFiles, files...), loader.resolvedModules, loader.sourceFileMetaDatas
95+
allFiles := append(libFiles, files...)
96+
97+
return processedFiles{
98+
files: allFiles,
99+
resolvedModules: resolvedModules,
100+
sourceFileMetaDatas: sourceFileMetaDatas,
101+
}
83102
}
84103

85104
func (p *fileLoader) addRootTasks(files []string, isLib bool) {
@@ -111,42 +130,41 @@ func (p *fileLoader) addAutomaticTypeDirectiveTasks() {
111130

112131
func (p *fileLoader) startTasks(tasks []*parseTask) {
113132
if len(tasks) > 0 {
114-
p.mu.Lock()
115-
defer p.mu.Unlock()
116133
for i, task := range tasks {
117-
// dedup tasks to ensure correct file order, regardless of which task would be started first
118-
if existingTask, ok := p.tasksByFileName[task.normalizedFilePath]; ok {
119-
tasks[i] = existingTask
134+
loadedTask, loaded := p.tasksByFileName.LoadOrStore(task.normalizedFilePath, task)
135+
if loaded {
136+
// dedup tasks to ensure correct file order, regardless of which task would be started first
137+
tasks[i] = loadedTask
120138
} else {
121-
p.tasksByFileName[task.normalizedFilePath] = task
122-
task.start(p)
139+
loadedTask.start(p)
123140
}
124141
}
125142
}
126143
}
127144

128145
func (p *fileLoader) collectTasks(tasks []*parseTask) iter.Seq[*parseTask] {
129146
return func(yield func(*parseTask) bool) {
130-
p.collectTasksWorker(tasks, yield)
147+
p.collectTasksWorker(tasks, core.Set[*parseTask]{}, yield)
131148
}
132149
}
133150

134-
func (p *fileLoader) collectTasksWorker(tasks []*parseTask, yield func(*parseTask) bool) bool {
151+
func (p *fileLoader) collectTasksWorker(tasks []*parseTask, seen core.Set[*parseTask], yield func(*parseTask) bool) bool {
135152
for _, task := range tasks {
136-
if _, ok := p.tasksByFileName[task.normalizedFilePath]; ok {
137-
// ensure we only walk each task once
138-
delete(p.tasksByFileName, task.normalizedFilePath)
153+
// ensure we only walk each task once
154+
if seen.Has(task) {
155+
continue
156+
}
157+
seen.Add(task)
139158

140-
if len(task.subTasks) > 0 {
141-
if !p.collectTasksWorker(task.subTasks, yield) {
142-
return false
143-
}
159+
if len(task.subTasks) > 0 {
160+
if !p.collectTasksWorker(task.subTasks, seen, yield) {
161+
return false
144162
}
163+
}
145164

146-
if task.file != nil {
147-
if !yield(task) {
148-
return false
149-
}
165+
if task.file != nil {
166+
if !yield(task) {
167+
return false
150168
}
151169
}
152170
}
@@ -184,11 +202,23 @@ type parseTask struct {
184202
file *ast.SourceFile
185203
isLib bool
186204
subTasks []*parseTask
205+
206+
metadata *ast.SourceFileMetaData
207+
resolutionsInFile module.ModeAwareCache[*module.ResolvedModule]
187208
}
188209

189210
func (t *parseTask) start(loader *fileLoader) {
211+
loader.totalFileCount.Add(1)
212+
if t.isLib {
213+
loader.libFileCount.Add(1)
214+
}
215+
190216
loader.wg.Queue(func() {
191217
file := loader.parseSourceFile(t.normalizedFilePath)
218+
t.file = file
219+
loader.wg.Queue(func() {
220+
t.metadata = loader.loadSourceFileMetaData(file.Path())
221+
})
192222

193223
// !!! if noResolve, skip all of this
194224
t.subTasks = make([]*parseTask, 0, len(file.ReferencedFiles)+len(file.Imports)+len(file.ModuleAugmentations))
@@ -215,42 +245,29 @@ func (t *parseTask) start(loader *fileLoader) {
215245
}
216246
}
217247

218-
for _, imp := range loader.resolveImportsAndModuleAugmentations(file) {
248+
importsAndAugmentations, resolutionsInFile := loader.resolveImportsAndModuleAugmentations(file)
249+
for _, imp := range importsAndAugmentations {
219250
t.addSubTask(imp, false)
220251
}
221252

222-
t.file = file
253+
t.resolutionsInFile = resolutionsInFile
254+
223255
loader.startTasks(t.subTasks)
224256
})
225257
}
226258

227-
func (p *fileLoader) loadSourceFileMetaData(path tspath.Path) {
228-
p.sourceFileMetaDatasMutex.RLock()
229-
_, ok := p.sourceFileMetaDatas[path]
230-
p.sourceFileMetaDatasMutex.RUnlock()
231-
if ok {
232-
return
233-
}
234-
259+
func (p *fileLoader) loadSourceFileMetaData(path tspath.Path) *ast.SourceFileMetaData {
235260
packageJsonType := p.resolver.GetPackageJsonTypeIfApplicable(string(path))
236261
impliedNodeFormat := ast.GetImpliedNodeFormatForFile(string(path), packageJsonType)
237-
metadata := &ast.SourceFileMetaData{
262+
return &ast.SourceFileMetaData{
238263
PackageJsonType: packageJsonType,
239264
ImpliedNodeFormat: impliedNodeFormat,
240265
}
241-
242-
p.sourceFileMetaDatasMutex.Lock()
243-
defer p.sourceFileMetaDatasMutex.Unlock()
244-
if p.sourceFileMetaDatas == nil {
245-
p.sourceFileMetaDatas = make(map[tspath.Path]*ast.SourceFileMetaData)
246-
}
247-
p.sourceFileMetaDatas[path] = metadata
248266
}
249267

250268
func (p *fileLoader) parseSourceFile(fileName string) *ast.SourceFile {
251269
path := tspath.ToPath(fileName, p.host.GetCurrentDirectory(), p.host.FS().UseCaseSensitiveFileNames())
252270
sourceFile := p.host.GetSourceFile(fileName, path, p.compilerOptions.GetEmitScriptTarget())
253-
p.loadSourceFileMetaData(path)
254271
return sourceFile
255272
}
256273

@@ -269,21 +286,14 @@ func (p *fileLoader) resolveTripleslashPathReference(moduleName string, containi
269286
return tspath.NormalizePath(referencedFileName)
270287
}
271288

272-
func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile) []string {
273-
toParse := make([]string, 0, len(file.Imports))
289+
func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile) ([]string, module.ModeAwareCache[*module.ResolvedModule]) {
274290
if len(file.Imports) > 0 || len(file.ModuleAugmentations) > 0 {
291+
toParse := make([]string, 0, len(file.Imports))
275292
moduleNames := getModuleNames(file)
276293
resolutions := p.resolveModuleNames(moduleNames, file)
277294

278295
resolutionsInFile := make(module.ModeAwareCache[*module.ResolvedModule], len(resolutions))
279296

280-
p.resolvedModulesMutex.Lock()
281-
defer p.resolvedModulesMutex.Unlock()
282-
if p.resolvedModules == nil {
283-
p.resolvedModules = make(map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule])
284-
}
285-
p.resolvedModules[file.Path()] = resolutionsInFile
286-
287297
for i, resolution := range resolutions {
288298
resolvedFileName := resolution.ResolvedFileName
289299
// TODO(ercornel): !!!: check if from node modules
@@ -315,8 +325,10 @@ func (p *fileLoader) resolveImportsAndModuleAugmentations(file *ast.SourceFile)
315325
toParse = append(toParse, resolvedFileName)
316326
}
317327
}
328+
329+
return toParse, resolutionsInFile
318330
}
319-
return toParse
331+
return nil, nil
320332
}
321333

322334
func (p *fileLoader) resolveModuleNames(entries []*ast.Node, file *ast.SourceFile) []*module.ResolvedModule {

‎internal/compiler/program.go‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,15 +41,13 @@ type Program struct {
4141
currentDirectory string
4242
configFileParsingDiagnostics []*ast.Diagnostic
4343

44-
resolver *module.Resolver
45-
resolvedModules map[tspath.Path]module.ModeAwareCache[*module.ResolvedModule]
44+
resolver *module.Resolver
4645

4746
comparePathsOptions tspath.ComparePathsOptions
4847

49-
files []*ast.SourceFile
50-
filesByPath map[tspath.Path]*ast.SourceFile
48+
processedFiles
5149

52-
sourceFileMetaDatas map[tspath.Path]*ast.SourceFileMetaData
50+
filesByPath map[tspath.Path]*ast.SourceFile
5351

5452
// The below settings are to track if a .js file should be add to the program if loaded via searching under node_modules.
5553
// This works as imported modules are discovered recursively in a depth first manner, specifically:
@@ -75,7 +73,6 @@ func NewProgram(options ProgramOptions) *Program {
7573
p.programOptions = options
7674
p.compilerOptions = options.Options
7775
p.configFileParsingDiagnostics = slices.Clip(options.ConfigFileParsingDiagnostics)
78-
p.sourceFileMetaDatas = make(map[tspath.Path]*ast.SourceFileMetaData)
7976
if p.compilerOptions == nil {
8077
p.compilerOptions = &core.CompilerOptions{}
8178
}
@@ -153,7 +150,7 @@ func NewProgram(options ProgramOptions) *Program {
153150
}
154151
}
155152

156-
p.files, p.resolvedModules, p.sourceFileMetaDatas = processAllProgramFiles(p.host, p.programOptions, p.compilerOptions, p.resolver, rootFiles, libs)
153+
p.processedFiles = processAllProgramFiles(p.host, p.programOptions, p.compilerOptions, p.resolver, rootFiles, libs)
157154
p.filesByPath = make(map[tspath.Path]*ast.SourceFile, len(p.files))
158155
for _, file := range p.files {
159156
p.filesByPath[file.Path()] = file

‎internal/compiler/program_test.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
package compiler
22

33
import (
4+
"path/filepath"
45
"slices"
56
"strings"
67
"testing"
78

89
"github.com/microsoft/typescript-go/internal/bundled"
910
"github.com/microsoft/typescript-go/internal/core"
11+
"github.com/microsoft/typescript-go/internal/repo"
12+
"github.com/microsoft/typescript-go/internal/tspath"
13+
"github.com/microsoft/typescript-go/internal/vfs/osvfs"
1014
"github.com/microsoft/typescript-go/internal/vfs/vfstest"
1115
"gotest.tools/v3/assert"
1216
)
@@ -242,3 +246,53 @@ func TestProgram(t *testing.T) {
242246
})
243247
}
244248
}
249+
250+
func BenchmarkNewProgram(b *testing.B) {
251+
if !bundled.Embedded {
252+
// Without embedding, we'd need to read all of the lib files out from disk into the MapFS.
253+
// Just skip this for now.
254+
b.Skip("bundled files are not embedded")
255+
}
256+
257+
for _, testCase := range programTestCases {
258+
b.Run(testCase.testName, func(b *testing.B) {
259+
fs := vfstest.FromMap[any](nil, false /*useCaseSensitiveFileNames*/)
260+
fs = bundled.WrapFS(fs)
261+
262+
for _, testFile := range testCase.files {
263+
_ = fs.WriteFile(testFile.fileName, testFile.contents, false)
264+
}
265+
266+
opts := core.CompilerOptions{Target: testCase.target}
267+
programOpts := ProgramOptions{
268+
RootFiles: []string{"c:/dev/src/index.ts"},
269+
Host: NewCompilerHost(&opts, "c:/dev/src", fs, bundled.LibPath()),
270+
Options: &opts,
271+
SingleThreaded: false,
272+
}
273+
274+
for b.Loop() {
275+
NewProgram(programOpts)
276+
}
277+
})
278+
}
279+
280+
b.Run("compiler", func(b *testing.B) {
281+
repo.SkipIfNoTypeScriptSubmodule(b)
282+
283+
compilerDir := tspath.NormalizeSlashes(filepath.Join(repo.TypeScriptSubmodulePath, "src", "compiler"))
284+
285+
fs := osvfs.FS()
286+
fs = bundled.WrapFS(fs)
287+
288+
opts := ProgramOptions{
289+
ConfigFileName: tspath.CombinePaths(compilerDir, "tsconfig.json"),
290+
Host: NewCompilerHost(nil, compilerDir, fs, bundled.LibPath()),
291+
SingleThreaded: false,
292+
}
293+
294+
for b.Loop() {
295+
NewProgram(opts)
296+
}
297+
})
298+
}

0 commit comments

Comments
 (0)