Skip to content

Commit a5833fe

Browse files
wweitaoJiatong Li
wweitao
and
Jiatong Li
authored
fix: SSPC dependency upload and watcher fixes (#1377)
* fix: resolve symlink of python dependency upload * fix: bundle events from dependency watcher * fix: correct watcher.close() with watcher.dispose() * fix: fix dependency watcher event * fix: add additional log for error cases of watcher and symlink resolver --------- Co-authored-by: Jiatong Li <[email protected]>
1 parent cf0f6b3 commit a5833fe

File tree

7 files changed

+184
-72
lines changed

7 files changed

+184
-72
lines changed

‎server/aws-lsp-codewhisperer/src/language-server/workspaceContext/artifactManager.ts‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import path = require('path')
44
import { URI } from 'vscode-uri'
55
import JSZip = require('jszip')
66
import { EclipseConfigGenerator, JavaProjectAnalyzer } from './javaManager'
7-
import { isDirectory, isEmptyDirectory } from './util'
7+
import { resolveSymlink, isDirectory, isEmptyDirectory } from './util'
88
import glob = require('fast-glob')
99
import { CodewhispererLanguage, getCodeWhispererLanguageIdFromPath } from '../../shared/languageDetection'
1010

@@ -47,6 +47,15 @@ const IGNORE_PATTERNS = [
4747
'**/target/**', // Maven/Gradle builds
4848
]
4949

50+
const IGNORE_DEPENDENCY_PATTERNS = [
51+
// Package management and git
52+
'**/.git/**',
53+
// Build outputs
54+
'**/dist/**',
55+
// Logs and temporary files
56+
'**/logs/**',
57+
]
58+
5059
interface FileSizeDetails {
5160
includedFileCount: number
5261
includedSize: number
@@ -161,15 +170,15 @@ export class ArtifactManager {
161170
const files = await glob(['**/*'], {
162171
cwd: filePath,
163172
dot: false,
164-
ignore: IGNORE_PATTERNS,
165-
followSymbolicLinks: false,
173+
ignore: IGNORE_DEPENDENCY_PATTERNS,
174+
followSymbolicLinks: true,
166175
absolute: false,
167176
onlyFiles: true,
168177
})
169178

170179
for (const relativePath of files) {
171-
const fullPath = path.join(filePath, relativePath)
172180
try {
181+
const fullPath = resolveSymlink(path.join(filePath, relativePath))
173182
const fileMetadata = await this.createFileMetadata(
174183
fullPath,
175184
path.join(filePathInZipOverride !== undefined ? filePathInZipOverride : '', relativePath),
@@ -178,7 +187,7 @@ export class ArtifactManager {
178187
)
179188
fileMetadataList.push(fileMetadata)
180189
} catch (error) {
181-
this.logging.warn(`Error processing file ${fullPath}: ${error}`)
190+
this.logging.warn(`Error processing file ${relativePath}: ${error}`)
182191
}
183192
}
184193
} else {

‎server/aws-lsp-codewhisperer/src/language-server/workspaceContext/dependency/dependencyHandler/DependencyWatcher.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import { Logging } from '@aws/language-server-runtimes/server-interface'
2+
import * as fs from 'fs'
3+
4+
export class DependencyWatcher {
5+
private eventQueue = new Set<string>()
6+
private processingTimeout: NodeJS.Timeout | null = null
7+
private isProcessing = false
8+
private watcher: fs.FSWatcher
9+
10+
constructor(
11+
private readonly path: string,
12+
private readonly callbackFunction: (events: string[]) => void,
13+
private readonly logging: Logging,
14+
private readonly interval: number = 1000
15+
) {
16+
this.watcher = this.setupWatcher()
17+
}
18+
19+
private setupWatcher(): fs.FSWatcher {
20+
try {
21+
const watcher = fs.watch(this.path, { recursive: false }, async (eventType, fileName) => {
22+
if (!fileName) return
23+
if (eventType === 'rename' || eventType === 'change') {
24+
this.eventQueue.add(fileName)
25+
if (this.processingTimeout) {
26+
clearTimeout(this.processingTimeout)
27+
}
28+
this.processingTimeout = setTimeout(() => {
29+
this.processEvents().catch(error => {
30+
this.logging.warn(`Error processing events: ${error}`)
31+
})
32+
}, this.interval)
33+
}
34+
})
35+
watcher.on('error', error => {
36+
this.logging.warn(`watcher error for ${this.path}: ${error}`)
37+
})
38+
return watcher
39+
} catch (error) {
40+
this.logging.warn(`Error setting up watcher for ${this.path}: ${error}`)
41+
throw error
42+
}
43+
}
44+
45+
private async processEvents(): Promise<void> {
46+
if (this.isProcessing) return
47+
this.isProcessing = true
48+
const events = Array.from(this.eventQueue)
49+
this.eventQueue.clear()
50+
try {
51+
this.callbackFunction(events)
52+
} catch (error) {
53+
this.logging.warn(`Error processing bundled events: ${error}`)
54+
} finally {
55+
this.isProcessing = false
56+
}
57+
}
58+
59+
getWatcher(): fs.FSWatcher {
60+
return this.watcher
61+
}
62+
63+
dispose(): void {
64+
if (this.processingTimeout) {
65+
clearTimeout(this.processingTimeout)
66+
}
67+
this.watcher.close()
68+
}
69+
}

‎server/aws-lsp-codewhisperer/src/language-server/workspaceContext/dependency/dependencyHandler/JSTSDependencyHandler.ts‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import * as path from 'path'
33
import * as fs from 'fs'
44
import { WorkspaceFolder } from '@aws/language-server-runtimes/server-interface'
55
import { FileMetadata } from '../../artifactManager'
6+
import { DependencyWatcher } from './DependencyWatcher'
67

78
interface JSTSDependencyInfo extends BaseDependencyInfo {
89
packageJsonPath: string
@@ -181,18 +182,22 @@ export class JSTSDependencyHandler extends LanguageDependencyHandler<JSTSDepende
181182
}
182183
this.logging.log(`Setting up Javascript/Typescript dependency watcher for ${packageJsonPath}`)
183184
try {
184-
const watcher = fs.watch(packageJsonPath, async eventType => {
185-
if (eventType === 'change') {
186-
this.logging.log(`Change detected in ${packageJsonPath}`)
187-
const updatedDependencyMap = this.generateDependencyMap(jstsDependencyInfo)
188-
let zips: FileMetadata[] = await this.compareAndUpdateDependencyMap(
189-
jstsDependencyInfo.workspaceFolder,
190-
updatedDependencyMap,
191-
true
192-
)
193-
this.emitDependencyChange(jstsDependencyInfo.workspaceFolder, zips)
194-
}
195-
})
185+
const callBackDependencyUpdate = async (events: string[]) => {
186+
this.logging.log(`Change detected in ${packageJsonPath}`)
187+
const updatedDependencyMap = this.generateDependencyMap(jstsDependencyInfo)
188+
let zips: FileMetadata[] = await this.compareAndUpdateDependencyMap(
189+
jstsDependencyInfo.workspaceFolder,
190+
updatedDependencyMap,
191+
true
192+
)
193+
this.emitDependencyChange(jstsDependencyInfo.workspaceFolder, zips)
194+
}
195+
const watcher = new DependencyWatcher(
196+
packageJsonPath,
197+
callBackDependencyUpdate,
198+
this.logging,
199+
this.DEPENDENCY_WATCHER_EVENT_BATCH_INTERVAL
200+
)
196201
this.dependencyWatchers.set(packageJsonPath, watcher)
197202
} catch (error) {
198203
this.logging.warn(`Error setting up watcher for ${packageJsonPath}: ${error}`)
@@ -214,7 +219,7 @@ export class JSTSDependencyHandler extends LanguageDependencyHandler<JSTSDepende
214219
const packageJsonPath = jstsDependencyInfo.packageJsonPath
215220
if (this.dependencyWatchers.has(packageJsonPath)) {
216221
this.logging.log(`Disposing dependency watcher for ${packageJsonPath}`)
217-
this.dependencyWatchers.get(packageJsonPath)?.close()
222+
this.dependencyWatchers.get(packageJsonPath)?.dispose()
218223
this.dependencyWatchers.delete(packageJsonPath)
219224
}
220225
}

‎server/aws-lsp-codewhisperer/src/language-server/workspaceContext/dependency/dependencyHandler/JavaDependencyHandler.ts‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import * as fs from 'fs'
44
import * as xml2js from 'xml2js'
55
import { FileMetadata } from '../../artifactManager'
66
import { WorkspaceFolder } from '@aws/language-server-runtimes/server-interface'
7+
import { DependencyWatcher } from './DependencyWatcher'
78

89
export interface JavaDependencyInfo extends BaseDependencyInfo {
910
dotClasspathPath: string
@@ -89,18 +90,22 @@ export class JavaDependencyHandler extends LanguageDependencyHandler<JavaDepende
8990
}
9091
this.logging.log(`Setting up Java dependency watcher for ${dotClasspathPath}`)
9192
try {
92-
const watcher = fs.watch(dotClasspathPath, async (eventType, filename) => {
93-
if (eventType === 'change') {
94-
this.logging.log(`Change detected in ${dotClasspathPath}`)
95-
const updatedDependencyMap = this.generateDependencyMap(javaDependencyInfo)
96-
let zips: FileMetadata[] = await this.compareAndUpdateDependencyMap(
97-
javaDependencyInfo.workspaceFolder,
98-
updatedDependencyMap,
99-
true
100-
)
101-
this.emitDependencyChange(javaDependencyInfo.workspaceFolder, zips)
102-
}
103-
})
93+
const callBackDependencyUpdate = async (events: string[]) => {
94+
this.logging.log(`Change detected in ${dotClasspathPath}`)
95+
const updatedDependencyMap = this.generateDependencyMap(javaDependencyInfo)
96+
let zips: FileMetadata[] = await this.compareAndUpdateDependencyMap(
97+
javaDependencyInfo.workspaceFolder,
98+
updatedDependencyMap,
99+
true
100+
)
101+
this.emitDependencyChange(javaDependencyInfo.workspaceFolder, zips)
102+
}
103+
const watcher = new DependencyWatcher(
104+
dotClasspathPath,
105+
callBackDependencyUpdate,
106+
this.logging,
107+
this.DEPENDENCY_WATCHER_EVENT_BATCH_INTERVAL
108+
)
104109
this.dependencyWatchers.set(dotClasspathPath, watcher)
105110
} catch (error) {
106111
this.logging.warn(`Error setting up watcher for ${dotClasspathPath}: ${error}`)
@@ -171,7 +176,7 @@ export class JavaDependencyHandler extends LanguageDependencyHandler<JavaDepende
171176
const dotClasspathPath = javaDependencyInfo.dotClasspathPath
172177
if (this.dependencyWatchers.has(dotClasspathPath)) {
173178
this.logging.log(`Disposing dependency watcher for ${dotClasspathPath}`)
174-
this.dependencyWatchers.get(dotClasspathPath)?.close()
179+
this.dependencyWatchers.get(dotClasspathPath)?.dispose()
175180
this.dependencyWatchers.delete(dotClasspathPath)
176181
}
177182
}

‎server/aws-lsp-codewhisperer/src/language-server/workspaceContext/dependency/dependencyHandler/LanguageDependencyHandler.ts‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { ArtifactManager, FileMetadata } from '../../artifactManager'
44
import path = require('path')
55
import { EventEmitter } from 'events'
66
import { CodewhispererLanguage } from '../../../../shared/languageDetection'
7+
import { isDirectory } from '../../util'
8+
import { DependencyWatcher } from './DependencyWatcher'
79

810
export interface Dependency {
911
name: string
@@ -28,12 +30,13 @@ export abstract class LanguageDependencyHandler<T extends BaseDependencyInfo> {
2830
protected dependencyMap = new Map<WorkspaceFolder, Map<string, Dependency>>()
2931
protected dependencyUploadedSizeMap = new Map<WorkspaceFolder, number>()
3032
protected dependencyUploadedSizeSum: Uint32Array<SharedArrayBuffer>
31-
protected dependencyWatchers: Map<string, fs.FSWatcher> = new Map<string, fs.FSWatcher>()
33+
protected dependencyWatchers: Map<string, DependencyWatcher> = new Map<string, DependencyWatcher>()
3234
protected artifactManager: ArtifactManager
3335
protected dependenciesFolderName: string
3436
protected eventEmitter: EventEmitter
3537
protected readonly MAX_SINGLE_DEPENDENCY_SIZE: number = 500 * 1024 * 1024 // 500 MB
3638
protected readonly MAX_WORKSPACE_DEPENDENCY_SIZE: number = 8 * 1024 * 1024 * 1024 // 8 GB
39+
protected readonly DEPENDENCY_WATCHER_EVENT_BATCH_INTERVAL: number = 1000
3740

3841
constructor(
3942
language: CodewhispererLanguage,
@@ -316,7 +319,7 @@ export abstract class LanguageDependencyHandler<T extends BaseDependencyInfo> {
316319
dispose(): void {
317320
this.dependencyMap.clear()
318321
this.dependencyUploadedSizeMap.clear()
319-
this.dependencyWatchers.forEach(watcher => watcher.close())
322+
this.dependencyWatchers.forEach(watcher => watcher.dispose())
320323
this.dependencyWatchers.clear()
321324
}
322325

@@ -339,19 +342,17 @@ export abstract class LanguageDependencyHandler<T extends BaseDependencyInfo> {
339342

340343
// For synchronous version if needed:
341344
protected getDirectorySize(directoryPath: string): number {
345+
if (!isDirectory(directoryPath)) {
346+
return fs.statSync(directoryPath).size
347+
}
342348
let totalSize = 0
343349
try {
344350
const files = fs.readdirSync(directoryPath)
345351

346352
for (const file of files) {
347353
const filePath = path.join(directoryPath, file)
348354
const stats = fs.statSync(filePath)
349-
350-
if (stats.isDirectory()) {
351-
totalSize += this.getDirectorySize(filePath)
352-
} else {
353-
totalSize += stats.size
354-
}
355+
totalSize += this.getDirectorySize(filePath)
355356
}
356357

357358
return totalSize

0 commit comments

Comments
 (0)