Skip to content

Commit c84343d

Browse files
Smartsheet-JB-BrownSmartsheet-JB-Brown
committed
reduce memory use during MetaDataScan
1 parent 422aed6 commit c84343d

File tree

4 files changed

+195
-116
lines changed

4 files changed

+195
-116
lines changed

‎jest.config.js‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,5 +45,21 @@ module.exports = {
4545
modulePathIgnorePatterns: [".vscode-test"],
4646
reporters: [["jest-simple-dot-reporter", {}]],
4747
setupFiles: ["<rootDir>/src/__mocks__/jest.setup.ts"],
48-
setupFilesAfterEnv: ["<rootDir>/src/integrations/terminal/__tests__/setupTerminalTests.ts"],
48+
setupFilesAfterEnv: [
49+
"<rootDir>/src/integrations/terminal/__tests__/setupTerminalTests.ts",
50+
"<rootDir>/src/__tests__/setupMemoryTests.ts",
51+
],
52+
// Increase test timeout to allow for GC
53+
testTimeout: 10000,
54+
// Run tests in series to better track memory
55+
maxConcurrency: 1,
56+
// Add memory tracking
57+
globals: {
58+
"ts-jest": {
59+
diagnostics: {
60+
warnOnly: true,
61+
ignoreCodes: [151001],
62+
},
63+
},
64+
},
4965
}

‎src/__tests__/setupMemoryTests.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
// Track memory usage before and after each test
2+
let startMemory: NodeJS.MemoryUsage
3+
4+
beforeEach(() => {
5+
if (global.gc) {
6+
global.gc()
7+
}
8+
startMemory = process.memoryUsage()
9+
})
10+
11+
afterEach(() => {
12+
if (global.gc) {
13+
global.gc()
14+
}
15+
const endMemory = process.memoryUsage()
16+
const diff = {
17+
heapUsed: endMemory.heapUsed - startMemory.heapUsed,
18+
heapTotal: endMemory.heapTotal - startMemory.heapTotal,
19+
external: endMemory.external - startMemory.external,
20+
rss: endMemory.rss - startMemory.rss,
21+
}
22+
23+
// Log if memory increase is significant (> 50MB)
24+
const SIGNIFICANT_INCREASE = 50 * 1024 * 1024 // 50MB in bytes
25+
if (diff.heapUsed > SIGNIFICANT_INCREASE) {
26+
console.warn(`\nSignificant memory increase detected in test:`)
27+
console.warn(`Heap Used: +${(diff.heapUsed / 1024 / 1024).toFixed(2)}MB`)
28+
console.warn(`Heap Total: +${(diff.heapTotal / 1024 / 1024).toFixed(2)}MB`)
29+
console.warn(`External: +${(diff.external / 1024 / 1024).toFixed(2)}MB`)
30+
console.warn(`RSS: +${(diff.rss / 1024 / 1024).toFixed(2)}MB\n`)
31+
}
32+
})
33+
34+
// Add global error handler to catch memory errors
35+
process.on("uncaughtException", (error) => {
36+
if (error.message.includes("heap out of memory")) {
37+
console.error("\nHeap out of memory error detected!")
38+
console.error("Current memory usage:")
39+
const usage = process.memoryUsage()
40+
console.error(`Heap Used: ${(usage.heapUsed / 1024 / 1024).toFixed(2)}MB`)
41+
console.error(`Heap Total: ${(usage.heapTotal / 1024 / 1024).toFixed(2)}MB`)
42+
console.error(`External: ${(usage.external / 1024 / 1024).toFixed(2)}MB`)
43+
console.error(`RSS: ${(usage.rss / 1024 / 1024).toFixed(2)}MB\n`)
44+
}
45+
throw error
46+
})

‎src/services/package-manager/MetadataScanner.ts‎

Lines changed: 72 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,9 @@ export class MetadataScanner {
2121
private readonly git?: SimpleGit
2222
private localizationOptions: LocalizationOptions
2323
private originalRootDir: string | null = null
24+
private static readonly MAX_DEPTH = 5 // Maximum directory depth
25+
private static readonly BATCH_SIZE = 50 // Number of items to process at once
26+
private static readonly CONCURRENT_SCANS = 3 // Number of concurrent directory scans
2427

2528
constructor(git?: SimpleGit, localizationOptions?: LocalizationOptions) {
2629
this.git = git
@@ -30,94 +33,94 @@ export class MetadataScanner {
3033
}
3134
}
3235

36+
/**
37+
* Generator function to yield items in batches
38+
*/
39+
private async *scanDirectoryBatched(
40+
rootDir: string,
41+
repoUrl: string,
42+
sourceName?: string,
43+
depth: number = 0,
44+
): AsyncGenerator<PackageManagerItem[]> {
45+
if (depth > MetadataScanner.MAX_DEPTH) {
46+
return
47+
}
48+
49+
const batch: PackageManagerItem[] = []
50+
const entries = await fs.readdir(rootDir, { withFileTypes: true })
51+
52+
for (const entry of entries) {
53+
if (!entry.isDirectory()) continue
54+
55+
const componentDir = path.join(rootDir, entry.name)
56+
const metadata = await this.loadComponentMetadata(componentDir)
57+
const localizedMetadata = metadata ? this.getLocalizedMetadata(metadata) : null
58+
59+
if (localizedMetadata) {
60+
const item = await this.createPackageManagerItem(
61+
localizedMetadata,
62+
componentDir,
63+
repoUrl,
64+
this.originalRootDir || rootDir,
65+
sourceName,
66+
)
67+
68+
if (item) {
69+
// If this is a package, scan for subcomponents
70+
if (this.isPackageMetadata(localizedMetadata)) {
71+
await this.scanPackageSubcomponents(componentDir, item)
72+
}
73+
74+
batch.push(item)
75+
if (batch.length >= MetadataScanner.BATCH_SIZE) {
76+
yield batch.splice(0)
77+
}
78+
}
79+
}
80+
81+
// Recursively scan subdirectories
82+
if (!localizedMetadata || !this.isPackageMetadata(localizedMetadata)) {
83+
const subGenerator = this.scanDirectoryBatched(componentDir, repoUrl, sourceName, depth + 1)
84+
for await (const subBatch of subGenerator) {
85+
batch.push(...subBatch)
86+
if (batch.length >= MetadataScanner.BATCH_SIZE) {
87+
yield batch.splice(0)
88+
}
89+
}
90+
}
91+
}
92+
93+
if (batch.length > 0) {
94+
yield batch
95+
}
96+
}
97+
3398
/**
3499
* Scans a directory for components
35100
* @param rootDir The root directory to scan
36101
* @param repoUrl The repository URL
37102
* @param sourceName Optional source repository name
38103
* @returns Array of discovered items
39104
*/
105+
/**
106+
* Scan a directory and return items in batches
107+
*/
40108
async scanDirectory(
41109
rootDir: string,
42110
repoUrl: string,
43111
sourceName?: string,
44112
isRecursiveCall: boolean = false,
45113
): Promise<PackageManagerItem[]> {
46-
const items: PackageManagerItem[] = []
47-
48114
// Only set originalRootDir on the first call
49115
if (!isRecursiveCall && !this.originalRootDir) {
50116
this.originalRootDir = rootDir
51117
}
52118

53-
try {
54-
const entries = await fs.readdir(rootDir, { withFileTypes: true })
55-
56-
// Process directories sequentially to avoid memory spikes
57-
for (const entry of entries) {
58-
if (!entry.isDirectory()) continue
59-
60-
const componentDir = path.join(rootDir, entry.name)
61-
const relativePath = path.relative(this.originalRootDir || rootDir, componentDir).replace(/\\/g, "/")
62-
63-
// Load metadata once
64-
const metadata = await this.loadComponentMetadata(componentDir)
65-
const localizedMetadata = metadata ? this.getLocalizedMetadata(metadata) : null
66-
67-
if (localizedMetadata) {
68-
// Create item if we have valid metadata
69-
const item = await this.createPackageManagerItem(
70-
localizedMetadata,
71-
componentDir,
72-
repoUrl,
73-
this.originalRootDir || rootDir,
74-
sourceName,
75-
)
76-
77-
if (item) {
78-
// Handle package items
79-
if (this.isPackageMetadata(localizedMetadata)) {
80-
// Process listed items sequentially
81-
if (localizedMetadata.items) {
82-
item.items = []
83-
for (const subItem of localizedMetadata.items) {
84-
const subPath = path.join(componentDir, subItem.path)
85-
const subMetadata = await this.loadComponentMetadata(subPath)
86-
const localizedSubMetadata = subMetadata
87-
? this.getLocalizedMetadata(subMetadata)
88-
: null
89-
90-
if (localizedSubMetadata) {
91-
item.items.push({
92-
type: subItem.type,
93-
path: subItem.path,
94-
metadata: localizedSubMetadata,
95-
lastUpdated: await this.getLastModifiedDate(subPath),
96-
})
97-
}
98-
}
99-
}
100-
101-
// Scan for unlisted components
102-
await this.scanPackageSubcomponents(componentDir, item)
103-
items.push(item)
104-
continue // Skip further recursion for package directories
105-
}
106-
107-
items.push(item)
108-
}
109-
}
119+
const items: PackageManagerItem[] = []
120+
const generator = this.scanDirectoryBatched(rootDir, repoUrl, sourceName)
110121

111-
// Only recurse if:
112-
// 1. No metadata was found, or
113-
// 2. Metadata was found but it's not a package
114-
if (!localizedMetadata || !this.isPackageMetadata(localizedMetadata)) {
115-
const subItems = await this.scanDirectory(componentDir, repoUrl, sourceName, true)
116-
items.push(...subItems)
117-
}
118-
}
119-
} catch (error) {
120-
console.error(`Error scanning directory ${rootDir}:`, error)
122+
for await (const batch of generator) {
123+
items.push(...batch)
121124
}
122125

123126
return items

0 commit comments

Comments
 (0)