Skip to content

Commit 3cc3dab

Browse files
Smartsheet-JB-BrownSmartsheet-JB-Brown
committed
change metadatascanner to be more memory efficient
1 parent b7f45f7 commit 3cc3dab

File tree

1 file changed

+83
-80
lines changed

1 file changed

+83
-80
lines changed

‎src/services/package-manager/MetadataScanner.ts‎

Lines changed: 83 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -45,82 +45,73 @@ export class MetadataScanner {
4545
): Promise<PackageManagerItem[]> {
4646
const items: PackageManagerItem[] = []
4747

48-
// Only set originalRootDir on the first call, not recursive calls
48+
// Only set originalRootDir on the first call
4949
if (!isRecursiveCall && !this.originalRootDir) {
5050
this.originalRootDir = rootDir
5151
}
5252

5353
try {
5454
const entries = await fs.readdir(rootDir, { withFileTypes: true })
5555

56+
// Process directories sequentially to avoid memory spikes
5657
for (const entry of entries) {
5758
if (!entry.isDirectory()) continue
5859

5960
const componentDir = path.join(rootDir, entry.name)
60-
// Always calculate paths relative to the original root directory
6161
const relativePath = path.relative(this.originalRootDir || rootDir, componentDir).replace(/\\/g, "/")
62-
const metadata = await this.loadComponentMetadata(componentDir)
63-
// If no metadata found, or metadata validation fails, try recursing
64-
if (!metadata || !this.getLocalizedMetadata(metadata)) {
65-
// Pass the current directory as the root for this recursive call
66-
const subItems = await this.scanDirectory(componentDir, repoUrl, sourceName, true)
67-
items.push(...subItems)
68-
continue
69-
}
70-
71-
// Get localized metadata with fallback
72-
const localizedMetadata = this.getLocalizedMetadata(metadata)
73-
if (!localizedMetadata) continue
7462

75-
// Always use the original root directory for path calculations
76-
const item = await this.createPackageManagerItem(
77-
localizedMetadata,
78-
componentDir,
79-
repoUrl,
80-
this.originalRootDir || rootDir,
81-
sourceName,
82-
)
83-
if (item) {
84-
// If this is a package, scan for subcomponents
85-
if (this.isPackageMetadata(localizedMetadata)) {
86-
// Load metadata for items listed in package metadata
87-
if (localizedMetadata.items) {
88-
const subcomponents = await Promise.all(
89-
localizedMetadata.items.map(async (subItem) => {
63+
// Load metadata once
64+
const metadata = await this.loadComponentMetadata(componentDir)
65+
const localizedMetadata = metadata ? this.getLocalizedMetadata(metadata) : null
66+
67+
if (localizedMetadata) {
68+
// Create item if we have valid metadata
69+
const item = await this.createPackageManagerItem(
70+
localizedMetadata,
71+
componentDir,
72+
repoUrl,
73+
this.originalRootDir || rootDir,
74+
sourceName,
75+
)
76+
77+
if (item) {
78+
// Handle package items
79+
if (this.isPackageMetadata(localizedMetadata)) {
80+
// Process listed items sequentially
81+
if (localizedMetadata.items) {
82+
item.items = []
83+
for (const subItem of localizedMetadata.items) {
9084
const subPath = path.join(componentDir, subItem.path)
91-
const subRelativePath = path.relative(rootDir, subPath).replace(/\\/g, "/")
9285
const subMetadata = await this.loadComponentMetadata(subPath)
93-
94-
// Skip if no metadata found
95-
if (!subMetadata) return null
96-
97-
// Get localized metadata with fallback
98-
const localizedSubMetadata = this.getLocalizedMetadata(subMetadata)
99-
if (!localizedSubMetadata) return null
100-
101-
return {
102-
type: subItem.type,
103-
path: subItem.path,
104-
metadata: localizedSubMetadata,
105-
lastUpdated: await this.getLastModifiedDate(subPath),
86+
const localizedSubMetadata = subMetadata
87+
? this.getLocalizedMetadata(subMetadata)
88+
: null
89+
90+
if (localizedSubMetadata) {
91+
item.items.push({
92+
type: subItem.type,
93+
path: subItem.path,
94+
metadata: localizedSubMetadata,
95+
lastUpdated: await this.getLastModifiedDate(subPath),
96+
})
10697
}
107-
}),
108-
)
109-
item.items = subcomponents.filter((sub): sub is NonNullable<typeof sub> => sub !== null)
98+
}
99+
}
100+
101+
// Scan for unlisted components
102+
await this.scanPackageSubcomponents(componentDir, item)
103+
items.push(item)
104+
continue // Skip further recursion for package directories
110105
}
111106

112-
// Also scan directory for unlisted subcomponents
113-
await this.scanPackageSubcomponents(componentDir, item)
114-
}
115-
items.push(item)
116-
// Skip recursion if this is a package directory
117-
if (this.isPackageMetadata(localizedMetadata)) {
118-
continue
107+
items.push(item)
119108
}
120109
}
121110

122-
// Recursively scan subdirectories only if not in a package
123-
if (!metadata || !this.isPackageMetadata(localizedMetadata)) {
111+
// Only recurse if:
112+
// 1. No metadata was found, or
113+
// 2. Metadata was found but it's not a package
114+
if (!localizedMetadata || !this.isPackageMetadata(localizedMetadata)) {
124115
const subItems = await this.scanDirectory(componentDir, repoUrl, sourceName, true)
125116
items.push(...subItems)
126117
}
@@ -297,39 +288,51 @@ export class MetadataScanner {
297288
packageItem: PackageManagerItem,
298289
parentPath: string = "",
299290
): Promise<void> {
300-
const entries = await fs.readdir(packageDir, { withFileTypes: true })
291+
try {
292+
const entries = await fs.readdir(packageDir, { withFileTypes: true })
301293

302-
for (const entry of entries) {
303-
if (!entry.isDirectory()) continue
294+
// Process directories sequentially
295+
for (const entry of entries) {
296+
if (!entry.isDirectory()) continue
304297

305-
const subPath = path.join(packageDir, entry.name)
306-
// Normalize path to use forward slashes
307-
const relativePath = parentPath ? `${parentPath}/${entry.name}` : entry.name
298+
const subPath = path.join(packageDir, entry.name)
299+
const relativePath = parentPath ? `${parentPath}/${entry.name}` : entry.name
308300

309-
// Try to load metadata directly
310-
const subMetadata = await this.loadComponentMetadata(subPath)
301+
// Try to load metadata directly
302+
const subMetadata = await this.loadComponentMetadata(subPath)
303+
if (!subMetadata) {
304+
// If no metadata found, recurse into directory
305+
await this.scanPackageSubcomponents(subPath, packageItem, relativePath)
306+
continue
307+
}
311308

312-
if (subMetadata) {
313309
// Get localized metadata with fallback
314310
const localizedSubMetadata = this.getLocalizedMetadata(subMetadata)
315-
if (localizedSubMetadata) {
316-
const isListed = packageItem.items?.some((i) => i.path === relativePath)
317-
318-
if (!isListed) {
319-
const subItem = {
320-
type: localizedSubMetadata.type,
321-
path: relativePath,
322-
metadata: localizedSubMetadata,
323-
lastUpdated: await this.getLastModifiedDate(subPath),
324-
}
325-
packageItem.items = packageItem.items || []
326-
packageItem.items.push(subItem)
327-
}
311+
if (!localizedSubMetadata) {
312+
// If no localized metadata, recurse into directory
313+
await this.scanPackageSubcomponents(subPath, packageItem, relativePath)
314+
continue
315+
}
316+
317+
// Check if this component is already listed
318+
const isListed = packageItem.items?.some((i) => i.path === relativePath)
319+
if (!isListed) {
320+
// Initialize items array if needed
321+
packageItem.items = packageItem.items || []
322+
323+
// Add new subcomponent
324+
packageItem.items.push({
325+
type: localizedSubMetadata.type,
326+
path: relativePath,
327+
metadata: localizedSubMetadata,
328+
lastUpdated: await this.getLastModifiedDate(subPath),
329+
})
328330
}
329-
}
330331

331-
// Recursively scan this directory
332-
await this.scanPackageSubcomponents(subPath, packageItem, relativePath)
332+
// Don't recurse into directories that have valid metadata
333+
}
334+
} catch (error) {
335+
console.error(`Error scanning package subcomponents in ${packageDir}:`, error)
333336
}
334337
}
335338

0 commit comments

Comments
 (0)