diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/Contracts/PackageLockV3Package.cs b/src/Microsoft.ComponentDetection.Detectors/npm/Contracts/PackageLockV3Package.cs index 6f93f0891..b61890c35 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/Contracts/PackageLockV3Package.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/Contracts/PackageLockV3Package.cs @@ -79,7 +79,12 @@ internal sealed record PackageLockV3Package [JsonPropertyName("license")] public string? License { get; init; } + /// + /// The engines this package supports. Can be a dictionary or array in practice. + /// Using PackageJsonEnginesConverter to handle both formats consistently. + /// [JsonPropertyName("engines")] + [JsonConverter(typeof(PackageJsonEnginesConverter))] public IDictionary? Engines { get; init; } [JsonPropertyName("dependencies")] diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfile3Detector.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfile3Detector.cs index f2916f10b..4e820b4ad 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfile3Detector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfile3Detector.cs @@ -1,7 +1,6 @@ namespace Microsoft.ComponentDetection.Detectors.Npm; using System.Collections.Generic; -using System.Linq; using System.Text.Json; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.TypedComponent; @@ -56,8 +55,38 @@ protected override void ProcessLockfile( return; } - // Build package lookup - keys are paths like "node_modules/lodash" or "node_modules/a/node_modules/b" + // Collect direct dependencies from package.json for explicit reference tracking + var directDependencies = new HashSet(System.StringComparer.Ordinal); + if (packageJson.Dependencies is not null) + { + foreach (var dep in packageJson.Dependencies.Keys) + { + directDependencies.Add(dep); + } + } + + if (packageJson.DevDependencies is not null) + { + foreach (var dep in packageJson.DevDependencies.Keys) + { + directDependencies.Add(dep); + } + } + + if (packageJson.OptionalDependencies is not null) + { + foreach (var dep in packageJson.OptionalDependencies.Keys) + { + directDependencies.Add(dep); + } + } + + // Build package lookup and component map - keys are paths like "node_modules/lodash" or "node_modules/a/node_modules/b" var packageLookup = new Dictionary(); + var componentMap = new Dictionary(); + var componentDevStatus = new Dictionary(); + + // First pass: Create all components and determine dev status foreach (var pkg in packagesElement.EnumerateObject()) { if (string.IsNullOrEmpty(pkg.Name)) @@ -66,138 +95,193 @@ protected override void ProcessLockfile( } var package = JsonSerializer.Deserialize(pkg.Value.GetRawText(), JsonOptions); - if (package is not null) + if (package is null) { - packageLookup[pkg.Name] = (pkg.Name, package); + continue; } - } - // Collect all top-level dependencies from package.json - var topLevelDependencies = new Queue<(string Path, PackageLockV3Package Package, TypedComponent? Parent)>(); + // Skip link packages (symbolic links to workspace packages) + if (package.Link == true) + { + continue; + } - this.EnqueueDependencies(topLevelDependencies, packageJson.Dependencies, packageLookup, null); - this.EnqueueDependencies(topLevelDependencies, packageJson.DevDependencies, packageLookup, null); - this.EnqueueDependencies(topLevelDependencies, packageJson.OptionalDependencies, packageLookup, null); + // Skip bundled dependencies (they are installed by their parent) + if (package.InBundle == true) + { + continue; + } - // Process each top-level dependency - while (topLevelDependencies.Count > 0) - { - var (path, lockPackage, _) = topLevelDependencies.Dequeue(); - var name = NpmComponentUtilities.GetModuleName(path); + packageLookup[pkg.Name] = (pkg.Name, package); + + // Derive package name from path + var name = NpmComponentUtilities.GetModuleName(pkg.Name); + if (string.IsNullOrEmpty(name) || string.IsNullOrEmpty(package.Version)) + { + continue; + } - var component = this.CreateComponent(name, lockPackage.Version, lockPackage.Integrity); + var component = this.CreateComponent(name, package.Version, package.Integrity); if (component is null) { continue; } - var previouslyAddedComponents = new HashSet { component.Id }; - var subQueue = new Queue<(string Path, PackageLockV3Package Package, TypedComponent Parent)>(); + // Check both Dev and DevOptional. In npm lockfiles, devOptional is set when a package has both peer: true and dev: true, + // and for detection purposes we treat devOptional packages as dev dependencies. + var isDevDependency = package.Dev == true || package.DevOptional == true; - // Record the top-level component - this.RecordComponent(singleFileComponentRecorder, component, lockPackage.Dev ?? false, component); + // Track component and its dev status + // If a component appears multiple times (at different paths), it's dev-only if ALL instances are dev + if (componentMap.TryGetValue(component.Id, out _)) + { + // Already seen this component - update dev status (if any is non-dev, it's not dev-only) + componentDevStatus[component.Id] = componentDevStatus[component.Id] && isDevDependency; + } + else + { + componentMap[component.Id] = component; + componentDevStatus[component.Id] = isDevDependency; + } + } - // Enqueue nested dependencies - this.EnqueueNestedDependencies(subQueue, path, lockPackage, packageLookup, singleFileComponentRecorder, component); + // Second pass: Register all components + foreach (var (componentId, component) in componentMap) + { + var isDevDependency = componentDevStatus[componentId]; - // Process sub-dependencies - while (subQueue.Count > 0) - { - var (subPath, subPackage, parentComponent) = subQueue.Dequeue(); - var subName = NpmComponentUtilities.GetModuleName(subPath); + // Check if this is a direct dependency from package.json + var npmComponent = (NpmComponent)component; + var isDirectDependency = directDependencies.Contains(npmComponent.Name); - var subComponent = this.CreateComponent(subName, subPackage.Version, subPackage.Integrity); - if (subComponent is null || previouslyAddedComponents.Contains(subComponent.Id)) - { - continue; - } + this.RecordComponent(singleFileComponentRecorder, component, isDevDependency, isDirectDependency); + } - previouslyAddedComponents.Add(subComponent.Id); + // Third pass: Build dependency graph edges using node-style resolution + foreach (var (path, (_, package)) in packageLookup) + { + if (package.Dependencies is null && package.OptionalDependencies is null) + { + continue; + } - this.RecordComponent(singleFileComponentRecorder, subComponent, subPackage.Dev ?? false, component, parentComponent.Id); + var name = NpmComponentUtilities.GetModuleName(path); + if (string.IsNullOrEmpty(name) || string.IsNullOrEmpty(package.Version)) + { + continue; + } - this.EnqueueNestedDependencies(subQueue, subPath, subPackage, packageLookup, singleFileComponentRecorder, subComponent); + var parentComponent = this.CreateComponent(name, package.Version, package.Integrity); + if (parentComponent is null || !componentMap.ContainsKey(parentComponent.Id)) + { + continue; } + + // Process regular dependencies + this.ProcessDependencyEdges(path, package.Dependencies, packageLookup, componentMap, singleFileComponentRecorder, parentComponent); + + // Process optional dependencies + this.ProcessDependencyEdges(path, package.OptionalDependencies, packageLookup, componentMap, singleFileComponentRecorder, parentComponent); } } - private void EnqueueDependencies( - Queue<(string Path, PackageLockV3Package Package, TypedComponent? Parent)> queue, - IDictionary? dependencies, - Dictionary packageLookup, - TypedComponent? parent) + /// + /// Resolves a dependency using node-style module resolution. + /// Walks up from the current path checking for the dependency in nested node_modules folders. + /// + private string? ResolveDependencyPath( + string fromPath, + string dependencyName, + Dictionary packageLookup) { - if (dependencies is null) + var basePath = fromPath; + + while (true) { - return; + // Build candidate path: either at top level or nested in current base + var candidate = string.IsNullOrEmpty(basePath) || basePath == NodeModules + ? $"{NodeModules}/{dependencyName}" + : $"{basePath}/{NodeModules}/{dependencyName}"; + + if (packageLookup.TryGetValue(candidate, out var pkg) && !string.IsNullOrEmpty(pkg.Package.Version)) + { + return candidate; + } + + // Move up to parent's node_modules + if (string.IsNullOrEmpty(basePath)) + { + return null; + } + + basePath = this.GetParentPackagePath(basePath); } + } - foreach (var (path, package) in dependencies.Keys - .Select(key => $"{NodeModules}/{key}") - .Where(packageLookup.ContainsKey) - .Select(path => packageLookup[path])) + /// + /// Gets the parent package path by removing the trailing /node_modules/pkg segment. + /// + private string? GetParentPackagePath(string packagePath) + { + // "node_modules/a/node_modules/b" -> "node_modules/a" + // "node_modules/@scope/a/node_modules/@scope/b" -> "node_modules/@scope/a" + const string marker = "/node_modules/"; + var idx = packagePath.LastIndexOf(marker, System.StringComparison.Ordinal); + if (idx < 0) { - queue.Enqueue((path, package, parent)); + return null; } + + var parent = packagePath[..idx]; + return string.IsNullOrEmpty(parent) ? null : parent; } - private void EnqueueNestedDependencies( - Queue<(string Path, PackageLockV3Package Package, TypedComponent Parent)> queue, - string currentPath, - PackageLockV3Package package, + /// + /// Processes dependency edges for a package, resolving each dependency using node-style resolution. + /// + private void ProcessDependencyEdges( + string fromPath, + IDictionary? dependencies, Dictionary packageLookup, + Dictionary componentMap, ISingleFileComponentRecorder componentRecorder, - TypedComponent parent) + TypedComponent parentComponent) { - if (package.Dependencies is null) + if (dependencies is null) { return; } - foreach (var dep in package.Dependencies) + foreach (var dep in dependencies) { - // First, check if there is an entry in the lockfile for this dependency nested in its ancestors - var ancestors = componentRecorder.DependencyGraph.GetAncestors(parent.Id); - ancestors.Add(parent.Id); - - // Remove version information from ancestor IDs - ancestors = ancestors.Select(x => x.Split(' ')[0]).ToList(); - - var found = false; - - // Depth-first search through ancestors - for (var i = 0; i < ancestors.Count && !found; i++) + var resolvedPath = this.ResolveDependencyPath(fromPath, dep.Key, packageLookup); + if (resolvedPath is null) { - var possiblePath = ancestors.Skip(i).ToList(); - var ancestorNodeModulesPath = string.Format( - "{0}/{1}/{0}/{2}", - NodeModules, - string.Join($"/{NodeModules}/", possiblePath), - dep.Key); - - if (packageLookup.TryGetValue(ancestorNodeModulesPath, out var nestedPkg)) - { - this.Logger.LogDebug("Found nested dependency {Dependency} in {AncestorNodeModulesPath}", dep.Key, ancestorNodeModulesPath); - queue.Enqueue((nestedPkg.Path, nestedPkg.Package, parent)); - found = true; - } + this.Logger.LogDebug("Could not resolve dependency {Dependency} from {FromPath}", dep.Key, fromPath); + continue; } - if (found) + if (!packageLookup.TryGetValue(resolvedPath, out var resolvedPkg)) { continue; } - // If not found in ancestors, check at the top level - var topLevelPath = $"{NodeModules}/{dep.Key}"; - if (packageLookup.TryGetValue(topLevelPath, out var topLevelPkg)) + var resolvedName = NpmComponentUtilities.GetModuleName(resolvedPath); + if (string.IsNullOrEmpty(resolvedName) || string.IsNullOrEmpty(resolvedPkg.Package.Version)) { - queue.Enqueue((topLevelPkg.Path, topLevelPkg.Package, parent)); + continue; } - else + + var childComponent = this.CreateComponent(resolvedName, resolvedPkg.Package.Version, resolvedPkg.Package.Integrity); + if (childComponent is null || !componentMap.ContainsKey(childComponent.Id)) { - this.Logger.LogWarning("Could not find dependency {Dependency} in lockfile", dep.Key); + continue; } + + // Register the dependency edge + componentRecorder.RegisterUsage( + new DetectedComponent(childComponent), + parentComponentId: parentComponent.Id); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs b/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs index 9c6e7c7e8..2b9d15e13 100644 --- a/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs +++ b/src/Microsoft.ComponentDetection.Detectors/npm/NpmLockfileDetectorBase.cs @@ -203,6 +203,22 @@ protected void RecordComponent( parentComponentId); } + protected void RecordComponent( + ISingleFileComponentRecorder recorder, + TypedComponent component, + bool isDevDependency, + bool isExplicitReferencedDependency) + { + // Intentionally keep isExplicitReferencedDependency for API consistency with other overloads. + // Callers should pass true when the component is an explicit dependency so we record it with the explicit flag. + NpmComponentUtilities.AddOrUpdateDetectedComponent( + recorder, + component, + isDevDependency, + parentComponentId: null, + isExplicitReferencedDependency); + } + private IObservable RemoveNodeModuleNestedFiles(IObservable componentStreams) { var directoryItemFacades = new List(); diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/NpmDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/NpmDetectorTests.cs index 2d575ee04..d14f55692 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/NpmDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/NpmDetectorTests.cs @@ -132,6 +132,61 @@ public async Task TestNpmDetector_AuthorNull_WhenAuthorMalformed_AuthorAsSingleS ((NpmComponent)detectedComponents.First().Component).Author.Should().BeNull(); } + [TestMethod] + public async Task TestNpmDetector_AuthorNull_WhenAuthorMalformed_EmailOnlyAsync() + { + var authorName = GetRandomString(); + var authorEmail = GetRandomString(); + var (packageJsonName, packageJsonContents, packageJsonPath) = + NpmTestUtilities.GetPackageJsonNoDependenciesMalformedAuthorAsSingleString(authorName, authorEmail, null); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(packageJsonName, packageJsonContents, this.packageJsonSearchPattern, fileLocation: packageJsonPath) + .ExecuteDetectorAsync(); + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + AssertDetectedComponentCount(detectedComponents, 1); + AssertNpmComponent(detectedComponents); + ((NpmComponent)detectedComponents.First().Component).Author.Should().BeNull(); + } + + [TestMethod] + public async Task TestNpmDetector_AuthorNull_WhenAuthorMalformed_UrlOnlyAsync() + { + var authorName = GetRandomString(); + var authorUrl = GetRandomString(); + var (packageJsonName, packageJsonContents, packageJsonPath) = + NpmTestUtilities.GetPackageJsonNoDependenciesMalformedAuthorAsSingleString(authorName, null, authorUrl); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(packageJsonName, packageJsonContents, this.packageJsonSearchPattern, fileLocation: packageJsonPath) + .ExecuteDetectorAsync(); + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + AssertDetectedComponentCount(detectedComponents, 1); + AssertNpmComponent(detectedComponents); + ((NpmComponent)detectedComponents.First().Component).Author.Should().BeNull(); + } + + [TestMethod] + public async Task TestNpmDetector_AuthorDetected_WhenAuthorMalformed_NameOnlyAsync() + { + var authorName = GetRandomString(); + var (packageJsonName, packageJsonContents, packageJsonPath) = + NpmTestUtilities.GetPackageJsonNoDependenciesMalformedAuthorAsSingleString(authorName, null, null); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(packageJsonName, packageJsonContents, this.packageJsonSearchPattern, fileLocation: packageJsonPath) + .ExecuteDetectorAsync(); + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + var detectedComponents = componentRecorder.GetDetectedComponents(); + AssertDetectedComponentCount(detectedComponents, 1); + AssertNpmComponent(detectedComponents); + + // When malformed, but it's just the name, it should still work + ((NpmComponent)detectedComponents.First().Component).Author.Name.Should().Be(authorName); + } + [TestMethod] public async Task TestNpmDetector_AuthorNameDetected_WhenEmailNotPresentAndUrlNotPresent_AuthorAsSingleStringAsync() { diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/NpmLockfile3DetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/NpmLockfile3DetectorTests.cs index 3d55199ab..20bab233e 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/NpmLockfile3DetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/NpmLockfile3DetectorTests.cs @@ -369,4 +369,541 @@ public async Task TestNpmDetector_PackageLockMissingPackagesProperty_ShouldNotTh var detectedComponents = componentRecorder.GetDetectedComponents(); detectedComponents.Should().BeEmpty(); // No dependencies should be detected since packages is missing } + + [TestMethod] + public async Task TestNpmDetector_PackageLockVersion3WithDevOptionalDependenciesReturnsValidAsync() + { + // Test for issue #1380: devOptional dependencies (peer + dev) should be marked as dev dependencies + var componentName0 = Guid.NewGuid().ToString("N"); + var version0 = NewRandomVersion(); + var componentName1 = Guid.NewGuid().ToString("N"); + var version1 = NewRandomVersion(); + + var (packageLockName, packageLockContents, packageLockPath) = NpmTestUtilities.GetWellFormedNestedPackageLock3WithDevOptionalDependencies(this.packageLockJsonFileName, componentName0, version0, componentName1, version1); + + var packagejson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""devDependencies"": {{ + ""{0}"": ""{1}"", + ""{2}"": ""{3}"" + }}, + ""peerDependencies"": {{ + ""{0}"": ""{1}"", + ""{2}"": ""{3}"" + }} + }}"; + + var packageJsonTemplate = string.Format(packagejson, componentName0, version0, componentName1, version1); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(packageLockName, packageLockContents, this.packageLockJsonSearchPatterns, fileLocation: packageLockPath) + .WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(2); + + // devOptional packages should be marked as dev dependencies + var component0 = detectedComponents.First(x => x.Component.Id.Contains(componentName0)); + componentRecorder.GetEffectiveDevDependencyValue(component0.Component.Id).Should().BeTrue(); + + var component1 = detectedComponents.First(x => x.Component.Id.Contains(componentName1)); + componentRecorder.GetEffectiveDevDependencyValue(component1.Component.Id).Should().BeTrue(); + + foreach (var component in detectedComponents) + { + ((NpmComponent)component.Component).Hash.Should().NotBeNullOrWhiteSpace(); + } + } + + [TestMethod] + public async Task TestNpmDetector_PackageLockVersion3WithPeerDependenciesReturnsValidAsync() + { + // Test that peer dependencies without dev flag should NOT be marked as dev dependencies + var componentName0 = Guid.NewGuid().ToString("N"); + var version0 = NewRandomVersion(); + var componentName1 = Guid.NewGuid().ToString("N"); + var version1 = NewRandomVersion(); + + var (packageLockName, packageLockContents, packageLockPath) = NpmTestUtilities.GetWellFormedNestedPackageLock3WithPeerDependencies(this.packageLockJsonFileName, componentName0, version0, componentName1, version1); + + var packagejson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"", + ""{2}"": ""{3}"" + }}, + ""peerDependencies"": {{ + ""{0}"": ""{1}"", + ""{2}"": ""{3}"" + }} + }}"; + + var packageJsonTemplate = string.Format(packagejson, componentName0, version0, componentName1, version1); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(packageLockName, packageLockContents, this.packageLockJsonSearchPatterns, fileLocation: packageLockPath) + .WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(2); + + // Peer-only packages (without dev flag) should NOT be marked as dev dependencies + var component0 = detectedComponents.First(x => x.Component.Id.Contains(componentName0)); + componentRecorder.GetEffectiveDevDependencyValue(component0.Component.Id).Should().BeFalse(); + + var component1 = detectedComponents.First(x => x.Component.Id.Contains(componentName1)); + componentRecorder.GetEffectiveDevDependencyValue(component1.Component.Id).Should().BeFalse(); + + foreach (var component in detectedComponents) + { + ((NpmComponent)component.Component).Hash.Should().NotBeNullOrWhiteSpace(); + } + } + + [TestMethod] + public async Task TestNpmDetector_PackageLockVersion3WithEnginesAsArray_DoesNotThrowAndReturnsSuccessAsync() + { + var componentName0 = Guid.NewGuid().ToString("N"); + var version0 = NewRandomVersion(); + + var packageLockJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""lockfileVersion"": 3, + ""requires"": true, + ""packages"": {{ + """": {{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""^{1}"" + }} + }}, + ""node_modules/{0}"": {{ + ""version"": ""{1}"", + ""resolved"": ""https://registry.npmjs.org/{0}/-/{0}-{1}.tgz"", + ""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="", + ""engines"": [ + ""node >= 18"" + ] + }} + }} + }}"; + + var packageLockTemplate = string.Format(packageLockJson, componentName0, version0); + + var packageJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}"; + + var packageJsonTemplate = string.Format(packageJson, componentName0, version0); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(this.packageLockJsonFileName, packageLockTemplate, this.packageLockJsonSearchPatterns) + .WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().ContainSingle(x => ((NpmComponent)x.Component).Name.Equals(componentName0)); + + foreach (var component in detectedComponents) + { + ((NpmComponent)component.Component).Hash.Should().NotBeNullOrWhiteSpace(); + } + } + + [TestMethod] + public async Task TestNpmDetector_PackageLockVersion3WithLinkPackages_ShouldSkipLinkPackagesAsync() + { + var componentName0 = Guid.NewGuid().ToString("N"); + var version0 = NewRandomVersion(); + var linkPackageName = "linked-package"; + + var packageLockJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""lockfileVersion"": 3, + ""packages"": {{ + """": {{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}, + ""node_modules/{0}"": {{ + ""version"": ""{1}"", + ""resolved"": ""https://registry.npmjs.org/{0}/-/{0}-{1}.tgz"", + ""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="" + }}, + ""node_modules/{2}"": {{ + ""resolved"": ""../local-workspace/{2}"", + ""link"": true + }} + }} + }}"; + + var packageLockTemplate = string.Format(packageLockJson, componentName0, version0, linkPackageName); + + var packageJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}"; + + var packageJsonTemplate = string.Format(packageJson, componentName0, version0); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(this.packageLockJsonFileName, packageLockTemplate, this.packageLockJsonSearchPatterns) + .WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(1); + detectedComponents.Should().ContainSingle(x => ((NpmComponent)x.Component).Name.Equals(componentName0)); + + // Link package should not be detected + detectedComponents.Should().NotContain(x => ((NpmComponent)x.Component).Name.Equals(linkPackageName)); + } + + [TestMethod] + public async Task TestNpmDetector_PackageLockVersion3WithBundledDependencies_ShouldSkipBundledAsync() + { + var componentName0 = Guid.NewGuid().ToString("N"); + var version0 = NewRandomVersion(); + var bundledName = "bundled-dep"; + var bundledVersion = NewRandomVersion(); + + var packageLockJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""lockfileVersion"": 3, + ""packages"": {{ + """": {{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}, + ""node_modules/{0}"": {{ + ""version"": ""{1}"", + ""resolved"": ""https://registry.npmjs.org/{0}/-/{0}-{1}.tgz"", + ""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="" + }}, + ""node_modules/{2}"": {{ + ""version"": ""{3}"", + ""inBundle"": true, + ""resolved"": ""https://registry.npmjs.org/{2}/-/{2}-{3}.tgz"", + ""integrity"": ""sha512-ABC123XYZ=="" + }} + }} + }}"; + + var packageLockTemplate = string.Format(packageLockJson, componentName0, version0, bundledName, bundledVersion); + + var packageJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}"; + + var packageJsonTemplate = string.Format(packageJson, componentName0, version0); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(this.packageLockJsonFileName, packageLockTemplate, this.packageLockJsonSearchPatterns) + .WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(1); + detectedComponents.Should().ContainSingle(x => ((NpmComponent)x.Component).Name.Equals(componentName0)); + + // Bundled dependency should not be detected + detectedComponents.Should().NotContain(x => ((NpmComponent)x.Component).Name.Equals(bundledName)); + } + + [TestMethod] + public async Task TestNpmDetector_PackageLockVersion3WithMissingVersions_ShouldSkipInvalidPackagesAsync() + { + var componentName0 = Guid.NewGuid().ToString("N"); + var version0 = NewRandomVersion(); + var invalidPackageName = "invalid-package"; + + var packageLockJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""lockfileVersion"": 3, + ""packages"": {{ + """": {{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}, + ""node_modules/{0}"": {{ + ""version"": ""{1}"", + ""resolved"": ""https://registry.npmjs.org/{0}/-/{0}-{1}.tgz"", + ""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="" + }}, + ""node_modules/{2}"": {{ + ""resolved"": ""https://registry.npmjs.org/{2}/-/{2}.tgz"", + ""integrity"": ""sha512-ABC123XYZ=="" + }} + }} + }}"; + + var packageLockTemplate = string.Format(packageLockJson, componentName0, version0, invalidPackageName); + + var packageJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}"; + + var packageJsonTemplate = string.Format(packageJson, componentName0, version0); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(this.packageLockJsonFileName, packageLockTemplate, this.packageLockJsonSearchPatterns) + .WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(1); + detectedComponents.Should().ContainSingle(x => ((NpmComponent)x.Component).Name.Equals(componentName0)); + + // Package without version should not be detected + detectedComponents.Should().NotContain(x => ((NpmComponent)x.Component).Name.Equals(invalidPackageName)); + } + + [TestMethod] + public async Task TestNpmDetector_PackageLockVersion3WithComponentAtMultiplePaths_ShouldTrackDevStatusCorrectlyAsync() + { + // Test that a component appearing multiple times is only dev if ALL instances are dev + var componentName = Guid.NewGuid().ToString("N"); + var version = NewRandomVersion(); + var depName = Guid.NewGuid().ToString("N"); + var depVersion = NewRandomVersion(); + + var packageLockJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""lockfileVersion"": 3, + ""packages"": {{ + """": {{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }}, + ""devDependencies"": {{ + ""{2}"": ""{3}"" + }} + }}, + ""node_modules/{0}"": {{ + ""version"": ""{1}"", + ""resolved"": ""https://registry.npmjs.org/{0}/-/{0}-{1}.tgz"", + ""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="", + ""dependencies"": {{ + ""{4}"": ""{5}"" + }} + }}, + ""node_modules/{2}"": {{ + ""version"": ""{3}"", + ""resolved"": ""https://registry.npmjs.org/{2}/-/{2}-{3}.tgz"", + ""integrity"": ""sha512-ABC123XYZ=="", + ""dev"": true, + ""dependencies"": {{ + ""{4}"": ""{5}"" + }} + }}, + ""node_modules/{4}"": {{ + ""version"": ""{5}"", + ""resolved"": ""https://registry.npmjs.org/{4}/-/{4}-{5}.tgz"", + ""integrity"": ""sha512-XYZ789ABC=="" + }} + }} + }}"; + + var sharedDepName = Guid.NewGuid().ToString("N"); + var sharedDepVersion = NewRandomVersion(); + var packageLockTemplate = string.Format(packageLockJson, componentName, version, depName, depVersion, sharedDepName, sharedDepVersion); + + var packageJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }}, + ""devDependencies"": {{ + ""{2}"": ""{3}"" + }} + }}"; + + var packageJsonTemplate = string.Format(packageJson, componentName, version, depName, depVersion); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(this.packageLockJsonFileName, packageLockTemplate, this.packageLockJsonSearchPatterns) + .WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(3); + + // The shared dependency appears in both dev and non-dev contexts, so should NOT be marked as dev + var sharedDep = detectedComponents.First(x => ((NpmComponent)x.Component).Name.Equals(sharedDepName)); + componentRecorder.GetEffectiveDevDependencyValue(sharedDep.Component.Id).Should().BeFalse(); + } + + [TestMethod] + public async Task TestNpmDetector_PackageLockVersion3WithUnresolvableDependency_ShouldHandleGracefullyAsync() + { + // Test that dependencies that cannot be resolved are logged but don't cause failure + var componentName = Guid.NewGuid().ToString("N"); + var version = NewRandomVersion(); + var unresolvedDep = "unresolved-dep"; + + var packageLockJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""lockfileVersion"": 3, + ""packages"": {{ + """": {{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}, + ""node_modules/{0}"": {{ + ""version"": ""{1}"", + ""resolved"": ""https://registry.npmjs.org/{0}/-/{0}-{1}.tgz"", + ""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="", + ""dependencies"": {{ + ""{2}"": ""1.0.0"" + }} + }} + }} + }}"; + + var packageLockTemplate = string.Format(packageLockJson, componentName, version, unresolvedDep); + + var packageJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}"; + + var packageJsonTemplate = string.Format(packageJson, componentName, version); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(this.packageLockJsonFileName, packageLockTemplate, this.packageLockJsonSearchPatterns) + .WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern) + .ExecuteDetectorAsync(); + + // Should not fail even though dependency is missing + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(1); + detectedComponents.Should().ContainSingle(x => ((NpmComponent)x.Component).Name.Equals(componentName)); + } + + [TestMethod] + public async Task TestNpmDetector_PackageLockVersion3WithScopedNestedDependencies_ShouldResolveCorrectlyAsync() + { + // Test nested node_modules resolution with scoped packages + var scopedPkg = "@scope/package"; + var version = NewRandomVersion(); + var nestedPkg = "nested-pkg"; + var nestedVersion = NewRandomVersion(); + + var packageLockJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""lockfileVersion"": 3, + ""packages"": {{ + """": {{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}, + ""node_modules/{0}"": {{ + ""version"": ""{1}"", + ""resolved"": ""https://registry.npmjs.org/{0}/-/{0}-{1}.tgz"", + ""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="", + ""dependencies"": {{ + ""{2}"": ""{3}"" + }} + }}, + ""node_modules/{0}/node_modules/{2}"": {{ + ""version"": ""{3}"", + ""resolved"": ""https://registry.npmjs.org/{2}/-/{2}-{3}.tgz"", + ""integrity"": ""sha512-ABC123XYZ=="" + }} + }} + }}"; + + var packageLockTemplate = string.Format(packageLockJson, scopedPkg, version, nestedPkg, nestedVersion); + + var packageJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""{1}"" + }} + }}"; + + var packageJsonTemplate = string.Format(packageJson, scopedPkg, version); + + var (scanResult, componentRecorder) = await this.DetectorTestUtility + .WithFile(this.packageLockJsonFileName, packageLockTemplate, this.packageLockJsonSearchPatterns) + .WithFile(this.packageJsonFileName, packageJsonTemplate, this.packageJsonSearchPattern) + .ExecuteDetectorAsync(); + + scanResult.ResultCode.Should().Be(ProcessingResultCode.Success); + + var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); + detectedComponents.Should().HaveCount(2); + + // Verify the dependency edge exists + var parentId = detectedComponents.First(x => ((NpmComponent)x.Component).Name.Equals(scopedPkg)).Component.Id; + var childId = detectedComponents.First(x => ((NpmComponent)x.Component).Name.Equals(nestedPkg)).Component.Id; + + var dependencyGraph = componentRecorder.GetDependencyGraphsByLocation().Values.First(); + dependencyGraph.GetDependenciesForComponent(parentId).Should().Contain(childId); + } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/NpmTestUtilities.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/NpmTestUtilities.cs index 314e8036c..5dfb64682 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/NpmTestUtilities.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/NpmTestUtilities.cs @@ -527,4 +527,106 @@ public static (string PackageJsonName, string PackageJsonContents, string Packag return (lockFileName, packageLockTemplate, Path.Combine(Path.GetTempPath(), lockFileName)); } + + /// + /// Creates a package-lock.json v3 with devOptional dependencies. + /// These are peer dependencies that are also dev dependencies (peer: true + dev: true = devOptional: true). + /// + /// A tuple containing the lockfile name, contents, and path. + public static (string PackageJsonName, string PackageJsonContents, string PackageJsonPath) GetWellFormedNestedPackageLock3WithDevOptionalDependencies(string lockFileName, string depName0 = null, string depVersion0 = null, string depName1 = null, string depVersion1 = null) + { + var packageLockJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""lockfileVersion"": 3, + ""requires"": true, + ""packages"": {{ + """": {{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""devDependencies"": {{ + ""{0}"": ""^{2}"", + ""{1}"": ""^{3}"" + }}, + ""peerDependencies"": {{ + ""{0}"": ""^{2}"", + ""{1}"": ""^{3}"" + }} + }}, + ""node_modules/{0}"": {{ + ""version"": ""{2}"", + ""resolved"": ""https://mseng.pkgs.visualstudio.com/_packaging/VsoMicrosoftExternals/npm/registry"", + ""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="", + ""devOptional"": true, + ""peer"": true + }}, + ""node_modules/{1}"": {{ + ""version"": ""{3}"", + ""resolved"": ""https://mseng.pkgs.visualstudio.com/_packaging/VsoMicrosoftExternals/npm/registry"", + ""integrity"": ""sha512-W86pkk7P9PAfARThHaD4fIjJ8QJUGMB2OhlCFsrueciPqlYZvDg/w62BmRm7PghVQcxGLbYoPN4+iykzP+0jRQ=="", + ""devOptional"": true, + ""peer"": true + }} + }} + }}"; + + var componentName0 = depName0 ?? Guid.NewGuid().ToString("N"); + var version0 = depVersion0 ?? NewRandomVersion(); + var componentName1 = depName1 ?? Guid.NewGuid().ToString("N"); + var version1 = depVersion1 ?? NewRandomVersion(); + + var packageLockTemplate = string.Format(packageLockJson, componentName0, componentName1, version0, version1); + + return (lockFileName, packageLockTemplate, Path.Combine(Path.GetTempPath(), lockFileName)); + } + + /// + /// Creates a package-lock.json v3 with peer dependencies (not dev). + /// These should be detected as production dependencies. + /// + /// A tuple containing the lockfile name, contents, and path. + public static (string PackageJsonName, string PackageJsonContents, string PackageJsonPath) GetWellFormedNestedPackageLock3WithPeerDependencies(string lockFileName, string depName0 = null, string depVersion0 = null, string depName1 = null, string depVersion1 = null) + { + var packageLockJson = @"{{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""lockfileVersion"": 3, + ""requires"": true, + ""packages"": {{ + """": {{ + ""name"": ""test"", + ""version"": ""0.0.0"", + ""dependencies"": {{ + ""{0}"": ""^{2}"", + ""{1}"": ""^{3}"" + }}, + ""peerDependencies"": {{ + ""{0}"": ""^{2}"", + ""{1}"": ""^{3}"" + }} + }}, + ""node_modules/{0}"": {{ + ""version"": ""{2}"", + ""resolved"": ""https://mseng.pkgs.visualstudio.com/_packaging/VsoMicrosoftExternals/npm/registry"", + ""integrity"": ""sha512-nAEMjKcB1LDrMyYnjNsDkxoewI2aexrwlT3UJeL+nlbd64FEQNmKgPGAYIieaLVgtpRiHE9OL6/rmHLlstQwnQ=="", + ""peer"": true + }}, + ""node_modules/{1}"": {{ + ""version"": ""{3}"", + ""resolved"": ""https://mseng.pkgs.visualstudio.com/_packaging/VsoMicrosoftExternals/npm/registry"", + ""integrity"": ""sha512-W86pkk7P9PAfARThHaD4fIjJ8QJUGMB2OhlCFsrueciPqlYZvDg/w62BmRm7PghVQcxGLbYoPN4+iykzP+0jRQ=="", + ""peer"": true + }} + }} + }}"; + + var componentName0 = depName0 ?? Guid.NewGuid().ToString("N"); + var version0 = depVersion0 ?? NewRandomVersion(); + var componentName1 = depName1 ?? Guid.NewGuid().ToString("N"); + var version1 = depVersion1 ?? NewRandomVersion(); + + var packageLockTemplate = string.Format(packageLockJson, componentName0, componentName1, version0, version1); + + return (lockFileName, packageLockTemplate, Path.Combine(Path.GetTempPath(), lockFileName)); + } }