Merge branch 'main' into users/jasonpaulos/add-linux-syft-scope-param

jasonpaulos · web-flow · commit 7a68c61a500c · 2026-01-13T16:43:29.000-05:00
diff --git a/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs b/src/Microsoft.ComponentDetection.Contracts/FileComponentDetector.cs
@@ -125,9 +125,9 @@ private async Task<IndividualDetectorScanResult> ProcessAsync(
                 MaxDegreeOfParallelism = threadsToUse,
             });
 
-        var preprocessedObserbable = await this.OnPrepareDetectionAsync(processRequests, detectorArgs, cancellationToken);
+        var preprocessedObservable = await this.OnPrepareDetectionAsync(processRequests, detectorArgs, cancellationToken);
 
-        await preprocessedObserbable.ForEachAsync(processRequest => processor.Post(processRequest));
+        await preprocessedObservable.ForEachAsync(processRequest => processor.Post(processRequest));
 
         processor.Complete();
 
diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs b/src/Microsoft.ComponentDetection.Detectors/pip/IPyPiClient.cs
@@ -207,7 +207,25 @@ public async Task<PythonProject> GetProjectAsync(PipDependencySpecification spec
         }
 
         var response = await request.Content.ReadAsStringAsync();
-        var project = JsonSerializer.Deserialize<PythonProject>(response);
+
+        // System.Text.Json throws on empty strings, unlike Newtonsoft.Json which returned null
+        if (string.IsNullOrWhiteSpace(response))
+        {
+            this.logger.LogWarning("Empty response from PyPI for {SpecName}", spec.Name);
+            return new PythonProject();
+        }
+
+        PythonProject project;
+        try
+        {
+            project = JsonSerializer.Deserialize<PythonProject>(response);
+        }
+        catch (JsonException ex)
+        {
+            this.logger.LogWarning(ex, "Invalid JSON response from PyPI for {SpecName}", spec.Name);
+            return new PythonProject();
+        }
+
         var versions = new PythonProject
         {
             Info = project.Info,
diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipCommandService.cs
@@ -215,7 +215,23 @@ private async Task<CommandLineExecutionResult> ExecuteCommandAsync(
             }
 
             var reportOutput = await this.fileUtilityService.ReadAllTextAsync(reportFile);
-            return (JsonSerializer.Deserialize<PipInstallationReport>(reportOutput), reportFile);
+
+            if (string.IsNullOrWhiteSpace(reportOutput))
+            {
+                throw new InvalidOperationException($"PipReport: Empty pip installation report for file {path}.");
+            }
+
+            PipInstallationReport report;
+            try
+            {
+                report = JsonSerializer.Deserialize<PipInstallationReport>(reportOutput);
+            }
+            catch (JsonException ex)
+            {
+                throw new InvalidOperationException($"PipReport: Invalid generated pip installation report JSON for {path}.", ex);
+            }
+
+            return (report, reportFile);
         }
         finally
         {
diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PipReportComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PipReportComponentDetector.cs
@@ -200,7 +200,24 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
                 {
                     this.Logger.LogInformation("PipReport: Using pre-generated pip report '{ReportFile}' for package file '{File}'.", existingReport.FullName, file.Location);
                     var reportOutput = await this.fileUtilityService.ReadAllTextAsync(existingReport);
-                    var report = JsonSerializer.Deserialize<PipInstallationReport>(reportOutput);
+
+                    // System.Text.Json throws on empty strings, unlike Newtonsoft.Json which returned null
+                    if (string.IsNullOrWhiteSpace(reportOutput))
+                    {
+                        this.Logger.LogDebug("PipReport: Empty report file '{ReportFile}' for package file '{File}'.", existingReport.FullName, file.Location);
+                        continue;
+                    }
+
+                    PipInstallationReport report;
+                    try
+                    {
+                        report = JsonSerializer.Deserialize<PipInstallationReport>(reportOutput);
+                    }
+                    catch (JsonException ex)
+                    {
+                        this.Logger.LogWarning(ex, "PipReport: Invalid JSON in report file '{ReportFile}' for package file '{File}'.", existingReport.FullName, file.Location);
+                        continue;
+                    }
 
                     if (await this.IsValidPreGeneratedReportAsync(report, pythonExePath, file.Location))
                     {
@@ -234,8 +251,16 @@ protected override async Task OnFileFoundAsync(ProcessRequest processRequest, ID
 
                 // Call pip executable to generate the installation report of a given project file.
                 (var report, var reportFile) = await this.pipCommandService.GenerateInstallationReportAsync(file.Location, pipExePath, pythonExePath, childCts.Token);
-                reports.Add(report);
-                reportFiles.Add(reportFile);
+
+                if (report is not null)
+                {
+                    reports.Add(report);
+                }
+
+                if (reportFile is not null)
+                {
+                    reportFiles.Add(reportFile);
+                }
             }
 
             if (reports.Count == 0)
diff --git a/src/Microsoft.ComponentDetection.Detectors/swiftpm/SwiftResolvedComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/swiftpm/SwiftResolvedComponentDetector.cs
@@ -62,6 +62,12 @@ private void ProcessPackageResolvedFile(ISingleFileComponentRecorder singleFileC
     {
         var parsedResolvedFile = this.ReadAndParseResolvedFile(componentStream.Stream);
 
+        if (parsedResolvedFile?.Pins == null)
+        {
+            this.Logger.LogWarning("Failed to parse Package.resolved file at {Location}", componentStream.Location);
+            return;
+        }
+
         foreach (var package in parsedResolvedFile.Pins)
         {
             try
@@ -109,6 +115,21 @@ private SwiftResolvedFile ReadAndParseResolvedFile(Stream stream)
             resolvedFile = reader.ReadToEnd();
         }
 
-        return JsonSerializer.Deserialize<SwiftResolvedFile>(resolvedFile);
+        // System.Text.Json throws on empty strings, unlike Newtonsoft.Json which returned null
+        if (string.IsNullOrWhiteSpace(resolvedFile))
+        {
+            this.Logger.LogDebug("Empty Package.resolved file encountered");
+            return null;
+        }
+
+        try
+        {
+            return JsonSerializer.Deserialize<SwiftResolvedFile>(resolvedFile);
+        }
+        catch (JsonException ex)
+        {
+            this.Logger.LogWarning(ex, "Invalid JSON in Package.resolved file");
+            return null;
+        }
     }
 }
diff --git a/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/vcpkg/VcpkgComponentDetector.cs
@@ -71,7 +71,7 @@ protected override async Task<IObservable<ProcessRequest>> OnPrepareDetectionAsy
     {
         var filteredProcessRequests = new List<ProcessRequest>();
 
-        await processRequests.ForEachAsync(async pr =>
+        await processRequests.ForEachAsync(pr =>
         {
             var fileLocation = pr.ComponentStream.Location;
             var fileName = Path.GetFileName(fileLocation);
@@ -80,18 +80,42 @@ await processRequests.ForEachAsync(async pr =>
             {
                 this.Logger.LogDebug("Discovered VCPKG package manifest file at: {Location}", pr.ComponentStream.Location);
 
-                using (var reader = new StreamReader(pr.ComponentStream.Stream))
+                using var reader = new StreamReader(pr.ComponentStream.Stream);
+
+                string contents;
+                try
+                {
+                    contents = reader.ReadToEnd();
+                }
+                catch (Exception ex)
                 {
-                    var contents = await reader.ReadToEndAsync().ConfigureAwait(false);
-                    var manifestData = JsonSerializer.Deserialize<ManifestInfo>(contents);
+                    this.Logger.LogDebug(ex, "Failed to read {ManifestInfoFile} at {Path}", ManifestInfoFile, pr.ComponentStream.Location);
+                    return;
+                }
 
-                    if (manifestData == null || string.IsNullOrWhiteSpace(manifestData.ManifestPath))
+                // System.Text.Json throws on empty strings, unlike Newtonsoft.Json which returned null
+                if (string.IsNullOrWhiteSpace(contents))
+                {
+                    this.Logger.LogDebug("Empty {ManifestInfoFile} file at {Path}", ManifestInfoFile, pr.ComponentStream.Location);
+                }
+                else
+                {
+                    try
                     {
-                        this.Logger.LogDebug("Failed to deserialize manifest-info.json or missing ManifestPath at {Path}", pr.ComponentStream.Location);
+                        var manifestData = JsonSerializer.Deserialize<ManifestInfo>(contents);
+
+                        if (string.IsNullOrWhiteSpace(manifestData?.ManifestPath))
+                        {
+                            this.Logger.LogDebug("Failed to deserialize {ManifestInfoFile} or missing ManifestPath at {Path}", ManifestInfoFile, pr.ComponentStream.Location);
+                        }
+                        else
+                        {
+                            this.manifestMappings.TryAdd(fileLocation, manifestData.ManifestPath);
+                        }
                     }
-                    else
+                    catch (JsonException ex)
                     {
-                        this.manifestMappings.TryAdd(fileLocation, manifestData.ManifestPath);
+                        this.Logger.LogWarning(ex, "Invalid JSON in {ManifestInfoFile} at {Path}", ManifestInfoFile, pr.ComponentStream.Location);
                     }
                 }
             }
@@ -211,7 +235,8 @@ private ISingleFileComponentRecorder GetManifestComponentRecorder(ISingleFileCom
             }
 
             this.Logger.LogDebug(
-                "No valid manifest-info.json found at either '{PreferredManifest}' or '{FallbackManifest}' for base location '{VcpkgInstalledDir}'. Returning original recorder.",
+                "No valid {ManifestInfoFile} found at either '{PreferredManifest}' or '{FallbackManifest}' for base location '{VcpkgInstalledDir}'. Returning original recorder.",
+                ManifestInfoFile,
                 preferredManifest,
                 fallbackManifest,
                 vcpkgInstalledDir);
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj b/test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj
@@ -57,6 +57,18 @@
         <None Update="Mocks\test.component-detection-pip-report.json">
           <CopyToOutputDirectory>Always</CopyToOutputDirectory>
         </None>
+        <None Update="Mocks\EmptyReport\requirements.txt">
+          <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </None>
+        <None Update="Mocks\EmptyReport\test.component-detection-pip-report.json">
+          <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </None>
+        <None Update="Mocks\InvalidJsonReport\requirements.txt">
+          <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </None>
+        <None Update="Mocks\InvalidJsonReport\test.component-detection-pip-report.json">
+          <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </None>
         <None Update="TestFiles\go_WithLocalReferences.mod">
           <CopyToOutputDirectory>Always</CopyToOutputDirectory>
         </None>
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Mocks/EmptyReport/requirements.txt b/test/Microsoft.ComponentDetection.Detectors.Tests/Mocks/EmptyReport/requirements.txt
@@ -0,0 +1 @@
+requests==2.28.1
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Mocks/EmptyReport/test.component-detection-pip-report.json b/test/Microsoft.ComponentDetection.Detectors.Tests/Mocks/EmptyReport/test.component-detection-pip-report.json
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Mocks/InvalidJsonReport/requirements.txt b/test/Microsoft.ComponentDetection.Detectors.Tests/Mocks/InvalidJsonReport/requirements.txt
@@ -0,0 +1 @@
+requests==2.28.1
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/Mocks/InvalidJsonReport/test.component-detection-pip-report.json b/test/Microsoft.ComponentDetection.Detectors.Tests/Mocks/InvalidJsonReport/test.component-detection-pip-report.json
@@ -0,0 +1 @@
+{ invalid json }
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipCommandServiceTests.cs
@@ -703,4 +703,70 @@ public async Task PipCommandService_CancelledAsync()
         var action = async () => await service.GenerateInstallationReportAsync(testPath, cancellationToken: cts.Token);
         await action.Should().ThrowAsync<InvalidOperationException>().WithMessage("PipReport: Cancelled*");
     }
+
+    [TestMethod]
+    public async Task PipCommandService_EmptyReportThrowsExceptionAsync()
+    {
+        var testPath = Path.Join(Directory.GetCurrentDirectory(), "requirements.txt");
+
+        this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync(
+            "pip",
+            It.IsAny<IEnumerable<string>>(),
+            "--version"))
+            .ReturnsAsync(true);
+
+        var service = new PipCommandService(
+            this.commandLineInvokationService.Object,
+            this.pathUtilityService,
+            this.fileUtilityService.Object,
+            this.envVarService.Object,
+            this.logger.Object);
+
+        this.commandLineInvokationService.Setup(x => x.ExecuteCommandAsync(
+            "pip",
+            It.IsAny<IEnumerable<string>>(),
+            It.Is<DirectoryInfo>(d => d.FullName.Contains(Directory.GetCurrentDirectory(), StringComparison.OrdinalIgnoreCase)),
+            It.IsAny<CancellationToken>(),
+            It.Is<string>(s => s.Contains("requirements.txt", StringComparison.OrdinalIgnoreCase))))
+            .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdErr = string.Empty, StdOut = string.Empty });
+
+        this.fileUtilityService.Setup(x => x.ReadAllTextAsync(It.IsAny<FileInfo>()))
+            .ReturnsAsync(string.Empty);
+
+        var action = async () => await service.GenerateInstallationReportAsync(testPath);
+        await action.Should().ThrowAsync<InvalidOperationException>().WithMessage("*Empty pip installation report*");
+    }
+
+    [TestMethod]
+    public async Task PipCommandService_InvalidJsonReportThrowsExceptionAsync()
+    {
+        var testPath = Path.Join(Directory.GetCurrentDirectory(), "requirements.txt");
+
+        this.commandLineInvokationService.Setup(x => x.CanCommandBeLocatedAsync(
+            "pip",
+            It.IsAny<IEnumerable<string>>(),
+            "--version"))
+            .ReturnsAsync(true);
+
+        var service = new PipCommandService(
+            this.commandLineInvokationService.Object,
+            this.pathUtilityService,
+            this.fileUtilityService.Object,
+            this.envVarService.Object,
+            this.logger.Object);
+
+        this.commandLineInvokationService.Setup(x => x.ExecuteCommandAsync(
+            "pip",
+            It.IsAny<IEnumerable<string>>(),
+            It.Is<DirectoryInfo>(d => d.FullName.Contains(Directory.GetCurrentDirectory(), StringComparison.OrdinalIgnoreCase)),
+            It.IsAny<CancellationToken>(),
+            It.Is<string>(s => s.Contains("requirements.txt", StringComparison.OrdinalIgnoreCase))))
+            .ReturnsAsync(new CommandLineExecutionResult { ExitCode = 0, StdErr = string.Empty, StdOut = string.Empty });
+
+        this.fileUtilityService.Setup(x => x.ReadAllTextAsync(It.IsAny<FileInfo>()))
+            .ReturnsAsync("{ invalid json }");
+
+        var action = async () => await service.GenerateInstallationReportAsync(testPath);
+        await action.Should().ThrowAsync<InvalidOperationException>().WithMessage("*Invalid generated pip installation report*");
+    }
 }
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PipReportComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PipReportComponentDetectorTests.cs
@@ -552,6 +552,13 @@ public async Task TestPipReportDetector_FallbackAsync()
             It.IsAny<CancellationToken>()))
             .ThrowsAsync(new InvalidOperationException("Want to fallback, so fail initial report generation"));
 
+        this.pipCommandService.Setup(x => x.GenerateInstallationReportAsync(
+            It.Is<string>(s => s.Contains("setup.py", StringComparison.OrdinalIgnoreCase)),
+            It.IsAny<string>(),
+            It.IsAny<string>(),
+            It.IsAny<CancellationToken>()))
+            .ThrowsAsync(new InvalidOperationException("Want to fallback for setup.py too, so fail initial report generation"));
+
         var (result, componentRecorder) = await this.DetectorTestUtility
             .WithFile("setup.py", string.Empty)
             .WithFile("requirements.txt", string.Empty)
@@ -709,6 +716,54 @@ public async Task TestPipReportDetector_InvalidPregeneratedFile_Async()
         idnaComponent.Version.Should().Be("3.7");
     }
 
+    [TestMethod]
+    public async Task TestPipReportDetector_EmptyPreGeneratedReportAsync()
+    {
+        this.mockEnvVarService.Setup(x => x.DoesEnvironmentVariableExist("DisablePipReportSkipRun")).Returns(false);
+
+        var file1 = Path.Join(Directory.GetCurrentDirectory(), "Mocks", "EmptyReport", "requirements.txt");
+
+        var (result, componentRecorder) = await this.DetectorTestUtility
+            .WithFile("requirements.txt", string.Empty, fileLocation: file1)
+            .ExecuteDetectorAsync();
+
+        result.ResultCode.Should().Be(ProcessingResultCode.Success);
+
+        // Verify that the empty pre-generated file was detected and logged
+        this.mockLogger.Verify(
+            x => x.Log(
+                LogLevel.Debug,
+                It.IsAny<EventId>(),
+                It.Is<It.IsAnyType>((o, t) => o.ToString().Contains("Empty report file")),
+                It.IsAny<Exception>(),
+                It.IsAny<Func<It.IsAnyType, Exception, string>>()),
+            Times.Once);
+    }
+
+    [TestMethod]
+    public async Task TestPipReportDetector_InvalidJsonPreGeneratedReportAsync()
+    {
+        this.mockEnvVarService.Setup(x => x.DoesEnvironmentVariableExist("DisablePipReportSkipRun")).Returns(false);
+
+        var file1 = Path.Join(Directory.GetCurrentDirectory(), "Mocks", "InvalidJsonReport", "requirements.txt");
+
+        var (result, componentRecorder) = await this.DetectorTestUtility
+            .WithFile("requirements.txt", string.Empty, fileLocation: file1)
+            .ExecuteDetectorAsync();
+
+        result.ResultCode.Should().Be(ProcessingResultCode.Success);
+
+        // Verify that the invalid JSON pre-generated file was detected and logged
+        this.mockLogger.Verify(
+            x => x.Log(
+                LogLevel.Warning,
+                It.IsAny<EventId>(),
+                It.Is<It.IsAnyType>((o, t) => o.ToString().Contains("Invalid JSON")),
+                It.IsAny<Exception>(),
+                It.IsAny<Func<It.IsAnyType, Exception, string>>()),
+            Times.Once);
+    }
+
     private List<(string PackageString, GitComponent Component)> ToGitTuple(IList<string> components)
     {
         return components.Select<string, (string, GitComponent)>(dep => (dep, null)).ToList();
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/PyPiClientTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/PyPiClientTests.cs
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/SwiftResolvedDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/SwiftResolvedDetectorTests.cs
diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/VcpkgComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/VcpkgComponentDetectorTests.cs
