Merge remote-tracking branch 'upstream/dev' into add_pnpm_log_if_err_install

Author: attiasas

cli/docs/flags.go

@@ -120,6 +120,7 @@ const (
 	RequirementsFile             = "requirements-file"
 	WorkingDirs                  = "working-dirs"
 	OutputDir                    = "output-dir"
+	SkipAutoInstall              = "skip-auto-install"
 
 	// Unique curation flags
 	CurationOutput = "curation-format"
@@ -156,7 +157,7 @@ var commandFlags = map[string][]string{
 		url, user, password, accessToken, ServerId, InsecureTls, Project, Watches, RepoPath, Licenses, OutputFormat, ExcludeTestDeps,
 		useWrapperAudit, DepType, RequirementsFile, Fail, ExtendedTable, WorkingDirs, ExclusionsAudit, Mvn, Gradle, Npm,
 		Pnpm, Yarn, Go, Nuget, Pip, Pipenv, Poetry, MinSeverity, FixableOnly, ThirdPartyContextualAnalysis, Threads,
-		Sca, Iac, Sast, Secrets, WithoutCA, ScanVuln, SecretValidation, OutputDir,
+		Sca, Iac, Sast, Secrets, WithoutCA, ScanVuln, SecretValidation, OutputDir, SkipAutoInstall,
 	},
 	CurationAudit: {
 		CurationOutput, WorkingDirs, Threads, RequirementsFile,
@@ -231,8 +232,9 @@ var flagsMap = map[string]components.Flag{
 		"Set to false if you wish to not use the gradle or maven wrapper.",
 		components.WithBoolDefaultValue(true),
 	),
-	WorkingDirs: components.NewStringFlag(WorkingDirs, "A comma-separated list of relative working directories, to determine audit targets locations."),
-	OutputDir:   components.NewStringFlag(OutputDir, "Target directory to save partial results to.", components.SetHiddenStrFlag()),
+	WorkingDirs:     components.NewStringFlag(WorkingDirs, "A comma-separated list of relative working directories, to determine audit targets locations."),
+	OutputDir:       components.NewStringFlag(OutputDir, "Target directory to save partial results to.", components.SetHiddenStrFlag()),
+	SkipAutoInstall: components.NewBoolFlag(SkipAutoInstall, "Set to true to skip auto-install of dependencies in un-built modules. Currently supported for Yarn and NPM only.", components.SetHiddenBoolFlag()),
 	ExclusionsAudit: components.NewStringFlag(
 		Exclusions,
 		"List of exclusions separated by semicolons, utilized to skip sub-projects from undergoing an audit. These exclusions may incorporate the * and ? wildcards.",

cli/scancommands.go

@@ -477,7 +477,8 @@ func CreateAuditCmd(c *components.Context) (*audit.AuditCommand, error) {
 		SetMinSeverityFilter(minSeverity).
 		SetFixableOnly(c.GetBoolFlagValue(flags.FixableOnly)).
 		SetThirdPartyApplicabilityScan(c.GetBoolFlagValue(flags.ThirdPartyContextualAnalysis)).
-		SetScansResultsOutputDir(scansOutputDir)
+		SetScansResultsOutputDir(scansOutputDir).
+		SetSkipAutoInstall(c.GetBoolFlagValue(flags.SkipAutoInstall))
 
 	if c.GetStringFlagValue(flags.Watches) != "" {
 		auditCmd.SetWatches(splitByCommaAndTrim(c.GetStringFlagValue(flags.Watches)))

commands/audit/sca/npm/npm.go

@@ -3,7 +3,6 @@ package npm
 import (
 	"errors"
 	"fmt"
-
 	biutils "github.com/jfrog/build-info-go/build/utils"
 	buildinfo "github.com/jfrog/build-info-go/entities"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/npm"
@@ -48,7 +47,7 @@ func BuildDependencyTree(params utils.AuditParams) (dependencyTrees []*xrayUtils
 	}()
 
 	// Calculate npm dependencies
-	dependenciesMap, err := biutils.CalculateDependenciesMap(npmExecutablePath, currentDir, packageInfo.BuildInfoModuleId(), treeDepsParam, log.Logger)
+	dependenciesMap, err := biutils.CalculateDependenciesMap(npmExecutablePath, currentDir, packageInfo.BuildInfoModuleId(), treeDepsParam, log.Logger, params.SkipAutoInstall())
 	if err != nil {
 		log.Info("Used npm version:", npmVersion.GetVersion())
 		return

commands/audit/sca/npm/npm_test.go

@@ -2,18 +2,19 @@ package npm
 
 import (
 	"encoding/json"
-	"os"
-	"path/filepath"
-	"testing"
-
+	bibuildutils "github.com/jfrog/build-info-go/build/utils"
+	buildinfo "github.com/jfrog/build-info-go/entities"
+	biutils "github.com/jfrog/build-info-go/utils"
+	"github.com/jfrog/jfrog-cli-core/v2/utils/tests"
 	"github.com/jfrog/jfrog-cli-security/commands/audit/sca"
 	"github.com/jfrog/jfrog-cli-security/utils"
+	"github.com/jfrog/jfrog-client-go/utils/io/fileutils"
 	xrayUtils "github.com/jfrog/jfrog-client-go/xray/services/utils"
-
-	biutils "github.com/jfrog/build-info-go/build/utils"
-	buildinfo "github.com/jfrog/build-info-go/entities"
-	"github.com/jfrog/jfrog-cli-core/v2/utils/tests"
 	"github.com/stretchr/testify/assert"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
 )
 
 func TestParseNpmDependenciesList(t *testing.T) {
@@ -25,7 +26,7 @@ func TestParseNpmDependenciesList(t *testing.T) {
 	var dependencies []buildinfo.Dependency
 	err = json.Unmarshal(dependenciesJson, &dependencies)
 	assert.NoError(t, err)
-	packageInfo := &biutils.PackageInfo{Name: "npmexmaple", Version: "0.1.0"}
+	packageInfo := &bibuildutils.PackageInfo{Name: "npmexmaple", Version: "0.1.0"}
 	looseEnvifyJsTokens := []*xrayUtils.GraphNode{{Id: "npm://loose-envify:1.4.0", Nodes: []*xrayUtils.GraphNode{{Id: "npm://js-tokens:4.0.0"}}}}
 	expectedTree := &xrayUtils.GraphNode{
 		Id: "npm://npmexmaple:0.1.0",
@@ -122,3 +123,66 @@ func TestIgnoreScripts(t *testing.T) {
 	_, _, err := BuildDependencyTree(params)
 	assert.NoError(t, err)
 }
+
+// This test checks that the tree construction is skipped when the project is not installed and the user prohibited installation
+func TestSkipBuildDepTreeWhenInstallForbidden(t *testing.T) {
+	testCases := []struct {
+		name                        string
+		testDir                     string
+		installCommand              string
+		shouldBeInstalled           bool
+		successfulTreeBuiltExpected bool
+	}{
+		{
+			name:                        "not installed | install required - install command",
+			testDir:                     filepath.Join("projects", "package-managers", "npm", "npm-no-lock"),
+			installCommand:              "npm install",
+			shouldBeInstalled:           false,
+			successfulTreeBuiltExpected: true,
+		},
+		{
+			name:                        "not installed | install required - install forbidden",
+			testDir:                     filepath.Join("projects", "package-managers", "npm", "npm-no-lock"),
+			shouldBeInstalled:           false,
+			successfulTreeBuiltExpected: false,
+		},
+		{
+			name:                        "installed | install not required",
+			testDir:                     filepath.Join("projects", "package-managers", "npm", "npm-project"),
+			shouldBeInstalled:           true,
+			successfulTreeBuiltExpected: true,
+		},
+	}
+
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			dirPath, cleanUp := sca.CreateTestWorkspace(t, test.testDir)
+			defer cleanUp()
+
+			exists, err := fileutils.IsFileExists(filepath.Join(dirPath, "package-lock.json"), false)
+			assert.NoError(t, err)
+
+			if !test.shouldBeInstalled && exists {
+				err = os.Remove(filepath.Join(dirPath, "package-lock.json"))
+				assert.NoError(t, err)
+			}
+
+			params := (&utils.AuditBasicParams{}).SetSkipAutoInstall(true)
+			if test.installCommand != "" {
+				splitInstallCommand := strings.Split(test.installCommand, " ")
+				params = params.SetInstallCommandName(splitInstallCommand[0]).SetInstallCommandArgs(splitInstallCommand[1:])
+			}
+			dependencyTrees, uniqueDeps, err := BuildDependencyTree(params)
+			if !test.successfulTreeBuiltExpected {
+				assert.Nil(t, dependencyTrees)
+				assert.Nil(t, uniqueDeps)
+				assert.Error(t, err)
+				assert.IsType(t, &biutils.ErrProjectNotInstalled{}, err)
+			} else {
+				assert.NotNil(t, dependencyTrees)
+				assert.NotNil(t, uniqueDeps)
+				assert.NoError(t, err)
+			}
+		})
+	}
+}

commands/audit/sca/yarn/yarn.go

@@ -3,12 +3,13 @@ package yarn
 import (
 	"errors"
 	"fmt"
+	biutils "github.com/jfrog/build-info-go/utils"
 	"path/filepath"
 
 	"golang.org/x/exp/maps"
 
 	"github.com/jfrog/build-info-go/build"
-	biutils "github.com/jfrog/build-info-go/build/utils"
+	bibuildutils "github.com/jfrog/build-info-go/build/utils"
 	"github.com/jfrog/gofrog/version"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/yarn"
 	"github.com/jfrog/jfrog-cli-core/v2/utils/config"
@@ -46,17 +47,17 @@ func BuildDependencyTree(params utils.AuditParams) (dependencyTrees []*xrayUtils
 	if err != nil {
 		return
 	}
-	executablePath, err := biutils.GetYarnExecutable()
+	executablePath, err := bibuildutils.GetYarnExecutable()
 	if errorutils.CheckError(err) != nil {
 		return
 	}
 
-	packageInfo, err := biutils.ReadPackageInfoFromPackageJsonIfExists(currentDir, nil)
+	packageInfo, err := bibuildutils.ReadPackageInfoFromPackageJsonIfExists(currentDir, nil)
 	if errorutils.CheckError(err) != nil {
 		return
 	}
 
-	installRequired, err := isInstallRequired(currentDir, params.InstallCommandArgs())
+	installRequired, err := isInstallRequired(currentDir, params.InstallCommandArgs(), params.SkipAutoInstall())
 	if err != nil {
 		return
 	}
@@ -70,7 +71,7 @@ func BuildDependencyTree(params utils.AuditParams) (dependencyTrees []*xrayUtils
 	}
 
 	// Calculate Yarn dependencies
-	dependenciesMap, root, err := biutils.GetYarnDependencies(executablePath, currentDir, packageInfo, log.Logger)
+	dependenciesMap, root, err := bibuildutils.GetYarnDependencies(executablePath, currentDir, packageInfo, log.Logger)
 	if err != nil {
 		return
 	}
@@ -89,7 +90,7 @@ func configureYarnResolutionServerAndRunInstall(params utils.AuditParams, curWd,
 		return runYarnInstallAccordingToVersion(curWd, yarnExecPath, params.InstallCommandArgs())
 	}
 
-	executableYarnVersion, err := biutils.GetVersion(yarnExecPath, curWd)
+	executableYarnVersion, err := bibuildutils.GetVersion(yarnExecPath, curWd)
 	if err != nil {
 		return
 	}
@@ -136,19 +137,23 @@ func configureYarnResolutionServerAndRunInstall(params utils.AuditParams, curWd,
 	return runYarnInstallAccordingToVersion(curWd, yarnExecPath, params.InstallCommandArgs())
 }
 
-func isInstallRequired(currentDir string, installCommandArgs []string) (installRequired bool, err error) {
+// We verify the project's installation status by examining the presence of the yarn.lock file and the presence of an installation command provided by the user.
+// If install command was provided - we install
+// If yarn.lock is missing, we should install unless the user has explicitly disabled auto-install. In this case we return an error
+// Notice!: If alterations are made manually in the package.json file, it necessitates a manual update to the yarn.lock file as well.
+func isInstallRequired(currentDir string, installCommandArgs []string, skipAutoInstall bool) (installRequired bool, err error) {
 	yarnLockExits, err := fileutils.IsFileExists(filepath.Join(currentDir, yarn.YarnLockFileName), false)
 	if err != nil {
 		err = fmt.Errorf("failed to check the existence of '%s' file: %s", filepath.Join(currentDir, yarn.YarnLockFileName), err.Error())
 		return
 	}
 
-	// We verify the project's installation status by examining the presence of the yarn.lock file and the presence of an installation command provided by the user.
-	// Notice!: If alterations are made manually in the package.json file, it necessitates a manual update to the yarn.lock file as well.
-	if len(installCommandArgs) > 0 || !yarnLockExits {
-		installRequired = true
+	if len(installCommandArgs) > 0 {
+		return true, nil
+	} else if !yarnLockExits && skipAutoInstall {
+		return false, &biutils.ErrProjectNotInstalled{UninstalledDir: currentDir}
 	}
-	return
+	return !yarnLockExits, nil
 }
 
 // Executes the user-defined 'install' command; if absent, defaults to running an 'install' command with specific flags suited to the current yarn version.
@@ -162,7 +167,7 @@ func runYarnInstallAccordingToVersion(curWd, yarnExecPath string, installCommand
 	}
 
 	installCommandArgs = []string{"install"}
-	executableVersionStr, err := biutils.GetVersion(yarnExecPath, curWd)
+	executableVersionStr, err := bibuildutils.GetVersion(yarnExecPath, curWd)
 	if err != nil {
 		return
 	}
@@ -200,13 +205,13 @@ func runYarnInstallAccordingToVersion(curWd, yarnExecPath string, installCommand
 }
 
 // Parse the dependencies into a Xray dependency tree format
-func parseYarnDependenciesMap(dependencies map[string]*biutils.YarnDependency, rootXrayId string) (*xrayUtils.GraphNode, []string) {
+func parseYarnDependenciesMap(dependencies map[string]*bibuildutils.YarnDependency, rootXrayId string) (*xrayUtils.GraphNode, []string) {
 	treeMap := make(map[string]xray.DepTreeNode)
 	for _, dependency := range dependencies {
 		xrayDepId := getXrayDependencyId(dependency)
 		var subDeps []string
 		for _, subDepPtr := range dependency.Details.Dependencies {
-			subDeps = append(subDeps, getXrayDependencyId(dependencies[biutils.GetYarnDependencyKeyFromLocator(subDepPtr.Locator)]))
+			subDeps = append(subDeps, getXrayDependencyId(dependencies[bibuildutils.GetYarnDependencyKeyFromLocator(subDepPtr.Locator)]))
 		}
 		if len(subDeps) > 0 {
 			treeMap[xrayDepId] = xray.DepTreeNode{Children: subDeps}
@@ -216,6 +221,6 @@ func parseYarnDependenciesMap(dependencies map[string]*biutils.YarnDependency, r
 	return graph, maps.Keys(uniqDeps)
 }
 
-func getXrayDependencyId(yarnDependency *biutils.YarnDependency) string {
+func getXrayDependencyId(yarnDependency *bibuildutils.YarnDependency) string {
 	return utils.NpmPackageTypeIdentifier + yarnDependency.Name() + ":" + yarnDependency.Details.Version
 }