diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7112fd6b1..a93ef3615 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,8 +47,8 @@ jobs: run: go install github.com/wadey/gocovmerge@latest - name: Install pre-commit run: | - pip install pre-commit - pre-commit install + pip install pre-commit + pre-commit install - name: Go Integration test shell: bash env: @@ -71,7 +71,7 @@ jobs: PR_GITHUB_NAMESPACE: "checkmarx" PR_GITHUB_REPO_NAME: "ast-cli" PR_GITHUB_NUMBER: 983 - PR_GITLAB_TOKEN : ${{ secrets.PR_GITLAB_TOKEN }} + PR_GITLAB_TOKEN: ${{ secrets.PR_GITLAB_TOKEN }} PR_GITLAB_NAMESPACE: ${{ secrets.PR_GITLAB_NAMESPACE }} PR_GITLAB_REPO_NAME: ${{ secrets.PR_GITLAB_REPO_NAME }} PR_GITLAB_PROJECT_ID: ${{ secrets.PR_GITLAB_PROJECT_ID }} @@ -127,14 +127,16 @@ jobs: with: go-version-file: go.mod - run: go version + - run: go mod tidy - name: golangci-lint uses: golangci/golangci-lint-action@3a919529898de77ec3da873e3063ca4b10e7f5cc #v3 with: skip-pkg-cache: true - version: v1.54.2 + version: v1.64.2 args: -c .golangci.yml --timeout 5m only-new-issues: true + govulncheck: runs-on: ubuntu-latest name: govulncheck @@ -163,7 +165,7 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Build the project - run: go build -o ./cx ./cmd + run: go build -o ./cx ./cmd - name: Build Docker image run: docker build -t ast-cli:${{ github.sha }} . - name: Run Trivy scanner without downloading DBs @@ -178,7 +180,7 @@ jobs: output: './trivy-image-results.txt' env: TRIVY_SKIP_JAVA_DB_UPDATE: true - + - name: Inspect action report if: always() shell: bash diff --git a/.golangci.yml b/.golangci.yml index a918e83b4..3501cb4d8 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,5 +1,43 @@ +# .golangci.yml + +run: + timeout: 5m + issues: + exclude-dirs: + - test/testdata_etc + - internal/cache + - internal/renameio + - internal/robustio + +linters: + disable-all: true + enable: + - bodyclose + - depguard + - dogsled + - dupl + - errcheck + - funlen + - gochecknoinits + - goconst + - gocritic + - gocyclo + - gofmt + - goimports + - ineffassign + - mnd # replacement for gomnd + - nakedret + - revive # replacement for golint + - rowserrcheck + - staticcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused # covers deadcode/varcheck/structcheck + - whitespace + linters-settings: - # https://golangci-lint.run/usage/linters/#depguard depguard: list-type: blacklist rules: @@ -9,6 +47,8 @@ linters-settings: - github.com/checkmarx/ast-cli/internal - github.com/gookit/color - github.com/CheckmarxDev/containers-resolver/pkg/containerResolver + - github.com/Checkmarx/manifest-parser/pkg/parser/models + - github.com/Checkmarx/manifest-parser/pkg/parser - github.com/Checkmarx/gen-ai-prompts/prompts/sast_result_remediation - github.com/spf13/viper - github.com/Checkmarx/gen-ai-wrapper @@ -36,7 +76,7 @@ linters-settings: - performance - style disabled-checks: - - dupImport # https://github.com/go-critic/go-critic/issues/845 + - dupImport # https://github.com/go-critic/go-critic/issues/845 - ifElseChain - octalLiteral - whyNoLint @@ -45,15 +85,16 @@ linters-settings: min-complexity: 15 goimports: local-prefixes: github.com/golangci/golangci-lint - golint: - min-confidence: 0 - gomnd: + mnd: settings: mnd: - # don't include the "operation" and "assign" checks: argument,case,condition,return + revive: + rules: + - name: exported + arguments: + - disableStutteringCheck govet: - check-shadowing: true settings: printf: funcs: @@ -67,71 +108,14 @@ linters-settings: suggest-new: true misspell: locale: US -linters: - # please, do not use `enable-all`: it's deprecated and will be removed soon. - # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint - disable-all: true - enable: - - bodyclose - - deadcode - - depguard - - dogsled - - dupl - - errcheck - - funlen - - gochecknoinits - - goconst - - gocritic - - gocyclo - - gofmt - - goimports - - golint - - gomnd - - goprintffuncname - - gosimple - - govet - - ineffassign - - interfacer - - lll - - misspell - - nakedret - - rowserrcheck - - scopelint - - staticcheck - - structcheck - - stylecheck - - typecheck - - unconvert - - unparam - - unused - - varcheck - - whitespace - # don't enable: - # - gochecknoglobals - # - gocognit - # - godox - # - maligned - # - prealloc issues: - # Excluding configuration per-path, per-linter, per-text and per-source exclude-rules: - path: _test\.go linters: - - gomnd -run: - skip-dirs: - - test/testdata_etc - - internal/cache - - internal/renameio - - internal/robustio - - # In case of linter atoi() erros - # go: '^1.21' + - mnd -# golangci.com configuration -# https://github.com/golangci/golangci/wiki/Configuration service: - golangci-lint-version: 1.54.2 # use the fixed version to not introduce new linters unexpectedly + golangci-lint-version: 1.64.2 prepare: - - echo "here I can run custom commands, but no preparation needed for this repo" + - echo "No special prep steps needed" diff --git a/cmd/main.go b/cmd/main.go index 9f9be2fcd..2bcecbf26 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -56,6 +56,7 @@ func main() { sastMetadataPath := viper.GetString(params.SastMetadataPathKey) accessManagementPath := viper.GetString(params.AccessManagementPathKey) byorPath := viper.GetString(params.ByorPathKey) + realtimeScannerPath := viper.GetString(params.RealtimeScannerPathKey) customStatesWrapper := wrappers.NewCustomStatesHTTPWrapper() scansWrapper := wrappers.NewHTTPScansWrapper(scans) @@ -91,6 +92,7 @@ func main() { accessManagementWrapper := wrappers.NewAccessManagementHTTPWrapper(accessManagementPath) byorWrapper := wrappers.NewByorHTTPWrapper(byorPath) containerResolverWrapper := wrappers.NewContainerResolverWrapper() + realTimeWrapper := wrappers.NewRealtimeScannerHTTPWrapper(realtimeScannerPath, jwtWrapper, featureFlagsWrapper) astCli := commands.NewAstCLI( applicationsWrapper, @@ -127,6 +129,7 @@ func main() { accessManagementWrapper, byorWrapper, containerResolverWrapper, + realTimeWrapper, ) exitListener() err = astCli.Execute() diff --git a/go.mod b/go.mod index efe986529..533a0e273 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Checkmarx/containers-resolver v1.0.9 github.com/Checkmarx/gen-ai-prompts v0.0.0-20240807143411-708ceec12b63 github.com/Checkmarx/gen-ai-wrapper v1.0.2 + github.com/Checkmarx/manifest-parser v0.0.4 github.com/Checkmarx/secret-detection v0.0.3-0.20250327150305-31c2c3be9edf github.com/MakeNowJust/heredoc v1.0.0 github.com/bouk/monkey v1.0.0 diff --git a/go.sum b/go.sum index f3a47569e..79be3e894 100644 --- a/go.sum +++ b/go.sum @@ -75,6 +75,8 @@ github.com/Checkmarx/gen-ai-prompts v0.0.0-20240807143411-708ceec12b63 h1:SCuTcE github.com/Checkmarx/gen-ai-prompts v0.0.0-20240807143411-708ceec12b63/go.mod h1:MI6lfLerXU+5eTV/EPTDavgnV3owz3GPT4g/msZBWPo= github.com/Checkmarx/gen-ai-wrapper v1.0.2 h1:T6X40+4hYnwfDsvkjWs9VIcE6s1O+8DUu0+sDdCY3GI= github.com/Checkmarx/gen-ai-wrapper v1.0.2/go.mod h1:xwRLefezwNNnRGu1EjGS6wNiR9FVV/eP9D+oXwLViVM= +github.com/Checkmarx/manifest-parser v0.0.4 h1:0UB+FTJu3A9YT/VeJDNvMrX7KBy4mYCVJVK8kNYkcaU= +github.com/Checkmarx/manifest-parser v0.0.4/go.mod h1:s11sV8akqWX+H0MwFK3XBF8H6JohAjoQe8ClvdDFziQ= github.com/Checkmarx/secret-detection v0.0.3-0.20250327150305-31c2c3be9edf h1:lKiogedU3WzWBc/xI6Xj1BhX2Gp1QBJj8C+czY7CcaE= github.com/Checkmarx/secret-detection v0.0.3-0.20250327150305-31c2c3be9edf/go.mod h1:mtAHOm1mHGh7MVu6JdYUyitANsLcHNLUTBIh9pTERNI= github.com/CycloneDX/cyclonedx-go v0.9.2 h1:688QHn2X/5nRezKe2ueIVCt+NRqf7fl3AVQk+vaFcIo= diff --git a/internal/commands/data/manifests/package.json b/internal/commands/data/manifests/package.json new file mode 100644 index 000000000..42bb2401a --- /dev/null +++ b/internal/commands/data/manifests/package.json @@ -0,0 +1,27 @@ +{ + "dependencies": { + "@CheckmarxDev/ast-cli-javascript-wrapper": "file:../ast-cli-javascript-wrapper/CheckmarxDev-ast-cli-javascript-wrapper-0.0.54.tgz", + "@checkmarxdev/ast-cli-javascript-wrapper": "0.0.54", + "copyfiles": "200", + "tree-kill": "^1.2.2" + }, + "description": "Beat vulnerabilities with more-secure code", + "devDependencies": { + "@types/chai": "4.3.1", + "@types/mocha": "9.1.1", + "@types/node": "^18.0.0", + "@types/vscode": "^1.50.0", + "@typescript-eslint/eslint-plugin": "^5.29.0", + "@typescript-eslint/parser": "^5.29.0", + "chai": "4.3.6", + "eslint": "^8.18.0", + "mocha": "10.0.0", + "typescript": "^4.7.4", + "vsce": "^2.9.2", + "vscode-extension-tester": "4.2.5", + "vscode-extension-tester-locators": "^1.62.2", + "webpack": "^5.73.0", + "webpack-cli": "^4.10.0" + }, + "version": "2.0.4" +} \ No newline at end of file diff --git a/internal/commands/data/manifests/requirements.txt b/internal/commands/data/manifests/requirements.txt new file mode 100644 index 000000000..013fdfd5a --- /dev/null +++ b/internal/commands/data/manifests/requirements.txt @@ -0,0 +1,95 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile +# +contourpy==1.3.1 + # via matplotlib + c==0.12.1 + # via matplotlib + fonttools==4.55.8 + # via matplotlib + kiwisolver==1.4.8 + # via matplotlib + matplotlib==3.10.0 + # via + # -r requirements.in + # seaborn +numpy==2.2.2 + # viaS + # -r requirements.in + # contourpy + # matplotlib + # pandas + # seaborn +packaging==24.2 + # via matplotlib +pandas==2.2.3 + # via + # -r requirements.in + # seaborn +pillow==11.1.0 + # via matplotlib +pyparsing==3.2.1 + # via matplotlib +python-dateutil==2.9.0.post0 + # via + # matplotlib + # pandas +pytz==2025.1 + # via pandas +seaborn==0.13.2 + # via -r requirements.in +six==1.17.0 + # via python-dateutil +tzdata==2025.1 + # via pandas + + + # Sample requirements.txt with various package specifiers + +# Exact version + +flask==1.1.2 + +# Range: greater than or equal and less than + +Django>=3.0,<4.0 + +# Less than or equal + +requests<=2.25.1 + +# Compatible release (PEP 440) + +urllib3\~=1.26.0 + +# Not equal + +numpy!=1.19.0 + +# Wildcard patch version + +pandas==1.2.\* + +# Extras + +package\_with\_extras\[security,docs]==0.1.0 + +# Environment marker (skip on Python>=3.8) + +scipy==1.5.2; python\_version < "3.8" + +# Combined ranges with comma + +celery>=4.0,<5.0 + +# Inline comment + +gevent==21.8.0 # pinned to a known-good version + +# Full-line comment below should be ignored + + + diff --git a/internal/commands/oss-realtime-engine.go b/internal/commands/oss-realtime-engine.go new file mode 100644 index 000000000..ccab105fc --- /dev/null +++ b/internal/commands/oss-realtime-engine.go @@ -0,0 +1,34 @@ +package commands + +import ( + "errors" + + "github.com/checkmarx/ast-cli/internal/commands/util/printer" + commonParams "github.com/checkmarx/ast-cli/internal/params" + "github.com/checkmarx/ast-cli/internal/services/ossrealtime" + "github.com/checkmarx/ast-cli/internal/wrappers" + "github.com/spf13/cobra" +) + +func RunScanOssRealtimeCommand(realtimeScannerWrapper wrappers.RealtimeScannerWrapper, + jwtWrapper wrappers.JWTWrapper, + featureFlagWrapper wrappers.FeatureFlagsWrapper) func(cmd *cobra.Command, args []string) error { + return func(cmd *cobra.Command, _ []string) error { + fileSourceFlag, _ := cmd.Flags().GetString(commonParams.SourcesFlag) + if fileSourceFlag == "" { + return errors.New("file source flag is required") + } + ossRealtimeService := ossrealtime.NewOssRealtimeService(jwtWrapper, featureFlagWrapper, realtimeScannerWrapper) + + packages, err := ossRealtimeService.RunOssRealtimeScan(fileSourceFlag) + if err != nil { + return errors.New("failed to run oss-realtime scan: " + err.Error()) + } + err = printer.Print(cmd.OutOrStdout(), packages, printer.FormatJSON) + if err != nil { + return err + } + + return nil + } +} diff --git a/internal/commands/oss-realtime-engine_test.go b/internal/commands/oss-realtime-engine_test.go new file mode 100644 index 000000000..630fcb577 --- /dev/null +++ b/internal/commands/oss-realtime-engine_test.go @@ -0,0 +1,43 @@ +package commands + +import ( + "testing" + + "github.com/checkmarx/ast-cli/internal/wrappers" + "github.com/checkmarx/ast-cli/internal/wrappers/mock" + "github.com/stretchr/testify/assert" +) + +func TestRunScanOssRealtimeCommand_RequirementsTxtFile_ScanSuccess(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + execCmdNilAssertion( + t, + "scan", "oss-realtime", "-s", "data/manifests/requirements.txt", + ) +} + +func TestRunScanOssRealtimeCommand_EmptyFilePath_ScanFailed(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + err := execCmdNotNilAssertion( + t, + "scan", "oss-realtime", "-s", "", + ) + assert.NotNil(t, err) +} + +func TestRunScanOssRealtimeCommand_PackageJsonFile_ScanSuccess(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + execCmdNilAssertion( + t, + "scan", "oss-realtime", "-s", "data/manifests/package.json", + ) +} + +func TestRunScanOssRealtimeCommand_UnsupportedFileType_ScanFailed(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + err := execCmdNotNilAssertion( + t, + "scan", "oss-realtime", "-s", "not-supported-extension.txt", + ) + assert.NotNil(t, err) +} diff --git a/internal/commands/root.go b/internal/commands/root.go index df74f27db..17124cf43 100644 --- a/internal/commands/root.go +++ b/internal/commands/root.go @@ -55,6 +55,7 @@ func NewAstCLI( accessManagementWrapper wrappers.AccessManagementWrapper, byorWrapper wrappers.ByorWrapper, containerResolverWrapper wrappers.ContainerResolverWrapper, + realTimeWrapper wrappers.RealtimeScannerWrapper, ) *cobra.Command { // Create the root rootCmd := &cobra.Command{ @@ -157,6 +158,7 @@ func NewAstCLI( accessManagementWrapper, featureFlagsWrapper, containerResolverWrapper, + realTimeWrapper, ) projectCmd := NewProjectCommand(applicationsWrapper, projectsWrapper, groupsWrapper, accessManagementWrapper, featureFlagsWrapper) diff --git a/internal/commands/root_test.go b/internal/commands/root_test.go index 7006b0e06..53b49be00 100644 --- a/internal/commands/root_test.go +++ b/internal/commands/root_test.go @@ -68,6 +68,7 @@ func createASTTestCommand() *cobra.Command { byorWrapper := &mock.ByorMockWrapper{} containerResolverMockWrapper := &mock.ContainerResolverMockWrapper{} customStatesMockWrapper := &mock.CustomStatesMockWrapper{} + realTimeWrapper := &mock.RealtimeScannerMockWrapper{} return NewAstCLI( applicationWrapper, scansMockWrapper, @@ -103,6 +104,7 @@ func createASTTestCommand() *cobra.Command { accessManagementWrapper, byorWrapper, containerResolverMockWrapper, + realTimeWrapper, ) } diff --git a/internal/commands/scan.go b/internal/commands/scan.go index 28f006eed..3cc25c9e7 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -159,6 +159,7 @@ func NewScanCommand( accessManagementWrapper wrappers.AccessManagementWrapper, featureFlagsWrapper wrappers.FeatureFlagsWrapper, containerResolverWrapper wrappers.ContainerResolverWrapper, + realtimeScannerWrapper wrappers.RealtimeScannerWrapper, ) *cobra.Command { scanCmd := &cobra.Command{ Use: "scan", @@ -211,6 +212,8 @@ func NewScanCommand( scaRealtimeCmd := scarealtime.NewScaRealtimeCommand(scaRealTimeWrapper) + ossRealtimeCmd := scanOssRealtimeSubCommand(realtimeScannerWrapper, jwtWrapper, featureFlagsWrapper) + addFormatFlagToMultipleCommands( []*cobra.Command{listScansCmd, showScanCmd, workflowScanCmd}, printer.FormatTable, printer.FormatList, printer.FormatJSON, @@ -230,6 +233,7 @@ func NewScanCommand( logsCmd, kicsRealtimeCmd, scaRealtimeCmd, + ossRealtimeCmd, ) return scanCmd } @@ -441,6 +445,36 @@ func scanASCASubCommand(jwtWrapper wrappers.JWTWrapper, featureFlagsWrapper wrap return scanASCACmd } +func scanOssRealtimeSubCommand(realtimeScannerWrapper wrappers.RealtimeScannerWrapper, jwtWrapper wrappers.JWTWrapper, featureFlagsWrapper wrappers.FeatureFlagsWrapper) *cobra.Command { + scanOssRealtimeCmd := &cobra.Command{ + Hidden: true, + Use: "oss-realtime", + Short: "Run a OSS-Realtime scan", + Long: "Running a OSS-Realtime scan is a fast and efficient way to identify malicious packages in a manifest file.", + Example: heredoc.Doc( + ` + $ cx scan oss-realtime -s + `, + ), + Annotations: map[string]string{ + "command:doc": heredoc.Doc( + ` + https://docs.checkmarx.com/en/34965-68625-checkmarx-one-cli-commands.html + `, + ), + }, + RunE: RunScanOssRealtimeCommand(realtimeScannerWrapper, jwtWrapper, featureFlagsWrapper), + } + + scanOssRealtimeCmd.PersistentFlags().StringP( + commonParams.SourcesFlag, + commonParams.SourcesFlagSh, + "", + "The file source should be the path to a single file or multiple files separated by commas", + ) + return scanOssRealtimeCmd +} + func scanListSubCommand(scansWrapper wrappers.ScansWrapper, sastMetadataWrapper wrappers.SastMetadataWrapper) *cobra.Command { listScansCmd := &cobra.Command{ Use: "list", diff --git a/internal/params/binds.go b/internal/params/binds.go index ee0cece70..50ef105d3 100644 --- a/internal/params/binds.go +++ b/internal/params/binds.go @@ -75,4 +75,5 @@ var EnvVarsBinds = []struct { {ScsRepoTokenKey, ScsRepoTokenEnv, ""}, {RiskManagementPathKey, RiskManagementPathEnv, "api/risk-management/projects/%s/results?scanID=%s"}, {ConfigFilePathKey, ConfigFilePathEnv, ""}, + {RealtimeScannerPathKey, RealtimeScannerPathEnv, "api/realtime-scanner"}, } diff --git a/internal/params/envs.go b/internal/params/envs.go index 94bd97b98..0d40abf87 100644 --- a/internal/params/envs.go +++ b/internal/params/envs.go @@ -74,4 +74,5 @@ const ( ScsRepoTokenEnv = "SCS_REPO_TOKEN" RiskManagementPathEnv = "CX_RISK_MANAGEMENT_PATH" ConfigFilePathEnv = "CX_CONFIG_FILE_PATH" + RealtimeScannerPathEnv = "CX_REALTIME_SCANNER_PATH" ) diff --git a/internal/params/keys.go b/internal/params/keys.go index ae8b7fb6c..7dde21449 100644 --- a/internal/params/keys.go +++ b/internal/params/keys.go @@ -74,4 +74,5 @@ var ( ScsRepoTokenKey = strings.ToLower(ScsRepoTokenEnv) RiskManagementPathKey = strings.ToLower(RiskManagementPathEnv) ConfigFilePathKey = strings.ToLower(ConfigFilePathEnv) + RealtimeScannerPathKey = strings.ToLower(RealtimeScannerPathEnv) ) diff --git a/internal/services/ossrealtime/config.go b/internal/services/ossrealtime/config.go new file mode 100644 index 000000000..720bddd48 --- /dev/null +++ b/internal/services/ossrealtime/config.go @@ -0,0 +1,19 @@ +package ossrealtime + +// OssPackage represents a package's details for OSS scanning. +type OssPackage struct { + PackageManager string `json:"PackageManager"` + PackageName string `json:"PackageName"` + PackageVersion string `json:"PackageVersion"` + FilePath string `json:"FilePath"` + LineStart int `json:"LineStart"` + LineEnd int `json:"LineEnd"` + StartIndex int `json:"StartIndex"` + EndIndex int `json:"EndIndex"` + Status string `json:"Status"` +} + +// OssPackageResults holds the results of an OSS scan. +type OssPackageResults struct { + Packages []OssPackage `json:"Packages"` +} diff --git a/internal/services/ossrealtime/oss-realtime.go b/internal/services/ossrealtime/oss-realtime.go new file mode 100644 index 000000000..17b7fcb6a --- /dev/null +++ b/internal/services/ossrealtime/oss-realtime.go @@ -0,0 +1,212 @@ +package ossrealtime + +import ( + "fmt" + "log" + + "github.com/Checkmarx/manifest-parser/pkg/parser" + "github.com/Checkmarx/manifest-parser/pkg/parser/models" + "github.com/checkmarx/ast-cli/internal/services/ossrealtime/osscache" + "github.com/checkmarx/ast-cli/internal/wrappers" + "github.com/pkg/errors" +) + +// OssRealtimeService is the service responsible for performing real-time OSS scanning. +type OssRealtimeService struct { + JwtWrapper wrappers.JWTWrapper + FeatureFlagWrapper wrappers.FeatureFlagsWrapper + RealtimeScannerWrapper wrappers.RealtimeScannerWrapper +} + +// NewOssRealtimeService creates a new OssRealtimeService. +func NewOssRealtimeService( + jwtWrapper wrappers.JWTWrapper, + featureFlagWrapper wrappers.FeatureFlagsWrapper, + realtimeScannerWrapper wrappers.RealtimeScannerWrapper, +) *OssRealtimeService { + return &OssRealtimeService{ + JwtWrapper: jwtWrapper, + FeatureFlagWrapper: featureFlagWrapper, + RealtimeScannerWrapper: realtimeScannerWrapper, + } +} + +// RunOssRealtimeScan performs an OSS real-time scan on the given manifest file. +func (o *OssRealtimeService) RunOssRealtimeScan(filePath string) (*OssPackageResults, error) { + if filePath == "" { + return nil, errors.New("file path is required") + } + + if enabled, err := o.isFeatureFlagEnabled(); err != nil || !enabled { + return nil, err + } + + if err := o.ensureLicense(); err != nil { + return nil, err + } + + pkgs, err := parseManifest(filePath) + if err != nil { + return nil, err + } + + response, toScan := prepareScan(pkgs) + + if len(toScan.Packages) > 0 { + packageMap := createPackageMap(pkgs) + result, err := o.scanAndCache(toScan) + if err != nil { + return nil, errors.Wrap(err, "scanning packages via realtime service") + } + enrichResponseWithRealtimeScannerResults(response, result, packageMap) + } + return response, nil +} + +func enrichResponseWithRealtimeScannerResults( + response *OssPackageResults, + result *wrappers.RealtimeScannerPackageResponse, + packageMap map[string]OssPackage, +) { + for _, pkg := range result.Packages { + entry := getPackageEntryFromPackageMap(packageMap, &pkg) + response.Packages = append(response.Packages, OssPackage{ + PackageManager: pkg.PackageManager, + PackageName: pkg.PackageName, + PackageVersion: pkg.Version, + FilePath: entry.FilePath, + LineStart: entry.LineStart, + LineEnd: entry.LineEnd, + StartIndex: entry.StartIndex, + EndIndex: entry.EndIndex, + Status: pkg.Status, + }) + } +} + +func getPackageEntryFromPackageMap( + packageMap map[string]OssPackage, + pkg *wrappers.RealtimeScannerResults, +) *OssPackage { + var entry OssPackage + if value, found := packageMap[generatePackageMapEntry(pkg.PackageManager, pkg.PackageName, pkg.Version)]; found { + entry = value + } else { + entry = packageMap[generatePackageMapEntry(pkg.PackageManager, pkg.PackageName, "latest")] + } + return &entry +} + +// isFeatureFlagEnabled checks if the OSS Realtime feature flag is enabled. +func (o *OssRealtimeService) isFeatureFlagEnabled() (bool, error) { + enabled, err := o.FeatureFlagWrapper.GetSpecificFlag(wrappers.OssRealtimeEnabled) + if err != nil { + return false, errors.Wrap(err, "failed to get feature flag") + } + return enabled.Status, nil +} + +// ensureLicense validates that a valid JWT wrapper is available. +func (o *OssRealtimeService) ensureLicense() error { + if o.JwtWrapper == nil { + return errors.New("jwt wrapper not provided") + } + return nil +} + +// parseManifest parses the manifest file and returns a list of packages. +func parseManifest(filePath string) ([]models.Package, error) { + manifestParser := parser.ParsersFactory(filePath) + if manifestParser == nil { + return nil, errors.Errorf("no parser available for file: %s", filePath) + } + pkgs, err := manifestParser.Parse(filePath) + if err != nil { + return nil, errors.Wrap(err, "parsing manifest file error") + } + return pkgs, nil +} + +// prepareScan processes the list of packages and separates cached and uncached packages. +func prepareScan(pkgs []models.Package) (*OssPackageResults, *wrappers.RealtimeScannerPackageRequest) { + var resp OssPackageResults + var req wrappers.RealtimeScannerPackageRequest + + resp.Packages = make([]OssPackage, 0, len(pkgs)) + + cache := osscache.ReadCache() + if cache == nil { + for _, pkg := range pkgs { + req.Packages = append(req.Packages, pkgToRequest(&pkg)) + } + return &resp, &req + } + + cacheMap := osscache.BuildCacheMap(*cache) + for _, pkg := range pkgs { + key := osscache.GenerateCacheKey(pkg.PackageManager, pkg.PackageName, pkg.Version) + if status, found := cacheMap[key]; found { + resp.Packages = append(resp.Packages, OssPackage{ + PackageManager: pkg.PackageManager, + PackageName: pkg.PackageName, + PackageVersion: pkg.Version, + LineStart: pkg.LineStart, + LineEnd: pkg.LineEnd, + FilePath: pkg.Filepath, + StartIndex: pkg.StartIndex, + Status: status, + }) + } else { + req.Packages = append(req.Packages, pkgToRequest(&pkg)) + } + } + return &resp, &req +} + +// createPackageMap generates a map of packages for quicker access during scanning. +func createPackageMap(pkgs []models.Package) map[string]OssPackage { + packageMap := make(map[string]OssPackage) + for _, pkg := range pkgs { + packageMap[generatePackageMapEntry(pkg.PackageManager, pkg.PackageName, pkg.Version)] = OssPackage{ + PackageManager: pkg.PackageManager, + PackageName: pkg.PackageName, + PackageVersion: pkg.Version, + FilePath: pkg.Filepath, + LineStart: pkg.LineStart, + LineEnd: pkg.LineEnd, + StartIndex: pkg.StartIndex, + EndIndex: pkg.EndIndex, + } + } + return packageMap +} + +// generatePackageMapEntry generates a unique key for the package map. +func generatePackageMapEntry(pkgManager, pkgName, pkgVersion string) string { + return fmt.Sprintf("%s_%s_%s", pkgManager, pkgName, pkgVersion) +} + +// scanAndCache performs a scan on the provided packages and caches the results. +func (o *OssRealtimeService) scanAndCache(requestPackages *wrappers.RealtimeScannerPackageRequest) (*wrappers.RealtimeScannerPackageResponse, error) { + result, err := o.RealtimeScannerWrapper.Scan(requestPackages) + if err != nil { + return nil, errors.Wrap(err, "scanning packages via realtime service") + } + if len(result.Packages) == 0 { + return nil, errors.New("empty response from oss-realtime scan") + } + + if err = osscache.AppendToCache(result); err != nil { + log.Printf("ossrealtime: failed to update cache: %v", err) + } + return result, nil +} + +// pkgToRequest transforms a parsed package into a scan request. +func pkgToRequest(pkg *models.Package) wrappers.RealtimeScannerPackage { + return wrappers.RealtimeScannerPackage{ + PackageManager: pkg.PackageManager, + PackageName: pkg.PackageName, + Version: pkg.Version, + } +} diff --git a/internal/services/ossrealtime/oss-realtime_test.go b/internal/services/ossrealtime/oss-realtime_test.go new file mode 100644 index 000000000..f5360e127 --- /dev/null +++ b/internal/services/ossrealtime/oss-realtime_test.go @@ -0,0 +1,264 @@ +package ossrealtime + +import ( + "os" + "testing" + "time" + + "github.com/Checkmarx/manifest-parser/pkg/parser/models" + "github.com/checkmarx/ast-cli/internal/services/ossrealtime/osscache" + "github.com/checkmarx/ast-cli/internal/wrappers" + "github.com/checkmarx/ast-cli/internal/wrappers/mock" + "github.com/stretchr/testify/assert" +) + +func setupPackages() []models.Package { + return []models.Package{ + {PackageManager: "npm", PackageName: "lodash", Version: "4.17.21"}, + {PackageManager: "npm", PackageName: "express", Version: "4.17.1"}, + {PackageManager: "npm", PackageName: "axios", Version: "0.21.1"}, + } +} + +func setupSinglePackage() []models.Package { + return []models.Package{ + {PackageManager: "npm", PackageName: "lodash", Version: "4.17.21"}, + } +} + +func setupCache(packages []osscache.PackageEntry, ttl time.Time) osscache.Cache { + return osscache.Cache{TTL: ttl, Packages: packages} +} + +func cleanCacheFile(t *testing.T) { + cacheFile := osscache.GetCacheFilePath() + _ = os.Remove(cacheFile) + t.Cleanup(func() { _ = os.Remove(cacheFile) }) +} + +func TestRunOssRealtimeScan_ValidLicenseAndManifest_ScanSuccess(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + ossRealtimeService := NewOssRealtimeService( + &mock.JWTMockWrapper{}, + &mock.FeatureFlagsMockWrapper{}, + &mock.RealtimeScannerMockWrapper{}, + ) + + const filePath = "../../commands/data/manifests/package.json" + + response, err := ossRealtimeService.RunOssRealtimeScan(filePath) + + assert.Nil(t, err) + assert.NotNil(t, response) + assert.Greater(t, len(response.Packages), 0) +} + +func TestRunOssRealtimeScan_InvalidLicenseAndValidManifest_ScanFail(t *testing.T) { + t.Skip() // Skip this test for now, no license check implemented + ossRealtimeService := NewOssRealtimeService( + &mock.JWTMockWrapper{AIEnabled: mock.AIProtectionDisabled}, + &mock.FeatureFlagsMockWrapper{}, + &mock.RealtimeScannerMockWrapper{}, + ) + + const filePath = "../../commands/data/manifests/package.json" + + response, err := ossRealtimeService.RunOssRealtimeScan(filePath) + + assert.NotNil(t, err) + assert.Nil(t, response) +} + +func TestRunOssRealtimeScan_ValidLicenseAndInvalidManifest_ScanFail(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + ossRealtimeService := NewOssRealtimeService( + &mock.JWTMockWrapper{}, + &mock.FeatureFlagsMockWrapper{}, + &mock.RealtimeScannerMockWrapper{}, + ) + + const filePath = "not-supported-manifest.ruby" + + response, err := ossRealtimeService.RunOssRealtimeScan(filePath) + + assert.NotNil(t, err) + assert.Nil(t, response) +} + +func TestPrepareScan_CacheExistsAndContainsPartialResults_RealtimeScannerRequestIsCalledPartially(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + cleanCacheFile(t) + allPackages := setupPackages() + storedPackages := []osscache.PackageEntry{ + {PackageManager: "npm", PackageName: "lodash", PackageVersion: "4.17.21", Status: "OK"}, + } + storedCache := setupCache(storedPackages, time.Now().Add(time.Hour)) + + _ = osscache.WriteCache(storedCache, &storedCache.TTL) + + resp, toScan := prepareScan(allPackages) + + assert.NotNil(t, resp) + assert.Len(t, resp.Packages, 1) + assert.Equal(t, "lodash", resp.Packages[0].PackageName) + assert.Equal(t, "4.17.21", resp.Packages[0].PackageVersion) + assert.Equal(t, "OK", resp.Packages[0].Status) + + assert.NotNil(t, toScan) + assert.Len(t, toScan.Packages, 2) +} + +func TestPrepareScan_CacheExpiredAndContainsPartialResults_RealtimeScannerRequestIsCalledFully(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + cleanCacheFile(t) + allPackages := setupPackages() + storedPackages := []osscache.PackageEntry{ + {PackageManager: "npm", PackageName: "lodash", PackageVersion: "4.17.21", Status: "OK"}, + } + storedCache := setupCache(storedPackages, time.Now()) + + _ = osscache.WriteCache(storedCache, &storedCache.TTL) + + resp, toScan := prepareScan(allPackages) + + assert.NotNil(t, resp) + assert.Len(t, resp.Packages, 0) + + assert.NotNil(t, toScan) + assert.Len(t, toScan.Packages, 3) +} + +func TestPrepareScan_NoCache_RealtimeScannerRequestIsCalledFully(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + cleanCacheFile(t) + allPackages := setupPackages() + + resp, toScan := prepareScan(allPackages) + + assert.NotNil(t, resp) + assert.Len(t, resp.Packages, 0) + + assert.NotNil(t, toScan) + assert.Len(t, toScan.Packages, 3) +} + +func TestPrepareScan_AllDataInCache_RealtimeScannerRequestIsEmpty(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + cleanCacheFile(t) + singlePackage := setupSinglePackage() + storedPackages := []osscache.PackageEntry{ + {PackageManager: "npm", PackageName: "lodash", PackageVersion: "4.17.21", Status: "OK"}, + } + storedCache := setupCache(storedPackages, time.Now().Add(time.Hour)) + + _ = osscache.WriteCache(storedCache, &storedCache.TTL) + + resp, toScan := prepareScan(singlePackage) + + assert.NotNil(t, resp) + assert.Len(t, resp.Packages, 1) + assert.Equal(t, "lodash", resp.Packages[0].PackageName) + assert.Equal(t, "4.17.21", resp.Packages[0].PackageVersion) + assert.Equal(t, "OK", resp.Packages[0].Status) + + assert.NotNil(t, toScan) + assert.Len(t, toScan.Packages, 0) +} + +func TestScanAndCache_NoCacheAndScanSuccess_CacheUpdated(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + cleanCacheFile(t) + ossRealtimeService := NewOssRealtimeService( + &mock.JWTMockWrapper{}, + &mock.FeatureFlagsMockWrapper{}, + &mock.RealtimeScannerMockWrapper{ + CustomScan: func(packages *wrappers.RealtimeScannerPackageRequest) (*wrappers.RealtimeScannerPackageResponse, error) { + var response wrappers.RealtimeScannerPackageResponse + for _, pkg := range packages.Packages { + response.Packages = append(response.Packages, wrappers.RealtimeScannerResults{ + PackageManager: pkg.PackageManager, + PackageName: pkg.PackageName, + Version: pkg.Version, + Status: "OK", + }) + } + return &response, nil + }, + }, + ) + + pkgs := setupSinglePackage() + + _, toScan := prepareScan(pkgs) + + _, err := ossRealtimeService.scanAndCache(toScan) + assert.Nil(t, err) + + cache := osscache.ReadCache() + assert.NotNil(t, cache) + assert.Len(t, cache.Packages, 1) + assert.Equal(t, "lodash", cache.Packages[0].PackageName) + assert.Equal(t, "4.17.21", cache.Packages[0].PackageVersion) +} + +func TestScanAndCache_CacheExistsAndScanSuccess_CacheUpdated(t *testing.T) { + mock.Flag = wrappers.FeatureFlagResponseModel{Name: wrappers.OssRealtimeEnabled, Status: true} + cleanCacheFile(t) + + ossRealtimeService := NewOssRealtimeService( + &mock.JWTMockWrapper{}, + &mock.FeatureFlagsMockWrapper{}, + &mock.RealtimeScannerMockWrapper{}, + ) + + pkgs := []models.Package{ + {PackageManager: "npm", PackageName: "lodash", Version: "4.17.21"}, + {PackageManager: "npm", PackageName: "express", Version: "4.17.1"}, + } + + storedPackages := []osscache.PackageEntry{ + {PackageManager: "npm", PackageName: "lodash", PackageVersion: "4.17.21", Status: "OK"}, + } + storedCache := setupCache(storedPackages, time.Now().Add(time.Hour)) + + err := osscache.WriteCache(storedCache, &storedCache.TTL) + assert.Nil(t, err) + + _, toScan := prepareScan(pkgs) + + ossRealtimeService.RealtimeScannerWrapper = &mock.RealtimeScannerMockWrapper{ + CustomScan: func(packages *wrappers.RealtimeScannerPackageRequest) (*wrappers.RealtimeScannerPackageResponse, error) { + var response wrappers.RealtimeScannerPackageResponse + for _, pkg := range packages.Packages { + status := "OK" + if pkg.PackageName == "express" { + status = "Malicious" + } + response.Packages = append(response.Packages, wrappers.RealtimeScannerResults{ + PackageManager: pkg.PackageManager, + PackageName: pkg.PackageName, + Version: pkg.Version, + Status: status, + }) + } + return &response, nil + }, + } + + _, err = ossRealtimeService.scanAndCache(toScan) + assert.Nil(t, err) + + cache := osscache.ReadCache() + assert.NotNil(t, cache) + assert.Len(t, cache.Packages, 2) + + assert.Equal(t, "npm", cache.Packages[0].PackageManager) + assert.Equal(t, "lodash", cache.Packages[0].PackageName) + assert.Equal(t, "4.17.21", cache.Packages[0].PackageVersion) + assert.Equal(t, "OK", cache.Packages[0].Status) + + assert.Equal(t, "npm", cache.Packages[1].PackageManager) + assert.Equal(t, "express", cache.Packages[1].PackageName) + assert.Equal(t, "4.17.1", cache.Packages[1].PackageVersion) + assert.Equal(t, "Malicious", cache.Packages[1].Status) +} diff --git a/internal/services/ossrealtime/osscache/oss-realtime-cache.go b/internal/services/ossrealtime/osscache/oss-realtime-cache.go new file mode 100644 index 000000000..ea5974d50 --- /dev/null +++ b/internal/services/ossrealtime/osscache/oss-realtime-cache.go @@ -0,0 +1,100 @@ +package osscache + +import ( + "encoding/json" + "fmt" + "os" + "time" + + "github.com/checkmarx/ast-cli/internal/wrappers" +) + +const ( + cacheFileName = "oss-realtime-cache.json" + ttlHoursNumber = 4 + ttl = ttlHoursNumber * time.Hour +) + +func ReadCache() *Cache { + tempFolder := os.TempDir() + cacheFilePath := fmt.Sprint(tempFolder, "/", cacheFileName) + if _, err := os.Stat(cacheFilePath); os.IsNotExist(err) { + return nil + } + file, err := os.Open(cacheFilePath) + if err != nil { + return nil + } + defer func(file *os.File) { + _ = file.Close() + }(file) + var cache Cache + if err = json.NewDecoder(file).Decode(&cache); err != nil { + return nil + } + if time.Now().After(cache.TTL) { + return nil + } + return &cache +} + +func WriteCache(cache Cache, cacheTTL *time.Time) error { + cacheFilePath := GetCacheFilePath() + file, err := os.Create(cacheFilePath) + if err != nil { + return fmt.Errorf("failed to create osscache file: %w", err) + } + defer func(file *os.File) { + _ = file.Close() + }(file) + if cacheTTL == nil { + cache.TTL = time.Now().Add(ttl) + } else { + cache.TTL = *cacheTTL + } + if err = json.NewEncoder(file).Encode(cache); err != nil { + return fmt.Errorf("failed to encode osscache file: %w", err) + } + return nil +} + +func AppendToCache(packages *wrappers.RealtimeScannerPackageResponse) error { + cache := ReadCache() + if cache == nil { + cache = &Cache{ + TTL: time.Now().Add(ttl), + Packages: make([]PackageEntry, 0), + } + } + + for _, pkg := range packages.Packages { + if pkg.Status != "Unknown" { + cache.Packages = append(cache.Packages, PackageEntry{ + PackageManager: pkg.PackageManager, + PackageName: pkg.PackageName, + PackageVersion: pkg.Version, + Status: pkg.Status, + }) + } + } + return WriteCache(*cache, &cache.TTL) +} + +func GetCacheFilePath() string { + tempFolder := os.TempDir() + return fmt.Sprint(tempFolder, "/", cacheFileName) +} + +// BuildCacheMap creates a lookup map from cache entries. +func BuildCacheMap(cache Cache) map[string]string { + m := make(map[string]string, len(cache.Packages)) + for _, pkg := range cache.Packages { + m[GenerateCacheKey(pkg.PackageManager, pkg.PackageName, pkg.PackageVersion)] = pkg.Status + } + return m +} + +// GenerateCacheKey constructs a unique key for a package. +func GenerateCacheKey(manager, name, version string) string { + return fmt.Sprintf("%s-%s-%s", manager, name, version) +} diff --git a/internal/services/ossrealtime/osscache/oss-realtime-cache_test.go b/internal/services/ossrealtime/osscache/oss-realtime-cache_test.go new file mode 100644 index 000000000..6cdb6aeb5 --- /dev/null +++ b/internal/services/ossrealtime/osscache/oss-realtime-cache_test.go @@ -0,0 +1,150 @@ +package osscache + +import ( + "os" + "reflect" + "testing" + "time" + + "github.com/checkmarx/ast-cli/internal/wrappers" + asserts "github.com/stretchr/testify/assert" + "gotest.tools/assert" +) + +func TestReadCache_Empty(t *testing.T) { + cacheFile := GetCacheFilePath() + // ensure no cache file exists + _ = os.Remove(cacheFile) + defer os.Remove(cacheFile) + + if got := ReadCache(); got != nil { + t.Errorf("ReadCache() = %v; want nil when no file", got) + } +} + +func TestWriteAndReadCache(t *testing.T) { + cacheFile := GetCacheFilePath() + defer os.Remove(cacheFile) + + // prepare a cache object + ttl := time.Now().Add(time.Hour).Truncate(time.Second) + want := Cache{ + TTL: ttl, + Packages: []PackageEntry{ + { + PackageManager: "npm", + PackageName: "lodash", + PackageVersion: "4.17.21", + Status: "OK", + }, + }, + } + + // write it + if err := WriteCache(want, &want.TTL); err != nil { + t.Fatalf("WriteCache() error = %v; want no error", err) + } + + // read it back + got := ReadCache() + if got == nil { + t.Fatal("ReadCache() returned nil; want non-nil") + } + assert.Equal(t, want.Packages[0].PackageName, got.Packages[0].PackageName) + assert.Equal(t, want.Packages[0].PackageVersion, got.Packages[0].PackageVersion) + assert.Equal(t, want.Packages[0].PackageManager, got.Packages[0].PackageManager) + assert.Equal(t, want.Packages[0].Status, got.Packages[0].Status) + asserts.True(t, want.TTL.Equal(got.TTL)) +} + +func TestAppendToCache(t *testing.T) { + cacheFile := GetCacheFilePath() + defer os.Remove(cacheFile) + + // first batch + first := &wrappers.RealtimeScannerPackageResponse{ + Packages: []wrappers.RealtimeScannerResults{ + {PackageManager: "npm", PackageName: "lodash", Version: "4.17.21", Status: "OK"}, + }, + } + if err := AppendToCache(first); err != nil { + t.Fatalf("AppendToCache(first) error = %v; want no error", err) + } + + // second batch + second := &wrappers.RealtimeScannerPackageResponse{ + Packages: []wrappers.RealtimeScannerResults{ + {PackageManager: "npm", PackageName: "express", Version: "4.17.1", Status: "Malicious"}, + }, + } + if err := AppendToCache(second); err != nil { + t.Fatalf("AppendToCache(second) error = %v; want no error", err) + } + + // now read & verify we have both entries + cache := ReadCache() + if cache == nil { + t.Fatal("ReadCache() returned nil; want non-nil") + } + + var got []wrappers.RealtimeScannerResults + for _, e := range cache.Packages { + got = append(got, wrappers.RealtimeScannerResults{ + PackageManager: e.PackageManager, + PackageName: e.PackageName, + Version: e.PackageVersion, + Status: e.Status, + }) + } + want := append([]wrappers.RealtimeScannerResults{}, first.Packages...) + want = append(want, second.Packages...) + if !reflect.DeepEqual(got, want) { + t.Errorf("cached packages = %+v; want %+v", got, want) + } + + if time.Now().After(cache.TTL) { + t.Errorf("cache TTL expired (%v); want TTL in the future", cache.TTL) + } +} + +func Test_buildCacheMap(t *testing.T) { + type args struct { + cache Cache + } + tests := []struct { + name string + args args + want map[string]string + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := BuildCacheMap(tt.args.cache); !reflect.DeepEqual(got, tt.want) { + t.Errorf("buildCacheMap() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_cacheKey(t *testing.T) { + type args struct { + manager string + name string + version string + } + tests := []struct { + name string + args args + want string + }{ + // TODO: Add test cases. + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GenerateCacheKey(tt.args.manager, tt.args.name, tt.args.version); got != tt.want { + t.Errorf("cacheKey() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/internal/services/ossrealtime/osscache/types.go b/internal/services/ossrealtime/osscache/types.go new file mode 100644 index 000000000..6405a35a6 --- /dev/null +++ b/internal/services/ossrealtime/osscache/types.go @@ -0,0 +1,23 @@ +package osscache + +import "time" + +type PackageEntry struct { + PackageManager string `json:"packageManager"` + PackageName string `json:"packageName"` + PackageVersion string `json:"packageVersion"` + Status string `json:"status"` +} + +type Cache struct { + TTL time.Time `json:"ttl"` + Packages []PackageEntry `json:"packages"` +} + +func (c *Cache) GetTTL() time.Time { + return c.TTL +} + +func (c *Cache) SetTTL(t time.Time) { + c.TTL = t +} diff --git a/internal/wrappers/feature-flags.go b/internal/wrappers/feature-flags.go index b584ffcc8..14df8fd61 100644 --- a/internal/wrappers/feature-flags.go +++ b/internal/wrappers/feature-flags.go @@ -16,6 +16,7 @@ const ContainerEngineCLIEnabled = "CONTAINER_ENGINE_CLI_ENABLED" const SCSEngineCLIEnabled = "NEW_2MS_SCORECARD_RESULTS_CLI_ENABLED" const NewScanReportEnabled = "NEW_SAST_SCAN_REPORT_ENABLED" const RiskManagementEnabled = "RISK_MANAGEMENT_IDES_PROJECT_RESULTS_SCORES_API_ENABLED" +const OssRealtimeEnabled = "OSS_REALTIME_ENABLED" const maxRetries = 3 var DefaultFFLoad bool = false diff --git a/internal/wrappers/mock/realtime-scanner-mock.go b/internal/wrappers/mock/realtime-scanner-mock.go new file mode 100644 index 000000000..2b7c63be1 --- /dev/null +++ b/internal/wrappers/mock/realtime-scanner-mock.go @@ -0,0 +1,46 @@ +package mock + +import ( + "crypto/rand" + "math/big" + + "github.com/checkmarx/ast-cli/internal/wrappers" +) + +type RealtimeScannerMockWrapper struct { + CustomScan func(packages *wrappers.RealtimeScannerPackageRequest) (*wrappers.RealtimeScannerPackageResponse, error) +} + +func NewRealtimeScannerMockWrapper() *RealtimeScannerMockWrapper { + return &RealtimeScannerMockWrapper{} +} + +func (r RealtimeScannerMockWrapper) Scan(packages *wrappers.RealtimeScannerPackageRequest) (*wrappers.RealtimeScannerPackageResponse, error) { + if r.CustomScan != nil { + return r.CustomScan(packages) + } + return generateMockResponse(packages), nil +} + +func generateMockResponse(packages *wrappers.RealtimeScannerPackageRequest) *wrappers.RealtimeScannerPackageResponse { + var response wrappers.RealtimeScannerPackageResponse + for _, pkg := range packages.Packages { + response.Packages = append(response.Packages, wrappers.RealtimeScannerResults{ + PackageManager: pkg.PackageManager, + PackageName: pkg.PackageName, + Version: pkg.Version, + Status: getRandomStatus(), + }) + } + return &response +} + +func getRandomStatus() string { + statuses := []string{"OK", "Malicious", "Unknown"} + // Randomly select a status from the list + randomIndex, err := rand.Int(rand.Reader, big.NewInt(int64(len(statuses)))) + if err != nil { + return "OK" // Fallback to "OK" in case of error + } + return statuses[randomIndex.Int64()] +} diff --git a/internal/wrappers/realtime-scanner-http.go b/internal/wrappers/realtime-scanner-http.go new file mode 100644 index 000000000..06c343942 --- /dev/null +++ b/internal/wrappers/realtime-scanner-http.go @@ -0,0 +1,59 @@ +package wrappers + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "time" + + commonParams "github.com/checkmarx/ast-cli/internal/params" + "github.com/pkg/errors" + "github.com/spf13/viper" +) + +type RealtimeScannerHTTPWrapper struct { + path string + jwtWrapper JWTWrapper + featureFlagWrapper FeatureFlagsWrapper +} + +func NewRealtimeScannerHTTPWrapper(path string, jwtWrapper JWTWrapper, featureFlagWrapper FeatureFlagsWrapper) *RealtimeScannerHTTPWrapper { + return &RealtimeScannerHTTPWrapper{ + path: path, + jwtWrapper: jwtWrapper, + featureFlagWrapper: featureFlagWrapper, + } +} + +func (r RealtimeScannerHTTPWrapper) Scan(packages *RealtimeScannerPackageRequest) (*RealtimeScannerPackageResponse, error) { + clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) + jsonBytes, err := json.Marshal(packages) + if err != nil { + return nil, err + } + + fn := func() (*http.Response, error) { + return SendHTTPRequest(http.MethodPost, fmt.Sprint(r.path, "/analyze-manifest"), bytes.NewBuffer(jsonBytes), true, clientTimeout) + } + resp, err := retryHTTPRequest(fn, retryAttempts, retryDelay*time.Millisecond) + if err != nil { + return nil, err + } + defer func() { + if err == nil { + _ = resp.Body.Close() + } + }() + decoder := json.NewDecoder(resp.Body) + switch resp.StatusCode { + case http.StatusBadRequest, http.StatusInternalServerError: + return nil, errors.Errorf("Failed to scan packages, status code: %s", resp.Status) + } + var model RealtimeScannerPackageResponse + err = decoder.Decode(&model) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse scan result") + } + return &model, nil +} diff --git a/internal/wrappers/realtime-scanner.go b/internal/wrappers/realtime-scanner.go new file mode 100644 index 000000000..43b7031d4 --- /dev/null +++ b/internal/wrappers/realtime-scanner.go @@ -0,0 +1,26 @@ +package wrappers + +type RealtimeScannerWrapper interface { + Scan(packages *RealtimeScannerPackageRequest) (*RealtimeScannerPackageResponse, error) +} + +type RealtimeScannerResults struct { + PackageManager string `json:"PackageManager"` + PackageName string `json:"PackageName"` + Version string `json:"PackageVersion"` + Status string `json:"Status,omitempty"` +} + +type RealtimeScannerPackageResponse struct { + Packages []RealtimeScannerResults `json:"Packages"` +} + +type RealtimeScannerPackage struct { + PackageManager string `json:"PackageManager"` + PackageName string `json:"PackageName"` + Version string `json:"PackageVersion"` +} + +type RealtimeScannerPackageRequest struct { + Packages []RealtimeScannerPackage `json:"Packages"` +} diff --git a/test/integration/data/manifests/package.json b/test/integration/data/manifests/package.json new file mode 100644 index 000000000..42bb2401a --- /dev/null +++ b/test/integration/data/manifests/package.json @@ -0,0 +1,27 @@ +{ + "dependencies": { + "@CheckmarxDev/ast-cli-javascript-wrapper": "file:../ast-cli-javascript-wrapper/CheckmarxDev-ast-cli-javascript-wrapper-0.0.54.tgz", + "@checkmarxdev/ast-cli-javascript-wrapper": "0.0.54", + "copyfiles": "200", + "tree-kill": "^1.2.2" + }, + "description": "Beat vulnerabilities with more-secure code", + "devDependencies": { + "@types/chai": "4.3.1", + "@types/mocha": "9.1.1", + "@types/node": "^18.0.0", + "@types/vscode": "^1.50.0", + "@typescript-eslint/eslint-plugin": "^5.29.0", + "@typescript-eslint/parser": "^5.29.0", + "chai": "4.3.6", + "eslint": "^8.18.0", + "mocha": "10.0.0", + "typescript": "^4.7.4", + "vsce": "^2.9.2", + "vscode-extension-tester": "4.2.5", + "vscode-extension-tester-locators": "^1.62.2", + "webpack": "^5.73.0", + "webpack-cli": "^4.10.0" + }, + "version": "2.0.4" +} \ No newline at end of file diff --git a/test/integration/data/manifests/requirements.txt b/test/integration/data/manifests/requirements.txt new file mode 100644 index 000000000..013fdfd5a --- /dev/null +++ b/test/integration/data/manifests/requirements.txt @@ -0,0 +1,95 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile +# +contourpy==1.3.1 + # via matplotlib + c==0.12.1 + # via matplotlib + fonttools==4.55.8 + # via matplotlib + kiwisolver==1.4.8 + # via matplotlib + matplotlib==3.10.0 + # via + # -r requirements.in + # seaborn +numpy==2.2.2 + # viaS + # -r requirements.in + # contourpy + # matplotlib + # pandas + # seaborn +packaging==24.2 + # via matplotlib +pandas==2.2.3 + # via + # -r requirements.in + # seaborn +pillow==11.1.0 + # via matplotlib +pyparsing==3.2.1 + # via matplotlib +python-dateutil==2.9.0.post0 + # via + # matplotlib + # pandas +pytz==2025.1 + # via pandas +seaborn==0.13.2 + # via -r requirements.in +six==1.17.0 + # via python-dateutil +tzdata==2025.1 + # via pandas + + + # Sample requirements.txt with various package specifiers + +# Exact version + +flask==1.1.2 + +# Range: greater than or equal and less than + +Django>=3.0,<4.0 + +# Less than or equal + +requests<=2.25.1 + +# Compatible release (PEP 440) + +urllib3\~=1.26.0 + +# Not equal + +numpy!=1.19.0 + +# Wildcard patch version + +pandas==1.2.\* + +# Extras + +package\_with\_extras\[security,docs]==0.1.0 + +# Environment marker (skip on Python>=3.8) + +scipy==1.5.2; python\_version < "3.8" + +# Combined ranges with comma + +celery>=4.0,<5.0 + +# Inline comment + +gevent==21.8.0 # pinned to a known-good version + +# Full-line comment below should be ignored + + + diff --git a/test/integration/oss-realtime_test.go b/test/integration/oss-realtime_test.go new file mode 100644 index 000000000..c48a705c9 --- /dev/null +++ b/test/integration/oss-realtime_test.go @@ -0,0 +1,39 @@ +//go:build integration + +package integration + +import ( + "os" + "testing" + + "github.com/checkmarx/ast-cli/internal/wrappers/configuration" + "github.com/stretchr/testify/assert" +) + +func TestOssRealtimeScan_RequirementsTxtFile_Success(t *testing.T) { + t.Skip() // Skip this test for now + configuration.LoadConfiguration() + _ = executeCmdNilAssertion(t, "Run OSS Realtime scan", "scan", "oss-realtime", "-s", "data/manifests/requirements.txt") + assert.True(t, validateCacheFileExist()) + defer deleteCacheFile() +} + +func TestOssRealtimeScan_PackageJsonFile_Success(t *testing.T) { + t.Skip() // Skip this test for now + configuration.LoadConfiguration() + _ = executeCmdNilAssertion(t, "Run OSS Realtime scan", "scan", "oss-realtime", "-s", "data/manifests/package.json") + assert.True(t, validateCacheFileExist()) + defer deleteCacheFile() +} + +func validateCacheFileExist() bool { + cacheFilePath := os.TempDir() + "/oss-realtime-cache.json" + if _, err := os.Stat(cacheFilePath); os.IsNotExist(err) { + return false + } + return true +} + +func deleteCacheFile() { + _ = os.Remove(os.TempDir() + "/oss-realtime-cache.json") +} diff --git a/test/integration/util_command.go b/test/integration/util_command.go index e648649cb..41f989c1c 100644 --- a/test/integration/util_command.go +++ b/test/integration/util_command.go @@ -86,6 +86,7 @@ func createASTIntegrationTestCommand(t *testing.T) *cobra.Command { sastIncrementalPath := viper.GetString(params.SastMetadataPathKey) accessManagementPath := viper.GetString(params.AccessManagementPathKey) byorPath := viper.GetString(params.ByorPathKey) + realtimeScannerPath := viper.GetString(params.RealtimeScannerPathKey) scansWrapper := wrappers.NewHTTPScansWrapper(scans) applicationsWrapper := wrappers.NewApplicationsHTTPWrapper(applications) @@ -121,6 +122,7 @@ func createASTIntegrationTestCommand(t *testing.T) *cobra.Command { accessManagementWrapper := wrappers.NewAccessManagementHTTPWrapper(accessManagementPath) ByorWrapper := wrappers.NewByorHTTPWrapper(byorPath) containerResolverWrapper := wrappers.NewContainerResolverWrapper() + realtimeScannerWrapper := wrappers.NewRealtimeScannerHTTPWrapper(realtimeScannerPath, jwtWrapper, featureFlagsWrapper) astCli := commands.NewAstCLI( applicationsWrapper, @@ -157,6 +159,7 @@ func createASTIntegrationTestCommand(t *testing.T) *cobra.Command { accessManagementWrapper, ByorWrapper, containerResolverWrapper, + realtimeScannerWrapper, ) return astCli }