diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e660fd9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +bin/ diff --git a/hack/regen-answers.sh b/hack/regen-answers.sh index a88b15e..1f49487 100755 --- a/hack/regen-answers.sh +++ b/hack/regen-answers.sh @@ -4,48 +4,22 @@ set -eux -o pipefail # Enter repo root cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../" -ANSWERS_FILE_PATH="${ANSWERS_FILE_PATH:-"data/answers.json"}" -TEST_IMAGE="${TEST_IMAGE:-"ghcr.io/chainguard-images/scanner-test"}" -TAG_SUFFIX="${TAG_SUFFIX:-"-wolfi"}" - -# Without this wolfictl with get us rate limited +# Check for required environment variables if [[ "${GITHUB_TOKEN:-}" == "" ]]; then - echo "Must set GITHUB_TOKEN or else you gon get rate limited. Exiting." - exit 1 + echo "Must set GITHUB_TOKEN or else you gon get rate limited. Exiting." + exit 1 fi -RESULTS_DIR="$(mktemp -d)" - # Avoid any auth issues accessing images (assume they are public) -export DOCKER_CONFIG="$RESULTS_DIR/docker-unauthed" +TEMP_DIR="$(mktemp -d)" +export DOCKER_CONFIG="$TEMP_DIR/docker-unauthed" mkdir "$DOCKER_CONFIG" -for testcase in $(jq -r .testCases[].testCaseName $ANSWERS_FILE_PATH | sort -u); do - for id in $(grype -q "${TEST_IMAGE}:${testcase}${TAG_SUFFIX}" -o json | jq -r .matches[].vulnerability.id | sort -u); do - # Note: the wolfictl output should be machine readable, we need this perl/col command to get rid of links in the output - idalias="$(set +o pipefail; wolfictl adv alias find $id | grep ' - ' | awk '{print $2}' | perl -pe 's/\e([^\[\]]|\[.*?[a-zA-Z]|\].*?\a)//g' | col -b | xargs)" - if [[ "$idalias" != "" ]]; then - if [[ $idalias > $id ]]; then - echo "[\"$id\", \"$idalias\"]" | tee -a $RESULTS_DIR/$testcase.out - else - echo "[\"$idalias\", \"$id\"]" | tee -a $RESULTS_DIR/$testcase.out - fi - else - echo "[\"$id\"]" | tee -a $RESULTS_DIR/$testcase.out - fi - done - if [[ -f $RESULTS_DIR/$testcase.out ]]; then - sort -u $RESULTS_DIR/$testcase.out > $RESULTS_DIR/$testcase.out.tmp - mv $RESULTS_DIR/$testcase.out.tmp $RESULTS_DIR/$testcase.out +# Build binary +(cd hack/regen-answers/ && go build -o ../../bin/regen-answers) - # If optionalVulnerabilities is being used, then append there, - # otherwise append to vulnerabilities - if [[ "$(jq -Mcr ".testCases[] | select(.testCaseName == \"$testcase\") | .optionalVulnerabilities" data/answers.json)" != "null" ]]; then - jq ".testCases[] |= if (.testCaseName == \"$testcase\") then (.optionalVulnerabilities = [$(cat $RESULTS_DIR/$testcase.out | tr '\n' ',' | sed 's/.$//' | sed 's/ //g')]) else . end" data/answers.json > "${ANSWERS_FILE_PATH}.tmp" - else - jq ".testCases[] |= if (.testCaseName == \"$testcase\") then (.vulnerabilities = [$(cat $RESULTS_DIR/$testcase.out | tr '\n' ',' | sed 's/.$//' | sed 's/ //g')]) else . end" data/answers.json > "${ANSWERS_FILE_PATH}.tmp" - fi +# Run it +bin/regen-answers - mv "${ANSWERS_FILE_PATH}.tmp" $ANSWERS_FILE_PATH - fi -done +# Cleanup +rm -rf "$TEMP_DIR" diff --git a/hack/regen-answers/go.mod b/hack/regen-answers/go.mod new file mode 100644 index 0000000..40c8e6f --- /dev/null +++ b/hack/regen-answers/go.mod @@ -0,0 +1,3 @@ +module github.com/chainguard-dev/vulnerability-scanner-support/hack/regen-answers + +go 1.24.4 diff --git a/hack/regen-answers/main.go b/hack/regen-answers/main.go new file mode 100644 index 0000000..e07dd81 --- /dev/null +++ b/hack/regen-answers/main.go @@ -0,0 +1,280 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "os/exec" + "sort" + "strings" +) + +type TestCase struct { + TestCaseName string `json:"testCaseName"` + Vulnerabilities [][]string `json:"vulnerabilities"` + OptionalVulnerabilities [][]string `json:"optionalVulnerabilities"` +} + +type AnswersFile struct { + TestCases []TestCase `json:"testCases"` +} + +type GrypeMatch struct { + Vulnerability struct { + ID string `json:"id"` + } `json:"vulnerability"` +} + +type GrypeOutput struct { + Matches []GrypeMatch `json:"matches"` +} + +type OSVEntry struct { + ID string `json:"id"` + Aliases []string `json:"aliases,omitempty"` + Package struct { + Name string `json:"name"` + Ecosystem string `json:"ecosystem"` + } `json:"package"` + DatabaseSpecific map[string]interface{} `json:"database_specific,omitempty"` + Affected []struct { + Package struct { + Name string `json:"name"` + Ecosystem string `json:"ecosystem"` + } `json:"package"` + Ranges []struct { + Type string `json:"type"` + Events []struct { + Fixed string `json:"fixed,omitempty"` + } `json:"events"` + } `json:"ranges"` + DatabaseSpecific map[string]interface{} `json:"database_specific,omitempty"` + } `json:"affected"` +} + +type OSVFeed struct { + Entries []OSVEntry +} + +func main() { + answersFilePath := os.Getenv("ANSWERS_FILE_PATH") + if answersFilePath == "" { + answersFilePath = "data/answers.json" + } + + testImage := os.Getenv("TEST_IMAGE") + if testImage == "" { + testImage = "ghcr.io/chainguard-images/scanner-test" + } + + tagSuffix := os.Getenv("TAG_SUFFIX") + if tagSuffix == "" { + tagSuffix = "-wolfi" + } + + if os.Getenv("GITHUB_TOKEN") == "" { + log.Fatal("Must set GITHUB_TOKEN or else you gon get rate limited. Exiting.") + } + + // Load existing answers file + answersData, err := os.ReadFile(answersFilePath) + if err != nil { + log.Fatalf("Failed to read answers file: %v", err) + } + + var answers AnswersFile + if err := json.Unmarshal(answersData, &answers); err != nil { + log.Fatalf("Failed to parse answers file: %v", err) + } + + // Fetch OSV feed for determining optional vulnerabilities + osvFeed, err := fetchOSVFeed() + if err != nil { + log.Fatalf("Failed to fetch OSV feed: %v", err) + } + + // Process each test case + for i := range answers.TestCases { + testCase := &answers.TestCases[i] + log.Printf("Processing test case: %s", testCase.TestCaseName) + + // Get vulnerabilities from grype + vulns, err := getVulnerabilitiesForTestCase(testCase.TestCaseName, testImage, tagSuffix) + if err != nil { + log.Fatalf("Error processing test case %s: %v", testCase.TestCaseName, err) + } + + // Initially place all vulnerabilities in optionalVulnerabilities + testCase.OptionalVulnerabilities = vulns + testCase.Vulnerabilities = [][]string{} + + // If we have OSV feed data, determine which vulnerabilities are not optional + if osvFeed != nil { + moveNonOptionalVulnerabilities(testCase, osvFeed) + } + } + + // Write updated answers file + updatedData, err := json.MarshalIndent(answers, "", " ") + if err != nil { + log.Fatalf("Failed to marshal updated answers: %v", err) + } + + if err := os.WriteFile(answersFilePath, updatedData, 0644); err != nil { + log.Fatalf("Failed to write updated answers file: %v", err) + } + + log.Printf("Successfully updated %s", answersFilePath) +} + +func getVulnerabilitiesForTestCase(testCaseName, testImage, tagSuffix string) ([][]string, error) { + imageTag := fmt.Sprintf("%s:%s%s", testImage, testCaseName, tagSuffix) + + // Run grype command + cmd := exec.Command("grype", "-q", imageTag, "-o", "json") + output, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("grype command failed: %v", err) + } + + var grypeOutput GrypeOutput + if err := json.Unmarshal(output, &grypeOutput); err != nil { + return nil, fmt.Errorf("failed to parse grype output: %v", err) + } + + // Collect unique vulnerability IDs + vulnMap := make(map[string]bool) + for _, match := range grypeOutput.Matches { + vulnMap[match.Vulnerability.ID] = true + } + + // Get aliases using wolfictl + var vulns [][]string + for id := range vulnMap { + alias, err := getVulnerabilityAlias(id) + if err != nil { + log.Printf("Warning: Failed to get alias for %s: %v", id, err) + continue + } + + if alias != "" { + // Order the pair with the "smaller" ID first + if alias < id { + vulns = append(vulns, []string{alias, id}) + } else { + vulns = append(vulns, []string{id, alias}) + } + } else { + // No alias found, just add the single ID + vulns = append(vulns, []string{id}) + } + } + + // Sort for consistent output + sort.Slice(vulns, func(i, j int) bool { + return vulns[i][0] < vulns[j][0] + }) + + return vulns, nil +} + +func getVulnerabilityAlias(id string) (string, error) { + cmd := exec.Command("wolfictl", "adv", "alias", "find", id) + output, err := cmd.Output() + if err != nil { + return "", err + } + + // Parse wolfictl output to find aliases + lines := strings.Split(string(output), "\n") + for _, line := range lines { + if strings.Contains(line, " - ") { + parts := strings.Fields(line) + if len(parts) >= 2 { + alias := cleanAlias(parts[1]) + if alias != "" && alias != id { + return alias, nil + } + } + } + } + + return "", nil +} + +func cleanAlias(alias string) string { + // Remove ANSI escape sequences and other formatting + // This is a simplified version - in production you might want a more robust solution + alias = strings.TrimSpace(alias) + + // Remove common ANSI escape sequences + for _, seq := range []string{"\x1b[", "\033[", "\u001b["} { + if idx := strings.Index(alias, seq); idx >= 0 { + end := strings.IndexAny(alias[idx:], "mGKHfJ") + if end > 0 { + alias = alias[:idx] + alias[idx+end+1:] + } + } + } + + return strings.TrimSpace(alias) +} + +func fetchOSVFeed() (*OSVFeed, error) { + resp, err := http.Get("https://packages.cgr.dev/chainguard/osv/all.json") + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var entries []OSVEntry + if err := json.Unmarshal(body, &entries); err != nil { + return nil, err + } + + return &OSVFeed{Entries: entries}, nil +} + +func moveNonOptionalVulnerabilities(testCase *TestCase, osvFeed *OSVFeed) { + // Create a map of vulnerability IDs that are in the OSV feed + osvVulns := make(map[string]bool) + for _, entry := range osvFeed.Entries { + osvVulns[entry.ID] = true + for _, alias := range entry.Aliases { + osvVulns[alias] = true + } + } + + stillOptional := [][]string{} + notOptional := [][]string{} + + for _, vuln := range testCase.OptionalVulnerabilities { + isOptional := true + + // Check if any ID in this vulnerability pair is in the OSV feed + for _, id := range vuln { + if osvVulns[id] { + // If it's in the OSV feed with fixed versions, it's not optional + isOptional = false + break + } + } + + if isOptional { + stillOptional = append(stillOptional, vuln) + } else { + notOptional = append(notOptional, vuln) + } + } + + testCase.OptionalVulnerabilities = stillOptional + testCase.Vulnerabilities = notOptional +}