Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
254 changes: 127 additions & 127 deletions .ci/check_new_rules.go
Original file line number Diff line number Diff line change
@@ -1,127 +1,127 @@
// Scripts to check if all the rules that exist in the latest version of "gitleaks" are included in our list of rules (in secret.go file)
package main

import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"regexp"
)

var (
regexGitleaksRules = regexp.MustCompile(`(?m)^[^/\n\r]\s*rules\.([a-zA-Z0-9_]+)\(`)
regex2msRules = regexp.MustCompile(`(?m)^[^/\n\r]\s*(?:// )?{Rule:\s*\*(?:rules\.)?([a-zA-Z0-9_]+)\(\),`)
)

func main() {

latestGitleaksRelease, err := fetchGitleaksLatestRelease()
if err != nil {
fmt.Printf("%s\n", err)
os.Exit(1)
}
fmt.Printf("Latest Gitleaks release: %s\n", latestGitleaksRelease)

gitleaksRules, err := fetchGitleaksRules(latestGitleaksRelease)
if err != nil {
fmt.Printf("%s\n", err)
os.Exit(1)
}

matchesGitleaksRules := regexGitleaksRules.FindAllStringSubmatch(string(gitleaksRules), -1)
if len(matchesGitleaksRules) == 0 {
fmt.Println("No rules found in the latest version of Gitleaks.")
os.Exit(1)
}
fmt.Printf("Total rules in the latest version of Gitleaks: %d\n", len(matchesGitleaksRules))

ourRules, err := fetchOurRules()
if err != nil {
fmt.Printf("%s\n", err)
os.Exit(1)
}
match2msRules := regex2msRules.FindAllStringSubmatch(string(ourRules), -1)
if len(match2msRules) == 0 {
fmt.Println("No rules found in 2ms.")
os.Exit(1)
}
fmt.Printf("Total rules in 2ms: %d\n", len(match2msRules))

map2msRules := make(map[string]bool)
for _, match := range match2msRules {
map2msRules[match[1]] = true
}

missingRulesIn2ms := []string{}
for _, rule := range matchesGitleaksRules {
if _, found := map2msRules[rule[1]]; !found {
missingRulesIn2ms = append(missingRulesIn2ms, rule[1])
}
}

if len(missingRulesIn2ms) > 0 {
fmt.Printf("%d rules exist in the latest version of Gitleaks but missing on 2ms: \n\n", len(missingRulesIn2ms))
for _, rule := range missingRulesIn2ms {
fmt.Printf("%s \n", rule)
}

fmt.Printf("\nLink to Gitleaks main.go file of version: %s:\n", latestGitleaksRelease)
fmt.Println(getGitleaksRulesRawURL(latestGitleaksRelease))

os.Exit(1)
} else {
fmt.Println("No differences found.")
os.Exit(0)
}
}

type Release struct {
TagName string `json:"tag_name"`
}

func fetchGitleaksLatestRelease() (string, error) {
var release Release

response, err := http.Get("https://api.github.com/repos/zricethezav/gitleaks/releases/latest")
if err != nil {
return "", fmt.Errorf("failed to get latest release: %w", err)
}
defer response.Body.Close()

decoder := json.NewDecoder(response.Body)
if err := decoder.Decode(&release); err != nil {
return "", fmt.Errorf("failed to decode latest release JSON: %w", err)
}

return release.TagName, nil
}

func fetchGitleaksRules(version string) ([]byte, error) {
rawURLGitleaksRules := getGitleaksRulesRawURL(version)
response, err := http.Get(rawURLGitleaksRules)
if err != nil {
return nil, fmt.Errorf("failed to fetch remote file: %w", err)
}
defer response.Body.Close()

content, err := io.ReadAll(response.Body)
if err != nil {
return nil, fmt.Errorf("failed to read remote file content: %w", err)
}

return content, nil
}

func getGitleaksRulesRawURL(version string) string {
return fmt.Sprintf("https://raw.githubusercontent.com/zricethezav/gitleaks/%s/cmd/generate/config/main.go", version)
}

func fetchOurRules() ([]byte, error) {
content, err := os.ReadFile("engine/rules/rules.go")
if err != nil {
return nil, fmt.Errorf("failed to read our file content: %w", err)
}
return content, nil
}
// Scripts to check if all the rules that exist in the latest version of "gitleaks" are included in our list of rules (in secret.go file)
package main
import (
"encoding/json"
"fmt"
"io"
"net/http"
"os"
"regexp"
)
var (
regexGitleaksRules = regexp.MustCompile(`(?m)^[^/\n\r]\s*rules\.([a-zA-Z0-9_]+)\(`)
regex2msRules = regexp.MustCompile(`(?m)^[^/\n\r]\s*(?:// )?{Rule:\s*\*(?:rules\.)?([a-zA-Z0-9_]+)\(\),`)
)
func main() {
latestGitleaksRelease, err := fetchGitleaksLatestRelease()
if err != nil {
fmt.Printf("%s\n", err)
os.Exit(1)
}
fmt.Printf("Latest Gitleaks release: %s\n", latestGitleaksRelease)
gitleaksRules, err := fetchGitleaksRules(latestGitleaksRelease)
if err != nil {
fmt.Printf("%s\n", err)
os.Exit(1)
}
matchesGitleaksRules := regexGitleaksRules.FindAllStringSubmatch(string(gitleaksRules), -1)
if len(matchesGitleaksRules) == 0 {
fmt.Println("No rules found in the latest version of Gitleaks.")
os.Exit(1)
}
fmt.Printf("Total rules in the latest version of Gitleaks: %d\n", len(matchesGitleaksRules))
ourRules, err := fetchOurRules()
if err != nil {
fmt.Printf("%s\n", err)
os.Exit(1)
}
match2msRules := regex2msRules.FindAllStringSubmatch(string(ourRules), -1)
if len(match2msRules) == 0 {
fmt.Println("No rules found in 2ms.")
os.Exit(1)
}
fmt.Printf("Total rules in 2ms: %d\n", len(match2msRules))
map2msRules := make(map[string]bool)
for _, match := range match2msRules {
map2msRules[match[1]] = true
}
missingRulesIn2ms := []string{}
for _, rule := range matchesGitleaksRules {
if _, found := map2msRules[rule[1]]; !found {
missingRulesIn2ms = append(missingRulesIn2ms, rule[1])
}
}
if len(missingRulesIn2ms) > 0 {
fmt.Printf("%d rules exist in the latest version of Gitleaks but missing on 2ms: \n\n", len(missingRulesIn2ms))
for _, rule := range missingRulesIn2ms {
fmt.Printf("%s \n", rule)
}
fmt.Printf("\nLink to Gitleaks main.go file of version: %s:\n", latestGitleaksRelease)
fmt.Println(getGitleaksRulesRawURL(latestGitleaksRelease))
os.Exit(1)
} else {
fmt.Println("No differences found.")
os.Exit(0)
}
}
type Release struct {
TagName string `json:"tag_name"`
}
func fetchGitleaksLatestRelease() (string, error) {
var release Release
response, err := http.Get("https://api.github.com/repos/zricethezav/gitleaks/releases/latest")
if err != nil {
return "", fmt.Errorf("failed to get latest release: %w", err)
}
defer response.Body.Close()
decoder := json.NewDecoder(response.Body)
if err := decoder.Decode(&release); err != nil {
return "", fmt.Errorf("failed to decode latest release JSON: %w", err)
}
return release.TagName, nil
}
func fetchGitleaksRules(version string) ([]byte, error) {
rawURLGitleaksRules := getGitleaksRulesRawURL(version)
response, err := http.Get(rawURLGitleaksRules)
if err != nil {
return nil, fmt.Errorf("failed to fetch remote file: %w", err)
}
defer response.Body.Close()
content, err := io.ReadAll(response.Body)
if err != nil {
return nil, fmt.Errorf("failed to read remote file content: %w", err)
}
return content, nil
}
func getGitleaksRulesRawURL(version string) string {
return fmt.Sprintf("https://raw.githubusercontent.com/zricethezav/gitleaks/%s/cmd/generate/config/main.go", version)
}
func fetchOurRules() ([]byte, error) {
content, err := os.ReadFile("engine/rules/rules.go")
if err != nil {
return nil, fmt.Errorf("failed to read our file content: %w", err)
}
return content, nil
}
60 changes: 30 additions & 30 deletions .ci/update-readme.sh
Original file line number Diff line number Diff line change
@@ -1,30 +1,30 @@
update_readme() {
output_file=$1
placeholder_name=$2
target_file=$3

sed -i "/<!-- $placeholder_name:start -->/,/<!-- $placeholder_name:end -->/{
/<!-- $placeholder_name:start -->/{
p
r $output_file
}
/<!-- $placeholder_name:end -->/!d
}" $target_file
}

# Update the README with the help message
help_message=$(go run .)

echo "" >output.txt
echo '```text' >>output.txt
echo "$help_message" >>output.txt
echo '```' >>output.txt
echo "" >>output.txt
update_readme "output.txt" "command-line" "README.md"
rm output.txt

go run . rules | awk 'BEGIN{FS = " *"}{print "| " $1 " | " $2 " | " $3 " | " $4 " |";}' >output.txt
update_readme "output.txt" "table" "./docs/list-of-rules.md"
rm output.txt

git --no-pager diff README.md ./docs/list-of-rules.md
update_readme() {
output_file=$1
placeholder_name=$2
target_file=$3
sed -i "/<!-- $placeholder_name:start -->/,/<!-- $placeholder_name:end -->/{
/<!-- $placeholder_name:start -->/{
p
r $output_file
}
/<!-- $placeholder_name:end -->/!d
}" $target_file
}
# Update the README with the help message
help_message=$(go run .)
echo "" >output.txt
echo '```text' >>output.txt
echo "$help_message" >>output.txt
echo '```' >>output.txt
echo "" >>output.txt
update_readme "output.txt" "command-line" "README.md"
rm output.txt
go run . rules | awk 'BEGIN{FS = " *"}{print "| " $1 " | " $2 " | " $3 " | " $4 " |";}' >output.txt
update_readme "output.txt" "table" "./docs/list-of-rules.md"
rm output.txt
git --no-pager diff README.md ./docs/list-of-rules.md
2 changes: 1 addition & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -1 +1 @@
* @Checkmarx/2ms-dev
* @Checkmarx/2ms-dev
40 changes: 20 additions & 20 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
<!--
Thanks for contributing to 2ms by offering a pull request.
-->

Closes #

**Proposed Changes**

<!--
Please describe the big picture of your changes here. If it fixes a bug or resolves a feature request, be sure to link to that issue.
-->

**Checklist**

- [ ] I covered my changes with tests.
- [ ] I Updated the documentation that is affected by my changes:
- [ ] Change in the CLI arguments
- [ ] Change in the configuration file

I submit this contribution under the Apache-2.0 license.
<!--
Thanks for contributing to 2ms by offering a pull request.
-->
Closes #
**Proposed Changes**
<!--
Please describe the big picture of your changes here. If it fixes a bug or resolves a feature request, be sure to link to that issue.
-->
**Checklist**
- [ ] I covered my changes with tests.
- [ ] I Updated the documentation that is affected by my changes:
- [ ] Change in the CLI arguments
- [ ] Change in the configuration file
I submit this contribution under the Apache-2.0 license.
Loading
Loading