Skip to content

Commit e5b742d

Browse files
cx-miguel-neivacx-miguel-neiva
committed
Updated Struct and added bucket upload to test
1 parent 61cf3b2 commit e5b742d

File tree

8 files changed

+216
-46
lines changed

8 files changed

+216
-46
lines changed

‎.github/workflows/bucket-upload.yaml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
on:
2+
pull_request:
3+
4+
jobs:
5+
bucket-upload-S3:
6+
runs-on: ubuntu-latest
7+
8+
steps:
9+
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
10+
with:
11+
ref: ${{ github.event.pull_request.head.sha }}
12+
13+
- uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
14+
with:
15+
go-version: "^1.22"
16+
17+
- name: Clone 2ms Repository and Checkout Commit SHA
18+
run: |
19+
git clone https://github.com/checkmarx/2ms.git $GITHUB_WORKSPACE/2ms
20+
cd $GITHUB_WORKSPACE/2ms
21+
git fetch --all
22+
git checkout ${{ github.event.pull_request.head.sha }}
23+
go build -o $GITHUB_WORKSPACE/2ms/dist/2ms main.go
24+
chmod +x $GITHUB_WORKSPACE/2ms/dist/2ms
25+
26+
- name: Load Repos from JSON and Clone Each Repo
27+
run: |
28+
curl -o /tmp/repos.json https://raw.githubusercontent.com/cx-miguel-neiva/2ms-github-action/main/repos.json
29+
REPOS_LIST=$(jq -r '.projects[]' /tmp/repos.json | tr '\n' ' ')
30+
echo "repos=$REPOS_LIST" >> $GITHUB_ENV
31+
for repo_url in $REPOS_LIST; do
32+
repo_name=$(basename "$repo_url" .git)
33+
mkdir -p "$GITHUB_WORKSPACE/repos/$repo_name"
34+
git clone "$repo_url" "$GITHUB_WORKSPACE/repos/$repo_name"
35+
done
36+
37+
- name: Run 2ms Scan for each repo
38+
run: |
39+
mkdir -p $GITHUB_WORKSPACE/results
40+
IFS=' ' read -r -a REPOS_ARRAY <<< "$repos"
41+
touch $GITHUB_WORKSPACE/scan_results.json
42+
echo "[" > $GITHUB_WORKSPACE/scan_results.json
43+
for repo_url in "${REPOS_ARRAY[@]}"; do
44+
repo_name=$(basename "$repo_url" .git)
45+
result_sarif="$GITHUB_WORKSPACE/results/$repo_name.sarif"
46+
start_time=$(date +%s.%N)
47+
if $GITHUB_WORKSPACE/2ms/dist/2ms filesystem --path "$GITHUB_WORKSPACE/repos/$repo_name" --ignore-on-exit results --report-path "$result_sarif"; then
48+
scan_status="success"
49+
else
50+
scan_status="failure"
51+
fi
52+
end_time=$(date +%s.%N)
53+
execution_time=$(echo "$end_time - $start_time" | bc)
54+
execution_time_formatted=$(printf "%.2f" "$execution_time")
55+
echo "{
56+
\"repo_name\": \"$repo_name\",
57+
\"scan_status\": \"$scan_status\",
58+
\"execution_time\": \"$execution_time_formatted\"
59+
}," >> $GITHUB_WORKSPACE/scan_results.json
60+
done
61+
sed -i '$ s/,$//' $GITHUB_WORKSPACE/scan_results.json
62+
echo "]" >> $GITHUB_WORKSPACE/scan_results.json
63+
cp -r $GITHUB_WORKSPACE/results $GITHUB_WORKSPACE/results_backup
64+
65+
- name: Get Results Directory
66+
id: get_results_dir
67+
run: |
68+
echo "results_dir=results" >> $GITHUB_ENV
69+
70+
- name: Get 2ms Version
71+
id: get_twoms_version
72+
run: |
73+
echo "twoms_version=$(curl -s https://api.github.com/repos/checkmarx/2ms/releases/latest | jq -r '.tag_name')" >> $GITHUB_ENV
74+
75+
- name: Set S3 Destination Path
76+
id: set_s3_path
77+
run: |
78+
BRANCH_NAME="${{ github.head_ref || github.ref_name }}"
79+
PR_NUMBER="${{ github.event.number }}"
80+
ENGINE="2ms"
81+
COMMIT_HASH="${{ github.sha }}"
82+
PR_OWNER="${{ github.actor }}"
83+
TARGET_BRANCH="master"
84+
DEST_DIR="${ENGINE}/${TARGET_BRANCH}/${BRANCH_NAME}/${{ env.twoms_version }}/pr-${PR_NUMBER}"
85+
echo "destination_dir=$DEST_DIR" >> $GITHUB_ENV
86+
echo "results_dir=${{ env.results_dir }}" >> $GITHUB_ENV
87+
88+
- name: Organize SARIF files
89+
run: |
90+
mkdir -p "${{ env.results_dir }}/pr-${{ github.event.number }}"
91+
for sarif_file in $GITHUB_WORKSPACE/results/*.sarif; do
92+
if [[ -f "$sarif_file" ]]; then
93+
project_name=$(basename "$sarif_file" .sarif)
94+
mkdir -p "${{ env.results_dir }}/pr-${{ github.event.number }}/$project_name"
95+
mv "$sarif_file" "${{ env.results_dir }}/pr-${{ github.event.number }}/$project_name/results.sarif"
96+
fi
97+
done
98+
99+
- name: Create Metadata File
100+
run: |
101+
COMMIT_TIMESTAMP=$(git log -1 --format=%ct)
102+
METADATA_PATH="${{ env.results_dir }}/pr-${{ github.event.number }}/metadata.json"
103+
echo '{
104+
"seq": "'"${COMMIT_TIMESTAMP}"'",
105+
"tag": "'"${{ github.event.number }}"'",
106+
"comment": "'"${{ github.event.pull_request.title }}"'",
107+
"commit": "'"${{ github.sha }}"'",
108+
"owner": "'"${{ github.actor }}"'",
109+
"branch": "'"${{ github.head_ref || github.ref_name }}"'",
110+
"engine": "2ms",
111+
"version": "'"${{ env.twoms_version }}"'"
112+
}' > "$METADATA_PATH"
113+
114+
- name: Upload results to S3
115+
run: |
116+
aws s3 cp --recursive "${{ env.results_dir }}/pr-${{ github.event.number }}" "s3://${{ secrets.CES_AWS_BUCKET }}/${{ env.destination_dir }}" \
117+
--storage-class STANDARD
118+
env:
119+
AWS_ACCESS_KEY_ID: ${{ secrets.CES_BUCKET_AWS_ACCESS_KEY }}
120+
AWS_SECRET_ACCESS_KEY: ${{ secrets.CES_BUCKET_AWS_SECRET_ACCESS_KEY }}
121+
122+
- name: Get Scan Results for Comment
123+
id: scan_results
124+
run: |
125+
echo "| Repository | Status | Execution Time (seconds) |" > $GITHUB_WORKSPACE/scan_results_table.md
126+
echo "|------------|--------|--------------------------|" >> $GITHUB_WORKSPACE/scan_results_table.md
127+
jq -r '
128+
.[] |
129+
"| \(.repo_name) | " +
130+
(if .scan_status == "success" then "✅" else "❌" end) +
131+
" | \(.execution_time) |"' $GITHUB_WORKSPACE/scan_results.json >> $GITHUB_WORKSPACE/scan_results_table.md
132+
echo "SCAN_RESULTS<<EOF" >> $GITHUB_ENV
133+
cat $GITHUB_WORKSPACE/scan_results_table.md >> $GITHUB_ENV
134+
echo "EOF" >> $GITHUB_ENV
135+
136+
- name: Create PR Comment with Job Summary in Table
137+
uses: peter-evans/create-or-update-comment@v2
138+
with:
139+
issue-number: ${{ github.event.pull_request.number }}
140+
body: |
141+
## 🛠 Scan Summary
142+
143+
144+
${{ env.SCAN_RESULTS }}
145+

‎engine/engine.go‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,15 @@ package engine
33
import (
44
"crypto/sha1"
55
"fmt"
6-
"github.com/checkmarx/2ms/engine/linecontent"
7-
"github.com/checkmarx/2ms/engine/score"
86
"os"
97
"regexp"
108
"strings"
119
"sync"
1210
"text/tabwriter"
1311

12+
"github.com/checkmarx/2ms/engine/linecontent"
13+
"github.com/checkmarx/2ms/engine/score"
14+
1415
"github.com/checkmarx/2ms/engine/rules"
1516
"github.com/checkmarx/2ms/engine/validation"
1617
"github.com/checkmarx/2ms/lib/secrets"
@@ -47,20 +48,23 @@ type EngineConfig struct {
4748

4849
func Init(engineConfig EngineConfig) (*Engine, error) {
4950
selectedRules := rules.FilterRules(engineConfig.SelectedList, engineConfig.IgnoreList, engineConfig.SpecialList)
51+
5052
if len(*selectedRules) == 0 {
5153
return nil, fmt.Errorf("no rules were selected")
5254
}
5355

5456
rulesToBeApplied := make(map[string]config.Rule)
5557
rulesBaseRiskScore := make(map[string]float64)
56-
keywords := []string{}
58+
keywords := map[string]struct{}{}
59+
5760
for _, rule := range *selectedRules {
5861
rulesToBeApplied[rule.Rule.RuleID] = rule.Rule
5962
rulesBaseRiskScore[rule.Rule.RuleID] = score.GetBaseRiskScore(rule.ScoreParameters.Category, rule.ScoreParameters.RuleType)
6063
for _, keyword := range rule.Rule.Keywords {
61-
keywords = append(keywords, strings.ToLower(keyword))
64+
keywords[strings.ToLower(keyword)] = struct{}{}
6265
}
6366
}
67+
6468
cfg.Rules = rulesToBeApplied
6569
cfg.Keywords = keywords
6670

‎engine/rules/authenticated_url.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,10 @@ func AuthenticatedURL() *config.Rule {
1414
Regex: regex,
1515
Keywords: []string{"://"},
1616
SecretGroup: 1,
17-
Allowlist: config.Allowlist{
18-
StopWords: []string{"password", "pass"},
17+
Allowlists: []config.Allowlist{
18+
{
19+
StopWords: []string{"password", "pass"},
20+
},
1921
},
2022
}
2123

‎engine/rules/hardcodedPassword.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,10 @@ func HardcodedPassword() *config.Rule {
3030
},
3131
Entropy: 0,
3232
SecretGroup: 1,
33-
Allowlist: config.Allowlist{
34-
StopWords: rules.DefaultStopWords,
33+
Allowlists: []config.Allowlist{
34+
{
35+
StopWords: rules.DefaultStopWords,
36+
},
3537
},
3638
}
3739

‎engine/rules/privateKey.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
package rules
22

33
import (
4-
"github.com/zricethezav/gitleaks/v8/config"
54
"regexp"
5+
6+
"github.com/zricethezav/gitleaks/v8/config"
67
)
78

89
func PrivateKey() *config.Rule {
@@ -19,9 +20,8 @@ func PrivateKey() *config.Rule {
1920
anything
2021
-----END PRIVATE KEY-----`,
2122
`-----BEGIN RSA PRIVATE KEY-----
22-
abcdefghijklmnopqrstuvwxyz
23-
-----END RSA PRIVATE KEY-----
24-
`,
23+
abcdefghijklmnopqrstuvwxz
24+
-----END RSA PRIVATE KEY-----`,
2525
`-----BEGIN PRIVATE KEY BLOCK-----
2626
anything
2727
-----END PRIVATE KEY BLOCK-----`,

‎engine/rules/rule.go‎

Lines changed: 45 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,9 @@ package rules
33
import (
44
"strings"
55

6-
"github.com/rs/zerolog/log"
6+
"github.com/zricethezav/gitleaks/v8/cmd/generate/config/base"
7+
"github.com/zricethezav/gitleaks/v8/logging"
8+
79
"github.com/zricethezav/gitleaks/v8/config"
810
"github.com/zricethezav/gitleaks/v8/detect"
911
)
@@ -20,37 +22,53 @@ type Rule struct {
2022
}
2123

2224
// Copied from https://github.com/gitleaks/gitleaks/blob/463d24618fa42fc7629dc30c9744ebe36c5df1ab/cmd/generate/config/rules/rule.go
23-
func validate(r config.Rule, truePositives []string, falsePositives []string) *config.Rule {
24-
// normalize keywords like in the config package
25-
var keywords []string
26-
for _, k := range r.Keywords {
27-
keywords = append(keywords, strings.ToLower(k))
28-
}
29-
r.Keywords = keywords
25+
func validate(rule config.Rule, truePositives []string, falsePositives []string) *config.Rule {
26+
r := &rule
27+
d := createSingleRuleDetector(r)
3028

31-
rules := make(map[string]config.Rule)
32-
rules[r.RuleID] = r
33-
d := detect.NewDetector(config.Config{
34-
Rules: rules,
35-
Keywords: keywords,
36-
})
3729
for _, tp := range truePositives {
38-
if len(d.DetectString(tp)) != 1 {
39-
log.Fatal(). // lint:ignore This Fatal happens in a test
40-
Str("rule", r.RuleID).
41-
Str("value", tp).
42-
Str("regex", r.Regex.String()).
43-
Msg("Failed to Validate. True positive was not detected by regex.")
30+
if len(d.DetectString(tp)) < 1 {
31+
logging.Fatal().
32+
Str("rule", r.RuleID).
33+
Str("value", tp).
34+
Str("regex", r.Regex.String()).
35+
Msg("Failed to Validate. True positive was not detected by regex.")
4436
}
4537
}
4638
for _, fp := range falsePositives {
47-
if len(d.DetectString(fp)) != 0 {
48-
log.Fatal(). // lint:ignore This Fatal happens in a test
49-
Str("rule", r.RuleID).
50-
Str("value", fp).
51-
Str("regex", r.Regex.String()).
52-
Msg("Failed to Validate. False positive was detected by regex.")
39+
findings := d.DetectString(fp)
40+
if len(findings) != 0 {
41+
logging.Fatal().
42+
Str("rule", r.RuleID).
43+
Str("value", fp).
44+
Str("regex", r.Regex.String()).
45+
Msg("Failed to Validate. False positive was detected by regex.")
46+
}
47+
}
48+
return r
49+
}
50+
51+
func createSingleRuleDetector(r *config.Rule) *detect.Detector {
52+
// normalize keywords like in the config package
53+
var (
54+
uniqueKeywords = make(map[string]struct{})
55+
keywords []string
56+
)
57+
for _, keyword := range r.Keywords {
58+
k := strings.ToLower(keyword)
59+
if _, ok := uniqueKeywords[k]; ok {
60+
continue
5361
}
62+
keywords = append(keywords, k)
63+
uniqueKeywords[k] = struct{}{}
64+
}
65+
r.Keywords = keywords
66+
67+
rules := map[string]config.Rule{
68+
r.RuleID: *r,
5469
}
55-
return &r
70+
cfg := base.CreateGlobalConfig()
71+
cfg.Rules = rules
72+
cfg.Keywords = uniqueKeywords
73+
return detect.NewDetector(cfg)
5674
}

‎engine/rules/rules.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,6 @@ func getDefaultRules() *[]Rule {
115115
{Rule: *rules.EasyPost(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryShipping, RuleType: 4}},
116116
{Rule: *rules.EasyPostTestAPI(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryShipping, RuleType: 4}},
117117
{Rule: *rules.EtsyAccessToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryECommercePlatform, RuleType: 4}},
118-
{Rule: *rules.Facebook(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategorySocialMedia, RuleType: 4}},
119118
{Rule: *rules.FastlyAPIToken(), Tags: []string{TagApiToken, TagApiKey}, ScoreParameters: ScoreParameters{Category: CategoryCDN, RuleType: 4}},
120119
{Rule: *rules.FinicityClientSecret(), Tags: []string{TagClientSecret}, ScoreParameters: ScoreParameters{Category: CategoryFinancialServices, RuleType: 4}},
121120
{Rule: *rules.FinicityAPIToken(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryFinancialServices, RuleType: 4}},
@@ -141,7 +140,6 @@ func getDefaultRules() *[]Rule {
141140
{Rule: *rules.GrafanaApiKey(), Tags: []string{TagApiKey}, ScoreParameters: ScoreParameters{Category: CategoryApplicationMonitoring, RuleType: 4}},
142141
{Rule: *rules.GrafanaCloudApiToken(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryApplicationMonitoring, RuleType: 4}},
143142
{Rule: *rules.GrafanaServiceAccountToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryApplicationMonitoring, RuleType: 4}},
144-
{Rule: *rules.Hashicorp(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryInfrastructureAsCode, RuleType: 4}},
145143
{Rule: *rules.HashicorpField(), Tags: []string{TagPassword}, ScoreParameters: ScoreParameters{Category: CategoryInfrastructureAsCode, RuleType: 4}},
146144
{Rule: *rules.Heroku(), Tags: []string{TagApiKey}, ScoreParameters: ScoreParameters{Category: CategorySaaS, RuleType: 4}},
147145
{Rule: *rules.HubSpot(), Tags: []string{TagApiToken, TagApiKey}, ScoreParameters: ScoreParameters{Category: CategoryMarketingAutomation, RuleType: 4}},
@@ -239,6 +237,7 @@ func getDefaultRules() *[]Rule {
239237
{Rule: *rules.YandexAccessToken(), Tags: []string{TagAccessToken}, ScoreParameters: ScoreParameters{Category: CategoryCloudPlatform, RuleType: 4}},
240238
{Rule: *rules.ZendeskSecretKey(), Tags: []string{TagSecretKey}, ScoreParameters: ScoreParameters{Category: CategoryCustomerSupport, RuleType: 4}},
241239
{Rule: *AuthenticatedURL(), Tags: []string{TagSensitiveUrl}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
240+
242241
// Rules to update
243242
{Rule: *rules.OnePasswordServiceAccountToken(), Tags: []string{TagSensitiveUrl}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
244243
{Rule: *rules.AzureActiveDirectoryClientSecret(), Tags: []string{TagSensitiveUrl}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
@@ -265,7 +264,7 @@ func getDefaultRules() *[]Rule {
265264
{Rule: *rules.GitlabScimToken(), Tags: []string{TagSensitiveUrl}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
266265
{Rule: *rules.GitlabSessionCookie(), Tags: []string{TagSensitiveUrl}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
267266
{Rule: *rules.HarnessApiKey(), Tags: []string{TagSensitiveUrl}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
268-
{Rule: *rules.HashiCorpTerraform(), Tags: []string{TagSensitiveUrl}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
267+
{Rule: *rules.HashiCorpTerraform(), Tags: []string{TagApiToken}, ScoreParameters: ScoreParameters{Category: CategoryInfrastructureAsCode, RuleType: 4}},
269268
{Rule: *rules.Intra42ClientSecret(), Tags: []string{TagSensitiveUrl}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
270269
{Rule: *rules.KubernetesSecret(), Tags: []string{TagSensitiveUrl}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
271270
{Rule: *rules.Meraki(), Tags: []string{TagSensitiveUrl}, ScoreParameters: ScoreParameters{Category: CategoryGeneralOrUnknown, RuleType: 4}},
@@ -337,6 +336,7 @@ func FilterRules(selectedList, ignoreList, specialList []string) *[]Rule {
337336
}
338337

339338
selectedRules := getDefaultRules()
339+
340340
if len(selectedList) > 0 {
341341
selectedRules = selectRules(selectedRules, selectedList)
342342
}

0 commit comments

Comments
 (0)