Skip to content

Commit 4d2aeb6

Browse files
jordanpadamsjordanpadams
authored
Merge pull request #37 from NASA-PDS/software-issues-repo#55
Add secrets detection
2 parents d17a81a + a92c74c commit 4d2aeb6

File tree

4 files changed

+266
-3
lines changed

4 files changed

+266
-3
lines changed

‎.github/workflows/secrets-detection.yaml‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
name: Secret Detection Workflow
2+
on:
3+
push:
4+
branches:
5+
- main
6+
pull_request:
7+
branches:
8+
- main
9+
10+
jobs:
11+
secret-detection:
12+
runs-on: ubuntu-latest
13+
steps:
14+
-
15+
name: Checkout code
16+
uses: actions/checkout@v4
17+
-
18+
name: Install necessary packages
19+
run: |
20+
pip install git+https://github.com/NASA-AMMOS/slim-detect-secrets.git@exp
21+
pip install jq
22+
23+
-
24+
name: Create an initial .secrets.baseline if .secrets.baseline does not exist
25+
run: |
26+
if [ ! -f .secrets.baseline ]; then
27+
# This generated baseline file will only be temporarily available on the GitHub side and will not appear in the user's local files.
28+
# Scanning an empty folder to generate an initial .secrets.baseline without secrets in the results.
29+
echo "⚠️ No existing .secrets.baseline file detected. Creating a new blank baseline file."
30+
mkdir empty-dir
31+
detect-secrets scan empty-dir > .secrets.baseline
32+
echo "✅ Blank .secrets.baseline file created successfully."
33+
rm -r empty-dir
34+
else
35+
echo "✅ Existing .secrets.baseline file detected. No new baseline file will be created."
36+
fi
37+
38+
-
39+
name: Scan repository for secrets
40+
run: |
41+
# scripts to scan repository for new secrets
42+
43+
# backup the list of known secrets
44+
cp .secrets.baseline .secrets.new
45+
46+
# find the secrets in the repository
47+
detect-secrets scan --disable-plugin AbsolutePathDetectorExperimental --baseline .secrets.new \
48+
--exclude-files '\.secrets..*' \
49+
--exclude-files '\.git.*' \
50+
--exclude-files '\.pre-commit-config\.yaml' \
51+
--exclude-files '\.mypy_cache' \
52+
--exclude-files '\.pytest_cache' \
53+
--exclude-files '\.tox' \
54+
--exclude-files '\.venv' \
55+
--exclude-files 'venv' \
56+
--exclude-files 'dist' \
57+
--exclude-files 'build' \
58+
--exclude-files '.*\.egg-info'
59+
60+
# if there is any difference between the known and newly detected secrets, break the build
61+
# Function to compare secrets without listing them
62+
compare_secrets() { diff <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "$1" | sort) <(jq -r '.results | keys[] as $key | "\($key),\(.[$key] | .[] | .hashed_secret)"' "$2" | sort) >/dev/null; }
63+
64+
# Check if there's any difference between the known and newly detected secrets
65+
if ! compare_secrets .secrets.baseline .secrets.new; then
66+
echo "⚠️ Attention Required! ⚠️" >&2
67+
echo "New secrets have been detected in your recent commit. Due to security concerns, we cannot display detailed information here and we cannot proceed until this issue is resolved." >&2
68+
echo "" >&2
69+
echo "Please follow the steps below on your local machine to reveal and handle the secrets:" >&2
70+
echo "" >&2
71+
echo "1️⃣ Run the 'detect-secrets' tool on your local machine. This tool will identify and clean up the secrets. You can find detailed instructions at this link: https://nasa-ammos.github.io/slim/continuous-testing/starter-kits/#detect-secrets" >&2
72+
echo "" >&2
73+
echo "2️⃣ After cleaning up the secrets, commit your changes and re-push your update to the repository." >&2
74+
echo "" >&2
75+
echo "Your efforts to maintain the security of our codebase are greatly appreciated!" >&2
76+
exit 1
77+
fi

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,3 +71,4 @@ bin/
7171
!.gitattributes
7272
!.gitignore
7373
!.gitkeep
74+
.venv

‎.pre-commit-config.yaml‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,13 @@ repos:
2828
- id: black
2929
files: ^src/|tests/
3030

31-
- repo: https://gitlab.com/pycqa/flake8
32-
rev: 3.9.2
31+
- repo: local
3332
hooks:
3433
- id: flake8
35-
files: ^src/|tests/
34+
name: flake8
35+
entry: flake8 src
36+
language: system
37+
pass_filenames: false
3638

3739
- repo: local
3840
hooks:
@@ -42,3 +44,23 @@ repos:
4244
language: system
4345
stages: [push]
4446
pass_filenames: false
47+
48+
- repo: https://github.com/NASA-AMMOS/slim-detect-secrets
49+
# using commit id for now, will change to tag when official version is released
50+
rev: 91e097ad4559ae6ab785c883dc5ed989202c7fbe
51+
hooks:
52+
- id: detect-secrets
53+
args:
54+
- '--baseline'
55+
- '.secrets.baseline'
56+
- --exclude-files '\.secrets..*'
57+
- --exclude-files '\.git.*'
58+
- --exclude-files '\.pre-commit-config\.yaml'
59+
- --exclude-files '\.mypy_cache'
60+
- --exclude-files '\.pytest_cache'
61+
- --exclude-files '\.tox'
62+
- --exclude-files '\.venv'
63+
- --exclude-files 'venv'
64+
- --exclude-files 'dist'
65+
- --exclude-files 'build'
66+
- --exclude-files '.*\.egg-info'

‎.secrets.baseline‎

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
{
2+
"version": "1.4.0",
3+
"plugins_used": [
4+
{
5+
"name": "ArtifactoryDetector"
6+
},
7+
{
8+
"name": "AWSKeyDetector"
9+
},
10+
{
11+
"name": "AWSSensitiveInfoDetectorExperimental"
12+
},
13+
{
14+
"name": "AzureStorageKeyDetector"
15+
},
16+
{
17+
"name": "Base64HighEntropyString",
18+
"limit": 4.5
19+
},
20+
{
21+
"name": "BasicAuthDetector"
22+
},
23+
{
24+
"name": "CloudantDetector"
25+
},
26+
{
27+
"name": "DiscordBotTokenDetector"
28+
},
29+
{
30+
"name": "EmailAddressDetector"
31+
},
32+
{
33+
"name": "GitHubTokenDetector"
34+
},
35+
{
36+
"name": "HexHighEntropyString",
37+
"limit": 3.0
38+
},
39+
{
40+
"name": "IbmCloudIamDetector"
41+
},
42+
{
43+
"name": "IbmCosHmacDetector"
44+
},
45+
{
46+
"name": "IPPublicDetector"
47+
},
48+
{
49+
"name": "JwtTokenDetector"
50+
},
51+
{
52+
"name": "KeywordDetector",
53+
"keyword_exclude": ""
54+
},
55+
{
56+
"name": "MailchimpDetector"
57+
},
58+
{
59+
"name": "NpmDetector"
60+
},
61+
{
62+
"name": "PrivateKeyDetector"
63+
},
64+
{
65+
"name": "SendGridDetector"
66+
},
67+
{
68+
"name": "SlackDetector"
69+
},
70+
{
71+
"name": "SoftlayerDetector"
72+
},
73+
{
74+
"name": "SquareOAuthDetector"
75+
},
76+
{
77+
"name": "StripeDetector"
78+
},
79+
{
80+
"name": "TwilioKeyDetector"
81+
}
82+
],
83+
"filters_used": [
84+
{
85+
"path": "detect_secrets.filters.allowlist.is_line_allowlisted"
86+
},
87+
{
88+
"path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies",
89+
"min_level": 2
90+
},
91+
{
92+
"path": "detect_secrets.filters.heuristic.is_indirect_reference"
93+
},
94+
{
95+
"path": "detect_secrets.filters.heuristic.is_likely_id_string"
96+
},
97+
{
98+
"path": "detect_secrets.filters.heuristic.is_lock_file"
99+
},
100+
{
101+
"path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string"
102+
},
103+
{
104+
"path": "detect_secrets.filters.heuristic.is_potential_uuid"
105+
},
106+
{
107+
"path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign"
108+
},
109+
{
110+
"path": "detect_secrets.filters.heuristic.is_sequential_string"
111+
},
112+
{
113+
"path": "detect_secrets.filters.heuristic.is_swagger_file"
114+
},
115+
{
116+
"path": "detect_secrets.filters.heuristic.is_templated_secret"
117+
},
118+
{
119+
"path": "detect_secrets.filters.regex.should_exclude_file",
120+
"pattern": [
121+
"\\.secrets..*",
122+
"\\.pre-commit-config\\.yaml",
123+
"\\.git.*",
124+
"\\.mypy_cache",
125+
"\\.pytest_cache",
126+
"\\.tox",
127+
"\\.venv",
128+
"venv",
129+
"dist",
130+
"build",
131+
".*\\.egg-info"
132+
]
133+
}
134+
],
135+
"results": {
136+
"data/export.ndjson": [
137+
{
138+
"type": "Hex High Entropy String",
139+
"filename": "data/export.ndjson",
140+
"hashed_secret": "15bf7bbaf9857f33dbeaef3ea59e20dd7515c7df",
141+
"is_verified": false,
142+
"line_number": 3
143+
},
144+
{
145+
"type": "Hex High Entropy String",
146+
"filename": "data/export.ndjson",
147+
"hashed_secret": "c67d18c26d910fc1615b6818adb3ebfc60f466a7",
148+
"is_verified": false,
149+
"line_number": 53
150+
}
151+
],
152+
"setup.cfg": [
153+
{
154+
"type": "Email Address",
155+
"filename": "setup.cfg",
156+
"hashed_secret": "3a6d7aa49a8e4a2fe32a5cd0e53da9cb96bd8d29",
157+
"is_verified": false,
158+
"line_number": 5
159+
}
160+
]
161+
},
162+
"generated_at": "2023-11-17T21:56:18Z"
163+
}

0 commit comments

Comments
 (0)