Merge branch 'dev' of github.com:dxa4481/truffleHog

Author: dxa4481

README.md

@@ -1,8 +1,8 @@
-# Truffle Hog
+# truffleHog
 Searches through git repositories for secrets, digging deep into commit history and branches. This is effective at finding secrets accidentally committed.
 
 ## NEW
-Trufflehog previously functioned by running entropy checks on git diffs. This functionality still exists, but high signal regex checks have been added, and the ability to surpress entropy checking has also been added.
+truffleHog previously functioned by running entropy checks on git diffs. This functionality still exists, but high signal regex checks have been added, and the ability to surpress entropy checking has also been added.
 
 These features help cut down on noise, and makes the tool easier to shove into a devops pipeline.
 
@@ -36,10 +36,10 @@ Things like subdomain enumeration, s3 bucket detection, and other useful regexes
 
 Feel free to also contribute high signal regexes upstream that you think will benifit the community. Things like Azure keys, Twilio keys, Google Compute keys, are welcome, provided a high signal regex can be constructed.
 
-Trufflehog's base rule set sources from https://github.com/dxa4481/truffleHogRegexes/blob/master/truffleHogRegexes/regexes.json
+trufflehog's base rule set sources from https://github.com/dxa4481/truffleHogRegexes/blob/master/truffleHogRegexes/regexes.json
 
 ## How it works
-This module will go through the entire commit history of each branch, and check each diff from each commit, and check for secrets. This is both by regex and by entropy. For entropy checks, trufflehog will evaluate the shannon entropy for both the base64 char set and hexidecimal char set for every blob of text greater than 20 characters comprised of those character sets in each diff. If at any point a high entropy string >20 characters is detected, it will print to the screen.
+This module will go through the entire commit history of each branch, and check each diff from each commit, and check for secrets. This is both by regex and by entropy. For entropy checks, truffleHog will evaluate the shannon entropy for both the base64 char set and hexidecimal char set for every blob of text greater than 20 characters comprised of those character sets in each diff. If at any point a high entropy string >20 characters is detected, it will print to the screen.
 
 ## Help
 

test_all.py

@@ -1,6 +1,11 @@
 import unittest
 import os
+import sys
+import json
+import io
 from truffleHog import truffleHog
+from mock import patch 
+from mock import MagicMock
 
 
 class TestStringMethods(unittest.TestCase):
@@ -22,5 +27,44 @@ def test_unicode_expection(self):
         except UnicodeEncodeError:
             self.fail("Unicode print error")
 
+    def test_return_correct_commit_hash(self):
+        # Start at commit d15627104d07846ac2914a976e8e347a663bbd9b, which 
+        # is immediately followed by a secret inserting commit:
+        # https://github.com/dxa4481/truffleHog/commit/9ed54617547cfca783e0f81f8dc5c927e3d1e345
+        since_commit = 'd15627104d07846ac2914a976e8e347a663bbd9b'
+        commit_w_secret = '9ed54617547cfca783e0f81f8dc5c927e3d1e345'
+        cross_valdiating_commit_w_secret_comment = 'OH no a secret'
+
+        json_result = ''
+        if sys.version_info >= (3,):
+            tmp_stdout = io.StringIO()
+        else:
+            tmp_stdout = io.BytesIO()
+        bak_stdout = sys.stdout
+
+        # Redirect STDOUT, run scan and re-establish STDOUT
+        sys.stdout = tmp_stdout
+        try:
+            truffleHog.find_strings("https://github.com/dxa4481/truffleHog.git", 
+                since_commit=since_commit, printJson=True, surpress_output=False)
+        finally:
+            sys.stdout = bak_stdout
+
+        json_result_list = tmp_stdout.getvalue().split('\n')
+        results = [json.loads(r) for r in json_result_list if bool(r.strip())]
+        filtered_results = list(filter(lambda r: r['commitHash'] == commit_w_secret, results))
+        self.assertEqual(1, len(filtered_results))
+        self.assertEqual(commit_w_secret, filtered_results[0]['commitHash'])
+        # Additionally, we cross-validate the commit comment matches the expected comment
+        self.assertEqual(cross_valdiating_commit_w_secret_comment, filtered_results[0]['commit'].strip())
+
+    @patch('truffleHog.truffleHog.clone_git_repo')
+    @patch('truffleHog.truffleHog.Repo')
+    def test_branch(self, repo_const_mock, clone_git_repo): 
+        repo = MagicMock()
+        repo_const_mock.return_value = repo
+        truffleHog.find_strings("test_repo", branch="testbranch")
+        repo.remotes.origin.fetch.assert_called_once_with("testbranch")
+
 if __name__ == '__main__':
     unittest.main()

truffleHog/truffleHog.py

@@ -27,12 +27,14 @@ def main():
     parser.add_argument("--entropy", dest="do_entropy", help="Enable entropy checks")
     parser.add_argument("--since_commit", dest="since_commit", help="Only scan from a given commit hash")
     parser.add_argument("--max_depth", dest="max_depth", help="The max commit depth to go back when searching for secrets")
+    parser.add_argument("--branch", dest="branch", help="Name of the branch to be scanned")
     parser.add_argument('git_url', type=str, help='URL for secret searching')
     parser.set_defaults(regex=False)
     parser.set_defaults(rules={})
     parser.set_defaults(max_depth=1000000)
     parser.set_defaults(since_commit=None)
     parser.set_defaults(entropy=True)
+    parser.set_defaults(branch=None)
     args = parser.parse_args()
     rules = {}
     if args.rules:
@@ -48,7 +50,7 @@ def main():
         for regex in rules:
             regexes[regex] = rules[regex]
     do_entropy = str2bool(args.do_entropy)
-    output = find_strings(args.git_url, args.since_commit, args.max_depth, args.output_json, args.do_regex, do_entropy, surpress_output=False)
+    output = find_strings(args.git_url, args.since_commit, args.max_depth, args.output_json, args.do_regex, do_entropy, surpress_output=False, branch=args.branch)
     project_path = output["project_path"]
     shutil.rmtree(project_path, onerror=del_rw)
     if output["foundIssues"]:
@@ -183,7 +185,7 @@ def find_entropy(printableDiff, commit_time, branch_name, prev_commit, blob, com
         entropicDiff['diff'] = blob.diff.decode('utf-8', errors='replace')
         entropicDiff['stringsFound'] = stringsFound
         entropicDiff['printDiff'] = printableDiff
-        entropicDiff['commitHash'] = commitHash
+        entropicDiff['commitHash'] = prev_commit.hexsha
         entropicDiff['reason'] = "High Entropy"
     return entropicDiff
 
@@ -207,7 +209,7 @@ def regex_check(printableDiff, commit_time, branch_name, prev_commit, blob, comm
             foundRegex['stringsFound'] = found_strings
             foundRegex['printDiff'] = found_diff
             foundRegex['reason'] = key
-            foundRegex['commitHash'] = commitHash
+            foundRegex['commitHash'] = prev_commit.hexsha
             regex_matches.append(foundRegex)
     return regex_matches
 
@@ -240,14 +242,19 @@ def handle_results(output, output_dir, foundIssues):
         output["foundIssues"].append(result_path)
     return output
 
-def find_strings(git_url, since_commit=None, max_depth=1000000, printJson=False, do_regex=False, do_entropy=True, surpress_output=True, custom_regexes={}):
+def find_strings(git_url, since_commit=None, max_depth=1000000, printJson=False, do_regex=False, do_entropy=True, surpress_output=True, custom_regexes={}, branch=None):
     output = {"foundIssues": []}
     project_path = clone_git_repo(git_url)
     repo = Repo(project_path)
     already_searched = set()
     output_dir = tempfile.mkdtemp()
 
-    for remote_branch in repo.remotes.origin.fetch():
+    if branch:
+        branches = repo.remotes.origin.fetch(branch)
+    else:
+        branches = repo.remotes.origin.fetch()
+
+    for remote_branch in branches:
         since_commit_reached = False
         branch_name = remote_branch.name
         prev_commit = None
@@ -281,7 +288,16 @@ def find_strings(git_url, since_commit=None, max_depth=1000000, printJson=False,
         output = handle_results(output, output_dir, foundIssues)
     output["project_path"] = project_path
     output["clone_uri"] = git_url
+    output["issues_path"] = output_dir
     return output
 
+def clean_up(output):
+    project_path = output.get("project_path", None)
+    if project_path and os.path.isdir(project_path):
+        shutil.rmtree(output["project_path"])
+    issues_path = output.get("issues_path", None)
+    if issues_path and os.path.isdir(issues_path):
+        shutil.rmtree(output["issues_path"])
+
 if __name__ == "__main__":
     main()