Allow arbitrarily encoded files to be checked with detect-aws-credentials

Author: asottile

pre_commit_hooks/detect_aws_credentials.py

@@ -69,7 +69,7 @@ def get_aws_secrets_from_file(credentials_file: str) -> Set[str]:
 
 def check_file_for_aws_keys(
         filenames: Sequence[str],
-        keys: Set[str],
+        keys: Set[bytes],
 ) -> List[BadFile]:
     """Check if files contain AWS secrets.
 
@@ -79,13 +79,14 @@ def check_file_for_aws_keys(
     bad_files = []
 
     for filename in filenames:
-        with open(filename, 'r') as content:
+        with open(filename, 'rb') as content:
             text_body = content.read()
             for key in keys:
                 # naively match the entire file, low chance of incorrect
                 # collision
                 if key in text_body:
-                    bad_files.append(BadFile(filename, key[:4].ljust(28, '*')))
+                    key_hidden = key.decode()[:4].ljust(28, '*')
+                    bad_files.append(BadFile(filename, key_hidden))
     return bad_files
 
 
@@ -137,7 +138,8 @@ def main(argv: Optional[Sequence[str]] = None) -> int:
         )
         return 2
 
-    bad_filenames = check_file_for_aws_keys(args.filenames, keys)
+    keys_b = {key.encode() for key in keys}
+    bad_filenames = check_file_for_aws_keys(args.filenames, keys_b)
     if bad_filenames:
         for bad_file in bad_filenames:
             print(f'AWS secret found in {bad_file.filename}: {bad_file.key}')

tests/detect_aws_credentials_test.py

@@ -117,6 +117,19 @@ def test_detect_aws_credentials(filename, expected_retval):
     assert ret == expected_retval
 
 
+def test_allows_arbitrarily_encoded_files(tmpdir):
+    src_ini = tmpdir.join('src.ini')
+    src_ini.write(
+        '[default]\n'
+        'aws_access_key_id=AKIASDFASDF\n'
+        'aws_secret_Access_key=9018asdf23908190238123\n',
+    )
+    arbitrary_encoding = tmpdir.join('f')
+    arbitrary_encoding.write_binary(b'\x12\x9a\xe2\xf2')
+    ret = main((str(arbitrary_encoding), '--credentials-file', str(src_ini)))
+    assert ret == 0
+
+
 @patch('pre_commit_hooks.detect_aws_credentials.get_aws_secrets_from_file')
 @patch('pre_commit_hooks.detect_aws_credentials.get_aws_secrets_from_env')
 def test_non_existent_credentials(mock_secrets_env, mock_secrets_file, capsys):