From 546a81d64e78210468e760682093de29df72b1b2 Mon Sep 17 00:00:00 2001 From: Aurelien Gateau Date: Mon, 24 Feb 2025 13:44:42 +0100 Subject: [PATCH 1/2] chore: make test_multi_content_scan() more readable - Remove VCR decorator: the code uses a VCR `with` block. - Do not wrap the wall test in the VCR `with` block: only wrap the call to `client.multi_content_scan()` - Early return if `expected` is None - Load `expected` into `example_dict` only once --- tests/test_client.py | 40 +++++++++++++++++----------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/tests/test_client.py b/tests/test_client.py index 2d93f8d0..e6a47fa9 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -341,7 +341,6 @@ def test_health_check(client: GGClient): ), ], ) -@my_vcr.use_cassette def test_multi_content_scan( client: GGClient, name: str, @@ -353,28 +352,23 @@ def test_multi_content_scan( with my_vcr.use_cassette(name + ".yaml"): multiscan = client.multi_content_scan(to_scan) - assert multiscan.status_code == 200 - if not isinstance(multiscan, MultiScanResult): - pytest.fail("multiscan is not a MultiScanResult") - return - - assert type(multiscan.to_dict()) == OrderedDict - assert type(multiscan.to_json()) == str - assert type(repr(multiscan)) == str - assert type(str(multiscan)) == str - assert multiscan.has_secrets == has_secrets - assert multiscan.has_policy_breaks == has_policy_breaks - - for i, scan_result in enumerate(multiscan.scan_results): - if expected: - example_dict = json.loads(expected) - assert all( - elem in example_dict[i]["policies"] for elem in scan_result.policies - ) - assert ( - scan_result.policy_break_count - == example_dict[i]["policy_break_count"] - ) + assert multiscan.status_code == 200 + assert isinstance(multiscan, MultiScanResult) + + assert type(multiscan.to_dict()) == OrderedDict + assert type(multiscan.to_json()) == str + assert type(repr(multiscan)) == str + assert type(str(multiscan)) == str + assert multiscan.has_secrets == has_secrets + assert multiscan.has_policy_breaks == has_policy_breaks + + if not expected: + return + + example_dict = json.loads(expected) + for i, scan_result in enumerate(multiscan.scan_results): + assert all(elem in example_dict[i]["policies"] for elem in scan_result.policies) + assert scan_result.policy_break_count == example_dict[i]["policy_break_count"] @patch("pygitguardian.config.DOCUMENT_SIZE_THRESHOLD_BYTES", 20) From a7661eaeec518d3b7c2199c6f0f9a5e9326edb7f Mon Sep 17 00:00:00 2001 From: Aurelien Gateau Date: Mon, 24 Feb 2025 13:52:16 +0100 Subject: [PATCH 2/2] test: fix tests failing when running without cassettes `.env` policy breaks are no longer reported by the API. Remove the code testing this. --- tests/cassettes/filename.yaml | 65 ------------------------- tests/cassettes/filename_secret.yaml | 68 -------------------------- tests/cassettes/with_breaks.yaml | 71 ++++++++++++---------------- tests/test_client.py | 70 ++------------------------- 4 files changed, 34 insertions(+), 240 deletions(-) delete mode 100644 tests/cassettes/filename.yaml delete mode 100644 tests/cassettes/filename_secret.yaml diff --git a/tests/cassettes/filename.yaml b/tests/cassettes/filename.yaml deleted file mode 100644 index 1b319690..00000000 --- a/tests/cassettes/filename.yaml +++ /dev/null @@ -1,65 +0,0 @@ -interactions: - - request: - body: '{"filename": ".env", "document": "normal"}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '42' - Content-Type: - - application/json - User-Agent: - - pygitguardian/1.3.4 (Linux;py3.8.10) - method: POST - uri: https://api.gitguardian.com/v1/scan - response: - body: - string: - '{"policy_break_count":1,"policies":["File extensions","Filenames","Secrets - detection"],"policy_breaks":[{"type":".env","policy":"Filenames","matches":[{"type":"filename","match":".env"}]}]}' - headers: - Access-Control-Expose-Headers: - - X-App-Version - Allow: - - POST, OPTIONS - Connection: - - keep-alive - Content-Length: - - '189' - Content-Type: - - application/json - Date: - - Thu, 30 Jun 2022 10:29:41 GMT - Referrer-Policy: - - strict-origin-when-cross-origin - Server: - - nginx - Set-Cookie: - - AWSALB=r/wiM/hKFEVK3tOj50oZhdhLH4pIuw/f+Q1PflUFnIyr9Z2L7nO9iaTmh6kYkaZ4s/G2i2BwoABBfYqLSAJZkv3RblCGnOVtxGA46Rjj0tnvn5RjartnScrhop+T; - Expires=Thu, 07 Jul 2022 10:29:41 GMT; Path=/ - - AWSALBCORS=r/wiM/hKFEVK3tOj50oZhdhLH4pIuw/f+Q1PflUFnIyr9Z2L7nO9iaTmh6kYkaZ4s/G2i2BwoABBfYqLSAJZkv3RblCGnOVtxGA46Rjj0tnvn5RjartnScrhop+T; - Expires=Thu, 07 Jul 2022 10:29:41 GMT; Path=/; SameSite=None; Secure - Strict-Transport-Security: - - max-age=31536000; includeSubDomains - Vary: - - Cookie - X-App-Version: - - v2.8.1 - X-Content-Type-Options: - - nosniff - - nosniff - X-Frame-Options: - - DENY - - SAMEORIGIN - X-Secrets-Engine-Version: - - 2.70.0 - X-XSS-Protection: - - 1; mode=block - status: - code: 200 - message: OK -version: 1 diff --git a/tests/cassettes/filename_secret.yaml b/tests/cassettes/filename_secret.yaml deleted file mode 100644 index 30aff053..00000000 --- a/tests/cassettes/filename_secret.yaml +++ /dev/null @@ -1,68 +0,0 @@ -interactions: - - request: - body: '{"filename": ".env", "document": "\n import urllib.request\n url - = ''http://jen_barber:correcthorsebatterystaple@cake.gitguardian.com/isreal.json''\n response - = urllib.request.urlopen(url)\n consume(response.read())\"\n"}' - headers: - Accept: - - '*/*' - Accept-Encoding: - - gzip, deflate - Connection: - - keep-alive - Content-Length: - - '231' - Content-Type: - - application/json - User-Agent: - - pygitguardian/1.3.4 (Linux;py3.8.10) - method: POST - uri: https://api.gitguardian.com/v1/scan - response: - body: - string: - '{"policy_break_count":2,"policies":["File extensions","Filenames","Secrets - detection"],"policy_breaks":[{"type":".env","policy":"Filenames","matches":[{"type":"filename","match":".env"}]},{"type":"Basic - Auth String","policy":"Secrets detection","matches":[{"type":"username","match":"jen_barber","index_start":40,"index_end":49,"line_start":2,"line_end":2},{"type":"password","match":"correcthorsebatterystaple","index_start":51,"index_end":75,"line_start":2,"line_end":2},{"type":"host","match":"cake.gitguardian.com","index_start":77,"index_end":96,"line_start":2,"line_end":2}],"validity":"no_checker"}]}' - headers: - Access-Control-Expose-Headers: - - X-App-Version - Allow: - - POST, OPTIONS - Connection: - - keep-alive - Content-Length: - - '607' - Content-Type: - - application/json - Date: - - Thu, 30 Jun 2022 10:29:36 GMT - Referrer-Policy: - - strict-origin-when-cross-origin - Server: - - nginx - Set-Cookie: - - AWSALB=jTRYMFmj073Osi11/9ey7j0QTb8zbBWbCaMGtyNxlrLzX+TeCdGhFSRFu5ulj9Z+jzovENbwssqCxQdxUx4/r/JMJ0ZPr05AybUNvX8qVIvtGfv3vfhsO9Ry0SZy; - Expires=Thu, 07 Jul 2022 10:29:36 GMT; Path=/ - - AWSALBCORS=jTRYMFmj073Osi11/9ey7j0QTb8zbBWbCaMGtyNxlrLzX+TeCdGhFSRFu5ulj9Z+jzovENbwssqCxQdxUx4/r/JMJ0ZPr05AybUNvX8qVIvtGfv3vfhsO9Ry0SZy; - Expires=Thu, 07 Jul 2022 10:29:36 GMT; Path=/; SameSite=None; Secure - Strict-Transport-Security: - - max-age=31536000; includeSubDomains - Vary: - - Cookie - X-App-Version: - - v2.8.1 - X-Content-Type-Options: - - nosniff - - nosniff - X-Frame-Options: - - DENY - - SAMEORIGIN - X-Secrets-Engine-Version: - - 2.70.0 - X-XSS-Protection: - - 1; mode=block - status: - code: 200 - message: OK -version: 1 diff --git a/tests/cassettes/with_breaks.yaml b/tests/cassettes/with_breaks.yaml index 7dc02080..3a9ea696 100644 --- a/tests/cassettes/with_breaks.yaml +++ b/tests/cassettes/with_breaks.yaml @@ -1,9 +1,7 @@ interactions: - request: - body: '[{"filename": ".env", "document": "\n import urllib.request\n url - = ''http://jen_barber:correcthorsebatterystaple@cake.gitguardian.com/isreal.json''\n response - = urllib.request.urlopen(url)\n consume(response.read())\"\n"}, {"document": - "\n import urllib.request\n url = ''http://jen_barber:correcthorsebatterystaple@cake.gitguardian.com/isreal.json''\n response + body: + '[{"document": "\n import urllib.request\n url = ''http://jen_barber:correcthorsebatterystaple@cake.gitguardian.com/isreal.json''\n response = urllib.request.urlopen(url)\n consume(response.read())\"\n"}, {"filename": "normal", "document": "normal"}]' headers: @@ -14,63 +12,54 @@ interactions: Connection: - keep-alive Content-Length: - - '492' + - '259' Content-Type: - application/json User-Agent: - - pygitguardian/1.3.4 (Linux;py3.8.10) + - pygitguardian/1.19.0 (Linux;py3.10.12) method: POST uri: https://api.gitguardian.com/v1/multiscan response: body: string: - '[{"policy_break_count":2,"policies":["File extensions","Filenames","Secrets - detection"],"policy_breaks":[{"type":".env","policy":"Filenames","matches":[{"type":"filename","match":".env"}]},{"type":"Basic - Auth String","policy":"Secrets detection","matches":[{"type":"username","match":"jen_barber","index_start":40,"index_end":49,"line_start":2,"line_end":2},{"type":"password","match":"correcthorsebatterystaple","index_start":51,"index_end":75,"line_start":2,"line_end":2},{"type":"host","match":"cake.gitguardian.com","index_start":77,"index_end":96,"line_start":2,"line_end":2}],"validity":"no_checker"}]},{"policy_break_count":1,"policies":["File - extensions","Filenames","Secrets detection"],"policy_breaks":[{"type":"Basic - Auth String","policy":"Secrets detection","matches":[{"type":"username","match":"jen_barber","index_start":40,"index_end":49,"line_start":2,"line_end":2},{"type":"password","match":"correcthorsebatterystaple","index_start":51,"index_end":75,"line_start":2,"line_end":2},{"type":"host","match":"cake.gitguardian.com","index_start":77,"index_end":96,"line_start":2,"line_end":2}],"validity":"no_checker"}]},{"policy_break_count":0,"policies":["File - extensions","Filenames","Secrets detection"],"policy_breaks":[]}]' + '[{"policy_break_count":1,"policies":["Secrets detection"],"policy_breaks":[{"type":"Basic + Auth String","policy":"Secrets detection","matches":[{"type":"username","match":"jen_barber","index_start":45,"index_end":54,"line_start":3,"line_end":3},{"type":"password","match":"correcthorsebatterystaple","index_start":56,"index_end":80,"line_start":3,"line_end":3},{"type":"host","match":"cake.gitguardian.com","index_start":82,"index_end":101,"line_start":3,"line_end":3}],"is_excluded":false,"exclude_reason":null,"incident_url":"https://dashboard.gitguardian.com/workspace/8/incidents/10114685","known_secret":true,"validity":"no_checker","diff_kind":null}],"is_diff":false},{"policy_break_count":0,"policies":["Secrets + detection"],"policy_breaks":[],"is_diff":false}]' headers: - Access-Control-Expose-Headers: + access-control-expose-headers: - X-App-Version - Allow: + allow: - POST, OPTIONS - Connection: - - keep-alive - Content-Type: + content-length: + - '766' + content-type: - application/json - Date: - - Thu, 30 Jun 2022 10:29:32 GMT - Referrer-Policy: + cross-origin-opener-policy: + - same-origin + date: + - Mon, 24 Feb 2025 12:49:30 GMT + referrer-policy: - strict-origin-when-cross-origin - Server: - - nginx - Set-Cookie: - - AWSALB=0NfNYw3UeyBFpnge2V9Eg/1mGjFw092I9Sz6A9XTSEcEwwY9QOaglYydWG0j9lP08vyA+Lbr2GKW5wEsC4ps3gWBR4xR/t2cWAZs9ce2iFiqaodevlC5KyXGb6kC; - Expires=Thu, 07 Jul 2022 10:29:32 GMT; Path=/ - - AWSALBCORS=0NfNYw3UeyBFpnge2V9Eg/1mGjFw092I9Sz6A9XTSEcEwwY9QOaglYydWG0j9lP08vyA+Lbr2GKW5wEsC4ps3gWBR4xR/t2cWAZs9ce2iFiqaodevlC5KyXGb6kC; - Expires=Thu, 07 Jul 2022 10:29:32 GMT; Path=/; SameSite=None; Secure - Strict-Transport-Security: + server: + - istio-envoy + strict-transport-security: - max-age=31536000; includeSubDomains - Transfer-Encoding: - - chunked - Vary: - - Accept-Encoding + vary: - Cookie - X-App-Version: - - v2.8.1 - X-Content-Type-Options: + x-app-version: + - v2.155.0 + x-content-type-options: - nosniff - nosniff - X-Frame-Options: + x-envoy-upstream-service-time: + - '490' + x-frame-options: - DENY - SAMEORIGIN - X-Secrets-Engine-Version: - - 2.70.0 - X-XSS-Protection: + x-secrets-engine-version: + - 2.132.2 + x-xss-protection: - 1; mode=block - content-length: - - '1240' status: code: 200 message: OK diff --git a/tests/test_client.py b/tests/test_client.py index e6a47fa9..508190c8 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -72,56 +72,6 @@ """ EXAMPLE_RESPONSE = """ [{ - "policy_break_count": 2, - "policies": [ - "Filenames", - "File extensions", - "Secrets detection" - ], - "policy_breaks": [ - { - "type": ".env", - "policy": "Filenames", - "matches": [ - { - "type": "filename", - "match": ".env" - } - ] - }, - { - "type": "Basic Auth String", - "policy": "Secrets detection", - "matches": [ - { - "type": "username", - "match": "jen_barber", - "index_start": 36, - "index_end": 45, - "line_start": 2, - "line_end": 2 - }, - { - "type": "password", - "match": "correcthorsebatterystaple", - "index_start": 47, - "index_end": 71, - "line_start": 2, - "line_end": 2 - }, - { - "type": "host", - "match": "cake.gitguardian.com", - "index_start": 73, - "index_end": 92, - "line_start": 2, - "line_end": 2 - } - ] - } - ] -}, -{ "policy_break_count": 1, "policies": [ "Filenames", @@ -322,7 +272,6 @@ def test_health_check(client: GGClient): pytest.param( "with_breaks", [ - {"filename": FILENAME, "document": DOCUMENT}, {"document": DOCUMENT}, {"filename": "normal", "document": "normal"}, ], @@ -432,15 +381,12 @@ def test_content_not_ok(): "name, to_scan, policy_break_count, has_secrets, has_policy_breaks", [ pytest.param( - "filename_secret", - {"filename": FILENAME, "document": DOCUMENT}, - 2, + "secret", + {"document": DOCUMENT}, + 1, True, True, - id="filename_secret", - ), - pytest.param( - "secret", {"document": DOCUMENT}, 1, True, True, id="secret (deprecated)" + id="secret (deprecated)", ), pytest.param( "secret_validity", @@ -458,14 +404,6 @@ def test_content_not_ok(): False, id="Document containing a 0 byte", ), - pytest.param( - "filename", - {"filename": FILENAME, "document": "normal"}, - 1, - False, - True, - id="filename", - ), pytest.param( "no_breaks", {"filename": "normal", "document": "normal"},