Merge pull request #33 from FireTail-io/hotfix/rate-limit-2

Various hotfixes

Author: rileyfiretail

src/lambda_handler.py

@@ -20,5 +20,5 @@ def handler(event, context):
         "message": "Scan complete",
         "repositories_scanned": list(repositories_scanned),
         "openapi_specs_discovered": openapi_specs_discovered,
-        "scan_duration": scan_duration
+        "scan_duration": scan_duration,
     }

src/local_handler.py

@@ -8,6 +8,13 @@ def handler():
     repositories_scanned, openapi_specs_discovered = scan()
     scan_duration = time.time() - start_time
 
+    if len(repositories_scanned) == 0:
+        logger.warning(
+            f"Scanned 0 repositories. Check your config.yml & access token permissions. "
+            f"Scan took {round(scan_duration, ndigits=3)} second(s)"
+        )
+        return
+
     logger.info(
         f"Scanned {len(repositories_scanned)} repositories: {', '.join(repositories_scanned)}. "
         f"{openapi_specs_discovered} OpenAPI spec(s) discovered. "

src/openapi/validation.py

@@ -4,7 +4,6 @@
 import prance  # type: ignore
 import yaml
 from prance.util.resolver import RESOLVE_INTERNAL  # type: ignore
-from prance.util.url import ResolutionError
 
 
 def resolve_and_validate_openapi_spec(file_contents: str) -> dict | None:
@@ -16,7 +15,7 @@ def resolve_and_validate_openapi_spec(file_contents: str) -> dict | None:
     )
     try:
         parser.parse()
-    except (prance.ValidationError, ResolutionError, AssertionError):
+    except:  # noqa: E722
         # In the future, maybe we can provide some proper details here.
         return None
     return parser.specification
@@ -38,6 +37,6 @@ def parse_resolve_and_validate_openapi_spec(file_path: str, get_file_contents: C
 
     else:
         return None
-    
+
     # If it was a valid JSON/YAML file, we can give it to Prance to load
     return resolve_and_validate_openapi_spec(get_file_contents())

src/scanning.py

@@ -101,7 +101,7 @@ def scan_repository_contents_recursive(
 def scan_repository_contents(
     github_client: GithubClient, repository: GithubRepository
 ) -> tuple[set[str], dict[str, dict]]:
-    repository_languages = list(respect_rate_limit(repository.get_languages, github_client).keys())
+    repository_languages = list(respect_rate_limit(lambda: repository.get_languages(), github_client).keys())
     logger.info(f"{repository.full_name}: Language(s) detected: {', '.join(repository_languages)}")
 
     language_analysers = get_language_analysers(repository_languages)
@@ -204,8 +204,8 @@ def scan_repositories(
 def get_organisations_of_user(github_client: GithubClient) -> set[GithubOrganisation]:
     organisations_to_scan = set()
 
-    for repo in github_client.get_user().get_repos():
-        organisations_to_scan.add(repo)
+    for org in github_client.get_user().get_orgs():
+        organisations_to_scan.add(org)
 
     return organisations_to_scan
 
@@ -250,13 +250,17 @@ def scan_with_config(
 
     # Get all of the repos belonging to users in the config
     for user, user_config in config.users.items():
-        repositories_to_scan.update(respect_rate_limit(get_repositories_of_user(github_client, user, user_config)))
+        repositories_to_scan.update(respect_rate_limit(
+            lambda: get_repositories_of_user(github_client, user, user_config),
+            github_client
+        ))
 
     # Get all of the repos beloning to orgs in the config
     for organisation, organisation_config in config.organisations.items():
-        repositories_to_scan.update(respect_rate_limit(get_repositories_of_organisation(
-            github_client, organisation, organisation_config
-        )))
+        repositories_to_scan.update(respect_rate_limit(
+            lambda: get_repositories_of_organisation(github_client, organisation, organisation_config),
+            github_client
+        ))
 
     # Filter out any repos which have been explicitly excluded
     repositories_to_scan = set(filter(lambda repo: not config.skip_repo(repo), repositories_to_scan))
@@ -287,7 +291,7 @@ def scan_with_config(
         return 0, 0
 
     return (
-        repositories_to_scan,
+        {respect_rate_limit(lambda: repository.full_name, github_client) for repository in repositories_to_scan},
         scan_repositories(github_client, firetail_app_token, firetail_api_url, repositories_to_scan)
     )
 
@@ -297,17 +301,21 @@ def scan_without_config(
 ) -> tuple[set[GithubRepository], int]:
     github_client = github.Github(github_token)
 
-    organisations_to_scan: set[GithubOrganisation] = respect_rate_limit(get_organisations_of_user(github_client))
+    organisations_to_scan: set[GithubOrganisation] = respect_rate_limit(
+        lambda: get_organisations_of_user(github_client),
+        github_client
+    )
 
     repositories_to_scan = set()
     for organisation in organisations_to_scan:
         logger.info(f"{organisation.login}: Getting repositories...")
-        repositories_to_scan.update(respect_rate_limit(get_repositories_of_organisation(
-            github_client, organisation.login, OrgConfig()
-        )))
+        repositories_to_scan.update(respect_rate_limit(
+            lambda: get_repositories_of_organisation(github_client, organisation.login, OrgConfig()),
+            github_client
+        ))
 
     return (
-        repositories_to_scan,
+        {respect_rate_limit(lambda: repository.full_name, github_client) for repository in repositories_to_scan},
         scan_repositories(github_client, firetail_app_token, firetail_api_url, repositories_to_scan)
     )
 
@@ -321,7 +329,7 @@ def scan() -> tuple[set[str], int]:
     for env_var_name, env_var_value in required_env_vars.items():
         if env_var_value in {None, ""}:
             logger.critical(f"{env_var_name} not set in environment. Cannot scan.")
-            return 0, 0
+            return set(), 0
 
     config_dict = None
     try:
@@ -342,4 +350,4 @@ def scan() -> tuple[set[str], int]:
             GITHUB_TOKEN, FIRETAIL_APP_TOKEN, FIRETAIL_API_URL
         )
 
-    return {repository.full_name for repository in repositories_scanned}, openapi_specs_discovered
+    return repositories_scanned, openapi_specs_discovered

src/static_analysis/python/analyse_flask.py

@@ -121,15 +121,12 @@ def analyse_flask(module: ast.Module) -> dict | None:
     # definition under each of the methods, but it's good enough for now.
     return {
         "openapi": "3.0.0",
-        "info": {
-            "title": "Static Analysis - Flask",
-            "version": get_datestamp()
-        },
+        "info": {"title": "Static Analysis - Flask", "version": get_datestamp()},
         "paths": {
             path: {
-                method: {
-                    "responses": {"default": {"description": "Discovered via static analysis"}}
-                } for method in methods
-            } for path, methods in paths.items()
+                method: {"responses": {"default": {"description": "Discovered via static analysis"}}}
+                for method in methods
+            }
+            for path, methods in paths.items()
         },
     }

tests/golang/test_analyse_golang.py

@@ -13,9 +13,7 @@ def test_analyse_net_http_hello_world():
             "openapi": "3.0.0",
             "info": {"title": "Static Analysis - Golang net/http"},
             "paths": {
-                "/hello": {
-                    "responses": {"default": {"description": "Discovered via static analysis"}}
-                },
+                "/hello": {"responses": {"default": {"description": "Discovered via static analysis"}}},
             },
         }
     }

tests/javascript/test_analyse_express.py

@@ -3,8 +3,12 @@
 import yaml
 
 from static_analysis.javascript.analyse_express import (
-    analyse_express, get_app_identifiers, get_express_identifiers,
-    get_paths_and_methods, get_router_identifiers)
+    analyse_express,
+    get_app_identifiers,
+    get_express_identifiers,
+    get_paths_and_methods,
+    get_router_identifiers,
+)
 from static_analysis.javascript.analyse_javascript import JS_PARSER
 
 
@@ -13,7 +17,8 @@ def patch_datetime_now(monkeypatch):
     class PatchedDatetime(datetime.datetime):
         def utcnow():
             return datetime.datetime(2000, 1, 1)
-    monkeypatch.setattr(datetime, 'datetime', PatchedDatetime)
+
+    monkeypatch.setattr(datetime, "datetime", PatchedDatetime)
 
 
 @pytest.mark.parametrize(

tests/javascript/test_analyse_javascript.py

@@ -2,17 +2,16 @@
 import pytest
 import yaml
 
-from static_analysis.javascript.analyse_javascript import (JS_PARSER,
-                                                           analyse_javascript,
-                                                           get_imports)
+from static_analysis.javascript.analyse_javascript import JS_PARSER, analyse_javascript, get_imports
 
 
 @pytest.fixture(autouse=True)
 def patch_datetime_now(monkeypatch):
     class PatchedDatetime(datetime.datetime):
         def utcnow():
             return datetime.datetime(2000, 1, 1)
-    monkeypatch.setattr(datetime, 'datetime', PatchedDatetime)
+
+    monkeypatch.setattr(datetime, "datetime", PatchedDatetime)
 
 
 @pytest.mark.parametrize(

tests/python/test_analyse_python.py

@@ -9,7 +9,8 @@ def patch_datetime_now(monkeypatch):
     class PatchedDatetime(datetime.datetime):
         def utcnow():
             return datetime.datetime(2000, 1, 1)
-    monkeypatch.setattr(datetime, 'datetime', PatchedDatetime)
+
+    monkeypatch.setattr(datetime, "datetime", PatchedDatetime)
 
 
 @pytest.mark.parametrize(
@@ -58,17 +59,8 @@ def test_analyse_flask_hello_world(test_file_contents):
     assert appspecs == {
         "static-analysis:flask:tests/python/example_apps/flask_hello_world.py": {
             "openapi": "3.0.0",
-            "info": {
-                "title": "Static Analysis - Flask",
-                "version": "2000-01-01 00:00:00"
-            },
-            "paths": {
-                "/": {
-                    "get": {
-                        "responses": {"default": {"description": "Discovered via static analysis"}}
-                    }
-                }
-            },
+            "info": {"title": "Static Analysis - Flask", "version": "2000-01-01 00:00:00"},
+            "paths": {"/": {"get": {"responses": {"default": {"description": "Discovered via static analysis"}}}}},
         }
     }
 
@@ -83,36 +75,19 @@ def test_analyse_flask_notes_app():
     assert appspecs == {
         "static-analysis:flask:tests/python/example_apps/flask_notes_app.py": {
             "openapi": "3.0.0",
-            "info": {
-                "title": "Static Analysis - Flask",
-                "version": "2000-01-01 00:00:00"
-                },
+            "info": {"title": "Static Analysis - Flask", "version": "2000-01-01 00:00:00"},
             "paths": {
-                "/": {
-                    "get": {
-                        "responses": {"default": {"description": "Discovered via static analysis"}}
-                    }
-                },
+                "/": {"get": {"responses": {"default": {"description": "Discovered via static analysis"}}}},
                 "/new": {
-                    "get": {
-                        "responses": {"default": {"description": "Discovered via static analysis"}}
-                    },
-                    "post": {
-                        "responses": {"default": {"description": "Discovered via static analysis"}}
-                    }
+                    "get": {"responses": {"default": {"description": "Discovered via static analysis"}}},
+                    "post": {"responses": {"default": {"description": "Discovered via static analysis"}}},
                 },
                 "/edit/<int:note_id>": {
-                    "get": {
-                        "responses": {"default": {"description": "Discovered via static analysis"}}
-                    },
-                    "post": {
-                        "responses": {"default": {"description": "Discovered via static analysis"}}
-                    }
+                    "get": {"responses": {"default": {"description": "Discovered via static analysis"}}},
+                    "post": {"responses": {"default": {"description": "Discovered via static analysis"}}},
                 },
                 "/delete/<int:note_id>": {
-                    "post": {
-                        "responses": {"default": {"description": "Discovered via static analysis"}}
-                    }
+                    "post": {"responses": {"default": {"description": "Discovered via static analysis"}}}
                 },
             },
         }