adding functionality to bucket the audit log requests

Author: rreddy2

nodestream_github/client/githubclient.py

@@ -6,7 +6,7 @@
 import json
 import logging
 from collections.abc import AsyncGenerator
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
 from enum import Enum
 from typing import Any
 
@@ -92,28 +92,36 @@ def validate_positive_int(value: int) -> int:
         exception_msg = "Formatting lookback period failed"
         raise ValueError(exception_msg) from e
 
+def generate_date_range(lookback_period: dict[str, int]) -> list[str]:
+    """
+    Generate a list of date strings in YYYY-MM-DD format for
+    the given lookback period.
+    """
+    if not lookback_period:
+        return []
+
+    end_date = datetime.now(tz=UTC).date()
+    start_date = (datetime.now(tz=UTC) - relativedelta(**lookback_period)).date()
+
+    delta_days = (end_date - start_date).days + 1
+    return [
+        (start_date + timedelta(days=i)).strftime("%Y-%m-%d")
+        for i in range(delta_days)
+    ]
 
 def build_search_phrase(
     actions: list[str],
     actors: list[str],
     exclude_actors: list[str],
-    lookback_period: dict[str, int],
+    target_date: str | None = None,
 ) -> str:
     # adding action-based filtering
     actions_phrase = ""
     if actions:
         actions_phrase = " ".join(f"action:{action}" for action in actions)
 
-    # adding lookback_period based filtering
-    date_filter = ""
-    if lookback_period:
-        lookback_period = validate_lookback_period(lookback_period)
-        date_filter = (
-            f"created:>={(datetime.now(tz=UTC) - relativedelta(**lookback_period))
-            .strftime('%Y-%m-%d')}"
-            if lookback_period
-            else ""
-        )
+    # adding date-based filtering for a specific date
+    date_filter = f"created:{target_date}" if target_date else ""
 
     # adding actor-based filtering
     actors_phrase = ""
@@ -412,19 +420,20 @@ async def fetch_enterprise_audit_log(
         https://docs.github.com/en/enterprise-cloud@latest/rest/enterprise-admin/audit-log?apiVersion=2022-11-28#get-the-audit-log-for-an-enterprise
         """
         try:
-            search_phrase = build_search_phrase(
-                actions=actions,
-                actors=actors,
-                exclude_actors=exclude_actors,
-                lookback_period=lookback_period,
-            )
-
-            params = {"phrase": search_phrase} if search_phrase else {}
-
-            async for audit in self._get_paginated(
-                f"enterprises/{enterprise_name}/audit-log", params=params
-            ):
-                yield audit
+            dates = generate_date_range(lookback_period) or [None]
+
+            for target_date in dates:
+                search_phrase = build_search_phrase(
+                    actions=actions,
+                    actors=actors,
+                    exclude_actors=exclude_actors,
+                    target_date=target_date,
+                )
+                params = {"phrase": search_phrase} if search_phrase else {}
+                async for audit in self._get_paginated(
+                    f"enterprises/{enterprise_name}/audit-log", params=params
+                ):
+                    yield audit
         except httpx.HTTPError as e:
             _fetch_problem("audit log", e)
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "nodestream-plugin-github"
-version = "0.14.1-beta.4"
+version = "0.14.1-beta.6"
 description = ""
 authors = [
     "Jon Bristow <[email protected]>",

tests/client/test_githubclient.py

@@ -1,8 +1,15 @@
 import httpx
 import pytest
+from freezegun import freeze_time
 from pytest_httpx import HTTPXMock
 
-from nodestream_github.client.githubclient import GithubRestApiClient, RateLimitedError
+from nodestream_github.client.githubclient import (
+    GithubRestApiClient,
+    RateLimitedError,
+    build_search_phrase,
+    generate_date_range,
+    validate_lookback_period,
+)
 from tests.mocks.githubrest import DEFAULT_BASE_URL, DEFAULT_HOSTNAME
 
 
@@ -127,3 +134,114 @@ async def test_pagination_truncate_warning(
 def test_all_null_args():
     # noinspection PyTypeChecker
     assert GithubRestApiClient(auth_token=None, github_hostname=None)
+
+# Test generate_date_range
+
+def test_generate_date_range_empty_lookback_period():
+    result = generate_date_range({})
+    assert result == []
+
+@freeze_time("2025-08-01")
+def test_generate_date_range_days_only():
+    result = generate_date_range({"days": 3})
+    expected = ["2025-07-29", "2025-07-30", "2025-07-31", "2025-08-01"]
+    assert result == expected
+
+@freeze_time("2025-08-01")
+def test_generate_date_range_zero_days():
+    """Test with zero days (same day only)."""
+    result = generate_date_range({"days": 0})
+    expected = ["2025-08-01"]
+    assert result == expected
+
+@freeze_time("2025-08-01")
+def test_generate_date_range_months_only():
+    result = generate_date_range({"months": 1})
+    # July 1 to August 1 = 32 days
+    assert len(result) == 32
+    assert result[0] == "2025-07-01"
+    assert result[-1] == "2025-08-01"
+
+@freeze_time("2025-08-01")
+def test_generate_date_range_years_only():
+    result = generate_date_range({"years": 1})
+    assert len(result) == 366
+    assert result[0] == "2024-08-01"
+    assert result[-1] == "2025-08-01"
+
+@freeze_time("2025-08-01")
+def test_generate_date_range_combined_periods():
+    """Test with combined periods."""
+    result = generate_date_range({"months": 1, "days": 5})
+    assert result[0] == "2025-06-26"
+    assert result[-1] == "2025-08-01"
+
+@freeze_time("2025-08-01")
+def test_generate_date_range_complex_combination():
+    """Test with years, months, and days."""
+    result = generate_date_range({"years": 1, "months": 2, "days": 10})
+    assert result[0] == "2024-05-22"
+    assert result[-1] == "2025-08-01"
+
+
+# Test validate_lookback_period
+
+def test_validate_lookback_period_valid_input():
+    result = validate_lookback_period({"days": 7, "months": 2, "years": 1})
+    expected = {"days": 7, "months": 2, "years": 1}
+    assert result == expected
+
+
+def test_validate_lookback_period_empty_dict():
+    result = validate_lookback_period({})
+    assert result == {}
+
+
+def test_validate_lookback_period_single_value():
+    result = validate_lookback_period({"days": 30})
+    assert result == {"days": 30}
+
+
+def test_validate_lookback_period_string_to_int_conversion():
+    result = validate_lookback_period({"days": "7", "months": "2"})
+    expected = {"days": 7, "months": 2}
+    assert result == expected
+
+def test_validate_lookback_period_zero_value():
+    with pytest.raises(ValueError, match="Formatting lookback period failed"):
+        validate_lookback_period({"days": 0})
+
+
+def test_validate_lookback_period_negative_value():
+    with pytest.raises(ValueError, match="Formatting lookback period failed"):
+        validate_lookback_period({"days": -5})
+
+
+def test_validate_lookback_period_multiple_negative_values():
+    with pytest.raises(ValueError, match="Formatting lookback period failed"):
+        validate_lookback_period({"days": 7, "months": -1, "years": 2})
+
+
+def test_validate_lookback_period_zero_string():
+    with pytest.raises(ValueError, match="Formatting lookback period failed"):
+        validate_lookback_period({"days": "0"})
+
+
+def test_validate_lookback_period_negative_string():
+    with pytest.raises(ValueError, match="Formatting lookback period failed"):
+        validate_lookback_period({"months": "-10"})
+
+
+def test_validate_lookback_period_invalid_string():
+    with pytest.raises(ValueError, match="Formatting lookback period failed"):
+        validate_lookback_period({"days": "invalid"})
+
+
+def test_validate_lookback_period_invalid_type():
+    with pytest.raises(ValueError, match="Formatting lookback period failed"):
+        validate_lookback_period({"days": []})
+
+
+def test_validate_lookback_period_none_value():
+    with pytest.raises(ValueError, match="Formatting lookback period failed"):
+        validate_lookback_period({"days": None})

tests/test_audit.py

@@ -137,23 +137,23 @@ async def test_get_audit_parameterized(
     [
         (
             {"days": 7},
-            "action:protected_branch.create created:>=2025-07-25",
+            "action:protected_branch.create created:2025-07-25",
         ),
         (
             {"months": 2},
-            "action:protected_branch.create created:>=2025-06-01",
+            "action:protected_branch.create created:2025-06-01",
         ),
         (
             {"years": 1},
-            "action:protected_branch.create created:>=2024-08-01",
+            "action:protected_branch.create created:2024-08-01",
         ),
         (
             {"days": 15, "months": 1},
-            "action:protected_branch.create created:>=2025-06-16",
+            "action:protected_branch.create created:2025-06-16",
         ),
         (
             {"days": 10, "months": 1, "years": 1},
-            "action:protected_branch.create created:>=2024-06-21",
+            "action:protected_branch.create created:2024-06-21",
         ),
     ],
 )
@@ -163,6 +163,8 @@ async def test_get_audit_lookback_periods(
     lookback_period: dict[str, int] | None,
     expected_path: str,
 ):
+    from nodestream_github.client.githubclient import generate_date_range
+    
     extractor = GithubAuditLogExtractor(
         auth_token="test-token",
         github_hostname=DEFAULT_HOSTNAME,
@@ -174,11 +176,34 @@ async def test_get_audit_lookback_periods(
         lookback_period=lookback_period,
     )
 
+    expected_dates = generate_date_range(lookback_period)
+    
+    # Mock the first date call with the expected_path
     gh_rest_mock.get_enterprise_audit_logs(
         status_code=200,
         search_phrase=expected_path,
         json=GITHUB_AUDIT,
     )
+    
+    # Mock additional dates if there are more than one
+    test_dates = expected_dates[:3] if len(expected_dates) > 3 else expected_dates
+    if len(test_dates) > 1:
+        for date in test_dates[1:]:
+            search_phrase = f"action:protected_branch.create created:{date}"
+            gh_rest_mock.get_enterprise_audit_logs(
+                status_code=200,
+                search_phrase=search_phrase,
+                json=GITHUB_AUDIT,
+            )
 
-    all_records = [record async for record in extractor.extract_records()]
-    assert all_records == GITHUB_EXPECTED_OUTPUT
+    # replacing generate_date_range with test dates so that we don't iterate through all dates
+    import nodestream_github.client.githubclient as client_module
+    original_generate = client_module.generate_date_range
+    client_module.generate_date_range = lambda x: test_dates
+    
+    try:
+        all_records = [record async for record in extractor.extract_records()]
+        expected_output = GITHUB_EXPECTED_OUTPUT * len(test_dates)
+        assert all_records == expected_output
+    finally:
+        client_module.generate_date_range = original_generate