Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 31 additions & 21 deletions src/sentry/integrations/github/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -430,23 +430,28 @@ def get_remaining_api_requests(self) -> int:
# This method is used by RepoTreesIntegration
# https://docs.github.com/en/rest/git/trees#get-a-tree
def get_tree(self, repo_full_name: str, tree_sha: str) -> list[dict[str, Any]]:
# We do not cache this call since it is a rather large object
contents: dict[str, Any] = self.get(
f"/repos/{repo_full_name}/git/trees/{tree_sha}",
# Will cause all objects or subtrees referenced by the tree specified in :tree_sha
params={"recursive": 1},
)
# If truncated is true in the response then the number of items in the tree array exceeded our maximum limit.
# If you need to fetch more items, use the non-recursive method of fetching trees, and fetch one sub-tree at a time.
# Note: The limit for the tree array is 100,000 entries with a maximum size of 7 MB when using the recursive parameter.
# XXX: We will need to improve this by iterating through trees without using the recursive parameter
if contents.get("truncated"):
# e.g. getsentry/DataForThePeople
logger.warning(
"The tree for %s has been truncated. Use different a approach for retrieving contents of tree.",
repo_full_name,
with SCMIntegrationInteractionEvent(
interaction_type=SCMIntegrationInteractionType.GET_REPO_TREE,
provider_key=self.integration_name,
integration_id=self.integration.id,
).capture():
# We do not cache this call since it is a rather large object
contents: dict[str, Any] = self.get(
f"/repos/{repo_full_name}/git/trees/{tree_sha}",
# Will cause all objects or subtrees referenced by the tree specified in :tree_sha
params={"recursive": 1},
)
return contents["tree"]
# If truncated is true in the response then the number of items in the tree array exceeded our maximum limit.
# If you need to fetch more items, use the non-recursive method of fetching trees, and fetch one sub-tree at a time.
# Note: The limit for the tree array is 100,000 entries with a maximum size of 7 MB when using the recursive parameter.
# XXX: We will need to improve this by iterating through trees without using the recursive parameter
if contents.get("truncated"):
# e.g. getsentry/DataForThePeople
logger.warning(
"The tree for %s has been truncated. Use different a approach for retrieving contents of tree.",
repo_full_name,
)
return contents["tree"]

# Used by RepoTreesIntegration
def should_count_api_error(self, error: ApiError, extra: dict[str, str]) -> bool:
Expand Down Expand Up @@ -493,11 +498,16 @@ def get_repos(self, page_number_limit: int | None = None) -> list[dict[str, Any]
It uses page_size from the base class to specify how many items per page.
The upper bound of requests is controlled with self.page_number_limit to prevent infinite requests.
"""
return self._get_with_pagination(
"/installation/repositories",
response_key="repositories",
page_number_limit=page_number_limit,
)
with SCMIntegrationInteractionEvent(
interaction_type=SCMIntegrationInteractionType.GET_REPOSITORIES,
provider_key=self.integration_name,
integration_id=self.integration.id,
).capture():
return self._get_with_pagination(
"/installation/repositories",
response_key="repositories",
page_number_limit=page_number_limit,
)

def search_repositories(self, query: bytes) -> Mapping[str, Sequence[Any]]:
"""
Expand Down
4 changes: 4 additions & 0 deletions src/sentry/integrations/source_code_management/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ class SCMIntegrationInteractionType(StrEnum):
# Rate Limiting
GET_RATE_LIMIT = "get_rate_limit"

# Repo Trees
GET_REPOSITORIES = "get_repositories"
GET_REPO_TREE = "get_repo_tree"


@dataclass
class SCMIntegrationInteractionEvent(IntegrationEventLifecycleMetric):
Expand Down
49 changes: 42 additions & 7 deletions tests/sentry/integrations/github/tasks/test_link_all_repos.py
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,12 @@ def test_link_all_repos(self, mock_record: MagicMock, _: MagicMock) -> None:
assert repos[0].name == "getsentry/sentry"
assert repos[1].name == "getsentry/snuba"

assert_slo_metric(mock_record, EventLifecycleOutcome.SUCCESS)
assert len(mock_record.mock_calls) == 4
start1, start2, end2, end1 = mock_record.mock_calls
assert start1.args[0] == EventLifecycleOutcome.STARTED
assert start2.args[0] == EventLifecycleOutcome.STARTED
assert end2.args[0] == EventLifecycleOutcome.SUCCESS
assert end1.args[0] == EventLifecycleOutcome.SUCCESS

@responses.activate
@patch("sentry.integrations.utils.metrics.EventLifecycle.record_event")
Expand Down Expand Up @@ -121,7 +126,12 @@ def test_link_all_repos_api_response_keyerror(

assert repos[0].name == "getsentry/snuba"

assert_slo_metric(mock_record, EventLifecycleOutcome.HALTED)
assert len(mock_record.mock_calls) == 4
start1, start2, end2, end1 = mock_record.mock_calls
assert start1.args[0] == EventLifecycleOutcome.STARTED
assert start2.args[0] == EventLifecycleOutcome.STARTED
assert end2.args[0] == EventLifecycleOutcome.SUCCESS
assert end1.args[0] == EventLifecycleOutcome.HALTED
assert_halt_metric(
mock_record, LinkAllReposHaltReason.REPOSITORY_NOT_CREATED.value
) # should be halt because it didn't complete successfully
Expand Down Expand Up @@ -155,7 +165,12 @@ def test_link_all_repos_api_response_keyerror_single_repo(
repos = Repository.objects.all()
assert len(repos) == 0

assert_slo_metric(mock_record, EventLifecycleOutcome.HALTED)
assert len(mock_record.mock_calls) == 4
start1, start2, end2, end1 = mock_record.mock_calls
assert start1.args[0] == EventLifecycleOutcome.STARTED
assert start2.args[0] == EventLifecycleOutcome.STARTED
assert end2.args[0] == EventLifecycleOutcome.SUCCESS
assert end1.args[0] == EventLifecycleOutcome.HALTED
assert_halt_metric(mock_record, LinkAllReposHaltReason.REPOSITORY_NOT_CREATED.value)

@patch("sentry.integrations.utils.metrics.EventLifecycle.record_event")
Expand Down Expand Up @@ -198,7 +213,12 @@ def test_link_all_repos_api_error(self, mock_record: MagicMock, _: MagicMock) ->
organization_id=self.organization.id,
)

assert_slo_metric(mock_record, EventLifecycleOutcome.FAILURE)
assert len(mock_record.mock_calls) == 4
start1, start2, end2, end1 = mock_record.mock_calls
assert start1.args[0] == EventLifecycleOutcome.STARTED
assert start2.args[0] == EventLifecycleOutcome.STARTED
assert end2.args[0] == EventLifecycleOutcome.FAILURE
assert end1.args[0] == EventLifecycleOutcome.FAILURE

@patch("sentry.integrations.utils.metrics.EventLifecycle.record_event")
@responses.activate
Expand All @@ -221,7 +241,12 @@ def test_link_all_repos_api_error_rate_limited(
organization_id=self.organization.id,
)

assert_slo_metric(mock_record, EventLifecycleOutcome.HALTED)
assert len(mock_record.mock_calls) == 4
start1, start2, end2, end1 = mock_record.mock_calls
assert start1.args[0] == EventLifecycleOutcome.STARTED
assert start2.args[0] == EventLifecycleOutcome.STARTED
assert end2.args[0] == EventLifecycleOutcome.FAILURE
assert end1.args[0] == EventLifecycleOutcome.HALTED
assert_halt_metric(mock_record, LinkAllReposHaltReason.RATE_LIMITED.value)

@patch("sentry.integrations.utils.metrics.EventLifecycle.record_event")
Expand All @@ -240,7 +265,12 @@ def test_link_all_repos_repo_creation_error(
organization_id=self.organization.id,
)

assert_slo_metric(mock_record, EventLifecycleOutcome.HALTED)
assert len(mock_record.mock_calls) == 4
start1, start2, end2, end1 = mock_record.mock_calls
assert start1.args[0] == EventLifecycleOutcome.STARTED
assert start2.args[0] == EventLifecycleOutcome.STARTED
assert end2.args[0] == EventLifecycleOutcome.SUCCESS
assert end1.args[0] == EventLifecycleOutcome.HALTED
assert_halt_metric(mock_record, LinkAllReposHaltReason.REPOSITORY_NOT_CREATED.value)

@patch("sentry.integrations.utils.metrics.EventLifecycle.record_event")
Expand All @@ -262,4 +292,9 @@ def test_link_all_repos_repo_creation_exception(
organization_id=self.organization.id,
)

assert_slo_metric(mock_record, EventLifecycleOutcome.FAILURE)
assert len(mock_record.mock_calls) == 4
start1, start2, end2, end1 = mock_record.mock_calls
assert start1.args[0] == EventLifecycleOutcome.STARTED
assert start2.args[0] == EventLifecycleOutcome.STARTED
assert end2.args[0] == EventLifecycleOutcome.SUCCESS
assert end1.args[0] == EventLifecycleOutcome.FAILURE
Loading