Skip to content
This repository was archived by the owner on Oct 16, 2024. It is now read-only.

cluster_metrics batch stops polling clusters after hitting exception #143

Open
akshaysharma096 wants to merge 14 commits into
base: master
Choose a base branch
from
Open

cluster_metrics batch stops polling clusters after hitting exception #143

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
0bbbfe6
CLUSTERMAN-599, add retries for fetching tags
akshaysharma096 Feb 9, 2022
f08ca46
CLUSTERMAN-599, add retries for fetching tags
akshaysharma096 Feb 9, 2022
299dc62
CLUSTERMNA-599, take care of nits
akshaysharma096 Feb 9, 2022
bfa5181
CLUSTERMNA-599, take care of nits and extra imports
akshaysharma096 Feb 9, 2022
8383a56
Fix failing pre-commits
Feb 9, 2022
e992db6
CLUSTERMAN-599, allow pod to die incase of aws error
akshaysharma096 Feb 21, 2022
37908da
Merge branch 'CLUSTERMAN-599_cluster_metrics_batch_stops_polling_clus…
akshaysharma096 Feb 21, 2022
0800372
CLUSTERMAN-599, roll back to old code
akshaysharma096 Feb 21, 2022
f301cff
Update auto_scaling_resource_group.py
akshaysharma096 Feb 21, 2022
589b8b2
CLUSTERMAN-599, roll back to old code
akshaysharma096 Feb 21, 2022
def2ee8
CLUSTERMAN-599, take care of linting
Feb 21, 2022
f82499e
CLUSTERMAN-599, add tests for batch
akshaysharma096 Feb 21, 2022
4d516e8
Merge branch 'CLUSTERMAN-599_cluster_metrics_batch_stops_polling_clus…
akshaysharma096 Feb 21, 2022
e3902ad
CLUSTERMAN-599, add tests for the code
Feb 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions clusterman/batch/cluster_metrics_collector.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

import colorlog
import staticconf
from botocore.exceptions import ClientError
from clusterman_metrics import ClustermanMetricsBotoClient
from clusterman_metrics import generate_key_with_dimensions
from clusterman_metrics import METADATA
Expand Down Expand Up @@ -130,6 +131,9 @@ def load_pool_managers(self) -> None:
try:
logger.info(f"Loading resource groups for {pool}.{scheduler} on {self.options.cluster}")
self.pool_managers[f"{pool}.{scheduler}"] = PoolManager(self.options.cluster, pool, scheduler)
except ClientError as error:
logger.exception(error)
raise
except Exception as e:
logger.exception(e)
continue
Expand Down
14 changes: 14 additions & 0 deletions tests/batch/cluster_metrics_collector_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import mock
import pytest
from botocore.exceptions import ClientError
from clusterman_metrics import ClustermanMetricsBotoClient

from clusterman.autoscaler.pool_manager import PoolManager
Expand Down Expand Up @@ -166,3 +167,16 @@ def mock_write_metrics(writer, generator, pools, schedulers):
mock.call(2),
mock.call(2),
]


def test_load_pool_managers():
ClusterMetricsCollector = mock.Mock()
batch = ClusterMetricsCollector()
batch.load_pool_managers.side_effect = ClientError(
{}, "when calling the DescribeTags operation (reached max retries: 4)"
)
batch.run_interval = 10
batch.metrics_client = mock.MagicMock(spec_set=ClustermanMetricsBotoClient)
batch.pools = {"mesos": ["pool-1", "pool-2"]}
with pytest.raises(ClientError):
batch.load_pool_managers()