Add spans to the longest running key methods (#90)

# Summary

The changes add spans to the longest running key methods. The most
important updates are grouped below by theme.

### Tracing Initialization:
* Added a `TRACER` instance using `trace.get_tracer("evergreen-agent")`
in `mongodb.py` and `opsmanager.py` to enable distributed tracing.
[[1]](diffhunk://#diff-3c2088327c59d8b95e3068cb652586e006cca037f3553dd6ad9d99a33e5f762bL32-R32)
[[2]](diffhunk://#diff-49b2d2368845a77541db91b50ade54101db603a4251fd6ed81adf9042751c557L43-R43)

### Tracing Key Method Calls:
* Added a span for the `wait_for` method in `MongoDBCommon` to trace its
execution.
* Added a span for the `in_desired_state` function to trace state
transitions.
* Added a span for the `assert_monitoring_data_exists` method in
`MongoDBOpsManager` to trace monitoring data validation.
* Added a span for the `wait_until_backup_pods_become_ready` method in
`MongoDBOpsManager` to trace the readiness of backup pods.

## Proof of Work

<!-- Enter your proof that it works here.-->

## Checklist
- [ ] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [x] Have you checked for release_note changes?

## Reminder (Please remove this when merging)
- Please try to Approve or Reject Changes the PR, keep PRs in review as
short as possible
- Our Short Guide for PRs:
[Link](https://docs.google.com/document/d/1T93KUtdvONq43vfTfUt8l92uo4e4SEEvFbIEKOxGr44/edit?tab=t.0)
- Remember the following Communication Standards - use comment prefixes
for clarity:
  * **blocking**: Must be addressed before approval.
  * **follow-up**: Can be addressed in a later PR or ticket.
  * **q**: Clarifying question.
  * **nit**: Non-blocking suggestions.
  * **note**: Side-note, non-actionable. Example: Praise 
  * --> no prefix is considered a question

Author: nammn

docker/mongodb-kubernetes-tests/kubetester/mongodb.py

@@ -29,6 +29,7 @@
 )
 
 logger = test_logger.get_test_logger(__name__)
+TRACER = trace.get_tracer("evergreen-agent")
 
 
 class Phase(Enum):
@@ -41,6 +42,8 @@ class Phase(Enum):
 
 
 class MongoDBCommon:
+    TRACER.start_as_current_span("wait_for")
+
     def wait_for(self, fn, timeout=None, should_raise=True):
         if timeout is None:
             timeout = 600
@@ -584,6 +587,9 @@ def get_pods(podname, qty) -> List[str]:
     return [podname.format(i) for i in range(qty)]
 
 
+TRACER.start_as_current_span("in_desired_state")
+
+
 def in_desired_state(
     current_state: Phase,
     desired_state: Phase,

docker/mongodb-kubernetes-tests/kubetester/opsmanager.py

@@ -40,6 +40,7 @@
 from tests.shardedcluster.conftest import read_deployment_state
 
 logger = test_logger.get_test_logger(__name__)
+TRACER = trace.get_tracer("evergreen-agent")
 
 
 class MongoDBOpsManager(CustomObject, MongoDBCommon):
@@ -170,6 +171,8 @@ def no_automation_agents_have_registered() -> bool:
 
         KubernetesTester.wait_until(no_automation_agents_have_registered, timeout=600, sleep_time=5)
 
+    TRACER.start_as_current_span("assert_monitoring_data_exists")
+
     def assert_monitoring_data_exists(
         self, database_name: str = "admin", period: str = "P1DT12H", timeout: int = 120, all_hosts: bool = True
     ):
@@ -589,6 +592,8 @@ def get_backup_daemon_container_status(
     ) -> kubernetes.client.V1ContainerStatus:
         return next(filter(lambda c: c.name == "mongodb-backup-daemon", backup_daemon_pod.status.container_statuses))
 
+    TRACER.start_as_current_span("wait_until_backup_pods_become_ready")
+
     def wait_until_backup_pods_become_ready(self, timeout=300):
         def backup_daemons_are_ready():
             try: