🔨 use arm host for unitest runner and update monitor auto disable

Author: Phinease

.github/workflows/auto-unit-test.yml

@@ -10,7 +10,7 @@ on:
       runner_label_json:
         description: 'runner array in json format (e.g. ["ubuntu-latest"] or ["self-hosted"])'
         required: false
-        default: '["ubuntu-latest"]'
+        default: '["ubuntu-24.04-arm"]'
   pull_request:
     branches: [develop]
     paths:
@@ -28,7 +28,7 @@ on:
 
 jobs:
   test:
-    runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-latest"]') }}
+    runs-on: ${{ fromJson(github.event.inputs.runner_label_json || '["ubuntu-24.04-arm"]') }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -45,7 +45,7 @@ jobs:
         run: |
           cd backend
           uv sync --extra data-process --extra test
-          uv pip install -e ../sdk
+          uv pip install -e "../sdk[dev]"
           cd ..
 
       - name: Run all tests and collect coverage

backend/apps/data_process_app.py

@@ -12,7 +12,6 @@
     TaskRequest,
 )
 from data_process.tasks import process_and_forward, process_sync
-from data_process.utils import get_task_info
 from services.data_process_service import get_data_process_service
 
 logger = logging.getLogger("data_process.app")

backend/database/client.py

@@ -4,8 +4,7 @@
 from typing import Any, BinaryIO, Dict, List, Optional, Tuple
 
 import boto3
-import psycopg2.extras
-from psycopg2 import extensions
+import psycopg2
 from botocore.client import Config
 from botocore.exceptions import ClientError
 from sqlalchemy import create_engine

sdk/nexent/monitor/monitoring.py

@@ -6,18 +6,29 @@
 integration with OpenTelemetry, Jaeger, Prometheus, and Grafana.
 
 This module uses a singleton pattern for consistent monitoring across the SDK.
+When OpenTelemetry dependencies are not available, the module gracefully degrades
+and disables monitoring functionality without breaking the application.
+
+Installation:
+- Basic: pip install nexent
+- With monitoring: pip install nexent[performance]
 """
 
-from opentelemetry.trace.status import Status, StatusCode
-from opentelemetry.exporter.prometheus import PrometheusMetricReader
-from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.instrumentation.requests import RequestsInstrumentor
-from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
-from opentelemetry.exporter.jaeger.thrift import JaegerExporter
-from opentelemetry import trace, metrics
-from opentelemetry.sdk.resources import Resource
+# Optional OpenTelemetry imports - gracefully handle missing dependencies
+try:
+    from opentelemetry.trace.status import Status, StatusCode
+    from opentelemetry.exporter.prometheus import PrometheusMetricReader
+    from opentelemetry.sdk.metrics import MeterProvider
+    from opentelemetry.sdk.trace.export import BatchSpanProcessor
+    from opentelemetry.sdk.trace import TracerProvider
+    from opentelemetry.instrumentation.requests import RequestsInstrumentor
+    from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
+    from opentelemetry.exporter.jaeger.thrift import JaegerExporter
+    from opentelemetry import trace, metrics
+    from opentelemetry.sdk.resources import Resource
+    OPENTELEMETRY_AVAILABLE = True
+except ImportError:
+    OPENTELEMETRY_AVAILABLE = False
 import logging
 import time
 import functools
@@ -27,11 +38,13 @@
 
 logger = logging.getLogger(__name__)
 
-# Import OpenTelemetry dependencies (assumed to be available in SDK)
-
 F = TypeVar('F', bound=Callable[..., Any])
 
 
+def is_opentelemetry_available() -> bool:
+    """Check if OpenTelemetry dependencies are available."""
+    return OPENTELEMETRY_AVAILABLE
+
 @dataclass
 class MonitoringConfig:
     """Configuration for monitoring system."""
@@ -42,6 +55,15 @@ class MonitoringConfig:
     telemetry_sample_rate: float = 1.0
     llm_slow_request_threshold_seconds: float = 5.0
     llm_slow_token_rate_threshold: float = 10.0
+    
+    def __post_init__(self):
+        """Validate configuration and adjust based on OpenTelemetry availability."""
+        if self.enable_telemetry and not OPENTELEMETRY_AVAILABLE:
+            logger.warning(
+                "OpenTelemetry dependencies not available. Disabling telemetry. "
+                "Install with: pip install nexent[performance]"
+            )
+            self.enable_telemetry = False
 
 
 class MonitoringManager:
@@ -90,6 +112,13 @@ def _init_telemetry(self) -> None:
             logger.info("Telemetry is disabled by configuration")
             return
 
+        if not OPENTELEMETRY_AVAILABLE:
+            logger.warning(
+                "OpenTelemetry dependencies not available. Telemetry initialization skipped. "
+                "Install with: pip install nexent[performance]"
+            )
+            return
+
         try:
             # Setup tracing with proper service name resource
             resource = Resource.create({
@@ -164,7 +193,9 @@ def _init_telemetry(self) -> None:
     @property
     def is_enabled(self) -> bool:
         """Check if monitoring is enabled."""
-        return self._config is not None and self._config.enable_telemetry
+        return (self._config is not None and 
+                self._config.enable_telemetry and 
+                OPENTELEMETRY_AVAILABLE)
 
     @property
     def tracer(self):
@@ -174,11 +205,16 @@ def tracer(self):
     def setup_fastapi_app(self, app) -> bool:
         """Setup monitoring for a FastAPI application."""
         try:
-            if self.is_enabled and app:
+            if self.is_enabled and app and OPENTELEMETRY_AVAILABLE:
                 FastAPIInstrumentor.instrument_app(app)
                 logger.info(
                     "FastAPI application monitoring initialized successfully")
                 return True
+            elif not OPENTELEMETRY_AVAILABLE:
+                logger.warning(
+                    "OpenTelemetry not available. FastAPI monitoring skipped. "
+                    "Install with: pip install nexent[performance]"
+                )
             return False
         except Exception as e:
             logger.error(f"Failed to initialize FastAPI monitoring: {e}")
@@ -187,9 +223,7 @@ def setup_fastapi_app(self, app) -> bool:
     @contextmanager
     def trace_llm_request(self, operation_name: str, model_name: str, **attributes: Any) -> Iterator[Optional[Any]]:
         """Context manager for tracing LLM requests with comprehensive metrics."""
-        if not self.is_enabled or not self._tracer:
-            logger.warning(
-                f"⚠️ trace_llm_request returning None: is_enabled={self.is_enabled}, has_tracer={self._tracer is not None}")
+        if not self.is_enabled or not OPENTELEMETRY_AVAILABLE or not self._tracer:
             yield None
             return
 
@@ -218,13 +252,13 @@ def trace_llm_request(self, operation_name: str, model_name: str, **attributes:
 
     def get_current_span(self) -> Optional[Any]:
         """Get the current active span."""
-        if not self.is_enabled:
+        if not self.is_enabled or not OPENTELEMETRY_AVAILABLE:
             return None
         return trace.get_current_span()
 
     def add_span_event(self, name: str, attributes: Optional[Dict[str, Any]] = None) -> None:
         """Add an event to the current span."""
-        if not self.is_enabled:
+        if not self.is_enabled or not OPENTELEMETRY_AVAILABLE:
             return
 
         span = trace.get_current_span()
@@ -233,7 +267,7 @@ def add_span_event(self, name: str, attributes: Optional[Dict[str, Any]] = None)
 
     def set_span_attributes(self, **attributes: Any) -> None:
         """Set attributes on the current span."""
-        if not self.is_enabled:
+        if not self.is_enabled or not OPENTELEMETRY_AVAILABLE:
             return
 
         span = trace.get_current_span()
@@ -246,7 +280,7 @@ def create_token_tracker(self, model_name: str, span: Optional[Any] = None) -> '
 
     def record_llm_metrics(self, metric_type: str, value: float, attributes: Dict[str, Any]) -> None:
         """Record LLM-specific metrics."""
-        if not self.is_enabled:
+        if not self.is_enabled or not OPENTELEMETRY_AVAILABLE:
             return
 
         if metric_type == "ttft" and self._llm_ttft_duration:
@@ -499,4 +533,5 @@ async def my_function():
     'MonitoringManager',
     'LLMTokenTracker',
     'get_monitoring_manager',
+    'is_opentelemetry_available',
 ]

sdk/nexent/vector_database/__init__.py

@@ -1,5 +1,5 @@
 import atexit
-from unittest.mock import patch, Mock
+from unittest.mock import patch, Mock, MagicMock
 import os
 import sys
 import types
@@ -14,19 +14,22 @@
 backend_dir = os.path.abspath(os.path.join(current_dir, "../../../backend"))
 sys.path.insert(0, backend_dir)
 
-from apps.agent_app import router
+# Mock boto3 before importing backend modules
+boto3_mock = MagicMock()
+sys.modules['boto3'] = boto3_mock
+
+# Import target endpoints with all external dependencies patched
+with patch('backend.database.client.MinioClient') as minio_mock, \
+     patch('elasticsearch.Elasticsearch', return_value=MagicMock()) as es_mock:
+    minio_mock.return_value = MagicMock()
+    
+    from apps.agent_app import router
 
 # Apply patches before importing any app modules (similar to test_base_app.py)
 
 patches = [
-    # Mock boto3 client
-    patch('boto3.client', return_value=Mock()),
-    # Mock boto3 resource
-    patch('boto3.resource', return_value=Mock()),
     # Mock database sessions
-    patch('backend.database.client.get_db_session', return_value=Mock()),
-    # Mock Elasticsearch to prevent connection errors
-    patch('elasticsearch.Elasticsearch', return_value=Mock())
+    patch('backend.database.client.get_db_session', return_value=Mock())
 ]
 
 for p in patches:

test/backend/app/test_agent_app.py

@@ -1,35 +1,11 @@
-import backend.services.agent_service as agent_service
-from backend.services.agent_service import update_agent_info_impl
-from backend.services.agent_service import get_creating_sub_agent_info_impl
-from backend.services.agent_service import list_all_agent_info_impl
-from backend.services.agent_service import get_agent_info_impl
-from backend.services.agent_service import get_creating_sub_agent_id_service
-from backend.services.agent_service import get_enable_tool_id_by_agent_id
-from backend.services.agent_service import (
-    get_agent_call_relationship_impl,
-    delete_agent_impl,
-    export_agent_impl,
-    export_agent_by_agent_id,
-    import_agent_by_agent_id,
-    insert_related_agent_impl,
-    load_default_agents_json_file,
-    clear_agent_memory,
-    import_agent_impl,
-    get_agent_id_by_name,
-    save_messages,
-    prepare_agent_run,
-    run_agent_stream,
-    stop_agent_tasks,
-)
-from fastapi import Request
-from consts.model import ExportAndImportAgentInfo, ExportAndImportDataFormat, MCPInfo, AgentRequest
 import sys
 import asyncio
 import json
 from unittest.mock import patch, MagicMock, mock_open, call, Mock, AsyncMock
 
 import pytest
 from fastapi.responses import StreamingResponse
+from fastapi import Request
 
 # Import the actual ToolConfig model for testing before any mocking
 from nexent.core.agents.agent_model import ToolConfig
@@ -38,17 +14,38 @@
 boto3_mock = MagicMock()
 sys.modules['boto3'] = boto3_mock
 
-# Mock Elasticsearch
+# Mock external dependencies before importing backend modules that might initialize them
+with patch('backend.database.client.MinioClient') as minio_mock, \
+     patch('elasticsearch.Elasticsearch', return_value=MagicMock()) as es_mock:
+    minio_mock.return_value = MagicMock()
+    
+    import backend.services.agent_service as agent_service
+    from backend.services.agent_service import update_agent_info_impl
+    from backend.services.agent_service import get_creating_sub_agent_info_impl
+    from backend.services.agent_service import list_all_agent_info_impl
+    from backend.services.agent_service import get_agent_info_impl
+    from backend.services.agent_service import get_creating_sub_agent_id_service
+    from backend.services.agent_service import get_enable_tool_id_by_agent_id
+    from backend.services.agent_service import (
+        get_agent_call_relationship_impl,
+        delete_agent_impl,
+        export_agent_impl,
+        export_agent_by_agent_id,
+        import_agent_by_agent_id,
+        insert_related_agent_impl,
+        load_default_agents_json_file,
+        clear_agent_memory,
+        import_agent_impl,
+        get_agent_id_by_name,
+        save_messages,
+        prepare_agent_run,
+        run_agent_stream,
+        stop_agent_tasks,
+    )
+    from consts.model import ExportAndImportAgentInfo, ExportAndImportDataFormat, MCPInfo, AgentRequest
+
+# Mock Elasticsearch (already done in the import section above, but keeping for reference)
 elasticsearch_client_mock = MagicMock()
-patch('elasticsearch._sync.client.Elasticsearch',
-      return_value=elasticsearch_client_mock).start()
-patch('elasticsearch.Elasticsearch',
-      return_value=elasticsearch_client_mock).start()
-
-# Mock ElasticSearchCore
-elasticsearch_core_mock = MagicMock()
-patch('sdk.nexent.vector_database.elasticsearch_core.ElasticSearchCore',
-      return_value=elasticsearch_core_mock).start()
 
 
 # Mock memory-related modules

test/backend/services/test_agent_service.py

@@ -1,6 +1,5 @@
 import os
 import subprocess
-import glob
 import sys
 import logging