Phase 4: openai/litellm/pydantic-ai を optional extra に隔離、dump/load を制御可能に

- requirements.txt から openai, litellm, pydantic-ai-slim を削除
- requirements/llm.txt に隔離（pip install rdagent[llm] で復元）
- pyproject.toml に llm optional extra 追加
- rdagent/oai/backend/__init__.py: トップレベル import 削除（動的ロードに委譲）
- rdagent/oai/utils/embedding.py: litellm を try/except ガード
- rdagent/scenarios/finetune/scen/utils.py: litellm を try/except ガード
- rdagent/log/ui/ds_trace.py: litellm を try/except ガード
- rdagent/app/utils/health_check.py: litellm を try/except ガード
- rdagent/utils/workflow/loop.py: use_pickle_session フラグ追加

Claude Code = LLM 自身なので Python レベルの SDK は不要。
Claudex factor シナリオは SDK なしで動作する。

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: 2 people

pyproject.toml

@@ -110,6 +110,7 @@ dependencies = {file = ["requirements.txt"]}
 [tool.setuptools.dynamic.optional-dependencies]
 docs = {file = ["requirements/docs.txt"]}
 lint = {file = ["requirements/lint.txt"]}
+llm = {file = ["requirements/llm.txt"]}  # legacy LLM SDK deps (litellm/openai/pydantic-ai); not needed for Claudex
 package = {file = ["requirements/package.txt"]}
 test = {file = ["requirements/test.txt"]}
 torch = {file = ["requirements/torch.txt"]} # some agent algorithms need torch.  pip install rdagent[torch]

rdagent/app/utils/health_check.py

@@ -3,10 +3,16 @@
 
 import docker
 import fire
-import litellm
 import typer
-from litellm import completion, embedding
-from litellm.utils import ModelResponse
+
+try:
+    import litellm
+    from litellm import completion, embedding
+    from litellm.utils import ModelResponse
+
+    _litellm_available = True
+except ImportError:
+    _litellm_available = False
 from typing_extensions import Annotated
 
 from rdagent.log import rdagent_logger as logger

rdagent/log/ui/ds_trace.py

@@ -10,7 +10,10 @@
 import pandas as pd
 import plotly.express as px
 import streamlit as st
-from litellm import get_valid_models
+try:
+    from litellm import get_valid_models
+except ImportError:
+    get_valid_models = lambda: []  # noqa: E731
 from streamlit import session_state as state
 
 from rdagent.app.data_science.loop import DataScienceRDLoop

rdagent/oai/backend/__init__.py

@@ -1 +1,2 @@
-from .litellm import LiteLLMAPIBackend
+# Backend is resolved dynamically via import_class(LLM_SETTINGS.backend).
+# No top-level imports here — litellm/openai are optional (pip install rdagent[llm]).

rdagent/oai/utils/embedding.py

@@ -4,9 +4,14 @@
 
 from typing import Optional
 
-from litellm import decode, encode, get_max_tokens, token_counter
-
 from rdagent.log import rdagent_logger as logger
+
+try:
+    from litellm import decode, encode, get_max_tokens, token_counter
+
+    _litellm_available = True
+except ImportError:
+    _litellm_available = False
 from rdagent.oai.llm_conf import LLM_SETTINGS
 
 # Common embedding model token limits
@@ -46,12 +51,13 @@ def get_embedding_max_tokens(model: str) -> int:
     model_name = model.split("/")[-1] if "/" in model else model
 
     # Level 1: Try litellm
-    try:
-        max_tokens = get_max_tokens(model_name)
-        if max_tokens and max_tokens > 0:
-            return max_tokens
-    except Exception as e:
-        logger.warning(f"Failed to get max tokens for {model_name}: {e}")
+    if _litellm_available:
+        try:
+            max_tokens = get_max_tokens(model_name)
+            if max_tokens and max_tokens > 0:
+                return max_tokens
+        except Exception as e:
+            logger.warning(f"Failed to get max tokens for {model_name}: {e}")
 
     # Level 2: Query mapping table
     if model_name in EMBEDDING_MODEL_LIMITS:
@@ -86,6 +92,14 @@ def trim_text_for_embedding(text: str, model: str, max_tokens: Optional[int] = N
     # Apply safety margin
     safe_max_tokens = int(max_tokens * 0.9)
 
+    if not _litellm_available:
+        # Without litellm, use a rough character-based truncation
+        char_limit = safe_max_tokens * 4  # ~4 chars per token heuristic
+        if len(text) > char_limit:
+            logger.warning(f"litellm not available; truncating text by character limit ({char_limit} chars)")
+            return text[:char_limit]
+        return text
+
     # Calculate current token count
     current_tokens = token_counter(model=model, text=text)
 

rdagent/scenarios/finetune/scen/utils.py

@@ -6,7 +6,13 @@
 
 import numpy as np
 import pandas as pd
-import litellm
+
+try:
+    import litellm
+
+    _litellm_available = True
+except ImportError:
+    _litellm_available = False
 
 from rdagent.app.finetune.llm.conf import FT_RD_SETTING
 from rdagent.core.utils import cache_with_pickle
@@ -101,13 +107,16 @@ def _compute_column_stats(data: list[dict]) -> dict[str, dict]:
                         texts.append(val)
 
         if texts:
-            try:
-                token_counts = [
-                    litellm.token_counter(model=_TOKENIZER_MODEL, text=t)
-                    for t in texts
-                ]
-            except Exception as e:
-                logger.warning(f"Token counting failed for column '{col}': {e}, falling back to char/4")
+            if _litellm_available:
+                try:
+                    token_counts = [
+                        litellm.token_counter(model=_TOKENIZER_MODEL, text=t)
+                        for t in texts
+                    ]
+                except Exception as e:
+                    logger.warning(f"Token counting failed for column '{col}': {e}, falling back to char/4")
+                    token_counts = [len(t) // 4 for t in texts]
+            else:
                 token_counts = [len(t) // 4 for t in texts]
 
             column_stats[col] = {

rdagent/utils/workflow/loop.py

@@ -112,7 +112,7 @@ class LoopTerminationError(Exception):
     class LoopResumeError(Exception):
         """Exception raised when loop conditions indicate the loop should stop all coroutines and resume"""
 
-    def __init__(self) -> None:
+    def __init__(self, *, use_pickle_session: bool = True) -> None:
         # progress control
         self.loop_idx: int = 0  # current loop index / next loop index to kickoff
         self.step_idx: defaultdict[int, int] = defaultdict(int)  # dict from loop index to next step index
@@ -133,6 +133,9 @@ def __init__(self) -> None:
 
         self.semaphores: dict[str, asyncio.Semaphore] = {}
 
+        # When False, skip pickle-based session dump/load (e.g. Claudex adapters use artifact JSON as SSOT)
+        self.use_pickle_session: bool = use_pickle_session
+
     def get_unfinished_loop_cnt(self, next_loop: int) -> int:
         n = 0
         for li in range(next_loop):
@@ -300,7 +303,7 @@ async def _run_step(self, li: int, force_subproc: bool = False) -> None:
                         # Save snapshot after completing the step;
                         # 1) It has to be after the step_idx is updated, so loading the snapshot will be on the right step.
                         # 2) Only save it when the step forward, withdraw does not worth saving.
-                        if name in self.loop_prev_out[li]:
+                        if self.use_pickle_session and name in self.loop_prev_out[li]:
                             # 3) Only dump the step if (so we don't have to redo the step when we load the session again)
                             # it has been executed successfully
                             self.dump(self.session_folder / f"{li}" / f"{si}_{name}")
@@ -403,6 +406,9 @@ async def run(self, step_n: int | None = None, loop_n: int | None = None, all_du
                 self.close_pbar()
 
     def withdraw_loop(self, loop_idx: int) -> None:
+        if not self.use_pickle_session:
+            logger.warning(f"Pickle session disabled; cannot withdraw loop {loop_idx}. Skipping.")
+            return
         prev_session_dir = self.session_folder / str(loop_idx - 1)
         prev_path = min(
             (p for p in prev_session_dir.glob("*_*") if p.is_file()),

requirements.txt

@@ -7,8 +7,6 @@ filelock
 loguru
 fire
 fuzzywuzzy
-openai
-litellm>=1.73  # to support `from litellm import get_valid_models`
 azure.identity
 pyarrow
 rich
@@ -70,7 +68,6 @@ azureml-mlflow
 types-pytz
 
 # Agent
-pydantic-ai-slim[mcp,openai,prefect]
 nest-asyncio
 
 # visualize SFT train

requirements/llm.txt

@@ -0,0 +1,7 @@
+# Legacy LLM SDK dependencies — only needed for non-Claudex scenarios
+# (kaggle, data_science, finetune) that still use Python-level LLM calls.
+# Claudex factor scenario does not require these.
+# Install with: pip install rdagent[llm]
+openai
+litellm>=1.73
+pydantic-ai-slim[mcp,openai,prefect]