Remove docling core test from lean imports, add transformers back and import semchunk and transformers in chunkers.py

Signed-off-by: Aakanksha Duggal <aduggal@redhat.com>

Author: aakankshaduggal

src/instructlab/sdg/utils/chunkers.py

@@ -22,6 +22,8 @@
 )
 from docling_core.transforms.chunker.hybrid_chunker import HybridChunker
 from langchain_text_splitters import Language, RecursiveCharacterTextSplitter
+import semchunk
+import transformers
 
 # First Party
 from instructlab.sdg.utils.model_formats import is_model_gguf, is_model_safetensors

tests/testdata/leanimports.py

@@ -4,18 +4,11 @@
 import sys
 
 # block slow imports
-for unwanted in ["deepspeed", "llama_cpp", "torch", "vllm"]:
+for unwanted in ["deepspeed", "llama_cpp", "torch", "transformers", "vllm"]:
     # importlib raises ModuleNotFound when sys.modules value is None.
     assert unwanted not in sys.modules
     sys.modules[unwanted] = None  # type: ignore[assignment]
 
-# Try to import in your PR to see if this works around the issue. If not, print an error
-try:
-    # Third Party
-    import docling_core
-except ImportError as e:
-    print(f"Could not import `docling_core` because: {e}")
-
 # First Party
 # This will trigger errors if any of the import chain tries to load
 # the unwanted modules