fix: remove pkuseg for install issue

Author: zhongyu09

.github/workflows/docs.yml

@@ -30,8 +30,6 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip setuptools wheel
-        pip install numpy
-        pip install pkuseg
         pip install -e ".[docs]"
 
     - name: Build documentation

openchatbi/catalog/retrival_helper.py

@@ -1,13 +1,10 @@
 """Helper functions for building column retrieval systems."""
 
-import pkuseg
 from rank_bm25 import BM25Okapi
 
 from openchatbi.llm.llm import get_embedding_model
-
-# Initialize pkuseg segmenter
-_segmenter = pkuseg.pkuseg()
 from openchatbi.utils import log, create_vector_db
+from openchatbi.text_segmenter import _segmenter
 
 
 def get_columns_metadata(catalog):

openchatbi/catalog/schema_retrival.py

@@ -4,12 +4,9 @@
 import re
 
 import Levenshtein
-import pkuseg
 
 from openchatbi import config
-
-# Initialize pkuseg segmenter
-_segmenter = pkuseg.pkuseg()
+from openchatbi.text_segmenter import _segmenter
 from openchatbi.catalog.retrival_helper import build_column_tables_mapping, build_columns_retriever
 from openchatbi.utils import log
 

openchatbi/text_segmenter.py

@@ -0,0 +1,40 @@
+"""Simple text segmentation utility."""
+
+import re
+import string
+from typing import List
+
+
+class SimpleSegmenter:
+    """A simple text segmenter that splits text by punctuation and whitespace.
+
+    This is a lightweight text segmentation tool that provides basic
+    functionality without external dependencies.
+    """
+
+    def __init__(self):
+        # Include both English and Chinese punctuation
+        chinese_punctuation = "，。！？；：" "''（）【】《》〈〉「」『』〔〕"
+        all_separators = string.punctuation + chinese_punctuation + " \t\n\r"
+        # Create regex pattern to split on any separator
+        self.split_pattern = "[" + re.escape(all_separators) + "]+"
+
+    def cut(self, text: str) -> List[str]:
+        """Segment text into tokens by splitting on punctuation and whitespace.
+
+        Args:
+            text: Input text to be segmented
+
+        Returns:
+            List of tokens
+        """
+        if not text:
+            return []
+
+        # Split by separators and filter empty strings
+        tokens = re.split(self.split_pattern, text)
+        return [token for token in tokens if token.strip()]
+
+
+# Global instance
+_segmenter = SimpleSegmenter()

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "openchatbi"
-version = "0.1.0"
+version = "0.1.1"
 description = "OpenChatBI - Natural language business intelligence powered by LLMs for intuitive data analysis and SQL generation"
 authors = [
     { name = "Yu Zhong", email = "[email protected]" },
@@ -72,7 +72,6 @@ dependencies = [
     "seaborn>=0.13.0,<1.0.0",
     "plotly>=5.17.0,<6.0.0",
     "json5>=0.10.0,<1.0.0",
-    "pkuseg>=0.0.25",
 ]
 
 [project.urls]
@@ -117,8 +116,6 @@ dev-dependencies = [
     "openchatbi[dev]",
 ]
 
-[tool.uv.extra-build-dependencies]
-pkuseg = ["numpy", "cython"]
 
 [build-system]
 requires = ["hatchling>=1.26.0"]