support thai_ftparser config (#26)

* fix image name tag

* fix:redis mode don't need caches

* tokenizer  init in ob container

* fix rag concurrent issue

* remove thai_tokenizer config

* fix

* choose redis,ob cluster name be difyai-redis

* support blocking lock

* fix lock expire and clean lock

* fix

* support parser config

* fix when using mysql cache

* fix

* fix

Author: longbingljw

api/Dockerfile

@@ -21,6 +21,7 @@ RUN apt-get update \
 
 # Install Python dependencies
 COPY pyproject.toml uv.lock ./
+
 RUN uv sync --index-url https://pypi.tuna.tsinghua.edu.cn/simple
 # RUN uv sync
 
@@ -76,6 +77,9 @@ RUN python -c "import tiktoken; tiktoken.encoding_for_model('gpt2')"
 # Copy source code
 COPY . /app/api/
 
+# Fix pyobvector bug
+RUN python /app/api/fix_pyobvector_bug.py
+
 # Copy entrypoint
 COPY docker/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh

api/commands.py

@@ -703,8 +703,7 @@ def ensure_caches_table_exists():
 @click.option("--directory", prompt=False, help="The target migration script directory.")
 def upgrade_db(directory: Optional[str] = None):
     click.echo("Preparing database migration...")
-
-    # 1. 先确保 caches 表存在（原子操作）
+   # 1. 在MySQL缓存模式下，确保 caches 表存在
     # 这样在MySQL缓存模式下，分布式锁可以正常工作
     if "mysql" in dify_config.SQLALCHEMY_DATABASE_URI_SCHEME and dify_config.CACHE_SCHEME == "mysql":
         try:
@@ -713,8 +712,9 @@ def upgrade_db(directory: Optional[str] = None):
             click.echo(click.style(f"Error: Failed to ensure caches table: {e}", fg="red"))
             click.echo(click.style("Migration stopped due to caches table creation failure.", fg="red"))
             raise Exception(f"Migration failed: {e}")
+   
+    # 2. 使用分布式锁
 
-    # 2. 然后使用分布式锁（可以安全使用了）
     lock = redis_client.lock(name="db_upgrade_lock", timeout=60)
     if lock.acquire(blocking=False):
         try:

api/configs/middleware/vdb/oceanbase_config.py

@@ -39,3 +39,15 @@ class OceanBaseVectorConfig(BaseSettings):
         "with older versions",
         default=False,
     )
+
+    OCEANBASE_FULLTEXT_PARSER: Optional[str] = Field(
+        description="Fulltext parser to use for text indexing. Options: 'thai_ftparser' (Thai), 'ik' (Chinese), "
+        "'auto' (automatic language detection). Default is 'ik'",
+        default="ik",
+    )
+
+    OCEANBASE_ENABLE_LANGUAGE_DETECTION: bool = Field(
+        description="Enable automatic language detection for choosing appropriate fulltext parser. "
+        "When disabled, uses OCEANBASE_FULLTEXT_PARSER setting",
+        default=True,
+    )

api/core/plugin/impl/plugin.py

@@ -37,6 +37,7 @@ def list_plugins(self, tenant_id: str) -> list[PluginEntity]:
             f"plugin/{tenant_id}/management/list",
             PluginListResponse,
             params={"page": 1, "page_size": 256, "response_type": "paged"},
+            
         )
         return result.list
 
@@ -46,6 +47,31 @@ def list_plugins_with_total(self, tenant_id: str, page: int, page_size: int) ->
             f"plugin/{tenant_id}/management/list",
             PluginListResponse,
             params={"page": page, "page_size": page_size, "response_type": "paged"},
+            # compatibility: adapt wrapped {code,message,data}, wrapped with data as list, bare {list,total}, or bare list
+            transformer=lambda r: (
+                r
+                if (
+                    isinstance(r, dict)
+                    and {"code", "message", "data"} <= set(r.keys())
+                    and isinstance(r.get("data"), dict)
+                    and {"list", "total"} <= set(r["data"].keys())
+                )
+                else (
+                    {
+                        "code": (r.get("code", 0) if isinstance(r, dict) else 0),
+                        "message": (r.get("message", "") if isinstance(r, dict) else ""),
+                        "data": (
+                            {"list": r["data"], "total": len(r["data"])}
+                            if (isinstance(r, dict) and isinstance(r.get("data"), list))
+                            else (
+                                r
+                                if (isinstance(r, dict) and {"list", "total"} <= set(r.keys()))
+                                else {"list": (r if isinstance(r, list) else []), "total": (len(r) if isinstance(r, list) else 0)}
+                            )
+                        ),
+                    }
+                )
+            ),
         )
 
     def upload_pkg(

api/core/rag/datasource/vdb/oceanbase/oceanbase_vector.py

@@ -117,16 +117,22 @@ def _create_collection(self) -> None:
                 columns=cols,
                 vidxs=vidx_params,
             )
-            try:
-                if self._hybrid_search_enabled:
+            if self._hybrid_search_enabled:
+                # Get parser from config or use default ik parser
+                parser_name = dify_config.OCEANBASE_FULLTEXT_PARSER or "ik"
+                    
+                logger.info(f"Creating fulltext index for collection '{self._collection_name}' using parser '{parser_name}'")
+                    
+                try:
                     self._client.perform_raw_text_sql(f"""ALTER TABLE {self._collection_name}
-                    ADD FULLTEXT INDEX fulltext_index_for_col_text (text) WITH PARSER ik""")
-            except Exception as e:
-                raise Exception(
-                    "Failed to add fulltext index to the target table, your OceanBase version must be 4.3.5.1 or above "
-                    + "to support fulltext index and vector index in the same table",
-                    e,
-                )
+                    ADD FULLTEXT INDEX fulltext_index_for_col_text (text) WITH PARSER {parser_name}""")
+                except Exception as e:
+                    raise Exception(
+                        "Failed to add fulltext index to the target table, your OceanBase version must be 4.3.5.1 or above "
+                        + "to support fulltext index and vector index in the same table",
+                        e,
+                    )
+
             redis_client.set(collection_exist_cache_key, 1, ex=3600)
 
     def _check_hybrid_search_support(self) -> bool: