fix: expand HTTP/Web/SDK/unit coverage

.github/workflows/tests.yml

@@ -235,6 +235,15 @@ jobs:
           done
           source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api 2>&1 | tee es_http_api_test.log
 
+      - name: Run unit tests against Elasticsearch
+        run: |
+          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null; do
+            echo "Waiting for service to be available..."
+            sleep 5
+          done
+          source .venv/bin/activate && set -o pipefail; pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_unit 2>&1 | tee es_unit_test.log
+
       - name: RAGFlow CLI retrieval test Elasticsearch
         env:
           PYTHONPATH: ${{ github.workspace }}
@@ -421,6 +430,15 @@ jobs:
           done
           source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_http_api 2>&1 | tee infinity_http_api_test.log
 
+      - name: Run unit tests against Infinity
+        run: |
+          export http_proxy=""; export https_proxy=""; export no_proxy=""; export HTTP_PROXY=""; export HTTPS_PROXY=""; export NO_PROXY=""
+          until sudo docker exec ${RAGFLOW_CONTAINER} curl -s --connect-timeout 5 ${HOST_ADDRESS}/v1/system/ping > /dev/null; do
+            echo "Waiting for service to be available..."
+            sleep 5
+          done
+          source .venv/bin/activate && set -o pipefail; DOC_ENGINE=infinity pytest -s --tb=short --level=${HTTP_API_TEST_LEVEL} test/testcases/test_unit 2>&1 | tee infinity_unit_test.log
+
       - name: RAGFlow CLI retrieval test Infinity
         env:
           PYTHONPATH: ${{ github.workspace }}

api/apps/document_app.py

@@ -760,7 +760,9 @@ async def change_parser():
 
     def reset_doc():
         nonlocal doc
-        e = DocumentService.update_by_id(doc.id, {"pipeline_id": req["pipeline_id"], "parser_id": req["parser_id"], "progress": 0, "progress_msg": "", "run": TaskStatus.UNSTART.value})
+        pipeline_id = req.get("pipeline_id", doc.pipeline_id)
+        parser_id = req.get("parser_id", doc.parser_id)
+        e = DocumentService.update_by_id(doc.id, {"pipeline_id": pipeline_id, "parser_id": parser_id, "progress": 0, "progress_msg": "", "run": TaskStatus.UNSTART.value})
         if not e:
             return get_data_error_result(message="Document not found!")
         if doc.token_num > 0:

api/apps/sdk/doc.py

@@ -1656,6 +1656,9 @@ async def retrieval_test(tenant_id):
                 rename_chunk[new_key] = value
             renamed_chunks.append(rename_chunk)
         ranks["chunks"] = renamed_chunks
+        if not highlight:
+            for chunk in ranks["chunks"]:
+                chunk.pop("highlight", None)
         return get_result(data=ranks)
     except Exception as e:
         if str(e).find("not_found") > 0:

rag/nlp/search.py

@@ -435,8 +435,10 @@ async def retrieval(
         sorted_idx = np.argsort(sim_np * -1)
 
         # When vector_similarity_weight is 0, similarity_threshold is not meaningful for term-only scores.
-        post_threshold = 0.0 if vector_similarity_weight <= 0 else similarity_threshold
-        valid_idx = [int(i) for i in sorted_idx if sim_np[i] >= post_threshold]
+        if vector_similarity_weight <= 0:
+            valid_idx = [int(i) for i in sorted_idx]
+        else:
+            valid_idx = [int(i) for i in sorted_idx if sim_np[i] >= similarity_threshold]
         filtered_count = len(valid_idx)
         ranks["total"] = int(filtered_count)
 

test/testcases/test_http_api/conftest.py

@@ -13,7 +13,9 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+from pathlib import Path
 from time import sleep
+import shutil
 
 import pytest
 from common import (
@@ -43,6 +45,13 @@
 )
 
 
+def pytest_sessionstart(session):
+    root = Path(__file__).resolve().parents[2]
+    for path in root.rglob("__pycache__"):
+        if path.is_dir():
+            shutil.rmtree(path, ignore_errors=True)
+
+
 @wait_for(30, 1, "Document parsing timeout")
 def condition(_auth, _dataset_id):
     res = list_documents(_auth, _dataset_id)
@@ -91,6 +100,7 @@ def clear_datasets(request, HttpApiAuth):
     def cleanup():
         delete_datasets(HttpApiAuth, {"ids": None})
 
+    delete_datasets(HttpApiAuth, {"ids": None})
     request.addfinalizer(cleanup)
 
 
@@ -108,9 +118,10 @@ def cleanup():
         for chat_assistant_id in chat_assistant_ids:
             delete_session_with_chat_assistants(HttpApiAuth, chat_assistant_id)
 
-    request.addfinalizer(cleanup)
-
     _, _, chat_assistant_ids = add_chat_assistants
+    for chat_assistant_id in chat_assistant_ids:
+        delete_session_with_chat_assistants(HttpApiAuth, chat_assistant_id)
+    request.addfinalizer(cleanup)
 
 
 @pytest.fixture(scope="class")