fix:external dataset weight rerank bug (#24533)

Co-authored-by: huangzhuo <huangzhuo1@xiaomi.com>

Author: huangzhuo1949

api/core/rag/rerank/weight_rerank.py

@@ -39,9 +39,16 @@ def run(
         unique_documents = []
         doc_ids = set()
         for document in documents:
-            if document.metadata is not None and document.metadata["doc_id"] not in doc_ids:
+            if (
+                document.provider == "dify"
+                and document.metadata is not None
+                and document.metadata["doc_id"] not in doc_ids
+            ):
                 doc_ids.add(document.metadata["doc_id"])
                 unique_documents.append(document)
+            else:
+                if document not in unique_documents:
+                    unique_documents.append(document)
 
         documents = unique_documents