fix Reranking if document contains null

nipeone · nipeone · commit 14ba50fe5959 · 2025-05-12T09:45:24.000+08:00
diff --git a/LLama/LLamaReranker.cs b/LLama/LLamaReranker.cs
@@ -72,7 +72,7 @@ public async Task<IReadOnlyList<float>> GetRelevanceScores(string input, IReadOn
 
         for(var idx = 0; idx < documents.Count; idx++)
         {
-            var docTokens = Context.Tokenize(documents[idx]);
+            var docTokens = Context.Tokenize(documents[idx] ?? "");
             LLamaToken[] tokens = [.. inputTokens, .. docTokens];
 
             if (batch.TokenCount + tokens.Length > Context.ContextSize)

Original file line number	Diff line number	Diff line change
`@@ -72,7 +72,7 @@ public async Task<IReadOnlyList<float>> GetRelevanceScores(string input, IReadOn`
`72`	`72`
`73`	`73`	`for(var idx = 0; idx < documents.Count; idx++)`
`74`	`74`	`{`
`75`		`- var docTokens = Context.Tokenize(documents[idx]);`
	`75`	`+ var docTokens = Context.Tokenize(documents[idx] ?? "");`
`76`	`76`	`LLamaToken[] tokens = [.. inputTokens, .. docTokens];`
`77`	`77`
`78`	`78`	`if (batch.TokenCount + tokens.Length > Context.ContextSize)`