fix: correctly handle UTF-8 and invalid chars

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

rag/engine/postgres.go

@@ -481,13 +481,16 @@ func (p *PostgresDB) StoreDocuments(s []string, metadata map[string]string) ([]R
 
 	// Insert documents
 	for i, content := range s {
+		// Sanitize content to prevent invalid UTF-8 from reaching PostgreSQL
+		content = strings.ToValidUTF8(content, " ")
+
 		embedding := resp.Data[i].Embedding
 		embeddingStr := formatVector(embedding)
 
 		// Extract title from metadata if available
-		title := metadata["title"]
+		title := strings.ToValidUTF8(metadata["title"], " ")
 		if title == "" {
-			title = metadata["source"]
+			title = strings.ToValidUTF8(metadata["source"], " ")
 		}
 
 		// Calculate word count

rag/persistency.go

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 
 	"os"
+	"strings"
 	"sync"
 
 	"github.com/dslipak/pdf"
@@ -473,7 +474,9 @@ func fileToText(fpath string) (string, error) {
 			return "", err
 		}
 		buf.ReadFrom(b)
-		return buf.String(), nil
+		// PDF extraction can produce invalid UTF-8 byte sequences that PostgreSQL rejects.
+		// Sanitize by replacing invalid sequences with the Unicode replacement character.
+		return strings.ToValidUTF8(buf.String(), " "), nil
 	case ".txt", ".md":
 		f, err := os.Open(fpath)
 		if err != nil {