feat: allow to manage external localrecall instances (#425)

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

Author: mudler

main.go

@@ -153,6 +153,7 @@ func main() {
 		webui.WithChunkOverlap(chunkOverlap),
 		webui.WithDatabaseURL(databaseURL),
 		webui.WithCollectionAPIKeys(collectionAPIKeys...),
+		webui.WithLocalRAGURL(localRAG),
 	)
 
 	// Single RAG provider: HTTP client when URL set, in-process when not

pkg/localrag/client.go

@@ -516,3 +516,105 @@ func (c *Client) Store(collection, filePath string) error {
 
 	return nil
 }
+
+// SourceInfo represents an external source for a collection (LocalRecall API contract).
+type SourceInfo struct {
+	URL            string `json:"url"`
+	UpdateInterval int    `json:"update_interval"` // minutes
+	LastUpdate     string `json:"last_update"`      // RFC3339
+}
+
+// AddSource registers an external source for a collection.
+func (c *Client) AddSource(collection, url string, updateIntervalMinutes int) error {
+	reqURL := fmt.Sprintf("%s/api/collections/%s/sources", c.BaseURL, collection)
+	var body struct {
+		URL            string `json:"url"`
+		UpdateInterval int    `json:"update_interval"`
+	}
+	body.URL = url
+	body.UpdateInterval = updateIntervalMinutes
+	if body.UpdateInterval < 1 {
+		body.UpdateInterval = 60
+	}
+	payload, err := json.Marshal(body)
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequest(http.MethodPost, reqURL, bytes.NewBuffer(payload))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	c.addAuthHeader(req)
+	resp, err := (&http.Client{}).Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(resp.Body)
+		return parseAPIError(resp, b, "failed to add source")
+	}
+	return nil
+}
+
+// RemoveSource removes an external source from a collection.
+func (c *Client) RemoveSource(collection, url string) error {
+	reqURL := fmt.Sprintf("%s/api/collections/%s/sources", c.BaseURL, collection)
+	payload, err := json.Marshal(map[string]string{"url": url})
+	if err != nil {
+		return err
+	}
+	req, err := http.NewRequest(http.MethodDelete, reqURL, bytes.NewBuffer(payload))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	c.addAuthHeader(req)
+	resp, err := (&http.Client{}).Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		b, _ := io.ReadAll(resp.Body)
+		return parseAPIError(resp, b, "failed to remove source")
+	}
+	return nil
+}
+
+// ListSources returns external sources for a collection.
+func (c *Client) ListSources(collection string) ([]SourceInfo, error) {
+	reqURL := fmt.Sprintf("%s/api/collections/%s/sources", c.BaseURL, collection)
+	req, err := http.NewRequest(http.MethodGet, reqURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	c.addAuthHeader(req)
+	resp, err := (&http.Client{}).Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, parseAPIError(resp, body, "failed to list sources")
+	}
+	var wrap apiResponse
+	if err := json.Unmarshal(body, &wrap); err != nil || !wrap.Success {
+		if wrap.Error != nil {
+			return nil, errors.New(wrap.Error.Message)
+		}
+		return nil, fmt.Errorf("invalid response: %w", err)
+	}
+	var data struct {
+		Sources []SourceInfo `json:"sources"`
+	}
+	if err := json.Unmarshal(wrap.Data, &data); err != nil {
+		return nil, err
+	}
+	return data.Sources, nil
+}

webui/collections_backend.go

@@ -0,0 +1,39 @@
+package webui
+
+import (
+	"io"
+	"time"
+)
+
+// CollectionSearchResult is a single search result (content + metadata) for API responses.
+type CollectionSearchResult struct {
+	Content    string            `json:"content"`
+	Metadata   map[string]string `json:"metadata,omitempty"`
+	ID         string            `json:"id,omitempty"`
+	Similarity float32           `json:"similarity,omitempty"`
+}
+
+// CollectionSourceInfo is a single external source for a collection.
+type CollectionSourceInfo struct {
+	URL             string    `json:"url"`
+	UpdateInterval  int       `json:"update_interval"` // minutes
+	LastUpdate      time.Time `json:"last_update"`
+}
+
+// CollectionsBackend is the interface used by REST handlers for collection operations.
+// It is implemented by in-process state (embedded) or by an HTTP client (when LocalRAG URL is set).
+type CollectionsBackend interface {
+	ListCollections() ([]string, error)
+	CreateCollection(name string) error
+	Upload(collection, filename string, fileBody io.Reader) error
+	ListEntries(collection string) ([]string, error)
+	GetEntryContent(collection, entry string) (content string, chunkCount int, err error)
+	Search(collection, query string, maxResults int) ([]CollectionSearchResult, error)
+	Reset(collection string) error
+	DeleteEntry(collection, entry string) (remainingEntries []string, err error)
+	AddSource(collection, url string, intervalMin int) error
+	RemoveSource(collection, url string) error
+	ListSources(collection string) ([]CollectionSourceInfo, error)
+	// EntryExists is used by upload handler to avoid duplicate entries.
+	EntryExists(collection, entry string) bool
+}

webui/collections_backend_http.go

@@ -0,0 +1,128 @@
+package webui
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/mudler/LocalAGI/pkg/localrag"
+)
+
+// collectionsBackendHTTP implements CollectionsBackend using the LocalRAG HTTP API.
+type collectionsBackendHTTP struct {
+	client *localrag.Client
+}
+
+var _ CollectionsBackend = (*collectionsBackendHTTP)(nil)
+
+// NewCollectionsBackendHTTP returns a CollectionsBackend that delegates to the given HTTP client.
+func NewCollectionsBackendHTTP(client *localrag.Client) CollectionsBackend {
+	return &collectionsBackendHTTP{client: client}
+}
+
+func (b *collectionsBackendHTTP) ListCollections() ([]string, error) {
+	return b.client.ListCollections()
+}
+
+func (b *collectionsBackendHTTP) CreateCollection(name string) error {
+	return b.client.CreateCollection(name)
+}
+
+func (b *collectionsBackendHTTP) Upload(collection, filename string, fileBody io.Reader) error {
+	tmpDir, err := os.MkdirTemp("", "localagi-upload")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmpDir)
+	tmpPath := filepath.Join(tmpDir, filename)
+	out, err := os.Create(tmpPath)
+	if err != nil {
+		return err
+	}
+	if _, err := io.Copy(out, fileBody); err != nil {
+		out.Close()
+		return err
+	}
+	if err := out.Close(); err != nil {
+		return err
+	}
+	return b.client.Store(collection, tmpPath)
+}
+
+func (b *collectionsBackendHTTP) ListEntries(collection string) ([]string, error) {
+	return b.client.ListEntries(collection)
+}
+
+func (b *collectionsBackendHTTP) GetEntryContent(collection, entry string) (string, int, error) {
+	return b.client.GetEntryContent(collection, entry)
+}
+
+func (b *collectionsBackendHTTP) Search(collection, query string, maxResults int) ([]CollectionSearchResult, error) {
+	if maxResults <= 0 {
+		maxResults = 5
+	}
+	results, err := b.client.Search(collection, query, maxResults)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]CollectionSearchResult, 0, len(results))
+	for _, r := range results {
+		out = append(out, CollectionSearchResult{
+			ID:         r.ID,
+			Content:    r.Content,
+			Metadata:   r.Metadata,
+			Similarity: r.Similarity,
+		})
+	}
+	return out, nil
+}
+
+func (b *collectionsBackendHTTP) Reset(collection string) error {
+	return b.client.Reset(collection)
+}
+
+func (b *collectionsBackendHTTP) DeleteEntry(collection, entry string) ([]string, error) {
+	return b.client.DeleteEntry(collection, entry)
+}
+
+func (b *collectionsBackendHTTP) AddSource(collection, url string, intervalMin int) error {
+	return b.client.AddSource(collection, url, intervalMin)
+}
+
+func (b *collectionsBackendHTTP) RemoveSource(collection, url string) error {
+	return b.client.RemoveSource(collection, url)
+}
+
+func (b *collectionsBackendHTTP) ListSources(collection string) ([]CollectionSourceInfo, error) {
+	srcs, err := b.client.ListSources(collection)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]CollectionSourceInfo, 0, len(srcs))
+	for _, s := range srcs {
+		var lastUpdate time.Time
+		if s.LastUpdate != "" {
+			lastUpdate, _ = time.Parse(time.RFC3339, s.LastUpdate)
+		}
+		out = append(out, CollectionSourceInfo{
+			URL:            s.URL,
+			UpdateInterval: s.UpdateInterval,
+			LastUpdate:     lastUpdate,
+		})
+	}
+	return out, nil
+}
+
+func (b *collectionsBackendHTTP) EntryExists(collection, entry string) bool {
+	entries, err := b.client.ListEntries(collection)
+	if err != nil {
+		return false
+	}
+	for _, e := range entries {
+		if e == entry {
+			return true
+		}
+	}
+	return false
+}