Add article summary feature with OpenAI integration

Implement MongoDB cache for summaries to reduce API costs and improve performance.
Rename parameters from OpenAI-specific to more generic API names.
Support configurable model selection through ModelType enum or direct model names.
Add comprehensive tests with mocks for summary generation and caching.
Update documentation in README with summary feature details.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

Author: paskal

README.md

@@ -10,9 +10,10 @@
 |--------------|-----------------|----------------|-------------------------------------------------------|
 | address      | UKEEPER_ADDRESS | all interfaces | web server listening address                          |
 | port         | UKEEPER_PORT    | `8080`         | web server port                                       |
-| mongo_uri    | MONGO_URI       | none           | MongoDB connection string, _required_                 |
-| openai_key   | OPENAI_KEY      | none           | OpenAI API key for summary generation                 |
-| frontend_dir | FRONTEND_DIR    | `/srv/web`     | directory with frontend files                         |
+| mongo-uri    | MONGO_URI       | none           | MongoDB connection string, _required_                 |
+| api-key      | API_KEY         | none           | OpenAI API key for summary generation                 |
+| model-type   | MODEL_TYPE      | `gpt-4o-mini`  | OpenAI model name for summary generation (e.g., gpt-4o, gpt-4o-mini) |
+| frontend-dir | FRONTEND_DIR    | `/srv/web`     | directory with frontend files                         |
 | token        | TOKEN           | none           | token for /content/v1/parser endpoint auth            |
 | mongo-delay  | MONGO_DELAY     | `0`            | mongo initial delay                                   |
 | mongo-db     | MONGO_DB        | `ureadability` | mongo database name                                   |
@@ -24,6 +25,30 @@
     GET /api/content/v1/parser?token=secret&summary=true&url=http://aa.com/blah - extract content (emulate Readability API parse call), summary is optional and requires OpenAI key and token to be enabled
     POST /api/v1/extract {url: http://aa.com/blah}  - extract content
 
+### Article Summary Feature
+
+The application can generate concise summaries of article content using OpenAI's GPT models:
+
+1. **Configuration**:
+   - Set `api-key` to your OpenAI API key
+   - Optionally set `model-type` to specify which model to use (e.g., `gpt-4o`, `gpt-4o-mini`)
+     - Default is `gpt-4o-mini` if not specified
+   - A server token must be configured for security reasons
+
+2. **Usage**:
+   - Add `summary=true` parameter to the `/api/content/v1/parser` endpoint
+   - Example: `/api/content/v1/parser?token=secret&summary=true&url=http://example.com/article`
+
+3. **Features**:
+   - Summaries are cached in MongoDB to reduce API costs and improve performance
+   - The cache stores:
+     - Content hash (to identify articles)
+     - Summary text
+     - Model used for generation
+     - Creation and update timestamps
+   - If the same content is requested again, the cached summary is returned
+   - The preview page automatically shows summaries when available
+
 ## Development
 
 ### Running tests

backend/datastore/mongo.go

@@ -37,15 +37,29 @@ func New(connectionURI, dbName string, delay time.Duration) (*MongoServer, error
 	return &MongoServer{client: client, dbName: dbName}, nil
 }
 
+// Stores contains all DAO instances
+type Stores struct {
+	Rules     RulesDAO
+	Summaries SummariesDAO
+}
+
 // GetStores initialize collections and make indexes
-func (m *MongoServer) GetStores() (rules RulesDAO) {
+func (m *MongoServer) GetStores() Stores {
 	rIndexes := []mongo.IndexModel{
 		{Keys: bson.D{{Key: "enabled", Value: 1}, {Key: "domain", Value: 1}}},
 		{Keys: bson.D{{Key: "user", Value: 1}, {Key: "domain", Value: 1}, {Key: "enabled", Value: 1}}},
 		{Keys: bson.D{{Key: "domain", Value: 1}, {Key: "match_urls", Value: 1}}},
 	}
-	rules = RulesDAO{Collection: m.collection("rules", rIndexes)}
-	return rules
+
+	sIndexes := []mongo.IndexModel{
+		{Keys: bson.D{{Key: "created_at", Value: 1}}},
+		{Keys: bson.D{{Key: "model", Value: 1}}},
+	}
+
+	return Stores{
+		Rules:     RulesDAO{Collection: m.collection("rules", rIndexes)},
+		Summaries: SummariesDAO{Collection: m.collection("summaries", sIndexes)},
+	}
 }
 
 // collection makes collection with indexes

backend/datastore/rules_test.go

@@ -19,7 +19,9 @@ func TestRules(t *testing.T) {
 	server, err := New("mongodb://localhost:27017/", "test_ureadability", 0)
 	require.NoError(t, err)
 	assert.NotNil(t, server.client)
-	rules := server.GetStores()
+	stores := server.GetStores()
+	assert.NotNil(t, stores)
+	rules := stores.Rules
 	assert.NotNil(t, rules)
 	rule := Rule{
 		Domain:  randStringBytesRmndr(42) + ".com",
@@ -74,7 +76,9 @@ func TestRulesCanceledContext(t *testing.T) {
 	server, err := New("mongodb://wrong", "", 0)
 	require.NoError(t, err)
 	assert.NotNil(t, server.client)
-	rules := server.GetStores()
+	stores := server.GetStores()
+	assert.NotNil(t, stores)
+	rules := stores.Rules
 	assert.NotNil(t, rules)
 
 	ctx, cancel := context.WithCancel(context.Background())

backend/datastore/summaries.go

@@ -0,0 +1,90 @@
+// Package datastore provides mongo implementation for store to keep and access summaries
+package datastore
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"time"
+
+	log "github.com/go-pkgz/lgr"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+// Summary contains information about a cached summary
+type Summary struct {
+	ID        string    `bson:"_id"`     // SHA256 hash of the content
+	Content   string    `bson:"content"` // original content that was summarized (could be truncated for storage efficiency)
+	Summary   string    `bson:"summary"` // generated summary
+	Model     string    `bson:"model"`   // openAI model used for summarization
+	CreatedAt time.Time `bson:"created_at"`
+	UpdatedAt time.Time `bson:"updated_at"`
+}
+
+// SummariesDAO handles database operations for article summaries
+type SummariesDAO struct {
+	Collection *mongo.Collection
+}
+
+// Get returns summary by content hash
+func (s SummariesDAO) Get(ctx context.Context, content string) (Summary, bool) {
+	contentHash := generateContentHash(content)
+	res := s.Collection.FindOne(ctx, bson.M{"_id": contentHash})
+	if res.Err() != nil {
+		if res.Err() == mongo.ErrNoDocuments {
+			return Summary{}, false
+		}
+		log.Printf("[WARN] can't get summary for hash %s: %v", contentHash, res.Err())
+		return Summary{}, false
+	}
+
+	summary := Summary{}
+	if err := res.Decode(&summary); err != nil {
+		log.Printf("[WARN] can't decode summary document for hash %s: %v", contentHash, err)
+		return Summary{}, false
+	}
+
+	return summary, true
+}
+
+// Save creates or updates summary in the database
+func (s SummariesDAO) Save(ctx context.Context, summary Summary) error {
+	if summary.ID == "" {
+		summary.ID = generateContentHash(summary.Content)
+	}
+
+	if summary.CreatedAt.IsZero() {
+		summary.CreatedAt = time.Now()
+	}
+	summary.UpdatedAt = time.Now()
+
+	opts := options.Update().SetUpsert(true)
+	_, err := s.Collection.UpdateOne(
+		ctx,
+		bson.M{"_id": summary.ID},
+		bson.M{"$set": summary},
+		opts,
+	)
+	if err != nil {
+		return fmt.Errorf("failed to save summary: %w", err)
+	}
+	return nil
+}
+
+// Delete removes summary from the database
+func (s SummariesDAO) Delete(ctx context.Context, contentHash string) error {
+	_, err := s.Collection.DeleteOne(ctx, bson.M{"_id": contentHash})
+	if err != nil {
+		return fmt.Errorf("failed to delete summary: %w", err)
+	}
+	return nil
+}
+
+// generateContentHash creates a hash for the content to use as an ID
+func generateContentHash(content string) string {
+	hash := sha256.Sum256([]byte(content))
+	return hex.EncodeToString(hash[:])
+}

backend/datastore/summaries_test.go

@@ -0,0 +1,88 @@
+package datastore
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSummariesDAO_SaveAndGet(t *testing.T) {
+	if _, ok := os.LookupEnv("ENABLE_MONGO_TESTS"); !ok {
+		t.Skip("ENABLE_MONGO_TESTS env variable is not set")
+	}
+
+	mdb, err := New("mongodb://localhost:27017", "test_ureadability", 0)
+	require.NoError(t, err)
+
+	// create a unique collection for this test to avoid conflicts
+	collection := mdb.client.Database(mdb.dbName).Collection("summaries_test")
+	defer func() {
+		_ = collection.Drop(context.Background())
+	}()
+
+	dao := SummariesDAO{Collection: collection}
+
+	content := "This is a test article content. It should generate a unique hash."
+	summary := Summary{
+		Content:   content,
+		Summary:   "This is a test summary of the article.",
+		Model:     "gpt-4o-mini",
+		CreatedAt: time.Now(),
+	}
+
+	// test saving a summary
+	err = dao.Save(context.Background(), summary)
+	require.NoError(t, err)
+
+	// test getting the summary
+	foundSummary, found := dao.Get(context.Background(), content)
+	assert.True(t, found)
+	assert.Equal(t, summary.Summary, foundSummary.Summary)
+	assert.Equal(t, summary.Model, foundSummary.Model)
+	assert.NotEmpty(t, foundSummary.ID)
+
+	// test getting a non-existent summary
+	_, found = dao.Get(context.Background(), "non-existent content")
+	assert.False(t, found)
+
+	// test updating an existing summary
+	updatedSummary := Summary{
+		ID:        foundSummary.ID,
+		Content:   content,
+		Summary:   "This is an updated summary.",
+		Model:     "gpt-4o-mini",
+		CreatedAt: foundSummary.CreatedAt,
+	}
+
+	err = dao.Save(context.Background(), updatedSummary)
+	require.NoError(t, err)
+
+	foundSummary, found = dao.Get(context.Background(), content)
+	assert.True(t, found)
+	assert.Equal(t, "This is an updated summary.", foundSummary.Summary)
+	assert.Equal(t, updatedSummary.CreatedAt, foundSummary.CreatedAt)
+	assert.NotEqual(t, updatedSummary.UpdatedAt, foundSummary.UpdatedAt) // UpdatedAt should be set by the DAO
+
+	// test deleting a summary
+	err = dao.Delete(context.Background(), foundSummary.ID)
+	require.NoError(t, err)
+
+	_, found = dao.Get(context.Background(), content)
+	assert.False(t, found)
+}
+
+func TestGenerateContentHash(t *testing.T) {
+	content1 := "This is a test content."
+	content2 := "This is a different test content."
+
+	hash1 := generateContentHash(content1)
+	hash2 := generateContentHash(content2)
+
+	assert.NotEqual(t, hash1, hash2)
+	assert.Equal(t, hash1, generateContentHash(content1)) // same content should produce same hash
+	assert.Equal(t, 64, len(hash1))                       // SHA-256 produces 64 character hex string
+}

backend/extractor/readability.go

@@ -3,6 +3,7 @@ package extractor
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -21,6 +22,8 @@ import (
 )
 
 //go:generate moq -out openai_mock.go . OpenAIClient
+
+// OpenAIClient defines interface for OpenAI API client
 type OpenAIClient interface {
 	CreateChatCompletion(ctx context.Context, request openai.ChatCompletionRequest) (openai.ChatCompletionResponse, error)
 }
@@ -34,14 +37,30 @@ type Rules interface {
 	All(ctx context.Context) []datastore.Rule
 }
 
+// Summaries interface with all methods to access summary cache
+//
+//go:generate moq -out summaries_mock.go . Summaries
+type Summaries interface {
+	Get(ctx context.Context, content string) (datastore.Summary, bool)
+	Save(ctx context.Context, summary datastore.Summary) error
+	Delete(ctx context.Context, contentHash string) error
+}
+
 // UReadability implements fetcher & extractor for local readability-like functionality
 type UReadability struct {
 	TimeOut     time.Duration
 	SnippetSize int
 	Rules       Rules
+	Summaries   Summaries
 	OpenAIKey   string
+	ModelType   string
+
+	apiClient OpenAIClient
+}
 
-	openAIClient OpenAIClient
+// SetAPIClient sets the API client for testing purposes
+func (f *UReadability) SetAPIClient(client OpenAIClient) {
+	f.apiClient = client
 }
 
 // Response from api calls
@@ -78,17 +97,31 @@ func (f *UReadability) ExtractByRule(ctx context.Context, reqURL string, rule *d
 	return f.extractWithRules(ctx, reqURL, rule)
 }
 
+// GenerateSummary creates a summary of the content using OpenAI
 func (f *UReadability) GenerateSummary(ctx context.Context, content string) (string, error) {
 	if f.OpenAIKey == "" {
-		return "", fmt.Errorf("OpenAI key is not set")
+		return "", errors.New("API key for summarization is not set")
+	}
+
+	if f.Summaries != nil {
+		if cachedSummary, found := f.Summaries.Get(ctx, content); found {
+			log.Printf("[DEBUG] using cached summary for content")
+			return cachedSummary.Summary, nil
+		}
+	}
+
+	if f.apiClient == nil {
+		f.apiClient = openai.NewClient(f.OpenAIKey)
 	}
-	if f.openAIClient == nil {
-		f.openAIClient = openai.NewClient(f.OpenAIKey)
+
+	model := openai.GPT4oMini
+	if f.ModelType != "" {
+		model = f.ModelType
 	}
-	resp, err := f.openAIClient.CreateChatCompletion(
+	resp, err := f.apiClient.CreateChatCompletion(
 		ctx,
 		openai.ChatCompletionRequest{
-			Model: openai.GPT4o,
+			Model: model,
 			Messages: []openai.ChatCompletionMessage{
 				{
 					Role:    openai.ChatMessageRoleSystem,
@@ -103,10 +136,28 @@ func (f *UReadability) GenerateSummary(ctx context.Context, content string) (str
 	)
 
 	if err != nil {
-		return "", err
+		log.Printf("[WARN] AI summarization failed: %v", err)
+		return "", fmt.Errorf("failed to generate summary: %w", err)
+	}
+
+	summary := resp.Choices[0].Message.Content
+
+	if f.Summaries != nil {
+		err = f.Summaries.Save(ctx, datastore.Summary{
+			Content:   content,
+			Summary:   summary,
+			Model:     model,
+			CreatedAt: time.Now(),
+		})
+
+		if err != nil {
+			log.Printf("[WARN] failed to cache summary: %v", err)
+		} else {
+			log.Printf("[DEBUG] summary cached successfully")
+		}
 	}
 
-	return resp.Choices[0].Message.Content, nil
+	return summary, nil
 }
 
 // ExtractWithRules is the core function that handles extraction with or without a specific rule