Add article summary feature with OpenAI integration

- Introduce 'summary' query parameter in /api/content/v1/parser endpoint
- Integrate OpenAI API for generating article summaries
- Add OpenAIKey field to Server struct and corresponding command-line
flag
- Update extractArticleEmulateReadability to handle summary requests
- Add generateSummary method using OpenAI's GPT-4o model (turns out to
be faster than even 4o mini)
- Add OpenAIClient interface and mock for testing
- Update README.md with new configuration options and API details

This feature allows users to request a summary of extracted articles
using OpenAI's GPT-4o model. To ensure secure usage, summary generation
requires a valid server token. The changes include comprehensive error
handling and test coverage for various scenarios, including token
validation and server misconfiguration.

# Conflicts:
#	backend/go.mod
#	backend/rest/server.go

Author: paskal

README.md

@@ -11,6 +11,7 @@
 | address      | UKEEPER_ADDRESS | all interfaces | web server listening address                          |
 | port         | UKEEPER_PORT    | `8080`         | web server port                                       |
 | mongo_uri    | MONGO_URI       | none           | MongoDB connection string, _required_                 |
+| openai_key   | OPENAI_KEY      | none           | OpenAI API key for summary generation                 |
 | frontend_dir | FRONTEND_DIR    | `/srv/web`     | directory with frontend files                         |
 | token        | TOKEN           | none           | token for /content/v1/parser endpoint auth            |
 | mongo-delay  | MONGO_DELAY     | `0`            | mongo initial delay                                   |
@@ -20,7 +21,7 @@
 
 ### API
 
-    GET /api/content/v1/parser?token=secret&url=http://aa.com/blah - extract content (emulate Readability API parse call)
+    GET /api/content/v1/parser?token=secret&summary=true&url=http://aa.com/blah - extract content (emulate Readability API parse call), summary is optional and requires OpenAI key and token to be enabled
     POST /api/v1/extract {url: http://aa.com/blah}  - extract content
 
 ## Development

backend/extractor/openai_mock.go

@@ -14,11 +14,17 @@ import (
 	"github.com/PuerkitoBio/goquery"
 	log "github.com/go-pkgz/lgr"
 	"github.com/mauidude/go-readability"
+	"github.com/sashabaranov/go-openai"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 
 	"github.com/ukeeper/ukeeper-redabilty/backend/datastore"
 )
 
+//go:generate moq -out openai_mock.go . OpenAIClient
+type OpenAIClient interface {
+	CreateChatCompletion(ctx context.Context, request openai.ChatCompletionRequest) (openai.ChatCompletionResponse, error)
+}
+
 // Rules interface with all methods to access datastore
 type Rules interface {
 	Get(ctx context.Context, rURL string) (datastore.Rule, bool)
@@ -33,10 +39,14 @@ type UReadability struct {
 	TimeOut     time.Duration
 	SnippetSize int
 	Rules       Rules
+	OpenAIKey   string
+
+	openAIClient OpenAIClient
 }
 
 // Response from api calls
 type Response struct {
+	Summary     string   `json:"summary,omitempty"`
 	Content     string   `json:"content"`
 	Rich        string   `json:"rich_content"`
 	Domain      string   `json:"domain"`
@@ -68,6 +78,37 @@ func (f *UReadability) ExtractByRule(ctx context.Context, reqURL string, rule *d
 	return f.extractWithRules(ctx, reqURL, rule)
 }
 
+func (f *UReadability) GenerateSummary(ctx context.Context, content string) (string, error) {
+	if f.OpenAIKey == "" {
+		return "", fmt.Errorf("OpenAI key is not set")
+	}
+	if f.openAIClient == nil {
+		f.openAIClient = openai.NewClient(f.OpenAIKey)
+	}
+	resp, err := f.openAIClient.CreateChatCompletion(
+		ctx,
+		openai.ChatCompletionRequest{
+			Model: openai.GPT4o,
+			Messages: []openai.ChatCompletionMessage{
+				{
+					Role:    openai.ChatMessageRoleSystem,
+					Content: "You are a helpful assistant that summarizes articles. Please summarize the main points in a few sentences as TLDR style (don't add a TLDR label). Then, list up to five detailed bullet points. Provide the response in plain text. Do not add any additional information. Do not add a Summary at the beginning of the response. If detailed bullet points are too similar to the summary, don't include them at all:",
+				},
+				{
+					Role:    openai.ChatMessageRoleUser,
+					Content: content,
+				},
+			},
+		},
+	)
+
+	if err != nil {
+		return "", err
+	}
+
+	return resp.Choices[0].Message.Content, nil
+}
+
 // ExtractWithRules is the core function that handles extraction with or without a specific rule
 func (f *UReadability) extractWithRules(ctx context.Context, reqURL string, rule *datastore.Rule) (*Response, error) {
 	log.Printf("[INFO] extract %s", reqURL)

backend/extractor/readability.go

@@ -11,6 +11,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/sashabaranov/go-openai"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.mongodb.org/mongo-driver/bson/primitive"
@@ -207,3 +208,61 @@ func TestGetContentCustom(t *testing.T) {
 	assert.Len(t, content, 6988)
 	assert.Len(t, rich, 7169)
 }
+
+func TestUReadability_GenerateSummary(t *testing.T) {
+	mockOpenAI := &OpenAIClientMock{
+		CreateChatCompletionFunc: func(ctx context.Context, request openai.ChatCompletionRequest) (openai.ChatCompletionResponse, error) {
+			return openai.ChatCompletionResponse{
+				Choices: []openai.ChatCompletionChoice{
+					{
+						Message: openai.ChatCompletionMessage{
+							Content: "This is a summary of the article.",
+						},
+					},
+				},
+			}, nil
+		},
+	}
+
+	tests := []struct {
+		name           string
+		content        string
+		openAIKey      string
+		expectedResult string
+		expectedError  string
+	}{
+		{
+			name:           "Valid OpenAI Key and content",
+			content:        "This is a test article content.",
+			openAIKey:      "test-key",
+			expectedResult: "This is a summary of the article.",
+			expectedError:  "",
+		},
+		{
+			name:           "No OpenAI Key",
+			content:        "This is a test article content.",
+			openAIKey:      "",
+			expectedResult: "",
+			expectedError:  "OpenAI key is not set",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			readability := UReadability{
+				OpenAIKey:    tt.openAIKey,
+				openAIClient: mockOpenAI,
+			}
+
+			result, err := readability.GenerateSummary(context.Background(), tt.content)
+
+			if tt.expectedError != "" {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), tt.expectedError)
+			} else {
+				require.NoError(t, err)
+				assert.Equal(t, tt.expectedResult, result)
+			}
+		})
+	}
+}

backend/extractor/readability_test.go

@@ -12,6 +12,7 @@ require (
 	github.com/jessevdk/go-flags v1.6.1
 	github.com/kennygrant/sanitize v1.2.4
 	github.com/mauidude/go-readability v0.0.0-20220221173116-a9b3620098b7
+	github.com/sashabaranov/go-openai v1.38.2
 	github.com/stretchr/testify v1.10.0
 	go.mongodb.org/mongo-driver v1.17.3
 	golang.org/x/net v0.38.0

backend/go.mod

@@ -168,6 +168,8 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sashabaranov/go-openai v1.38.2 h1:akrssjj+6DY3lWuDwHv6cBvJ8Z+FZDM9XEaaYFt0Auo=
+github.com/sashabaranov/go-openai v1.38.2/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

backend/go.sum

@@ -27,6 +27,7 @@ var opts struct {
 	MongoURI    string            `short:"m" long:"mongo_uri" env:"MONGO_URI" required:"true" description:"MongoDB connection string"`
 	MongoDelay  time.Duration     `long:"mongo-delay" env:"MONGO_DELAY" default:"0" description:"mongo initial delay"`
 	MongoDB     string            `long:"mongo-db" env:"MONGO_DB" default:"ureadability" description:"mongo database name"`
+	OpenAIKey   string            `long:"openai_key" env:"OPENAI_KEY" description:"OpenAI API key for summary generation"`
 	Debug       bool              `long:"dbg" env:"DEBUG" description:"debug mode"`
 }
 
@@ -47,7 +48,7 @@ func main() {
 		log.Fatalf("[ERROR] can't connect to mongo %v", err)
 	}
 	srv := rest.Server{
-		Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300, Rules: db.GetStores()},
+		Readability: extractor.UReadability{TimeOut: 30 * time.Second, SnippetSize: 300, Rules: db.GetStores(), OpenAIKey: opts.OpenAIKey},
 		Token:       opts.Token,
 		Credentials: opts.Credentials,
 		Version:     revision,

backend/main.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -176,11 +177,27 @@ func (s *Server) extractArticle(w http.ResponseWriter, r *http.Request) {
 // if token is not set for application, it won't be checked
 func (s *Server) extractArticleEmulateReadability(w http.ResponseWriter, r *http.Request) {
 	token := r.URL.Query().Get("token")
+	summary, _ := strconv.ParseBool(r.URL.Query().Get("summary"))
+
 	if s.Token != "" && token == "" {
 		rest.SendErrorJSON(w, r, log.Default(), http.StatusExpectationFailed, nil, "no token passed")
 		return
 	}
 
+	// Check if summary is requested but token is not provided, or OpenAI key is not set
+	if summary {
+		if s.Readability.OpenAIKey == "" {
+			render.Status(r, http.StatusBadRequest)
+			render.JSON(w, r, JSON{"error": "OpenAI key is not set"})
+			return
+		}
+		if s.Token == "" {
+			render.Status(r, http.StatusBadRequest)
+			render.JSON(w, r, JSON{"error": "summary generation requires token, but token is not set for the server"})
+			return
+		}
+	}
+
 	if s.Token != "" && s.Token != token {
 		rest.SendErrorJSON(w, r, log.Default(), http.StatusUnauthorized, nil, "wrong token passed")
 		return
@@ -198,6 +215,16 @@ func (s *Server) extractArticleEmulateReadability(w http.ResponseWriter, r *http
 		return
 	}
 
+	if summary {
+		summaryText, err := s.Readability.GenerateSummary(r.Context(), res.Content)
+		if err != nil {
+			render.Status(r, http.StatusInternalServerError)
+			render.JSON(w, r, JSON{"error": fmt.Sprintf("failed to generate summary: %v", err)})
+			return
+		}
+		res.Summary = summaryText
+	}
+
 	rest.RenderJSON(w, &res)
 }
 
@@ -237,6 +264,13 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 
+		if s.Readability.OpenAIKey != "" {
+			result.Summary, e = s.Readability.GenerateSummary(r.Context(), result.Content)
+			if e != nil {
+				log.Printf("[WARN] failed to generate summary for preview of %s: %v", url, e)
+			}
+		}
+
 		responses = append(responses, *result)
 	}
 
@@ -247,6 +281,7 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
 		Excerpt string
 		Rich    template.HTML
 		Content string
+		Summary template.HTML
 	}
 
 	results := make([]result, 0, len(responses))
@@ -258,6 +293,8 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
 			//nolint:gosec // this content is escaped by Extractor, so it's safe to use it as is
 			Rich:    template.HTML(r.Rich),
 			Content: r.Content,
+			//nolint: gosec // we do not expect CSS from OpenAI response
+			Summary: template.HTML(strings.ReplaceAll(r.Summary, "\n", "<br>")),
 		})
 	}