Add article summary feature with OpenAI integration

- Introduce 'summary' query parameter in /api/content/v1/parser endpoint
- Integrate OpenAI API for generating article summaries
- Add OpenAIKey field to Server struct and corresponding command-line
flag
- Update extractArticleEmulateReadability to handle summary requests
- Add generateSummary method using OpenAI's GPT-4o model (turns out to
be faster than even 4o mini)
- Add OpenAIClient interface and mock for testing
- Update README.md with new configuration options and API details

This feature allows users to request a summary of extracted articles
using OpenAI's GPT-4o model. To ensure secure usage, summary generation
requires a valid server token. The changes include comprehensive error
handling and test coverage for various scenarios, including token
validation and server misconfiguration.

Author: paskal

README.md

@@ -11,6 +11,7 @@
 | address      | UKEEPER_ADDRESS | all interfaces | web server listening address                          |
 | port         | UKEEPER_PORT    | `8080`         | web server port                                       |
 | mongo_uri    | MONGO_URI       | none           | MongoDB connection string, _required_                 |
+| openai_key   | OPENAI_KEY      | none           | OpenAI API key for summary generation                 |
 | frontend_dir | FRONTEND_DIR    | `/srv/web`     | directory with frontend files                         |
 | token        | TOKEN           | none           | token for /content/v1/parser endpoint auth            |
 | mongo-delay  | MONGO_DELAY     | `0`            | mongo initial delay                                   |
@@ -20,74 +21,9 @@
 
 ### API
 
-    GET /api/content/v1/parser?token=secret&url=http://aa.com/blah - extract content (emulate Readability API parse call)
+    GET /api/content/v1/parser?token=secret&summary=true&url=http://aa.com/blah - extract content (emulate Readability API parse call), summary is optional and requires OpenAI key and token to be enabled
     POST /api/v1/extract {url: http://aa.com/blah}  - extract content
 
-    POST /api/v1/rule {"domain": "aa.com", content="#content p"} - add/update custom rule
-    DELETE /api/v1/rule/:id - delete (disable) rule by ID
-    GET /api/v1/rule?url=http://blah.com/aaa - get rule for url
-    GET /api/v1/rules - get all rules, enabled and disabled
-
-#### testing
-
-on master (dev version) prefix /ureadability should be added
-
-<details><summary>HTTP calls</summary>
-
-    http POST "master.radio-t.com:8780/ureadability/api/v1/rule" domain=blah.ukeeper.com content="#content p" enabled:=true
-    HTTP/1.1 200 OK
-    Access-Control-Allow-Headers: Content-Type, Authorization, X-Requested-With
-    Access-Control-Allow-Methods: GET, PUT, POST, DELETE, OPTIONS
-    Access-Control-Allow-Origin: *
-    Application-Name: ureadability
-    Connection: keep-alive
-    Content-Length: 110
-    Content-Type: application/json; charset=utf-8
-    Date: Mon, 11 Jan 2016 02:38:13 GMT
-    Org: Umputun
-    Server: nginx/1.9.7
-
-    {
-        "content": "#content p",
-        "domain": "blah.ukeeper.com",
-        "enabled": true,
-        "id": "56931595daa6d301279ba801",
-        "user": ""
-    }
-
-
-    http  "master.radio-t.com:8780/ureadability/api/v1/rules"
-    HTTP/1.1 200 OK
-    Access-Control-Allow-Headers: Content-Type, Authorization, X-Requested-With
-    Access-Control-Allow-Methods: GET, PUT, POST, DELETE, OPTIONS
-    Access-Control-Allow-Origin: *
-    Application-Name: ureadability
-    Connection: keep-alive
-    Content-Length: 219
-    Content-Type: application/json; charset=utf-8
-    Date: Mon, 11 Jan 2016 02:38:34 GMT
-    Org: Umputun
-    Server: nginx/1.9.7
-
-    [
-        {
-            "content": "#content p",
-            "domain": "p.ukeeper.com",
-            "enabled": true,
-            "id": "5693123fdaa6d301279ba800",
-            "user": ""
-        },
-        {
-            "content": "#content p",
-            "domain": "blah.ukeeper.com",
-            "enabled": true,
-            "id": "56931595daa6d301279ba801",
-            "user": ""
-        }
-    ]
-
-</details>
-
 ## Development
 
 ### Running tests

backend/extractor/readability.go

@@ -37,6 +37,7 @@ type UReadability struct {
 
 // Response from api calls
 type Response struct {
+	Summary     string   `json:"summary,omitempty"`
 	Content     string   `json:"content"`
 	Rich        string   `json:"rich_content"`
 	Domain      string   `json:"domain"`

backend/go.mod

@@ -15,6 +15,7 @@ require (
 	github.com/jessevdk/go-flags v1.6.1
 	github.com/kennygrant/sanitize v1.2.4
 	github.com/mauidude/go-readability v0.0.0-20220221173116-a9b3620098b7
+	github.com/sashabaranov/go-openai v1.28.1
 	github.com/stretchr/testify v1.9.0
 	go.mongodb.org/mongo-driver v1.16.1
 	golang.org/x/net v0.28.0

backend/go.sum

@@ -183,6 +183,8 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sashabaranov/go-openai v1.28.1 h1:aREx6faUTeOZNMDTNGAY8B9vNmmN7qoGvDV0Ke2J1Mc=
+github.com/sashabaranov/go-openai v1.28.1/go.mod h1:lj5b/K+zjTSFxVLijLSTDZuP7adOgerWeFyZLUhAKRg=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

backend/main.go

@@ -26,6 +26,7 @@ var opts struct {
 	MongoURI    string            `short:"m" long:"mongo_uri" env:"MONGO_URI" required:"true" description:"MongoDB connection string"`
 	MongoDelay  time.Duration     `long:"mongo-delay" env:"MONGO_DELAY" default:"0" description:"mongo initial delay"`
 	MongoDB     string            `long:"mongo-db" env:"MONGO_DB" default:"ureadability" description:"mongo database name"`
+	OpenAIKey   string            `long:"openai_key" env:"OPENAI_KEY" description:"OpenAI API key for summary generation"`
 	Debug       bool              `long:"dbg" env:"DEBUG" description:"debug mode"`
 }
 
@@ -45,6 +46,7 @@ func main() {
 		Token:       opts.Token,
 		Credentials: opts.Credentials,
 		Version:     revision,
+		OpenAIKey:   opts.OpenAIKey,
 	}
 
 	ctx, cancel := context.WithCancel(context.Background())

backend/rest/openai_mock.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -20,21 +21,29 @@ import (
 	log "github.com/go-pkgz/lgr"
 	UM "github.com/go-pkgz/rest"
 	"github.com/go-pkgz/rest/logger"
+	"github.com/sashabaranov/go-openai"
 	"go.mongodb.org/mongo-driver/bson/primitive"
 
 	"github.com/ukeeper/ukeeper-redabilty/backend/datastore"
 	"github.com/ukeeper/ukeeper-redabilty/backend/extractor"
 )
 
+//go:generate moq -out openai_mock.go . OpenAIClient
+type OpenAIClient interface {
+	CreateChatCompletion(ctx context.Context, request openai.ChatCompletionRequest) (openai.ChatCompletionResponse, error)
+}
+
 // Server is a basic rest server providing access to store and invoking parser
 type Server struct {
 	Readability extractor.UReadability
 	Version     string
 	Token       string
 	Credentials map[string]string
+	OpenAIKey   string
 
-	indexPage *template.Template
-	rulePage  *template.Template
+	openAIClient OpenAIClient
+	indexPage    *template.Template
+	rulePage     *template.Template
 }
 
 // JSON is a map alias, just for convenience
@@ -181,16 +190,32 @@ func (s *Server) extractArticle(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, &res)
 }
 
-// emulate readability API parse - https://www.readability.com/api/content/v1/parser?token=%s&url=%s
+// extractArticleEmulateReadability emulates readability API parse - https://www.readability.com/api/content/v1/parser?token=%s&url=%s
 // if token is not set for application, it won't be checked
 func (s *Server) extractArticleEmulateReadability(w http.ResponseWriter, r *http.Request) {
 	token := r.URL.Query().Get("token")
+	summary, _ := strconv.ParseBool(r.URL.Query().Get("summary"))
+
 	if s.Token != "" && token == "" {
 		render.Status(r, http.StatusExpectationFailed)
 		render.JSON(w, r, JSON{"error": "no token passed"})
 		return
 	}
 
+	// Check if summary is requested but token is not provided, or OpenAI key is not set
+	if summary {
+		if s.OpenAIKey == "" {
+			render.Status(r, http.StatusBadRequest)
+			render.JSON(w, r, JSON{"error": "OpenAI key is not set"})
+			return
+		}
+		if s.Token == "" {
+			render.Status(r, http.StatusBadRequest)
+			render.JSON(w, r, JSON{"error": "summary generation requires token, but token is not set for the server"})
+			return
+		}
+	}
+
 	if s.Token != "" && s.Token != token {
 		render.Status(r, http.StatusUnauthorized)
 		render.JSON(w, r, JSON{"error": "wrong token passed"})
@@ -211,6 +236,16 @@ func (s *Server) extractArticleEmulateReadability(w http.ResponseWriter, r *http
 		return
 	}
 
+	if summary {
+		summaryText, err := s.generateSummary(r.Context(), res.Content)
+		if err != nil {
+			render.Status(r, http.StatusInternalServerError)
+			render.JSON(w, r, JSON{"error": fmt.Sprintf("failed to generate summary: %v", err)})
+			return
+		}
+		res.Summary = summaryText
+	}
+
 	render.JSON(w, r, &res)
 }
 
@@ -250,6 +285,13 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 
+		if s.OpenAIKey != "" {
+			result.Summary, e = s.generateSummary(r.Context(), result.Content)
+			if e != nil {
+				log.Printf("[WARN] failed to generate summary for preview of %s: %v", url, e)
+			}
+		}
+
 		responses = append(responses, *result)
 	}
 
@@ -260,6 +302,7 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
 		Excerpt string
 		Rich    template.HTML
 		Content string
+		Summary template.HTML
 	}
 
 	var results []result
@@ -270,6 +313,7 @@ func (s *Server) handlePreview(w http.ResponseWriter, r *http.Request) {
 			//nolint: gosec // this content is escaped by Extractor, so it's safe to use it as is
 			Rich:    template.HTML(r.Rich),
 			Content: r.Content,
+			Summary: template.HTML(strings.ReplaceAll(r.Summary, "\n", "<br>")),
 		})
 	}
 
@@ -354,6 +398,34 @@ func (s *Server) authFake(w http.ResponseWriter, r *http.Request) {
 	render.JSON(w, r, JSON{"pong": t.Format("20060102150405")})
 }
 
+func (s *Server) generateSummary(ctx context.Context, content string) (string, error) {
+	if s.openAIClient == nil {
+		s.openAIClient = openai.NewClient(s.OpenAIKey)
+	}
+	resp, err := s.openAIClient.CreateChatCompletion(
+		ctx,
+		openai.ChatCompletionRequest{
+			Model: openai.GPT4o,
+			Messages: []openai.ChatCompletionMessage{
+				{
+					Role:    openai.ChatMessageRoleSystem,
+					Content: "You are a helpful assistant that summarizes articles. Please summarize the main points in a few sentences as TLDR style (don't add a TLDR label). Then, list up to five detailed bullet points. Provide the response in plain text. Do not add any additional information. Do not add a Summary at the beginning of the response. If detailed bullet points are too similar to the summary, don't include them at all:",
+				},
+				{
+					Role:    openai.ChatMessageRoleUser,
+					Content: content,
+				},
+			},
+		},
+	)
+
+	if err != nil {
+		return "", err
+	}
+
+	return resp.Choices[0].Message.Content, nil
+}
+
 func getBid(id string) primitive.ObjectID {
 	bid, err := primitive.ObjectIDFromHex(id)
 	if err != nil {