fix: RSS parsing with comments from stacker.news and HTML entities encoding

Author: piraces

pkg/feed/feed.go

@@ -7,17 +7,17 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"github.com/piraces/rsslay/pkg/helpers"
-	"log"
-	"net/http"
-	"strings"
-	"time"
-
 	"github.com/PuerkitoBio/goquery"
 	strip "github.com/grokify/html-strip-tags-go"
 	"github.com/mmcdole/gofeed"
 	"github.com/nbd-wtf/go-nostr"
+	"github.com/piraces/rsslay/pkg/helpers"
 	"github.com/rif/cache2go"
+	"html"
+	"log"
+	"net/http"
+	"strings"
+	"time"
 )
 
 var (
@@ -145,8 +145,14 @@ func ItemToTextNote(pubkey string, item *gofeed.Item, feed *gofeed.Feed, default
 	if item.Title != "" {
 		content = "**" + item.Title + "**\n\n"
 	}
+
 	description := strip.StripTags(item.Description)
 
+	// Handle stacker.news comments
+	if strings.Contains(feed.Link, "stacker.news") {
+		description += fmt.Sprintf(": %s", item.GUID)
+	}
+
 	if !strings.EqualFold(item.Title, description) {
 		content += description
 	}
@@ -176,6 +182,7 @@ func ItemToTextNote(pubkey string, item *gofeed.Item, feed *gofeed.Feed, default
 		content += description
 	}
 
+	content = html.UnescapeString(content)
 	if len(content) > 250 {
 		content = content[0:249] + "…"
 	}
@@ -198,7 +205,7 @@ func ItemToTextNote(pubkey string, item *gofeed.Item, feed *gofeed.Feed, default
 		CreatedAt: createdAt,
 		Kind:      nostr.KindTextNote,
 		Tags:      nostr.Tags{},
-		Content:   content,
+		Content:   strings.ToValidUTF8(content, ""),
 	}
 	evt.ID = string(evt.Serialize())
 

pkg/feed/feed_test.go

@@ -42,6 +42,16 @@ var sampleNitterFeed = gofeed.Feed{
 	},
 }
 
+var sampleStackerNewsFeed = gofeed.Feed{
+	Title:           "Stacker News",
+	Description:     "Like Hacker News, but we pay you Bitcoin.",
+	Link:            "https://stacker.news",
+	FeedLink:        "https://stacker.news/rss",
+	Links:           []string{"https://blog.cryptographyengineering.com/2014/11/zero-knowledge-proofs-illustrated-primer.html"},
+	PublishedParsed: &actualTime,
+	Language:        "en",
+}
+
 var sampleNitterFeedRTItem = gofeed.Item{
 	Title:           "RT by @coldplay: TOMORROW",
 	Description:     "Sample description",
@@ -81,6 +91,16 @@ var sampleDefaultFeedItem = gofeed.Item{
 var sampleDefaultFeedItemExpectedContent = fmt.Sprintf("**%s**\n\n%s", sampleDefaultFeedItem.Title, sampleDefaultFeedItem.Description)
 var sampleDefaultFeedItemExpectedContentSubstring = sampleDefaultFeedItemExpectedContent[0:249]
 
+var sampleStackerNewsFeedItem = gofeed.Item{
+	Title:           "Zero Knowledge Proofs: An illustrated primer",
+	Description:     "<a href=\"https://stacker.news/items/131533\">Comments</a>",
+	Content:         "Sample content",
+	Link:            "https://blog.cryptographyengineering.com/2014/11/zero-knowledge-proofs-illustrated-primer.html",
+	UpdatedParsed:   &actualTime,
+	PublishedParsed: &actualTime,
+	GUID:            "https://stacker.news/items/131533",
+}
+
 var sampleDefaultFeed = gofeed.Feed{
 	Title:           "Golang Weekly",
 	Description:     "A weekly newsletter about the Go programming language",
@@ -216,6 +236,14 @@ func TestItemToTextNote(t *testing.T) {
 			originalUrl:      sampleDefaultFeed.FeedLink,
 			expectedContent:  sampleDefaultFeedItemExpectedContentSubstring + "…" + "\n\n" + sampleDefaultFeedItem.Link,
 		},
+		{
+			pubKey:           samplePubKey,
+			item:             &sampleStackerNewsFeedItem,
+			feed:             &sampleStackerNewsFeed,
+			defaultCreatedAt: actualTime,
+			originalUrl:      sampleStackerNewsFeed.FeedLink,
+			expectedContent:  fmt.Sprintf("**%s**\n\nComments: %s\n\n%s", sampleStackerNewsFeedItem.Title, sampleStackerNewsFeedItem.GUID, sampleStackerNewsFeedItem.Link),
+		},
 	}
 	for _, tc := range testCases {
 		event := ItemToTextNote(tc.pubKey, tc.item, tc.feed, tc.defaultCreatedAt, tc.originalUrl)