Skip to content

Commit 23dac43

Browse files
arpitharpith
authored
feat: Port feed utilities to Go with shared test data
Ports all JavaScript feed utility functions to Go with comprehensive test coverage. ## Changes - Created feedfetcher package with all utility functions - Hash(), Score(), BuildRequestHeaders(), BuildRedisKeys(), etc. - All functions tested using shared YAML test data - Added Go tests to GitHub Actions CI ## Test Coverage - 38 Go tests passing ✓ - 29 JavaScript tests passing ✓ - All tests use shared testdata/test-cases.yaml 🤖 Generated with [Claude Code](https://claude.com/claude-code)
1 parent 69dd981 commit 23dac43

File tree

10 files changed

+8401
-50
lines changed

10 files changed

+8401
-50
lines changed

‎.github/workflows/test.yml‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ on:
77
branches: [ master ]
88

99
jobs:
10-
test:
10+
test-nodejs:
1111
runs-on: ubuntu-latest
1212

1313
strategy:
@@ -30,3 +30,21 @@ jobs:
3030

3131
- name: Run feed utility tests
3232
run: node src/lib/feedUtils.test.js
33+
34+
test-go:
35+
runs-on: ubuntu-latest
36+
37+
strategy:
38+
matrix:
39+
go-version: ['1.22', '1.23']
40+
41+
steps:
42+
- uses: actions/checkout@v3
43+
44+
- name: Set up Go ${{ matrix.go-version }}
45+
uses: actions/setup-go@v4
46+
with:
47+
go-version: ${{ matrix.go-version }}
48+
49+
- name: Run Go tests
50+
run: go test -v ./feedfetcher/...

‎feedfetcher/articles.go‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package feedfetcher
2+
3+
import (
4+
"crypto/md5"
5+
"encoding/hex"
6+
"time"
7+
)
8+
9+
// Article represents a feed article with required fields
10+
type Article struct {
11+
GUID string `json:"guid" yaml:"guid"`
12+
Title string `json:"title,omitempty" yaml:"title,omitempty"`
13+
Description string `json:"description,omitempty" yaml:"description,omitempty"`
14+
PubDate string `json:"pubDate,omitempty" yaml:"pubDate,omitempty"`
15+
Pubdate string `json:"pubdate,omitempty" yaml:"pubdate,omitempty"`
16+
Date string `json:"date,omitempty" yaml:"date,omitempty"`
17+
Hash string `json:"hash,omitempty" yaml:"hash,omitempty"`
18+
Score int64 `json:"score,omitempty" yaml:"score,omitempty"`
19+
FeedURL string `json:"feedurl,omitempty" yaml:"feedurl,omitempty"`
20+
}
21+
22+
// Hash generates MD5 hash of article GUID
23+
// Reference: api/src/lib/articleUtils.js hash() function
24+
func Hash(article Article) string {
25+
hasher := md5.New()
26+
hasher.Write([]byte(article.GUID))
27+
return hex.EncodeToString(hasher.Sum(nil))
28+
}
29+
30+
// Score generates score (timestamp) for article
31+
// Reference: api/src/lib/articleUtils.js score() function
32+
func Score(article Article) int64 {
33+
// Try pubDate, pubdate, date in order
34+
articleDate := article.PubDate
35+
if articleDate == "" {
36+
articleDate = article.Pubdate
37+
}
38+
if articleDate == "" {
39+
articleDate = article.Date
40+
}
41+
42+
// Try to parse the date
43+
if articleDate != "" {
44+
// Try RFC3339 format first (ISO 8601)
45+
t, err := time.Parse(time.RFC3339, articleDate)
46+
if err == nil {
47+
return t.UnixMilli()
48+
}
49+
}
50+
51+
// If no valid date, return current time
52+
return time.Now().UnixMilli()
53+
}

‎feedfetcher/articles_test.go‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package feedfetcher
2+
3+
import (
4+
"os"
5+
"testing"
6+
7+
"gopkg.in/yaml.v3"
8+
)
9+
10+
type TestCases struct {
11+
HashFunctionTests []struct {
12+
Description string `yaml:"description"`
13+
Input struct {
14+
GUID string `yaml:"guid"`
15+
} `yaml:"input"`
16+
Expected string `yaml:"expected"`
17+
} `yaml:"hash_function_tests"`
18+
19+
ScoreFunctionTests []struct {
20+
Description string `yaml:"description"`
21+
Input Article `yaml:"input"`
22+
Expected int64 `yaml:"expected"`
23+
ExpectedType string `yaml:"expected_type"`
24+
} `yaml:"score_function_tests"`
25+
}
26+
27+
func TestHash(t *testing.T) {
28+
// Load test cases from YAML
29+
data, err := os.ReadFile("../testdata/test-cases.yaml")
30+
if err != nil {
31+
t.Fatalf("Failed to read test data: %v", err)
32+
}
33+
34+
var testCases TestCases
35+
if err := yaml.Unmarshal(data, &testCases); err != nil {
36+
t.Fatalf("Failed to parse test data: %v", err)
37+
}
38+
39+
for _, tc := range testCases.HashFunctionTests {
40+
t.Run(tc.Description, func(t *testing.T) {
41+
article := Article{GUID: tc.Input.GUID}
42+
result := Hash(article)
43+
if result != tc.Expected {
44+
t.Errorf("Hash mismatch: got %s, expected %s", result, tc.Expected)
45+
}
46+
})
47+
}
48+
}
49+
50+
func TestScore(t *testing.T) {
51+
// Load test cases from YAML
52+
data, err := os.ReadFile("../testdata/test-cases.yaml")
53+
if err != nil {
54+
t.Fatalf("Failed to read test data: %v", err)
55+
}
56+
57+
var testCases TestCases
58+
if err := yaml.Unmarshal(data, &testCases); err != nil {
59+
t.Fatalf("Failed to parse test data: %v", err)
60+
}
61+
62+
for _, tc := range testCases.ScoreFunctionTests {
63+
t.Run(tc.Description, func(t *testing.T) {
64+
result := Score(tc.Input)
65+
if tc.ExpectedType == "timestamp" {
66+
// For invalid dates that fallback to time.Now(), just check it's positive
67+
if result <= 0 {
68+
t.Errorf("Score should be positive: got %d", result)
69+
}
70+
} else {
71+
if result != tc.Expected {
72+
t.Errorf("Score mismatch: got %d, expected %d", result, tc.Expected)
73+
}
74+
}
75+
})
76+
}
77+
}

‎feedfetcher/utils.go‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package feedfetcher
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"strconv"
7+
)
8+
9+
// Redis key prefixes - used for building and parsing keys
10+
const (
11+
RedisFeedPrefix = "feed:"
12+
RedisArticlesPrefix = "articles:"
13+
RedisArticlePrefix = "article:"
14+
)
15+
16+
// BuildRequestHeaders builds request headers for conditional GET requests
17+
// Reference: api/src/lib/feedUtils.js buildRequestHeaders() function
18+
func BuildRequestHeaders(lastModified, etag string) map[string]string {
19+
headers := map[string]string{
20+
"user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36",
21+
}
22+
23+
if lastModified != "" {
24+
headers["If-Modified-Since"] = lastModified
25+
}
26+
27+
if etag != "" {
28+
headers["If-None-Match"] = etag
29+
}
30+
31+
return headers
32+
}
33+
34+
// RedisKeys represents Redis key names for a feed
35+
type RedisKeys struct {
36+
FeedKey string `json:"feedKey" yaml:"feedKey"`
37+
ArticlesKey string `json:"articlesKey" yaml:"articlesKey"`
38+
}
39+
40+
// BuildRedisKeys builds Redis key names for a feed
41+
// Reference: api/src/lib/feedUtils.js buildRedisKeys() function
42+
func BuildRedisKeys(feedURI string) RedisKeys {
43+
return RedisKeys{
44+
FeedKey: RedisFeedPrefix + feedURI,
45+
ArticlesKey: RedisArticlesPrefix + feedURI,
46+
}
47+
}
48+
49+
// BuildArticleKey builds the article key for Redis sorted set
50+
// Reference: api/src/lib/feedUtils.js buildArticleKey() function
51+
func BuildArticleKey(hash string) string {
52+
return RedisArticlePrefix + hash
53+
}
54+
55+
// ProcessArticle processes an article by adding computed fields
56+
// Reference: api/src/lib/feedUtils.js processArticle() function
57+
func ProcessArticle(article Article, feedURI string) Article {
58+
processed := article
59+
processed.Hash = Hash(article)
60+
processed.Score = Score(article)
61+
processed.FeedURL = feedURI
62+
return processed
63+
}
64+
65+
// ShouldStoreArticle determines if an article should be stored in S3
66+
// Reference: api/src/lib/feedUtils.js shouldStoreArticle() function
67+
func ShouldStoreArticle(oldScore *string, newScore int64) bool {
68+
// Store if article is new (oldScore is nil)
69+
if oldScore == nil {
70+
return true
71+
}
72+
73+
// Parse oldScore as int64
74+
oldScoreInt, err := strconv.ParseInt(*oldScore, 10, 64)
75+
if err != nil {
76+
// If parsing fails, treat as different score
77+
return true
78+
}
79+
80+
// Store if score has changed
81+
return newScore != oldScoreInt
82+
}
83+
84+
// IsValidArticle validates that an article has required fields
85+
// Reference: api/src/lib/feedUtils.js isValidArticle() function
86+
func IsValidArticle(article *Article) bool {
87+
return article != nil && article.GUID != "" && article.Description != ""
88+
}
89+
90+
// ExtractArticleIds extracts article IDs (hashes) from Redis keys by removing the "article:" prefix
91+
// Reference: api/src/lib/feedUtils.js extractArticleIds() function
92+
func ExtractArticleIds(articleKeys []string) []string {
93+
prefixLength := len(RedisArticlePrefix)
94+
ids := make([]string, len(articleKeys))
95+
for i, key := range articleKeys {
96+
if len(key) > prefixLength {
97+
ids[i] = key[prefixLength:]
98+
} else {
99+
ids[i] = key
100+
}
101+
}
102+
return ids
103+
}
104+
105+
// GenerateArticleBody generates JSON body for S3 storage
106+
// Reference: api/src/lib/feedUtils.js generateArticleBody() function
107+
func GenerateArticleBody(article Article) (string, error) {
108+
body, err := json.Marshal(article)
109+
if err != nil {
110+
return "", fmt.Errorf("failed to marshal article: %w", err)
111+
}
112+
return string(body), nil
113+
}

0 commit comments

Comments
 (0)