feat(stars): Achieve sampling mode to obtain star historical data from a large database repository (#248)

- Add GitHub Max sample pages configuration option
- Implement Link header parsing to get the actual maximum number of pages
- Add a sampling algorithm to evenly distribute the page fetching strategy
- Support pagination sampling data fetching for large repositories
- Optimize concurrent request limits and error handling mechanisms
- Change the Y-axis data of the chart to use actual counts instead of index values

Author: lik0914

README.md

@@ -6,6 +6,22 @@
 
 Plot your repo stars over time!
 
+## Features
+
+### Smart Sampling Mode (Large Repository Optimization)
+
+For large repositories with massive amounts of stars, this project uses **Smart Sampling Mode** to efficiently fetch star history data and render trend charts.
+
+**How it works:**
+
+1. **Auto Detection**: First requests the first page of GitHub API data and parses the `Link` Header to get total page count
+2. **Mode Switching**:
+   - When total pages ≤ `maxSamplePages` (default 15 pages, ~1500 stars), fetches all data
+   - When total pages > `maxSamplePages`, automatically switches to sampling mode
+3. **Uniform Sampling**: Evenly selects sample points across all pages to ensure coverage of the complete star growth timeline
+4. **Data Point Extraction**: Extracts the timestamp and corresponding star count from the first Stargazer of each sampled page
+5. **Trend Completion**: Adds current time and total star count as the final data point to ensure the chart extends to the latest state
+
 ## Usage
 
 ```console
@@ -14,6 +30,19 @@ go run main.go
 
 Then browse http://localhost:3000/me/myrepo .
 
-Example chart:
+## Configuration
+
+Configure via environment variables:
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `REDIS_URL` | `redis://localhost:6379` | Redis cache URL |
+| `GITHUB_TOKENS` | - | GitHub API Token (supports multiple, comma-separated) |
+| `GITHUB_PAGE_SIZE` | `100` | Number of stars per page |
+| `GITHUB_MAX_SAMPLE_PAGES` | `15` | Max sample pages (triggers sampling mode when exceeded) |
+| `GITHUB_MAX_RATE_LIMIT_USAGE` | `80` | API Rate Limit usage threshold percentage |
+| `LISTEN` | `127.0.0.1:3000` | Server listen address |
+
+## Example
 
 [![starcharts stargazers over time](https://starchart.cc/caarlos0/starcharts.svg)](https://starchart.cc/caarlos0/starcharts)

config/config.go

@@ -11,6 +11,7 @@ type Config struct {
 	GitHubTokens          []string `env:"GITHUB_TOKENS"`
 	GitHubPageSize        int      `env:"GITHUB_PAGE_SIZE" envDefault:"100"`
 	GitHubMaxRateUsagePct int      `env:"GITHUB_MAX_RATE_LIMIT_USAGE" envDefault:"80"`
+	GitHubMaxSamplePages  int      `env:"GITHUB_MAX_SAMPLE_PAGES" envDefault:"15"`
 	Listen                string   `env:"LISTEN" envDefault:"127.0.0.1:3000"`
 }
 

controller/chart.go

@@ -65,7 +65,13 @@ func GetRepoChart(gh *github.GitHub, cache *cache.Redis) http.Handler {
 		}
 		for i, star := range stargazers {
 			series.XValues = append(series.XValues, star.StarredAt)
-			series.YValues = append(series.YValues, float64(i+1))
+			// If star.Count > 0, use the actual count from sampling mode
+			// Otherwise use index+1 (non-sampling mode, continuous data)
+			if star.Count > 0 {
+				series.YValues = append(series.YValues, float64(star.Count))
+			} else {
+				series.YValues = append(series.YValues, float64(i+1))
+			}
 		}
 		if len(series.XValues) < 2 {
 			log.Info("not enough results, adding some fake ones")

internal/github/github.go

@@ -24,6 +24,7 @@ var ErrGitHubAPI = errors.New("failed to talk with github api")
 type GitHub struct {
 	tokens          roundrobin.RoundRobiner
 	pageSize        int
+	maxSamplePages  int
 	cache           *cache.Redis
 	maxRateUsagePct int
 }
@@ -66,9 +67,10 @@ func init() {
 func New(config config.Config, cache *cache.Redis) *GitHub {
 	tokensCount.Set(float64(len(config.GitHubTokens)))
 	return &GitHub{
-		tokens:   roundrobin.New(config.GitHubTokens),
-		pageSize: config.GitHubPageSize,
-		cache:    cache,
+		tokens:         roundrobin.New(config.GitHubTokens),
+		pageSize:       config.GitHubPageSize,
+		maxSamplePages: config.GitHubMaxSamplePages,
+		cache:          cache,
 	}
 }
 

internal/github/stars.go

@@ -6,8 +6,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"net/http"
+	"regexp"
 	"sort"
+	"strconv"
 	"sync"
 	"time"
 
@@ -16,28 +19,128 @@ import (
 )
 
 var (
-	errNoMorePages  = errors.New("no more pages to get")
-	ErrTooManyStars = errors.New("repo has too many stargazers, github won't allow us to list all stars")
+	errNoMorePages = errors.New("no more pages to get")
+	// linkLastPageRegex is used to parse the last page number from the Link header
+	linkLastPageRegex = regexp.MustCompile(`[&?]page=(\d+)[^>]*>;\s*rel="last"`)
 )
 
+// maxConcurrentRequests is the maximum number of concurrent requests to GitHub API
+const maxConcurrentRequests = 5
+
 // Stargazer is a star at a given time.
 type Stargazer struct {
 	StarredAt time.Time `json:"starred_at"`
+	// Count represents the actual position/count of this star (used in sampling mode).
+	// If 0, use index+1 as count (non-sampling mode).
+	Count int `json:"-"`
 }
 
 // Stargazers returns all the stargazers of a given repo.
+// If star count is too large, it uses sampling mode to fetch data points.
 func (gh *GitHub) Stargazers(ctx context.Context, repo Repository) (stars []Stargazer, err error) {
-	if gh.totalPages(repo) > 400 {
-		return stars, ErrTooManyStars
+	// First request the first page to get the actual max page count (via Link header)
+	firstPageStars, lastPage, err := gh.getFirstPageAndLastPage(ctx, repo)
+	if err != nil {
+		return nil, err
+	}
+
+	log.WithField("repo", repo.FullName).
+		WithField("lastPage", lastPage).
+		WithField("starCount", repo.StargazersCount).
+		Debug("got pagination info from API")
+
+	// If only one page or page count is less than max sample pages, fetch all pages
+	if lastPage <= gh.maxSamplePages {
+		return gh.getAllStargazersWithFirstPage(ctx, repo, firstPageStars, lastPage)
+	}
+
+	// Otherwise use sampling mode
+	return gh.getSampledStargazers(ctx, repo, firstPageStars, lastPage)
+}
+
+// getFirstPageAndLastPage requests the first page and parses the Link header to get the max page count.
+func (gh *GitHub) getFirstPageAndLastPage(ctx context.Context, repo Repository) ([]Stargazer, int, error) {
+	log := log.WithField("repo", repo.FullName)
+
+	resp, err := gh.makeStarPageRequest(ctx, repo, 1, "")
+	if err != nil {
+		return nil, 0, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusForbidden {
+		rateLimits.Inc()
+		log.Warn("rate limit hit")
+		return nil, 0, ErrRateLimit
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		bts, _ := io.ReadAll(resp.Body)
+		return nil, 0, fmt.Errorf("%w: %v", ErrGitHubAPI, string(bts))
+	}
+
+	bts, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	var stars []Stargazer
+	if err := json.Unmarshal(bts, &stars); err != nil {
+		return nil, 0, err
+	}
+
+	// Parse Link header to get the max page count
+	linkHeader := resp.Header.Get("Link")
+	lastPage := gh.parseLastPageFromLink(linkHeader)
+
+	// If no Link header or parsing failed, there is only one page
+	if lastPage == 0 {
+		lastPage = 1
+	}
+
+	log.WithField("lastPage", lastPage).Debug("parsed last page from Link header")
+
+	return stars, lastPage, nil
+}
+
+// parseLastPageFromLink parses the max page count from the Link header.
+// Link header format: <url>; rel="next", <url>; rel="last"
+func (gh *GitHub) parseLastPageFromLink(linkHeader string) int {
+	if linkHeader == "" {
+		return 0
+	}
+
+	matches := linkLastPageRegex.FindStringSubmatch(linkHeader)
+	if len(matches) < 2 {
+		return 0
+	}
+
+	lastPage, err := strconv.Atoi(matches[1])
+	if err != nil {
+		return 0
+	}
+
+	return lastPage
+}
+
+// getAllStargazersWithFirstPage fetches all stargazers (used for small repositories).
+// firstPageStars is the already fetched first page data.
+func (gh *GitHub) getAllStargazersWithFirstPage(ctx context.Context, repo Repository, firstPageStars []Stargazer, lastPage int) (stars []Stargazer, err error) {
+	stars = append(stars, firstPageStars...)
+
+	// If only one page, return directly
+	if lastPage <= 1 {
+		return stars, nil
 	}
 
 	var (
 		wg   errgroup.Group
 		lock sync.Mutex
 	)
 
-	wg.SetLimit(4)
-	for page := 1; page <= gh.lastPage(repo); page++ {
+	wg.SetLimit(maxConcurrentRequests)
+	// Start fetching from page 2 (page 1 is already fetched)
+	for page := 2; page <= lastPage; page++ {
 		page := page
 		wg.Go(func() error {
 			result, err := gh.getStargazersPage(ctx, repo, page)
@@ -61,6 +164,134 @@ func (gh *GitHub) Stargazers(ctx context.Context, repo Repository) (stars []Star
 	return
 }
 
+// getSampledStargazers fetches stargazers using sampling mode (used for large repositories).
+// Inspired by star-history project's sampling logic.
+// firstPageStars is the already fetched first page data, lastPage is the actual max page count parsed from Link header.
+func (gh *GitHub) getSampledStargazers(ctx context.Context, repo Repository, firstPageStars []Stargazer, lastPage int) (stars []Stargazer, err error) {
+	log.WithField("repo", repo.FullName).
+		WithField("lastPage", lastPage).
+		Info("using sampling mode for large repo")
+
+	// Calculate sample page numbers, evenly distributed across all pages
+	samplePages := gh.calculateSamplePages(lastPage, gh.maxSamplePages)
+
+	type pageResult struct {
+		page      int
+		star      Stargazer
+		starCount int // the actual count position of this star
+	}
+
+	var (
+		wg      errgroup.Group
+		lock    sync.Mutex
+		results []pageResult
+	)
+
+	// First page is already fetched, add it to results directly
+	if len(firstPageStars) > 0 {
+		results = append(results, pageResult{
+			page:      1,
+			star:      firstPageStars[0],
+			starCount: 1,
+		})
+	}
+
+	wg.SetLimit(maxConcurrentRequests)
+	for _, page := range samplePages {
+		// Skip first page (already fetched)
+		if page == 1 {
+			continue
+		}
+		page := page
+		wg.Go(func() error {
+			result, err := gh.getStargazersPage(ctx, repo, page)
+			if errors.Is(err, errNoMorePages) {
+				return nil
+			}
+			if err != nil {
+				return err
+			}
+			if len(result) == 0 {
+				return nil
+			}
+
+			// Calculate the actual position of the first star on this page (based on page number and page size)
+			// The 1st star on page 1 is star #1
+			// The 1st star on page N is star #(N-1)*pageSize + 1
+			starCount := (page-1)*gh.pageSize + 1
+
+			lock.Lock()
+			defer lock.Unlock()
+			results = append(results, pageResult{
+				page:      page,
+				star:      result[0],
+				starCount: starCount,
+			})
+			return nil
+		})
+	}
+
+	if err = wg.Wait(); err != nil {
+		return nil, err
+	}
+
+	// Sort results by page number
+	sort.Slice(results, func(i, j int) bool {
+		return results[i].page < results[j].page
+	})
+
+	// Extract the first star from each sampled page as a data point and set Count
+	for _, r := range results {
+		star := r.star
+		star.Count = r.starCount
+		stars = append(stars, star)
+	}
+
+	// Add the last data point (current time and total star count)
+	// This ensures the chart extends to the current time point
+	stars = append(stars, Stargazer{
+		StarredAt: time.Now(),
+		Count:     repo.StargazersCount,
+	})
+
+	return stars, nil
+}
+
+// calculateSamplePages calculates the page numbers to sample.
+// Evenly distributed across all pages, ensuring the first page is included.
+func (gh *GitHub) calculateSamplePages(totalPages, maxSamples int) []int {
+	pages := make([]int, 0, maxSamples)
+
+	for i := 1; i <= maxSamples; i++ {
+		// Calculate evenly distributed page numbers
+		page := int(math.Round(float64(i*totalPages) / float64(maxSamples)))
+		if page < 1 {
+			page = 1
+		}
+		if page > totalPages {
+			page = totalPages
+		}
+		pages = append(pages, page)
+	}
+
+	// Ensure first page is included (important for displaying start time)
+	if len(pages) > 0 && pages[0] != 1 {
+		pages[0] = 1
+	}
+
+	// Deduplicate (may have duplicates in edge cases)
+	seen := make(map[int]bool)
+	uniquePages := make([]int, 0, len(pages))
+	for _, p := range pages {
+		if !seen[p] {
+			seen[p] = true
+			uniquePages = append(uniquePages, p)
+		}
+	}
+
+	return uniquePages
+}
+
 // - get last modified from cache
 //   - if exists, hit api with it
 //     - if it returns 304, get from cache
@@ -139,14 +370,6 @@ func (gh *GitHub) getStargazersPage(ctx context.Context, repo Repository, page i
 	}
 }
 
-func (gh *GitHub) totalPages(repo Repository) int {
-	return repo.StargazersCount / gh.pageSize
-}
-
-func (gh *GitHub) lastPage(repo Repository) int {
-	return gh.totalPages(repo) + 1
-}
-
 func (gh *GitHub) makeStarPageRequest(ctx context.Context, repo Repository, page int, etag string) (*http.Response, error) {
 	url := fmt.Sprintf(
 		"https://api.github.com/repos/%s/stargazers?page=%d&per_page=%d",