Improve Metadata for Feeds based on YT playlists

pkg/builder/builder.go

@@ -13,10 +13,10 @@ type Builder interface {
 	Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error)
 }
 
-func New(ctx context.Context, provider model.Provider, key string) (Builder, error) {
+func New(ctx context.Context, provider model.Provider, key string, downloader Downloader) (Builder, error) {
 	switch provider {
 	case model.ProviderYoutube:
-		return NewYouTubeBuilder(key)
+		return NewYouTubeBuilder(key, downloader)
 	case model.ProviderVimeo:
 		return NewVimeoBuilder(ctx, key)
 	case model.ProviderSoundcloud:

pkg/builder/youtube.go

@@ -16,8 +16,13 @@ import (
 	"google.golang.org/api/youtube/v3"
 
 	"github.com/mxpv/podsync/pkg/model"
+	"github.com/mxpv/podsync/pkg/ytdl"
 )
 
+type Downloader interface {
+	PlaylistMetadata(ctx context.Context, url string) (metadata ytdl.PlaylistMetadata, err error)
+}
+
 const (
 	maxYoutubeResults       = 50
 	hdBytesPerSecond        = 350000
@@ -33,8 +38,9 @@ func (key apiKey) Get() (string, string) {
 }
 
 type YouTubeBuilder struct {
-	client *youtube.Service
-	key    apiKey
+	client     *youtube.Service
+	key        apiKey
+	downloader Downloader
 }
 
 // Cost: 5 units (call method: 1, snippet: 2, contentDetails: 2)
@@ -212,21 +218,30 @@ func (yt *YouTubeBuilder) queryFeed(ctx context.Context, feed *model.Feed, info
 			return err
 		}
 
-		feed.Title = fmt.Sprintf("%s: %s", playlist.Snippet.ChannelTitle, playlist.Snippet.Title)
+		feed.Title = playlist.Snippet.Title
 		feed.Description = playlist.Snippet.Description
 
 		feed.ItemURL = fmt.Sprintf("https://youtube.com/playlist?list=%s", playlist.Id)
 		feed.ItemID = playlist.Id
 
-		feed.Author = "<notfound>"
+		feed.Author = playlist.Snippet.ChannelTitle
 
 		if date, err := yt.parseDate(playlist.Snippet.PublishedAt); err != nil {
 			return err
 		} else { // nolint:golint
 			feed.PubDate = date
 		}
-
-		thumbnails = playlist.Snippet.Thumbnails
+		metadata, err := yt.downloader.PlaylistMetadata(ctx, feed.ItemURL)
+		if err != nil {
+			return errors.Wrapf(err, "failed to get playlist metadata for %s", feed.ItemURL)
+		}
+		log.Infof("Playlist metadata: %v", metadata)
+		if len(metadata.Thumbnails) > 0 {
+			// best qualtiy thumbnail is the last one
+			feed.CoverArt = metadata.Thumbnails[len(metadata.Thumbnails)-1].Url
+		} else {
+			thumbnails = playlist.Snippet.Thumbnails
+		}
 
 	default:
 		return errors.New("unsupported link format")
@@ -235,9 +250,9 @@ func (yt *YouTubeBuilder) queryFeed(ctx context.Context, feed *model.Feed, info
 	if feed.Description == "" {
 		feed.Description = fmt.Sprintf("%s (%s)", feed.Title, feed.PubDate)
 	}
-
-	feed.CoverArt = yt.selectThumbnail(thumbnails, feed.CoverArtQuality, "")
-
+	if feed.CoverArt == "" {
+		feed.CoverArt = yt.selectThumbnail(thumbnails, feed.CoverArtQuality, "")
+	}
 	return nil
 }
 
@@ -309,7 +324,8 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(ctx context.Context, playlist m
 			// Parse date added to playlist / publication date
 			dateStr := ""
 			playlistItem, ok := playlist[video.Id]
-			if ok {
+			if ok && playlistItem.PublishedAt > snippet.PublishedAt {
+				// Use playlist item publish date if it's more recent
 				dateStr = playlistItem.PublishedAt
 			} else {
 				dateStr = snippet.PublishedAt
@@ -456,7 +472,7 @@ func (yt *YouTubeBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F
 	return _feed, nil
 }
 
-func NewYouTubeBuilder(key string) (*YouTubeBuilder, error) {
+func NewYouTubeBuilder(key string, ytdlp Downloader) (*YouTubeBuilder, error) {
 	if key == "" {
 		return nil, errors.New("empty YouTube API key")
 	}
@@ -466,5 +482,5 @@ func NewYouTubeBuilder(key string) (*YouTubeBuilder, error) {
 		return nil, errors.Wrap(err, "failed to create youtube client")
 	}
 
-	return &YouTubeBuilder{client: yt, key: apiKey(key)}, nil
+	return &YouTubeBuilder{client: yt, key: apiKey(key), downloader: ytdlp}, nil
 }

pkg/builder/youtube_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/mxpv/podsync/pkg/feed"
+	"github.com/mxpv/podsync/pkg/ytdl"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -17,12 +18,30 @@ var (
 	ytKey   = os.Getenv("YOUTUBE_TEST_API_KEY")
 )
 
+// Mock downloader for testing - doesn't require youtube-dl to be installed
+type mockDownloader struct{}
+
+func (m *mockDownloader) PlaylistMetadata(ctx context.Context, url string) (metadata ytdl.PlaylistMetadata, err error) {
+	return ytdl.PlaylistMetadata{
+		Id:          "test-id",
+		Title:       "Test Playlist",
+		Description: "Test Description",
+		Channel:     "Test Channel",
+		ChannelId:   "test-channel-id",
+		ChannelUrl:  "https://youtube.com/channel/test-channel-id",
+		WebpageUrl:  url,
+	}, nil
+}
+
 func TestYT_QueryChannel(t *testing.T) {
 	if ytKey == "" {
 		t.Skip("YouTube API key is not provided")
 	}
 
-	builder, err := NewYouTubeBuilder(ytKey)
+	// Use mock downloader to avoid requiring youtube-dl installation
+	downloader := &mockDownloader{}
+
+	builder, err := NewYouTubeBuilder(ytKey, downloader)
 	require.NoError(t, err)
 
 	channel, err := builder.listChannels(testCtx, model.TypeChannel, "UC2yTVSttx7lxAOAzx1opjoA", "id")
@@ -39,7 +58,10 @@ func TestYT_BuildFeed(t *testing.T) {
 		t.Skip("YouTube API key is not provided")
 	}
 
-	builder, err := NewYouTubeBuilder(ytKey)
+	// Use mock downloader to avoid requiring youtube-dl installation
+	downloader := &mockDownloader{}
+
+	builder, err := NewYouTubeBuilder(ytKey, downloader)
 	require.NoError(t, err)
 
 	urls := []string{
@@ -79,7 +101,10 @@ func TestYT_GetVideoCount(t *testing.T) {
 		t.Skip("YouTube API key is not provided")
 	}
 
-	builder, err := NewYouTubeBuilder(ytKey)
+	// Use mock downloader to avoid requiring youtube-dl installation
+	downloader := &mockDownloader{}
+
+	builder, err := NewYouTubeBuilder(ytKey, downloader)
 	require.NoError(t, err)
 
 	feeds := []*model.Info{

pkg/feed/xml.go

@@ -38,12 +38,16 @@ func Build(_ctx context.Context, feed *model.Feed, cfg *Config, hostname string)
 
 	var (
 		now         = time.Now().UTC()
-		author      = feed.Title
+		author      = feed.Author
 		title       = feed.Title
 		description = feed.Description
 		feedLink    = feed.ItemURL
 	)
 
+	if author == "<notfound>" {
+		author = feed.Title
+	}
+
 	if cfg.Custom.Author != "" {
 		author = cfg.Custom.Author
 	}

pkg/ytdl/ytdl.go

@@ -2,6 +2,7 @@ package ytdl
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -24,6 +25,25 @@ const (
 	UpdatePeriod           = 24 * time.Hour
 )
 
+type PlaylistMetadataThumbnail struct {
+	Id         string `json:"id"`
+	Url        string `json:"url"`
+	Resolution string `json:"resolution"`
+	Width      int    `json:"width"`
+	Height     int    `json:"height"`
+}
+
+type PlaylistMetadata struct {
+	Id          string                      `json:"id"`
+	Title       string                      `json:"title"`
+	Description string                      `json:"description"`
+	Thumbnails  []PlaylistMetadataThumbnail `json:"thumbnails"`
+	Channel     string                      `json:"channel"`
+	ChannelId   string                      `json:"channel_id"`
+	ChannelUrl  string                      `json:"channel_url"`
+	WebpageUrl  string                      `json:"webpage_url"`
+}
+
 var (
 	ErrTooManyRequests = errors.New(http.StatusText(http.StatusTooManyRequests))
 )
@@ -156,6 +176,35 @@ func (dl *YoutubeDl) Update(ctx context.Context) error {
 	return nil
 }
 
+func (dl *YoutubeDl) PlaylistMetadata(ctx context.Context, url string) (metadata PlaylistMetadata, err error) {
+	log.Info("getting playlist metadata for: ", url)
+	args := []string{
+		"--playlist-items", "0",
+		"-J",            // JSON output
+		"-q",            // quiet mode
+		"--no-warnings", // suppress warnings
+		url,
+	}
+	dl.updateLock.Lock()
+	defer dl.updateLock.Unlock()
+	output, err := dl.exec(ctx, args...)
+	if err != nil {
+		log.WithError(err).Errorf("youtube-dl error: %s", url)
+
+		// YouTube might block host with HTTP Error 429: Too Many Requests
+		if strings.Contains(output, "HTTP Error 429") {
+			return PlaylistMetadata{}, ErrTooManyRequests
+		}
+
+		log.Error(output)
+		return PlaylistMetadata{}, errors.New(output)
+	}
+
+	var playlistMetadata PlaylistMetadata
+	json.Unmarshal([]byte(output), &playlistMetadata)
+	return playlistMetadata, nil
+}
+
 func (dl *YoutubeDl) Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (r io.ReadCloser, err error) {
 	tmpDir, err := os.MkdirTemp("", "podsync-")
 	if err != nil {

services/update/updater.go

@@ -23,6 +23,7 @@ import (
 
 type Downloader interface {
 	Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (io.ReadCloser, error)
+	PlaylistMetadata(ctx context.Context, url string) (metadata ytdl.PlaylistMetadata, err error)
 }
 
 type TokenList []string
@@ -101,7 +102,7 @@ func (u *Manager) updateFeed(ctx context.Context, feedConfig *feed.Config) error
 	}
 
 	// Create an updater for this feed type
-	provider, err := builder.New(ctx, info.Provider, keyProvider.Get())
+	provider, err := builder.New(ctx, info.Provider, keyProvider.Get(), u.downloader)
 	if err != nil {
 		return err
 	}