diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 42b6d0a1..07f5181f 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -13,10 +13,10 @@ type Builder interface { Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) } -func New(ctx context.Context, provider model.Provider, key string) (Builder, error) { +func New(ctx context.Context, provider model.Provider, key string, downloader Downloader) (Builder, error) { switch provider { case model.ProviderYoutube: - return NewYouTubeBuilder(key) + return NewYouTubeBuilder(key, downloader) case model.ProviderVimeo: return NewVimeoBuilder(ctx, key) case model.ProviderSoundcloud: diff --git a/pkg/builder/youtube.go b/pkg/builder/youtube.go index 528b384c..73714197 100644 --- a/pkg/builder/youtube.go +++ b/pkg/builder/youtube.go @@ -16,8 +16,13 @@ import ( "google.golang.org/api/youtube/v3" "github.com/mxpv/podsync/pkg/model" + "github.com/mxpv/podsync/pkg/ytdl" ) +type Downloader interface { + PlaylistMetadata(ctx context.Context, url string) (metadata ytdl.PlaylistMetadata, err error) +} + const ( maxYoutubeResults = 50 hdBytesPerSecond = 350000 @@ -33,8 +38,9 @@ func (key apiKey) Get() (string, string) { } type YouTubeBuilder struct { - client *youtube.Service - key apiKey + client *youtube.Service + key apiKey + downloader Downloader } // Cost: 5 units (call method: 1, snippet: 2, contentDetails: 2) @@ -212,21 +218,30 @@ func (yt *YouTubeBuilder) queryFeed(ctx context.Context, feed *model.Feed, info return err } - feed.Title = fmt.Sprintf("%s: %s", playlist.Snippet.ChannelTitle, playlist.Snippet.Title) + feed.Title = playlist.Snippet.Title feed.Description = playlist.Snippet.Description feed.ItemURL = fmt.Sprintf("https://youtube.com/playlist?list=%s", playlist.Id) feed.ItemID = playlist.Id - feed.Author = "" + feed.Author = playlist.Snippet.ChannelTitle if date, err := yt.parseDate(playlist.Snippet.PublishedAt); err != nil { return err } else { // nolint:golint feed.PubDate = date } - - thumbnails = playlist.Snippet.Thumbnails + metadata, err := yt.downloader.PlaylistMetadata(ctx, feed.ItemURL) + if err != nil { + return errors.Wrapf(err, "failed to get playlist metadata for %s", feed.ItemURL) + } + log.Infof("Playlist metadata: %v", metadata) + if len(metadata.Thumbnails) > 0 { + // best qualtiy thumbnail is the last one + feed.CoverArt = metadata.Thumbnails[len(metadata.Thumbnails)-1].Url + } else { + thumbnails = playlist.Snippet.Thumbnails + } default: return errors.New("unsupported link format") @@ -235,9 +250,9 @@ func (yt *YouTubeBuilder) queryFeed(ctx context.Context, feed *model.Feed, info if feed.Description == "" { feed.Description = fmt.Sprintf("%s (%s)", feed.Title, feed.PubDate) } - - feed.CoverArt = yt.selectThumbnail(thumbnails, feed.CoverArtQuality, "") - + if feed.CoverArt == "" { + feed.CoverArt = yt.selectThumbnail(thumbnails, feed.CoverArtQuality, "") + } return nil } @@ -309,7 +324,8 @@ func (yt *YouTubeBuilder) queryVideoDescriptions(ctx context.Context, playlist m // Parse date added to playlist / publication date dateStr := "" playlistItem, ok := playlist[video.Id] - if ok { + if ok && playlistItem.PublishedAt > snippet.PublishedAt { + // Use playlist item publish date if it's more recent dateStr = playlistItem.PublishedAt } else { dateStr = snippet.PublishedAt @@ -456,7 +472,7 @@ func (yt *YouTubeBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F return _feed, nil } -func NewYouTubeBuilder(key string) (*YouTubeBuilder, error) { +func NewYouTubeBuilder(key string, ytdlp Downloader) (*YouTubeBuilder, error) { if key == "" { return nil, errors.New("empty YouTube API key") } @@ -466,5 +482,5 @@ func NewYouTubeBuilder(key string) (*YouTubeBuilder, error) { return nil, errors.Wrap(err, "failed to create youtube client") } - return &YouTubeBuilder{client: yt, key: apiKey(key)}, nil + return &YouTubeBuilder{client: yt, key: apiKey(key), downloader: ytdlp}, nil } diff --git a/pkg/builder/youtube_test.go b/pkg/builder/youtube_test.go index 5c2c5057..8ba9d5d5 100644 --- a/pkg/builder/youtube_test.go +++ b/pkg/builder/youtube_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/mxpv/podsync/pkg/feed" + "github.com/mxpv/podsync/pkg/ytdl" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,12 +18,30 @@ var ( ytKey = os.Getenv("YOUTUBE_TEST_API_KEY") ) +// Mock downloader for testing - doesn't require youtube-dl to be installed +type mockDownloader struct{} + +func (m *mockDownloader) PlaylistMetadata(ctx context.Context, url string) (metadata ytdl.PlaylistMetadata, err error) { + return ytdl.PlaylistMetadata{ + Id: "test-id", + Title: "Test Playlist", + Description: "Test Description", + Channel: "Test Channel", + ChannelId: "test-channel-id", + ChannelUrl: "https://youtube.com/channel/test-channel-id", + WebpageUrl: url, + }, nil +} + func TestYT_QueryChannel(t *testing.T) { if ytKey == "" { t.Skip("YouTube API key is not provided") } - builder, err := NewYouTubeBuilder(ytKey) + // Use mock downloader to avoid requiring youtube-dl installation + downloader := &mockDownloader{} + + builder, err := NewYouTubeBuilder(ytKey, downloader) require.NoError(t, err) channel, err := builder.listChannels(testCtx, model.TypeChannel, "UC2yTVSttx7lxAOAzx1opjoA", "id") @@ -39,7 +58,10 @@ func TestYT_BuildFeed(t *testing.T) { t.Skip("YouTube API key is not provided") } - builder, err := NewYouTubeBuilder(ytKey) + // Use mock downloader to avoid requiring youtube-dl installation + downloader := &mockDownloader{} + + builder, err := NewYouTubeBuilder(ytKey, downloader) require.NoError(t, err) urls := []string{ @@ -79,7 +101,10 @@ func TestYT_GetVideoCount(t *testing.T) { t.Skip("YouTube API key is not provided") } - builder, err := NewYouTubeBuilder(ytKey) + // Use mock downloader to avoid requiring youtube-dl installation + downloader := &mockDownloader{} + + builder, err := NewYouTubeBuilder(ytKey, downloader) require.NoError(t, err) feeds := []*model.Info{ diff --git a/pkg/feed/xml.go b/pkg/feed/xml.go index 51b4e0d0..f1f92072 100644 --- a/pkg/feed/xml.go +++ b/pkg/feed/xml.go @@ -38,12 +38,16 @@ func Build(_ctx context.Context, feed *model.Feed, cfg *Config, hostname string) var ( now = time.Now().UTC() - author = feed.Title + author = feed.Author title = feed.Title description = feed.Description feedLink = feed.ItemURL ) + if author == "" { + author = feed.Title + } + if cfg.Custom.Author != "" { author = cfg.Custom.Author } diff --git a/pkg/ytdl/ytdl.go b/pkg/ytdl/ytdl.go index 538bfb4b..70c7c526 100644 --- a/pkg/ytdl/ytdl.go +++ b/pkg/ytdl/ytdl.go @@ -2,6 +2,7 @@ package ytdl import ( "context" + "encoding/json" "fmt" "io" "net/http" @@ -24,6 +25,25 @@ const ( UpdatePeriod = 24 * time.Hour ) +type PlaylistMetadataThumbnail struct { + Id string `json:"id"` + Url string `json:"url"` + Resolution string `json:"resolution"` + Width int `json:"width"` + Height int `json:"height"` +} + +type PlaylistMetadata struct { + Id string `json:"id"` + Title string `json:"title"` + Description string `json:"description"` + Thumbnails []PlaylistMetadataThumbnail `json:"thumbnails"` + Channel string `json:"channel"` + ChannelId string `json:"channel_id"` + ChannelUrl string `json:"channel_url"` + WebpageUrl string `json:"webpage_url"` +} + var ( ErrTooManyRequests = errors.New(http.StatusText(http.StatusTooManyRequests)) ) @@ -156,6 +176,35 @@ func (dl *YoutubeDl) Update(ctx context.Context) error { return nil } +func (dl *YoutubeDl) PlaylistMetadata(ctx context.Context, url string) (metadata PlaylistMetadata, err error) { + log.Info("getting playlist metadata for: ", url) + args := []string{ + "--playlist-items", "0", + "-J", // JSON output + "-q", // quiet mode + "--no-warnings", // suppress warnings + url, + } + dl.updateLock.Lock() + defer dl.updateLock.Unlock() + output, err := dl.exec(ctx, args...) + if err != nil { + log.WithError(err).Errorf("youtube-dl error: %s", url) + + // YouTube might block host with HTTP Error 429: Too Many Requests + if strings.Contains(output, "HTTP Error 429") { + return PlaylistMetadata{}, ErrTooManyRequests + } + + log.Error(output) + return PlaylistMetadata{}, errors.New(output) + } + + var playlistMetadata PlaylistMetadata + json.Unmarshal([]byte(output), &playlistMetadata) + return playlistMetadata, nil +} + func (dl *YoutubeDl) Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (r io.ReadCloser, err error) { tmpDir, err := os.MkdirTemp("", "podsync-") if err != nil { diff --git a/services/update/updater.go b/services/update/updater.go index 727b9884..efb89c28 100644 --- a/services/update/updater.go +++ b/services/update/updater.go @@ -23,6 +23,7 @@ import ( type Downloader interface { Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (io.ReadCloser, error) + PlaylistMetadata(ctx context.Context, url string) (metadata ytdl.PlaylistMetadata, err error) } type TokenList []string @@ -101,7 +102,7 @@ func (u *Manager) updateFeed(ctx context.Context, feedConfig *feed.Config) error } // Create an updater for this feed type - provider, err := builder.New(ctx, info.Provider, keyProvider.Get()) + provider, err := builder.New(ctx, info.Provider, keyProvider.Get(), u.downloader) if err != nil { return err }