diff --git a/docs/how_to_nebula.md b/docs/how_to_nebula.md new file mode 100644 index 00000000..1b800081 --- /dev/null +++ b/docs/how_to_nebula.md @@ -0,0 +1,43 @@ +# How to get token +1. Log in to nebula.tv +2. Open Network tab in developer console +3. Refresh page +5. Find request to `/api/v1/authorization` +6. Under request headers find `Authorization Token: ` +7. Copy the value of `` and use in `config.toml` + +# How to get nebula feed +Setup your config.toml as below. + +For each feed, you must use url format `https://nebula.tv/`. Only channels are supported at this time. + +Depending on the video length and chosen quality, videos may take a long time to download. If they exceed the timeout time, they fail. To avoid this, use a high download timeout value (65 minutes seems to be good for 1080p jetlag). + +An example config.toml for Nebula: +``` +[server] +port = 8080 +hostname = "http://localhost:8080" + +[storage] + [storage.local] + data_dir = "./app/data/" + +[downloader] + timeout = 65 + +[tokens] +nebula = "" + +[feeds] + [feeds.jetlag] + youtube_dl_args = ["--embed-subs"] + url = "https://nebula.tv/jetlag" + max_height = 1080 + page_size = 3 + opml = true +``` + +I like to use feed option +`youtube_dl_args = ["--embed-subs"]` +to get subtitles for jetlag but that's up to you. \ No newline at end of file diff --git a/go.mod b/go.mod index a2df4fea..794a1743 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,16 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) +require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + golang.org/x/text v0.14.0 // indirect +) + require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/cespare/xxhash v1.1.0 // indirect @@ -34,6 +44,7 @@ require ( github.com/grafov/m3u8 v0.11.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/mmcdole/gofeed v1.3.0 github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.21.0 // indirect diff --git a/go.sum b/go.sum index 648f0c90..ad708a70 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,10 @@ github.com/BrianHicks/finch v0.0.0-20140409222414-419bd73c29ec/go.mod h1:+hWo/MW github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.44.144 h1:mMWdnYL8HZsobrQe1mwvQ18Xt8UbOVhWgipjuma5Mkg= github.com/aws/aws-sdk-go v1.44.144/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= @@ -39,6 +43,7 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/grafov/m3u8 v0.11.1 h1:igZ7EBIB2IAsPPazKwRKdbhxcoBKO3lO1UY57PZDeNA= github.com/grafov/m3u8 v0.11.1/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -53,6 +58,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -61,6 +68,15 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= +github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -87,6 +103,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -107,6 +124,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= @@ -124,6 +142,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -137,6 +156,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 42b6d0a1..c2835f62 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -13,7 +13,7 @@ type Builder interface { Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) } -func New(ctx context.Context, provider model.Provider, key string) (Builder, error) { +func New(ctx context.Context, provider model.Provider, key string, durationGetter fnGetDuration) (Builder, error) { switch provider { case model.ProviderYoutube: return NewYouTubeBuilder(key) @@ -21,6 +21,8 @@ func New(ctx context.Context, provider model.Provider, key string) (Builder, err return NewVimeoBuilder(ctx, key) case model.ProviderSoundcloud: return NewSoundcloudBuilder() + case model.ProviderNebula: + return newNebulaBuilder(key, durationGetter) default: return nil, errors.Errorf("unsupported provider %q", provider) } diff --git a/pkg/builder/nebula.go b/pkg/builder/nebula.go new file mode 100644 index 00000000..dc53edec --- /dev/null +++ b/pkg/builder/nebula.go @@ -0,0 +1,149 @@ +package builder + +import ( + "context" + "encoding/json" + "io" + "net/http" + "regexp" + "strings" + "time" + + "github.com/mmcdole/gofeed" + "github.com/mxpv/podsync/pkg/feed" + "github.com/mxpv/podsync/pkg/model" + "github.com/pkg/errors" +) + +type fnGetDuration func(ctx context.Context, url string, extra_args ...string) (duration int64, err error) + +type NebulaBuilder struct { + token string + getDuration fnGetDuration +} + +func splitThumbnail(desc string) string { + a := strings.Split(desc, "") + c := strings.Split(b[0], "?format") + + return c[0] +} + +func getChannelAvatarURL(channelName string) (string, error) { + // Make the GET request + resp, err := http.Get("https://content.api.nebula.app/content/" + channelName) + if err != nil { + return "", err + } + defer resp.Body.Close() + + // Read the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + // Define a struct to unmarshal the JSON into + var data struct { + Images struct { + Avatar struct { + Src string `json:"src"` + } `json:"avatar"` + } `json:"images"` + } + + // Parse the JSON + err = json.Unmarshal(body, &data) + if err != nil { + return "", err + } + + return data.Images.Avatar.Src, nil +} + +func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) { + info, err := ParseURL(cfg.URL) + if err != nil { + return nil, err + } + + // add authorisation token to yt-dlp arguments + cfg.YouTubeDLArgs = append(cfg.YouTubeDLArgs, "--add-headers", "Authorization: Token "+neb.token) + + fp := gofeed.Parser{} + rssFeed, err := fp.ParseURL("https://rss.nebula.app/video/channels/" + info.ItemID + ".rss") + if err != nil { + return nil, err + } + + _feed := &model.Feed{ + ItemID: info.ItemID, + Provider: info.Provider, + LinkType: info.LinkType, + Format: cfg.Format, + Quality: cfg.Quality, + PageSize: cfg.PageSize, + UpdatedAt: time.Now().UTC(), + + Title: rssFeed.Title, + ItemURL: rssFeed.Link, + Description: rssFeed.Description, + PubDate: *rssFeed.UpdatedParsed, + } + + avatarSrc, err := getChannelAvatarURL(info.ItemID) + if err == nil { + _feed.CoverArt = avatarSrc + } + + // setup episodes and add to feed + added := 0 + for _, item := range rssFeed.Items { + newEpisode := &model.Episode{ + ID: item.GUID, + Title: item.Title, + Description: item.Description, + VideoURL: item.Link, + PubDate: *item.PublishedParsed, + // Size: , + Status: model.EpisodeNew, + } + + dur, err := neb.getDuration(ctx, item.Link, "--add-headers", "Authorization: Token "+neb.token) + if err != nil { + newEpisode.Duration = dur + } + + thumbnailURL := splitThumbnail(item.Description) + if thumbnailURL != "" { + newEpisode.Thumbnail = thumbnailURL + pattern := `]+>` + re := regexp.MustCompile(pattern) + newEpisode.Description = re.ReplaceAllString(newEpisode.Description, "") + } + + _feed.Episodes = append(_feed.Episodes, newEpisode) + + added++ + + if added >= _feed.PageSize { + break + } + } + + return _feed, nil +} + +func newNebulaBuilder(key string, durationGetter fnGetDuration) (*NebulaBuilder, error) { + if key == "" { + return nil, errors.New("invalid key given for Nebula") + } + + return &NebulaBuilder{token: key, getDuration: durationGetter}, nil +} diff --git a/pkg/builder/url.go b/pkg/builder/url.go index b3c173dd..261f01b3 100755 --- a/pkg/builder/url.go +++ b/pkg/builder/url.go @@ -56,6 +56,19 @@ func ParseURL(link string) (model.Info, error) { return info, nil } + if strings.HasSuffix(parsed.Host, "nebula.tv") { + kind, id, err := parseNebulaURL(parsed) + if err != nil { + return model.Info{}, err + } + + info.Provider = model.ProviderNebula + info.LinkType = kind + info.ItemID = id + + return info, nil + } + return model.Info{}, errors.New("unsupported URL host") } @@ -186,3 +199,13 @@ func parseSoundcloudURL(parsed *url.URL) (model.Type, string, error) { return kind, id, nil } + +func parseNebulaURL(parsed *url.URL) (model.Type, string, error) { + parts := strings.Split(parsed.EscapedPath(), "/") + + if len(parts) != 2 { + return "", "", errors.New("invalid nebula link path") + } + + return model.TypeChannel, parts[1], nil +} diff --git a/pkg/feed/config.go b/pkg/feed/config.go index d45f0f7c..a938bbad 100644 --- a/pkg/feed/config.go +++ b/pkg/feed/config.go @@ -41,7 +41,7 @@ type Config struct { // Included in OPML file OPML bool `toml:"opml"` // Private feed (not indexed by podcast aggregators) - PrivateFeed bool `toml:"private_feed"` + PrivateFeed *bool `toml:"private_feed"` // Playlist sort PlaylistSort model.Sorting `toml:"playlist_sort"` } diff --git a/pkg/feed/xml.go b/pkg/feed/xml.go index cf64f8ac..05217fdb 100644 --- a/pkg/feed/xml.go +++ b/pkg/feed/xml.go @@ -66,7 +66,7 @@ func Build(_ctx context.Context, feed *model.Feed, cfg *Config, hostname string) p.IAuthor = author p.AddSummary(description) - if feed.PrivateFeed { + if feed.PrivateFeed == nil || *feed.PrivateFeed { p.IBlock = "yes" } diff --git a/pkg/model/feed.go b/pkg/model/feed.go index fb6cc40b..fff680a8 100644 --- a/pkg/model/feed.go +++ b/pkg/model/feed.go @@ -64,7 +64,7 @@ type Feed struct { Episodes []*Episode `json:"-"` // Array of episodes UpdatedAt time.Time `json:"updated_at"` PlaylistSort Sorting `json:"playlist_sort"` - PrivateFeed bool `json:"private_feed"` + PrivateFeed *bool `json:"private_feed"` } type EpisodeStatus string diff --git a/pkg/model/link.go b/pkg/model/link.go index 095e738b..55fd8a43 100755 --- a/pkg/model/link.go +++ b/pkg/model/link.go @@ -15,11 +15,12 @@ const ( ProviderYoutube = Provider("youtube") ProviderVimeo = Provider("vimeo") ProviderSoundcloud = Provider("soundcloud") + ProviderNebula = Provider("nebula") ) // Info represents data extracted from URL type Info struct { LinkType Type // Either group, channel or user - Provider Provider // Youtube, Vimeo, or SoundCloud + Provider Provider // Youtube, Vimeo, SoundCloud, or Nebula ItemID string } diff --git a/pkg/ytdl/ytdl.go b/pkg/ytdl/ytdl.go index d641a657..ee071b37 100644 --- a/pkg/ytdl/ytdl.go +++ b/pkg/ytdl/ytdl.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "sync" "time" @@ -156,6 +157,50 @@ func (dl *YoutubeDl) Update(ctx context.Context) error { return nil } +func (dl *YoutubeDl) GetDuration(ctx context.Context, url string, extra_args ...string) (duration int64, err error) { + var args []string + + // extra args i.e. for authentication + args = append(args, extra_args...) + + args = append(args, "--get-duration", url) + + output, err := dl.exec(ctx, args...) + + if err != nil { + return 0, err + } + + parts := strings.Split(strings.Split(output, "\n")[0], ":") + + // seconds + num, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + return 0, err + } + duration += int64(num) + + // minutes + if len(parts) > 1 { + num, err = strconv.Atoi(parts[len(parts)-2]) + if err != nil { + return 0, err + } + duration += int64(num) * 60 + } + + // hours + if len(parts) > 2 { + num, err = strconv.Atoi(parts[len(parts)-3]) + if err != nil { + return 0, err + } + duration += int64(num) * 60 * 60 + } + + return duration, nil +} + func (dl *YoutubeDl) Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (r io.ReadCloser, err error) { tmpDir, err := os.MkdirTemp("", "podsync-") if err != nil { @@ -230,12 +275,12 @@ func buildArgs(feedConfig *feed.Config, episode *model.Episode, outputFilePath s if feedConfig.Format == model.FormatVideo { // Video, mp4, high by default - format := "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[ext=mp4][vcodec^=avc1]/best[ext=mp4]/best" + format := "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio/best[ext=mp4][vcodec^=avc1]/best[ext=mp4]/best" if feedConfig.Quality == model.QualityLow { - format = "worstvideo[ext=mp4][vcodec^=avc1]+worstaudio[ext=m4a]/worst[ext=mp4][vcodec^=avc1]/worst[ext=mp4]/worst" + format = "worstvideo[ext=mp4][vcodec^=avc1]+worstaudio/worst[ext=mp4][vcodec^=avc1]/worst[ext=mp4]/worst" } else if feedConfig.Quality == model.QualityHigh && feedConfig.MaxHeight > 0 { - format = fmt.Sprintf("bestvideo[height<=%d][ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[height<=%d][ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", feedConfig.MaxHeight, feedConfig.MaxHeight) + format = fmt.Sprintf("bestvideo[height<=%d][ext=mp4][vcodec^=avc1]+bestaudio/best[height<=%d][ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", feedConfig.MaxHeight, feedConfig.MaxHeight) } args = append(args, "--format", format) @@ -248,9 +293,11 @@ func buildArgs(feedConfig *feed.Config, episode *model.Episode, outputFilePath s args = append(args, "--extract-audio", "--audio-format", "mp3", "--format", format) } else { - args = append(args, "--audio-format", feedConfig.CustomFormat.Extension, "--format", feedConfig.CustomFormat.YouTubeDLFormat) + args = append(args, "--format", feedConfig.CustomFormat.YouTubeDLFormat) } + args = append(args, "-S", "ext:mp4:m4a") + // Insert additional per-feed youtube-dl arguments args = append(args, feedConfig.YouTubeDLArgs...) diff --git a/services/update/updater.go b/services/update/updater.go index 727b9884..f148da3b 100644 --- a/services/update/updater.go +++ b/services/update/updater.go @@ -23,6 +23,7 @@ import ( type Downloader interface { Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (io.ReadCloser, error) + GetDuration(ctx context.Context, url string, extra_args ...string) (duration int64, err error) } type TokenList []string @@ -101,7 +102,7 @@ func (u *Manager) updateFeed(ctx context.Context, feedConfig *feed.Config) error } // Create an updater for this feed type - provider, err := builder.New(ctx, info.Provider, keyProvider.Get()) + provider, err := builder.New(ctx, info.Provider, keyProvider.Get(), u.downloader.GetDuration) if err != nil { return err } @@ -239,6 +240,8 @@ func (u *Manager) downloadEpisodes(ctx context.Context, feedConfig *feed.Config) if err == ytdl.ErrTooManyRequests { logger.Warn("server responded with a 'Too Many Requests' error") break + } else { + logger.Warn(err) } if err := u.db.UpdateEpisode(feedID, episode.ID, func(episode *model.Episode) error {