From 9b3f3e2162978a31aaad3348f7162079d713dffd Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 19 Apr 2025 15:57:41 +1200 Subject: [PATCH 01/11] Added Nebula support --- docs/how_to_nebula.md | 29 ++++++++++++++++++ go.mod | 11 +++++++ pkg/builder/builder.go | 2 ++ pkg/builder/nebula.go | 69 ++++++++++++++++++++++++++++++++++++++++++ pkg/builder/url.go | 23 ++++++++++++++ pkg/model/link.go | 3 +- pkg/ytdl/ytdl.go | 8 ++--- 7 files changed, 140 insertions(+), 5 deletions(-) create mode 100644 docs/how_to_nebula.md create mode 100644 pkg/builder/nebula.go diff --git a/docs/how_to_nebula.md b/docs/how_to_nebula.md new file mode 100644 index 00000000..c67ff02c --- /dev/null +++ b/docs/how_to_nebula.md @@ -0,0 +1,29 @@ +# How to get nebula feed +Setup your config.toml as below. + +Under tokens put `nebula = "nil"`. +For each feed, include +`youtube_dl_args = ["--username", "", "--password", "", "--embed-subs"]`. +You must use url format `https://nebula.tv/`. Only channels are supported at this time. + +An example config.toml for Nebula: +``` +[server] +port = 8080 +hostname = "http://localhost:8080" + +[storage] + [storage.local] + data_dir = "./app/data/" + +[tokens] +nebula = "nil" + +[feeds] + [feeds.wendover] + youtube_dl_args = ["--username", "", "--password", "", "--embed-subs"] + url = "https://nebula.tv/wendover" + max_height = 1080 + page_size = 4 + opml = true +``` \ No newline at end of file diff --git a/go.mod b/go.mod index a2df4fea..794a1743 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,16 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) +require ( + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + golang.org/x/text v0.14.0 // indirect +) + require ( github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect github.com/cespare/xxhash v1.1.0 // indirect @@ -34,6 +44,7 @@ require ( github.com/grafov/m3u8 v0.11.1 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/mmcdole/gofeed v1.3.0 github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.21.0 // indirect diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 42b6d0a1..947ca3ae 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -21,6 +21,8 @@ func New(ctx context.Context, provider model.Provider, key string) (Builder, err return NewVimeoBuilder(ctx, key) case model.ProviderSoundcloud: return NewSoundcloudBuilder() + case model.ProviderNebula: + return newNebulaBuilder() default: return nil, errors.Errorf("unsupported provider %q", provider) } diff --git a/pkg/builder/nebula.go b/pkg/builder/nebula.go new file mode 100644 index 00000000..d8c84eb7 --- /dev/null +++ b/pkg/builder/nebula.go @@ -0,0 +1,69 @@ +package builder + +import ( + "context" + "time" + + "github.com/mmcdole/gofeed" + "github.com/mxpv/podsync/pkg/feed" + "github.com/mxpv/podsync/pkg/model" +) + +type NebulaBuilder struct { +} + +func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) { + info, err := ParseURL(cfg.URL) + if err != nil { + return nil, err + } + + fp := gofeed.Parser{} + rssFeed, err := fp.ParseURL("https://rss.nebula.app/video/channels/" + info.ItemID + ".rss") + if err != nil { + return nil, err + } + + _feed := &model.Feed{ + ItemID: info.ItemID, + Provider: info.Provider, + LinkType: info.LinkType, + Format: cfg.Format, + Quality: cfg.Quality, + PageSize: cfg.PageSize, + UpdatedAt: time.Now().UTC(), + + Title: rssFeed.Title, + ItemURL: rssFeed.Link, + Description: rssFeed.Description, + PubDate: *rssFeed.UpdatedParsed, + } + + // setup episodes and add to feed + added := 0 + for _, item := range rssFeed.Items { + _feed.Episodes = append(_feed.Episodes, &model.Episode{ + ID: item.GUID, + Title: item.Title, + Description: item.Description, + VideoURL: item.Link, + PubDate: *item.PublishedParsed, + // Thumbnail: item., + // Duration: item.length, + // Size: , + Status: model.EpisodeNew, + }) + + added++ + + if added >= _feed.PageSize { + break + } + } + + return _feed, nil +} + +func newNebulaBuilder() (*NebulaBuilder, error) { + return &NebulaBuilder{}, nil +} diff --git a/pkg/builder/url.go b/pkg/builder/url.go index b3c173dd..261f01b3 100755 --- a/pkg/builder/url.go +++ b/pkg/builder/url.go @@ -56,6 +56,19 @@ func ParseURL(link string) (model.Info, error) { return info, nil } + if strings.HasSuffix(parsed.Host, "nebula.tv") { + kind, id, err := parseNebulaURL(parsed) + if err != nil { + return model.Info{}, err + } + + info.Provider = model.ProviderNebula + info.LinkType = kind + info.ItemID = id + + return info, nil + } + return model.Info{}, errors.New("unsupported URL host") } @@ -186,3 +199,13 @@ func parseSoundcloudURL(parsed *url.URL) (model.Type, string, error) { return kind, id, nil } + +func parseNebulaURL(parsed *url.URL) (model.Type, string, error) { + parts := strings.Split(parsed.EscapedPath(), "/") + + if len(parts) != 2 { + return "", "", errors.New("invalid nebula link path") + } + + return model.TypeChannel, parts[1], nil +} diff --git a/pkg/model/link.go b/pkg/model/link.go index 095e738b..55fd8a43 100755 --- a/pkg/model/link.go +++ b/pkg/model/link.go @@ -15,11 +15,12 @@ const ( ProviderYoutube = Provider("youtube") ProviderVimeo = Provider("vimeo") ProviderSoundcloud = Provider("soundcloud") + ProviderNebula = Provider("nebula") ) // Info represents data extracted from URL type Info struct { LinkType Type // Either group, channel or user - Provider Provider // Youtube, Vimeo, or SoundCloud + Provider Provider // Youtube, Vimeo, SoundCloud, or Nebula ItemID string } diff --git a/pkg/ytdl/ytdl.go b/pkg/ytdl/ytdl.go index d641a657..0351374f 100644 --- a/pkg/ytdl/ytdl.go +++ b/pkg/ytdl/ytdl.go @@ -230,12 +230,12 @@ func buildArgs(feedConfig *feed.Config, episode *model.Episode, outputFilePath s if feedConfig.Format == model.FormatVideo { // Video, mp4, high by default - format := "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[ext=mp4][vcodec^=avc1]/best[ext=mp4]/best" + format := "bestvideo[ext=mp4][vcodec^=avc1]+bestaudio/best[ext=mp4][vcodec^=avc1]/best[ext=mp4]/best" if feedConfig.Quality == model.QualityLow { - format = "worstvideo[ext=mp4][vcodec^=avc1]+worstaudio[ext=m4a]/worst[ext=mp4][vcodec^=avc1]/worst[ext=mp4]/worst" + format = "worstvideo[ext=mp4][vcodec^=avc1]+worstaudio/worst[ext=mp4][vcodec^=avc1]/worst[ext=mp4]/worst" } else if feedConfig.Quality == model.QualityHigh && feedConfig.MaxHeight > 0 { - format = fmt.Sprintf("bestvideo[height<=%d][ext=mp4][vcodec^=avc1]+bestaudio[ext=m4a]/best[height<=%d][ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", feedConfig.MaxHeight, feedConfig.MaxHeight) + format = fmt.Sprintf("bestvideo[height<=%d][ext=mp4][vcodec^=avc1]+bestaudio/best[height<=%d][ext=mp4][vcodec^=avc1]/best[ext=mp4]/best", feedConfig.MaxHeight, feedConfig.MaxHeight) } args = append(args, "--format", format) @@ -248,7 +248,7 @@ func buildArgs(feedConfig *feed.Config, episode *model.Episode, outputFilePath s args = append(args, "--extract-audio", "--audio-format", "mp3", "--format", format) } else { - args = append(args, "--audio-format", feedConfig.CustomFormat.Extension, "--format", feedConfig.CustomFormat.YouTubeDLFormat) + args = append(args, "--format", feedConfig.CustomFormat.YouTubeDLFormat) } // Insert additional per-feed youtube-dl arguments From 4748f5005e276f4eb10b4b9a30ca2f42aab00f5d Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 20 Apr 2025 01:09:31 +1200 Subject: [PATCH 02/11] Fix nebula cause it broke youtube downloaded file type --- go.sum | 20 ++++++++++++++++++++ pkg/ytdl/ytdl.go | 2 ++ services/update/updater.go | 2 ++ 3 files changed, 24 insertions(+) diff --git a/go.sum b/go.sum index 648f0c90..ad708a70 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,10 @@ github.com/BrianHicks/finch v0.0.0-20140409222414-419bd73c29ec/go.mod h1:+hWo/MW github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.44.144 h1:mMWdnYL8HZsobrQe1mwvQ18Xt8UbOVhWgipjuma5Mkg= github.com/aws/aws-sdk-go v1.44.144/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= @@ -39,6 +43,7 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/grafov/m3u8 v0.11.1 h1:igZ7EBIB2IAsPPazKwRKdbhxcoBKO3lO1UY57PZDeNA= github.com/grafov/m3u8 v0.11.1/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= @@ -53,6 +58,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -61,6 +68,15 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4= +github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk= +github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -87,6 +103,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -107,6 +124,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= @@ -124,6 +142,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -137,6 +156,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= diff --git a/pkg/ytdl/ytdl.go b/pkg/ytdl/ytdl.go index 0351374f..51396268 100644 --- a/pkg/ytdl/ytdl.go +++ b/pkg/ytdl/ytdl.go @@ -251,6 +251,8 @@ func buildArgs(feedConfig *feed.Config, episode *model.Episode, outputFilePath s args = append(args, "--format", feedConfig.CustomFormat.YouTubeDLFormat) } + args = append(args, "-S", "ext:mp4:m4a") + // Insert additional per-feed youtube-dl arguments args = append(args, feedConfig.YouTubeDLArgs...) diff --git a/services/update/updater.go b/services/update/updater.go index 727b9884..12d87be5 100644 --- a/services/update/updater.go +++ b/services/update/updater.go @@ -239,6 +239,8 @@ func (u *Manager) downloadEpisodes(ctx context.Context, feedConfig *feed.Config) if err == ytdl.ErrTooManyRequests { logger.Warn("server responded with a 'Too Many Requests' error") break + } else { + logger.Warn(err) } if err := u.db.UpdateEpisode(feedID, episode.ID, func(episode *model.Episode) error { From 452831477a30516efa41c40c95d61400c9a617cc Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 20 Apr 2025 12:23:11 +1200 Subject: [PATCH 03/11] Added nebula authentication by token --- pkg/builder/builder.go | 2 +- pkg/builder/nebula.go | 13 +++++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 947ca3ae..2a64b58b 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -22,7 +22,7 @@ func New(ctx context.Context, provider model.Provider, key string) (Builder, err case model.ProviderSoundcloud: return NewSoundcloudBuilder() case model.ProviderNebula: - return newNebulaBuilder() + return newNebulaBuilder(key) default: return nil, errors.Errorf("unsupported provider %q", provider) } diff --git a/pkg/builder/nebula.go b/pkg/builder/nebula.go index d8c84eb7..4bec0454 100644 --- a/pkg/builder/nebula.go +++ b/pkg/builder/nebula.go @@ -7,9 +7,11 @@ import ( "github.com/mmcdole/gofeed" "github.com/mxpv/podsync/pkg/feed" "github.com/mxpv/podsync/pkg/model" + "github.com/pkg/errors" ) type NebulaBuilder struct { + token string } func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) { @@ -18,6 +20,9 @@ func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F return nil, err } + // add authorisation token to yt-dlp arguments + cfg.YouTubeDLArgs = append(cfg.YouTubeDLArgs, "--add-headers", "Authorization: Token "+neb.token) + fp := gofeed.Parser{} rssFeed, err := fp.ParseURL("https://rss.nebula.app/video/channels/" + info.ItemID + ".rss") if err != nil { @@ -64,6 +69,10 @@ func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F return _feed, nil } -func newNebulaBuilder() (*NebulaBuilder, error) { - return &NebulaBuilder{}, nil +func newNebulaBuilder(key string) (*NebulaBuilder, error) { + if key == "" { + return nil, errors.New("invalid key given for Nebula") + } + + return &NebulaBuilder{token: key}, nil } From bf2b5bd3eca96de5b7bfdfb85f4ee7bae6ec69ba Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 20 Apr 2025 16:42:27 +1200 Subject: [PATCH 04/11] Added getting video duration from Nebula using yt-dlp --- pkg/builder/builder.go | 4 ++-- pkg/builder/nebula.go | 21 +++++++++++++----- pkg/ytdl/ytdl.go | 45 ++++++++++++++++++++++++++++++++++++++ services/update/updater.go | 3 ++- 4 files changed, 64 insertions(+), 9 deletions(-) diff --git a/pkg/builder/builder.go b/pkg/builder/builder.go index 2a64b58b..c2835f62 100644 --- a/pkg/builder/builder.go +++ b/pkg/builder/builder.go @@ -13,7 +13,7 @@ type Builder interface { Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) } -func New(ctx context.Context, provider model.Provider, key string) (Builder, error) { +func New(ctx context.Context, provider model.Provider, key string, durationGetter fnGetDuration) (Builder, error) { switch provider { case model.ProviderYoutube: return NewYouTubeBuilder(key) @@ -22,7 +22,7 @@ func New(ctx context.Context, provider model.Provider, key string) (Builder, err case model.ProviderSoundcloud: return NewSoundcloudBuilder() case model.ProviderNebula: - return newNebulaBuilder(key) + return newNebulaBuilder(key, durationGetter) default: return nil, errors.Errorf("unsupported provider %q", provider) } diff --git a/pkg/builder/nebula.go b/pkg/builder/nebula.go index 4bec0454..dc55a3bf 100644 --- a/pkg/builder/nebula.go +++ b/pkg/builder/nebula.go @@ -10,8 +10,11 @@ import ( "github.com/pkg/errors" ) +type fnGetDuration func(ctx context.Context, url string, extra_args ...string) (duration int64, err error) + type NebulaBuilder struct { - token string + token string + getDuration fnGetDuration } func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) { @@ -47,17 +50,23 @@ func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F // setup episodes and add to feed added := 0 for _, item := range rssFeed.Items { - _feed.Episodes = append(_feed.Episodes, &model.Episode{ + newEpisode := &model.Episode{ ID: item.GUID, Title: item.Title, Description: item.Description, VideoURL: item.Link, PubDate: *item.PublishedParsed, // Thumbnail: item., - // Duration: item.length, // Size: , Status: model.EpisodeNew, - }) + } + + dur, err := neb.getDuration(ctx, item.Link, "--add-headers", "Authorization: Token "+neb.token) + if err != nil { + newEpisode.Duration = dur + } + + _feed.Episodes = append(_feed.Episodes, newEpisode) added++ @@ -69,10 +78,10 @@ func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F return _feed, nil } -func newNebulaBuilder(key string) (*NebulaBuilder, error) { +func newNebulaBuilder(key string, durationGetter fnGetDuration) (*NebulaBuilder, error) { if key == "" { return nil, errors.New("invalid key given for Nebula") } - return &NebulaBuilder{token: key}, nil + return &NebulaBuilder{token: key, getDuration: durationGetter}, nil } diff --git a/pkg/ytdl/ytdl.go b/pkg/ytdl/ytdl.go index 51396268..ee071b37 100644 --- a/pkg/ytdl/ytdl.go +++ b/pkg/ytdl/ytdl.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "sync" "time" @@ -156,6 +157,50 @@ func (dl *YoutubeDl) Update(ctx context.Context) error { return nil } +func (dl *YoutubeDl) GetDuration(ctx context.Context, url string, extra_args ...string) (duration int64, err error) { + var args []string + + // extra args i.e. for authentication + args = append(args, extra_args...) + + args = append(args, "--get-duration", url) + + output, err := dl.exec(ctx, args...) + + if err != nil { + return 0, err + } + + parts := strings.Split(strings.Split(output, "\n")[0], ":") + + // seconds + num, err := strconv.Atoi(parts[len(parts)-1]) + if err != nil { + return 0, err + } + duration += int64(num) + + // minutes + if len(parts) > 1 { + num, err = strconv.Atoi(parts[len(parts)-2]) + if err != nil { + return 0, err + } + duration += int64(num) * 60 + } + + // hours + if len(parts) > 2 { + num, err = strconv.Atoi(parts[len(parts)-3]) + if err != nil { + return 0, err + } + duration += int64(num) * 60 * 60 + } + + return duration, nil +} + func (dl *YoutubeDl) Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (r io.ReadCloser, err error) { tmpDir, err := os.MkdirTemp("", "podsync-") if err != nil { diff --git a/services/update/updater.go b/services/update/updater.go index 12d87be5..f148da3b 100644 --- a/services/update/updater.go +++ b/services/update/updater.go @@ -23,6 +23,7 @@ import ( type Downloader interface { Download(ctx context.Context, feedConfig *feed.Config, episode *model.Episode) (io.ReadCloser, error) + GetDuration(ctx context.Context, url string, extra_args ...string) (duration int64, err error) } type TokenList []string @@ -101,7 +102,7 @@ func (u *Manager) updateFeed(ctx context.Context, feedConfig *feed.Config) error } // Create an updater for this feed type - provider, err := builder.New(ctx, info.Provider, keyProvider.Get()) + provider, err := builder.New(ctx, info.Provider, keyProvider.Get(), u.downloader.GetDuration) if err != nil { return err } From 285387e67c53fe0ce48b604d0332b373565fbecd Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 20 Apr 2025 17:01:18 +1200 Subject: [PATCH 05/11] Updated how to for nebula --- docs/how_to_nebula.md | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/docs/how_to_nebula.md b/docs/how_to_nebula.md index c67ff02c..07d51093 100644 --- a/docs/how_to_nebula.md +++ b/docs/how_to_nebula.md @@ -1,10 +1,15 @@ +# How to get token +1. Log in to nebula.tv +2. Open Network tab in developer console +3. Refresh page +5. Find request to `/api/v1/authorization` +6. Under request headers find `Authorization Token: ` +7. Copy the value of `` and use in `config.toml` + # How to get nebula feed Setup your config.toml as below. -Under tokens put `nebula = "nil"`. -For each feed, include -`youtube_dl_args = ["--username", "", "--password", "", "--embed-subs"]`. -You must use url format `https://nebula.tv/`. Only channels are supported at this time. +For each feed, you must use url format `https://nebula.tv/`. Only channels are supported at this time. An example config.toml for Nebula: ``` @@ -17,13 +22,17 @@ hostname = "http://localhost:8080" data_dir = "./app/data/" [tokens] -nebula = "nil" +nebula = "" [feeds] - [feeds.wendover] - youtube_dl_args = ["--username", "", "--password", "", "--embed-subs"] - url = "https://nebula.tv/wendover" + [feeds.jetlag] + youtube_dl_args = ["--embed-subs"] + url = "https://nebula.tv/jetlag" max_height = 1080 - page_size = 4 + page_size = 3 opml = true -``` \ No newline at end of file +``` + +I like to use feed option +`youtube_dl_args = ["--embed-subs"]` +to get subtitles for jetlag but that's up to you. \ No newline at end of file From 40b456be43630a8b1f3972081850b2b337ecd665 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 20 Apr 2025 22:15:12 +1200 Subject: [PATCH 06/11] Added thumbnails for Nebula --- pkg/builder/nebula.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/builder/nebula.go b/pkg/builder/nebula.go index dc55a3bf..7cfff7c3 100644 --- a/pkg/builder/nebula.go +++ b/pkg/builder/nebula.go @@ -2,6 +2,7 @@ package builder import ( "context" + "strings" "time" "github.com/mmcdole/gofeed" @@ -17,6 +18,19 @@ type NebulaBuilder struct { getDuration fnGetDuration } +func splitThumbnail(desc string) (string, error) { + a := strings.Split(desc, "") + c := strings.Split(b[0], "?format") + + return c[0], nil +} + func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) { info, err := ParseURL(cfg.URL) if err != nil { @@ -56,7 +70,6 @@ func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F Description: item.Description, VideoURL: item.Link, PubDate: *item.PublishedParsed, - // Thumbnail: item., // Size: , Status: model.EpisodeNew, } @@ -66,6 +79,11 @@ func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F newEpisode.Duration = dur } + thumbnailURL, err := splitThumbnail(item.Description) + if err == nil { + newEpisode.Thumbnail = thumbnailURL + } + _feed.Episodes = append(_feed.Episodes, newEpisode) added++ From 40649825bbf7df7f9969f3b1dedc59d3612a3366 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 20 Apr 2025 22:15:12 +1200 Subject: [PATCH 07/11] Added thumbnails for Nebula --- pkg/builder/nebula.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/builder/nebula.go b/pkg/builder/nebula.go index dc55a3bf..8a38add1 100644 --- a/pkg/builder/nebula.go +++ b/pkg/builder/nebula.go @@ -2,6 +2,7 @@ package builder import ( "context" + "strings" "time" "github.com/mmcdole/gofeed" @@ -17,6 +18,19 @@ type NebulaBuilder struct { getDuration fnGetDuration } +func splitThumbnail(desc string) string { + a := strings.Split(desc, "") + c := strings.Split(b[0], "?format") + + return c[0] +} + func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) { info, err := ParseURL(cfg.URL) if err != nil { @@ -56,7 +70,6 @@ func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F Description: item.Description, VideoURL: item.Link, PubDate: *item.PublishedParsed, - // Thumbnail: item., // Size: , Status: model.EpisodeNew, } @@ -66,6 +79,11 @@ func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F newEpisode.Duration = dur } + thumbnailURL := splitThumbnail(item.Description) + if thumbnailURL != "" { + newEpisode.Thumbnail = thumbnailURL + } + _feed.Episodes = append(_feed.Episodes, newEpisode) added++ From 8912a3e18f3e7da49b42226adcd77bbf8334e1bc Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 23 Apr 2025 21:00:16 +1200 Subject: [PATCH 08/11] Make feeds default to private --- pkg/feed/config.go | 2 +- pkg/feed/xml.go | 2 +- pkg/model/feed.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/feed/config.go b/pkg/feed/config.go index d45f0f7c..a938bbad 100644 --- a/pkg/feed/config.go +++ b/pkg/feed/config.go @@ -41,7 +41,7 @@ type Config struct { // Included in OPML file OPML bool `toml:"opml"` // Private feed (not indexed by podcast aggregators) - PrivateFeed bool `toml:"private_feed"` + PrivateFeed *bool `toml:"private_feed"` // Playlist sort PlaylistSort model.Sorting `toml:"playlist_sort"` } diff --git a/pkg/feed/xml.go b/pkg/feed/xml.go index cf64f8ac..05217fdb 100644 --- a/pkg/feed/xml.go +++ b/pkg/feed/xml.go @@ -66,7 +66,7 @@ func Build(_ctx context.Context, feed *model.Feed, cfg *Config, hostname string) p.IAuthor = author p.AddSummary(description) - if feed.PrivateFeed { + if feed.PrivateFeed == nil || *feed.PrivateFeed { p.IBlock = "yes" } diff --git a/pkg/model/feed.go b/pkg/model/feed.go index fb6cc40b..fff680a8 100644 --- a/pkg/model/feed.go +++ b/pkg/model/feed.go @@ -64,7 +64,7 @@ type Feed struct { Episodes []*Episode `json:"-"` // Array of episodes UpdatedAt time.Time `json:"updated_at"` PlaylistSort Sorting `json:"playlist_sort"` - PrivateFeed bool `json:"private_feed"` + PrivateFeed *bool `json:"private_feed"` } type EpisodeStatus string From eb2c907315c34a61d66ef14f1519ce9d01e11c4b Mon Sep 17 00:00:00 2001 From: Daniel Date: Wed, 23 Apr 2025 21:15:48 +1200 Subject: [PATCH 09/11] Remove Nebula episode image from description --- pkg/builder/nebula.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkg/builder/nebula.go b/pkg/builder/nebula.go index 8a38add1..6b661572 100644 --- a/pkg/builder/nebula.go +++ b/pkg/builder/nebula.go @@ -2,6 +2,7 @@ package builder import ( "context" + "regexp" "strings" "time" @@ -82,6 +83,9 @@ func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F thumbnailURL := splitThumbnail(item.Description) if thumbnailURL != "" { newEpisode.Thumbnail = thumbnailURL + pattern := `]+>` + re := regexp.MustCompile(pattern) + newEpisode.Description = re.ReplaceAllString(newEpisode.Description, "") } _feed.Episodes = append(_feed.Episodes, newEpisode) From 61023807eff60d00b748db53c0772d5a6816d42e Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 24 Apr 2025 20:58:36 +1200 Subject: [PATCH 10/11] Added cover art for Nebula channel feeds --- pkg/builder/nebula.go | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pkg/builder/nebula.go b/pkg/builder/nebula.go index 6b661572..dc53edec 100644 --- a/pkg/builder/nebula.go +++ b/pkg/builder/nebula.go @@ -2,6 +2,9 @@ package builder import ( "context" + "encoding/json" + "io" + "net/http" "regexp" "strings" "time" @@ -32,6 +35,38 @@ func splitThumbnail(desc string) string { return c[0] } +func getChannelAvatarURL(channelName string) (string, error) { + // Make the GET request + resp, err := http.Get("https://content.api.nebula.app/content/" + channelName) + if err != nil { + return "", err + } + defer resp.Body.Close() + + // Read the response body + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + // Define a struct to unmarshal the JSON into + var data struct { + Images struct { + Avatar struct { + Src string `json:"src"` + } `json:"avatar"` + } `json:"images"` + } + + // Parse the JSON + err = json.Unmarshal(body, &data) + if err != nil { + return "", err + } + + return data.Images.Avatar.Src, nil +} + func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) { info, err := ParseURL(cfg.URL) if err != nil { @@ -62,6 +97,11 @@ func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.F PubDate: *rssFeed.UpdatedParsed, } + avatarSrc, err := getChannelAvatarURL(info.ItemID) + if err == nil { + _feed.CoverArt = avatarSrc + } + // setup episodes and add to feed added := 0 for _, item := range rssFeed.Items { From 85869f9cfd9e54524a272a35af5cc4107dab5a53 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 26 Apr 2025 21:54:38 +1200 Subject: [PATCH 11/11] Added timeout note to how_to_nebula --- docs/how_to_nebula.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/how_to_nebula.md b/docs/how_to_nebula.md index 07d51093..1b800081 100644 --- a/docs/how_to_nebula.md +++ b/docs/how_to_nebula.md @@ -11,6 +11,8 @@ Setup your config.toml as below. For each feed, you must use url format `https://nebula.tv/`. Only channels are supported at this time. +Depending on the video length and chosen quality, videos may take a long time to download. If they exceed the timeout time, they fail. To avoid this, use a high download timeout value (65 minutes seems to be good for 1080p jetlag). + An example config.toml for Nebula: ``` [server] @@ -21,6 +23,9 @@ hostname = "http://localhost:8080" [storage.local] data_dir = "./app/data/" +[downloader] + timeout = 65 + [tokens] nebula = ""