Update to match upstream

docs/how_to_nebula.md

@@ -0,0 +1,43 @@
+# How to get token
+1. Log in to nebula.tv
+2. Open Network tab in developer console
+3. Refresh page
+5. Find request to `/api/v1/authorization`
+6. Under request headers find `Authorization Token: <token>`
+7. Copy the value of `<token>` and use in `config.toml`
+
+# How to get nebula feed
+Setup your config.toml as below.
+
+For each feed, you must use url format `https://nebula.tv/<channel name>`. Only channels are supported at this time.
+
+Depending on the video length and chosen quality, videos may take a long time to download. If they exceed the timeout time, they fail. To avoid this, use a high download timeout value (65 minutes seems to be good for 1080p jetlag).
+
+An example config.toml for Nebula:
+```
+[server]
+port = 8080
+hostname = "http://localhost:8080"
+
+[storage]
+  [storage.local]
+  data_dir = "./app/data/"
+
+[downloader]
+  timeout = 65
+
+[tokens]
+nebula = "<your token>"
+
+[feeds]
+    [feeds.jetlag]
+    youtube_dl_args = ["--embed-subs"]
+    url = "https://nebula.tv/jetlag"
+    max_height = 1080
+    page_size = 3
+    opml = true
+```
+
+I like to use feed option
+`youtube_dl_args = ["--embed-subs"]`
+to get subtitles for jetlag but that's up to you.

go.mod

@@ -24,6 +24,16 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 )
 
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	golang.org/x/text v0.14.0 // indirect
+)
+
 require (
 	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
@@ -34,6 +44,7 @@ require (
 	github.com/grafov/m3u8 v0.11.1 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/mmcdole/gofeed v1.3.0
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect

go.sum

@@ -5,6 +5,10 @@ github.com/BrianHicks/finch v0.0.0-20140409222414-419bd73c29ec/go.mod h1:+hWo/MW
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U=
+github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
+github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c=
+github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/aws/aws-sdk-go v1.44.144 h1:mMWdnYL8HZsobrQe1mwvQ18Xt8UbOVhWgipjuma5Mkg=
 github.com/aws/aws-sdk-go v1.44.144/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
@@ -39,6 +43,7 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/grafov/m3u8 v0.11.1 h1:igZ7EBIB2IAsPPazKwRKdbhxcoBKO3lO1UY57PZDeNA=
 github.com/grafov/m3u8 v0.11.1/go.mod h1:nqzOkfBiZJENr52zTVd/Dcl03yzphIMbJqkXGu+u080=
 github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
@@ -53,6 +58,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -61,6 +68,15 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mmcdole/gofeed v1.3.0 h1:5yn+HeqlcvjMeAI4gu6T+crm7d0anY85+M+v6fIFNG4=
+github.com/mmcdole/gofeed v1.3.0/go.mod h1:9TGv2LcJhdXePDzxiuMnukhV2/zb6VtnZt1mS+SjkLE=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23 h1:Zr92CAlFhy2gL+V1F+EyIuzbQNbSgP4xhTODZtrXUtk=
+github.com/mmcdole/goxpp v1.1.1-0.20240225020742-a0c311522b23/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
@@ -87,6 +103,7 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
@@ -107,6 +124,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
@@ -124,6 +142,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -137,6 +156,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=

pkg/builder/builder.go

@@ -13,14 +13,16 @@ type Builder interface {
 	Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error)
 }
 
-func New(ctx context.Context, provider model.Provider, key string) (Builder, error) {
+func New(ctx context.Context, provider model.Provider, key string, durationGetter fnGetDuration) (Builder, error) {
 	switch provider {
 	case model.ProviderYoutube:
 		return NewYouTubeBuilder(key)
 	case model.ProviderVimeo:
 		return NewVimeoBuilder(ctx, key)
 	case model.ProviderSoundcloud:
 		return NewSoundcloudBuilder()
+	case model.ProviderNebula:
+		return newNebulaBuilder(key, durationGetter)
 	default:
 		return nil, errors.Errorf("unsupported provider %q", provider)
 	}

pkg/builder/nebula.go

@@ -0,0 +1,149 @@
+package builder
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/mmcdole/gofeed"
+	"github.com/mxpv/podsync/pkg/feed"
+	"github.com/mxpv/podsync/pkg/model"
+	"github.com/pkg/errors"
+)
+
+type fnGetDuration func(ctx context.Context, url string, extra_args ...string) (duration int64, err error)
+
+type NebulaBuilder struct {
+	token       string
+	getDuration fnGetDuration
+}
+
+func splitThumbnail(desc string) string {
+	a := strings.Split(desc, "<img src=")
+
+	if len(a) < 2 {
+		return ""
+	}
+
+	b := strings.Split(a[1], ">")
+	c := strings.Split(b[0], "?format")
+
+	return c[0]
+}
+
+func getChannelAvatarURL(channelName string) (string, error) {
+	// Make the GET request
+	resp, err := http.Get("https://content.api.nebula.app/content/" + channelName)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+
+	// Read the response body
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	// Define a struct to unmarshal the JSON into
+	var data struct {
+		Images struct {
+			Avatar struct {
+				Src string `json:"src"`
+			} `json:"avatar"`
+		} `json:"images"`
+	}
+
+	// Parse the JSON
+	err = json.Unmarshal(body, &data)
+	if err != nil {
+		return "", err
+	}
+
+	return data.Images.Avatar.Src, nil
+}
+
+func (neb *NebulaBuilder) Build(ctx context.Context, cfg *feed.Config) (*model.Feed, error) {
+	info, err := ParseURL(cfg.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	// add authorisation token to yt-dlp arguments
+	cfg.YouTubeDLArgs = append(cfg.YouTubeDLArgs, "--add-headers", "Authorization: Token "+neb.token)
+
+	fp := gofeed.Parser{}
+	rssFeed, err := fp.ParseURL("https://rss.nebula.app/video/channels/" + info.ItemID + ".rss")
+	if err != nil {
+		return nil, err
+	}
+
+	_feed := &model.Feed{
+		ItemID:    info.ItemID,
+		Provider:  info.Provider,
+		LinkType:  info.LinkType,
+		Format:    cfg.Format,
+		Quality:   cfg.Quality,
+		PageSize:  cfg.PageSize,
+		UpdatedAt: time.Now().UTC(),
+
+		Title:       rssFeed.Title,
+		ItemURL:     rssFeed.Link,
+		Description: rssFeed.Description,
+		PubDate:     *rssFeed.UpdatedParsed,
+	}
+
+	avatarSrc, err := getChannelAvatarURL(info.ItemID)
+	if err == nil {
+		_feed.CoverArt = avatarSrc
+	}
+
+	// setup episodes and add to feed
+	added := 0
+	for _, item := range rssFeed.Items {
+		newEpisode := &model.Episode{
+			ID:          item.GUID,
+			Title:       item.Title,
+			Description: item.Description,
+			VideoURL:    item.Link,
+			PubDate:     *item.PublishedParsed,
+			// Size: ,
+			Status: model.EpisodeNew,
+		}
+
+		dur, err := neb.getDuration(ctx, item.Link, "--add-headers", "Authorization: Token "+neb.token)
+		if err != nil {
+			newEpisode.Duration = dur
+		}
+
+		thumbnailURL := splitThumbnail(item.Description)
+		if thumbnailURL != "" {
+			newEpisode.Thumbnail = thumbnailURL
+			pattern := `<img src=` + regexp.QuoteMeta(thumbnailURL) + `\?[^>]+>`
+			re := regexp.MustCompile(pattern)
+			newEpisode.Description = re.ReplaceAllString(newEpisode.Description, "")
+		}
+
+		_feed.Episodes = append(_feed.Episodes, newEpisode)
+
+		added++
+
+		if added >= _feed.PageSize {
+			break
+		}
+	}
+
+	return _feed, nil
+}
+
+func newNebulaBuilder(key string, durationGetter fnGetDuration) (*NebulaBuilder, error) {
+	if key == "" {
+		return nil, errors.New("invalid key given for Nebula")
+	}
+
+	return &NebulaBuilder{token: key, getDuration: durationGetter}, nil
+}

pkg/builder/url.go

@@ -56,6 +56,19 @@ func ParseURL(link string) (model.Info, error) {
 		return info, nil
 	}
 
+	if strings.HasSuffix(parsed.Host, "nebula.tv") {
+		kind, id, err := parseNebulaURL(parsed)
+		if err != nil {
+			return model.Info{}, err
+		}
+
+		info.Provider = model.ProviderNebula
+		info.LinkType = kind
+		info.ItemID = id
+
+		return info, nil
+	}
+
 	return model.Info{}, errors.New("unsupported URL host")
 }
 
@@ -186,3 +199,13 @@ func parseSoundcloudURL(parsed *url.URL) (model.Type, string, error) {
 
 	return kind, id, nil
 }
+
+func parseNebulaURL(parsed *url.URL) (model.Type, string, error) {
+	parts := strings.Split(parsed.EscapedPath(), "/")
+
+	if len(parts) != 2 {
+		return "", "", errors.New("invalid nebula link path")
+	}
+
+	return model.TypeChannel, parts[1], nil
+}

pkg/feed/config.go

@@ -41,7 +41,7 @@ type Config struct {
 	// Included in OPML file
 	OPML bool `toml:"opml"`
 	// Private feed (not indexed by podcast aggregators)
-	PrivateFeed bool `toml:"private_feed"`
+	PrivateFeed *bool `toml:"private_feed"`
 	// Playlist sort
 	PlaylistSort model.Sorting `toml:"playlist_sort"`
 }

pkg/feed/xml.go

@@ -66,7 +66,7 @@ func Build(_ctx context.Context, feed *model.Feed, cfg *Config, hostname string)
 	p.IAuthor = author
 	p.AddSummary(description)
 
-	if feed.PrivateFeed {
+	if feed.PrivateFeed == nil || *feed.PrivateFeed {
 		p.IBlock = "yes"
 	}
 

pkg/model/feed.go

@@ -64,7 +64,7 @@ type Feed struct {
 	Episodes        []*Episode `json:"-"`        // Array of episodes
 	UpdatedAt       time.Time  `json:"updated_at"`
 	PlaylistSort    Sorting    `json:"playlist_sort"`
-	PrivateFeed     bool       `json:"private_feed"`
+	PrivateFeed     *bool      `json:"private_feed"`
 }
 
 type EpisodeStatus string

pkg/model/link.go

@@ -15,11 +15,12 @@ const (
 	ProviderYoutube    = Provider("youtube")
 	ProviderVimeo      = Provider("vimeo")
 	ProviderSoundcloud = Provider("soundcloud")
+	ProviderNebula     = Provider("nebula")
 )
 
 // Info represents data extracted from URL
 type Info struct {
 	LinkType Type     // Either group, channel or user
-	Provider Provider // Youtube, Vimeo, or SoundCloud
+	Provider Provider // Youtube, Vimeo, SoundCloud, or Nebula
 	ItemID   string
 }