Skip to content

Commit 3930c13

Browse files
matthiasharzermatthiasharzer
authored
fix: Adjust Patreon-API handling (#12)
* chore: adjust api response handling to be a little more generic but overall more precise to the realworld patreon-API * test: add tests for unmarshal_response.go * chore: adjust spelling * fix: close response body to prevent resource leak * chore: adjust spelling
1 parent 2ca835d commit 3930c13

File tree

14 files changed

+871
-173
lines changed

14 files changed

+871
-173
lines changed

‎Makefile‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,18 +7,24 @@ endif
77

88
BUILD_VERSION ?= "unknown"
99

10-
build:
10+
clean:
1111
@rm -rf build/
1212

13+
build: clean
1314
@GOOS=windows GOARCH=amd64 go build -o ./build/patreon-crawler.exe -ldflags "-X main.version=$(BUILD_VERSION)" ./main.go
1415
@GOOS=linux GOARCH=amd64 go build -o ./build/patreon-crawler -ldflags "-X main.version=$(BUILD_VERSION)" ./main.go
1516

16-
qa: analyze
17+
qa: analyze test
1718

1819
analyze:
1920
@go vet
2021
@go run honnef.co/go/tools/cmd/staticcheck@latest --checks=all
2122

23+
test:
24+
@go test -failfast -cover ./...
25+
2226
.PHONY: build \
2327
analyze \
24-
qa
28+
qa \
29+
test \
30+
clean

‎README.md‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,12 +39,12 @@ If you do not wish do be prompted, you can also use the `--cookie` and `--downlo
3939

4040
The `patreon-crawler` supports the following command line flags.
4141

42-
| Argument | Description |
43-
|-------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
44-
| `creator` | The creator ID to download media from. You can find this in the URL when visiting a creators page: `patreon.com/c/<creator-id>/...` |
45-
| `cookie` | The cookie from the Patreon website to authenticate against the Patreon API |
46-
| `download-dir` | The base directory to download media to. All files will be located in `<download-dir>/<creator>` |
47-
| `download-limit` | The maximum number of media files to download. |
48-
| `download-inaccessible-media` | Whether to download media that is inaccessible (blurred images) |
49-
| `grouping` | The strategy for grouping post media into folders. <br>`none` - Puts all media into the same folder (per creator)<br>`by-post` - Creates a folder for each post, containing its media |
42+
| Argument | Description |
43+
|---------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
44+
| `--creator <creator-id>` | The creator ID to download media from. You can find this in the URL when visiting a creators page: `patreon.com/c/<creator-id>/...` |
45+
| `--cookie <cookie-string>` | The cookie from the Patreon website to authenticate against the Patreon API |
46+
| `--download-dir <directory>` | The base directory to download media to. All files will be located in `<download-dir>/<creator>` |
47+
| `--download-limit <number>` | The maximum number of media files to download. |
48+
| `--download-inaccessible-media` | Whether to download media that is inaccessible (blurred images) |
49+
| `--grouping <none \| by-post>` | The strategy for grouping post media into folders. <br>`none` - Puts all media into the same folder (per creator)<br>`by-post` - Creates a folder for each post, containing its media |
5050

‎go.mod‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
11
module patreon-crawler
22

33
go 1.24
4+
5+
require github.com/stretchr/testify v1.10.0
6+
7+
require (
8+
github.com/davecgh/go-spew v1.1.1 // indirect
9+
github.com/pmezard/go-difflib v1.0.0 // indirect
10+
gopkg.in/yaml.v3 v3.0.1 // indirect
11+
)

‎go.sum‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
2+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
3+
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
4+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
5+
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
6+
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
7+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
8+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
9+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
10+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎main.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,7 @@ func getAPIClient() (*api.Client, error) {
121121
if authenticated {
122122
return apiClient, nil
123123
}
124-
return nil, fmt.Errorf("failed to authenticate with provided cookie via --cookie")
124+
return nil, fmt.Errorf("failed to authenticate with cookie provided via --cookie")
125125
}
126126

127127
cookie, err := readCookieFromFile()
@@ -170,7 +170,7 @@ func main() {
170170
flag.PrintDefaults()
171171

172172
fmt.Println("\nUsage:")
173-
fmt.Println(" patreon-crawler --creator <creator ID> --download-dir <download directory>")
173+
fmt.Println(" patreon-crawler --creator <creator-dir> --download-dir <directory>")
174174
return
175175
}
176176

‎patreon/api/client.go‎

Lines changed: 37 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,12 @@ import (
44
"bytes"
55
"encoding/json"
66
"fmt"
7-
"io"
87
"net/http"
9-
"regexp"
108
"strings"
119
)
1210

13-
var patreonCampaignRegex = regexp.MustCompile(`patreon-media/p/campaign/(\d+)/.`)
14-
1511
const apiURL = "https://www.patreon.com/api"
1612

17-
func creatorURL(creatorID string) string {
18-
return "https://www.patreon.com/" + creatorID
19-
}
20-
2113
type Client struct {
2214
cookie string
2315
}
@@ -59,23 +51,42 @@ func (c *Client) doAPIRequest(path string, options map[string]string) (*http.Res
5951
}
6052

6153
func (c *Client) GetCampaignID(creatorID string) (string, error) {
62-
campaignURL := creatorURL(creatorID)
63-
response, err := http.Get(campaignURL)
54+
currentUser, err := c.GetCurrentUser()
6455
if err != nil {
65-
return "", fmt.Errorf("failed to get campaign ID: %s", err)
56+
return "", err
57+
}
58+
59+
for _, include := range currentUser.Included {
60+
switch include := include.(type) {
61+
case ResponseCampaign:
62+
if strings.EqualFold(include.Attributes.Vanity, creatorID) {
63+
return include.ID, nil
64+
}
65+
}
6666
}
6767

68-
body, err := io.ReadAll(response.Body)
68+
return "", fmt.Errorf("failed to find campaign ID")
69+
}
70+
71+
func (c *Client) GetCurrentUser() (UserResponse, error) {
72+
options := map[string]string{
73+
"include": "active_memberships.campaign",
74+
"fields[campaign]": "name,published_at,url,vanity",
75+
"json-api-version": "1.0",
76+
}
77+
response, err := c.doAPIRequest("/current_user", options)
6978
if err != nil {
70-
return "", fmt.Errorf("failed to read campaign ID response: %s", err)
79+
return UserResponse{}, err
7180
}
81+
defer response.Body.Close()
7282

73-
matches := patreonCampaignRegex.FindStringSubmatch(string(body))
74-
if len(matches) == 0 {
75-
return "", fmt.Errorf("failed to find campaign ID")
83+
var userResponse UserResponse
84+
err = UnmarshalResponse(response.Body, &userResponse)
85+
if err != nil {
86+
return UserResponse{}, err
7687
}
7788

78-
return matches[1], nil
89+
return userResponse, nil
7990
}
8091

8192
func (c *Client) GetPosts(campaignID string, cursor *string) (PostsResponse, error) {
@@ -97,9 +108,10 @@ func (c *Client) GetPosts(campaignID string, cursor *string) (PostsResponse, err
97108
if err != nil {
98109
return PostsResponse{}, err
99110
}
111+
defer response.Body.Close()
100112

101113
var postsResponse PostsResponse
102-
err = json.NewDecoder(response.Body).Decode(&postsResponse)
114+
err = UnmarshalResponse(response.Body, &postsResponse)
103115
if err != nil {
104116
return PostsResponse{}, err
105117
}
@@ -108,25 +120,19 @@ func (c *Client) GetPosts(campaignID string, cursor *string) (PostsResponse, err
108120
}
109121

110122
func (c *Client) IsAuthenticated() (bool, error) {
111-
options := map[string]string{
112-
"include": "active_memberships.campaign",
113-
"fields[campaign]": "avatar_photo_image_urls,name,published_at,url,vanity,is_nsfw,url_for_current_user",
114-
"fields[member]": "is_free_member,is_free_trial",
115-
"json-api-version": "1.0",
116-
}
117-
118-
response, err := c.doAPIRequest("/current_user", options)
123+
response, err := c.doAPIRequest("/current_user", nil)
119124
if err != nil {
120-
return false, err
125+
return false, fmt.Errorf("failed to get current user: %w", err)
121126
}
127+
defer response.Body.Close()
122128

123-
var userErrorResponse UserErrorResponse
124-
err = json.NewDecoder(response.Body).Decode(&userErrorResponse)
129+
var errorResponse ErrorResponse
130+
err = json.NewDecoder(response.Body).Decode(&errorResponse)
125131
if err != nil {
126-
return false, err
132+
return false, fmt.Errorf("failed to decode response: %w", err)
127133
}
128134

129-
if len(userErrorResponse.Errors) > 0 {
135+
if len(errorResponse.Errors) > 0 {
130136
return false, nil
131137
}
132138

‎patreon/api/posts_response.go‎

Lines changed: 1 addition & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -1,104 +1,3 @@
11
package api
22

3-
type PostsResponse struct {
4-
Data []ResponsePost `json:"data"`
5-
Includes []ResponseInclude `json:"included"`
6-
Meta ResponseMeta `json:"meta"`
7-
Links ResponseLinks `json:"links"`
8-
}
9-
10-
type ResponsePost struct {
11-
ID string `json:"id"`
12-
Type string `json:"type"`
13-
Attributes ResponsePostAttributes `json:"attributes"`
14-
RelationShips ResponsePostRelationships `json:"relationships"`
15-
}
16-
17-
type ResponsePostAttributes struct {
18-
PostType string `json:"post_type"`
19-
Title string `json:"title"`
20-
PublishedAt string `json:"published_at"`
21-
URL string `json:"url"`
22-
CurrentUserCanView bool `json:"current_user_can_view"`
23-
TeaserText string `json:"teaser_text"`
24-
PostMetaData ResponsePostMetaData `json:"post_metadata"`
25-
}
26-
27-
type ResponsePostMetaData struct {
28-
ImageOrder []string `json:"image_order"`
29-
}
30-
31-
type ResponsePostRelationships struct {
32-
Attachments ResponsePostRelationshipsAttachments `json:"attachments"`
33-
Images ResponsePostRelationshipsImages `json:"images"`
34-
Media ResponsePostRelationshipsMedia `json:"media"`
35-
}
36-
37-
type ResponsePostRelationshipsAttachments struct {
38-
Data []any `json:"data"`
39-
}
40-
41-
type ResponsePostRelationshipsImages struct {
42-
Data []ResponseMedia `json:"data"`
43-
}
44-
45-
type ResponsePostRelationshipsMedia struct {
46-
Data []ResponseMedia `json:"data"`
47-
}
48-
49-
type ResponseMedia struct {
50-
ID string `json:"id"`
51-
Type string `json:"type"`
52-
}
53-
54-
type ResponseInclude struct {
55-
ResponseMedia
56-
Attributes ResponseMediaAttributes `json:"attributes"`
57-
}
58-
59-
type ResponseMediaAttributes struct {
60-
SizeBytes int `json:"size_bytes"`
61-
MimeType string `json:"mimetype"`
62-
DownloadURL string `json:"download_url"`
63-
ImageURLs ResponseMediaImageURLs `json:"image_urls"`
64-
Metadata ResponseMediaMetadata `json:"metadata"`
65-
}
66-
67-
type ResponseMediaImageURLs struct {
68-
URL string `json:"url"`
69-
Original string `json:"original"`
70-
Default string `json:"default"`
71-
DefaultBlurred string `json:"default_blurred"`
72-
DefaultSmall string `json:"default_small"`
73-
DefaultLarge string `json:"default_large"`
74-
DefaultBlurredSmall string `json:"default_blurred_small"`
75-
Thumbnail string `json:"thumbnail"`
76-
ThumbnailLarge string `json:"thumbnail_large"`
77-
ThumbnailSmall string `json:"thumbnail_small"`
78-
}
79-
80-
type ResponseMediaMetadata struct {
81-
Dimensions ResponseMediaDimensions `json:"dimensions"`
82-
}
83-
84-
type ResponseMediaDimensions struct {
85-
W int `json:"w"`
86-
H int `json:"h"`
87-
}
88-
89-
type ResponseMeta struct {
90-
Pagination ResponsePagination `json:"pagination"`
91-
}
92-
93-
type ResponsePagination struct {
94-
Total int `json:"total"`
95-
Cursors ResponseCursors `json:"cursors"`
96-
}
97-
98-
type ResponseCursors struct {
99-
Next string `json:"next"`
100-
}
101-
102-
type ResponseLinks struct {
103-
Next string `json:"next"`
104-
}
3+
type PostsResponse = Response[[]ResponsePost]

0 commit comments

Comments
 (0)