Skip to content

Commit dca0aed

Browse files
yuval-cloudinaryyuval-cloudinary
committed
cloudinary: automatically add/remove known media files extensions rclone#8416 
1 parent 0ccf650 commit dca0aed

File tree

5 files changed

+82
-17
lines changed

5 files changed

+82
-17
lines changed

‎backend/cloudinary/api/types.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ type CloudinaryEncoder interface {
1818
ToStandardPath(string) string
1919
// ToStandardName takes name in this encoding and converts
2020
// it in Standard encoding.
21-
ToStandardName(string) string
21+
ToStandardName(string, string) string
2222
// Encoded root of the remote (as passed into NewFs)
2323
FromStandardFullPath(string) string
2424
}

‎backend/cloudinary/cloudinary.go‎

Lines changed: 56 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ import (
88
"fmt"
99
"io"
1010
"net/http"
11+
"net/url"
1112
"path"
13+
"slices"
1214
"strconv"
1315
"strings"
1416
"time"
@@ -103,19 +105,39 @@ func init() {
103105
Advanced: true,
104106
Help: "Wait N seconds for eventual consistency of the databases that support the backend operation",
105107
},
108+
{
109+
Name: "adjust_media_files_extensions",
110+
Default: true,
111+
Advanced: true,
112+
Help: "Cloudinary handles media formats as a file attribute and strips it from the name, which is unlike most other file systems",
113+
},
114+
{
115+
Name: "media_extensions",
116+
Default: []string{
117+
"3ds", "3g2", "3gp", "ai", "arw", "avi", "avif", "bmp", "bw",
118+
"cr2", "cr3", "djvu", "dng", "eps3", "fbx", "flif", "flv", "gif",
119+
"glb", "gltf", "hdp", "heic", "heif", "ico", "indd", "jp2", "jpe",
120+
"jpeg", "jpg", "jxl", "jxr", "m2ts", "mov", "mp4", "mpeg", "mts",
121+
"mxf", "obj", "ogv", "pdf", "ply", "png", "psd", "svg", "tga",
122+
"tif", "tiff", "ts", "u3ma", "usdz", "wdp", "webm", "webp", "wmv"},
123+
Advanced: true,
124+
Help: "Cloudinary supported media extensions",
125+
},
106126
},
107127
})
108128
}
109129

110130
// Options defines the configuration for this backend
111131
type Options struct {
112-
CloudName string `config:"cloud_name"`
113-
APIKey string `config:"api_key"`
114-
APISecret string `config:"api_secret"`
115-
UploadPrefix string `config:"upload_prefix"`
116-
UploadPreset string `config:"upload_preset"`
117-
Enc encoder.MultiEncoder `config:"encoding"`
118-
EventuallyConsistentDelay fs.Duration `config:"eventually_consistent_delay"`
132+
CloudName string `config:"cloud_name"`
133+
APIKey string `config:"api_key"`
134+
APISecret string `config:"api_secret"`
135+
UploadPrefix string `config:"upload_prefix"`
136+
UploadPreset string `config:"upload_preset"`
137+
Enc encoder.MultiEncoder `config:"encoding"`
138+
EventuallyConsistentDelay fs.Duration `config:"eventually_consistent_delay"`
139+
MediaExtensions []string `config:"media_extensions"`
140+
AdjustMediaFilesExtensions bool `config:"adjust_media_files_extensions"`
119141
}
120142

121143
// Fs represents a remote cloudinary server
@@ -203,6 +225,18 @@ func (f *Fs) FromStandardPath(s string) string {
203225

204226
// FromStandardName implementation of the api.CloudinaryEncoder
205227
func (f *Fs) FromStandardName(s string) string {
228+
if f.opt.AdjustMediaFilesExtensions {
229+
parsedURL, err := url.Parse(s)
230+
ext := ""
231+
if err != nil {
232+
fs.Logf(nil, "Error parsing URL: %v", err)
233+
} else {
234+
ext = path.Ext(parsedURL.Path)
235+
if slices.Contains(f.opt.MediaExtensions, strings.ToLower(strings.TrimPrefix(ext, "."))) {
236+
s = strings.TrimSuffix(parsedURL.Path, ext)
237+
}
238+
}
239+
}
206240
return strings.ReplaceAll(f.opt.Enc.FromStandardName(s), "&", "\uFF06")
207241
}
208242

@@ -212,8 +246,20 @@ func (f *Fs) ToStandardPath(s string) string {
212246
}
213247

214248
// ToStandardName implementation of the api.CloudinaryEncoder
215-
func (f *Fs) ToStandardName(s string) string {
216-
return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&")
249+
func (f *Fs) ToStandardName(s string, assetUrl string) string {
250+
ext := ""
251+
if f.opt.AdjustMediaFilesExtensions {
252+
parsedURL, err := url.Parse(assetUrl)
253+
if err != nil {
254+
fs.Logf(nil, "Error parsing URL: %v", err)
255+
} else {
256+
ext = path.Ext(parsedURL.Path)
257+
if !slices.Contains(f.opt.MediaExtensions, strings.ToLower(strings.TrimPrefix(ext, "."))) {
258+
ext = ""
259+
}
260+
}
261+
}
262+
return strings.ReplaceAll(f.opt.Enc.ToStandardName(s), "\uFF06", "&") + ext
217263
}
218264

219265
// FromStandardFullPath encodes a full path to Cloudinary standard
@@ -331,10 +377,7 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
331377
}
332378

333379
for _, asset := range results.Assets {
334-
remote := api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName)
335-
if dir != "" {
336-
remote = path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName))
337-
}
380+
remote := path.Join(dir, api.CloudinaryEncoder.ToStandardName(f, asset.DisplayName, asset.SecureURL))
338381
o := &Object{
339382
fs: f,
340383
remote: remote,

‎docs/content/cloudinary.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,28 @@ Properties:
206206
- Type: Duration
207207
- Default: 0s
208208

209+
#### --cloudinary-adjust-media-files-extensions
210+
211+
Cloudinary handles media formats as a file attribute and strips it from the name, which is unlike most other file systems
212+
213+
Properties:
214+
215+
- Config: adjust_media_files_extensions
216+
- Env Var: RCLONE_CLOUDINARY_ADJUST_MEDIA_FILES_EXTENSIONS
217+
- Type: bool
218+
- Default: true
219+
220+
#### --cloudinary-media-extensions
221+
222+
Cloudinary supported media extensions
223+
224+
Properties:
225+
226+
- Config: media_extensions
227+
- Env Var: RCLONE_CLOUDINARY_MEDIA_EXTENSIONS
228+
- Type: stringArray
229+
- Default: [3ds 3g2 3gp ai arw avi avif bmp bw cr2 cr3 djvu dng eps3 fbx flif flv gif glb gltf hdp heic heif ico indd jp2 jpe jpeg jpg jxl jxr m2ts mov mp4 mpeg mts mxf obj ogv pdf ply png psd svg tga tif tiff ts u3ma usdz wdp webm webp wmv]
230+
209231
#### --cloudinary-description
210232

211233
Description of the remote.

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ require (
2424
github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2
2525
github.com/aws/smithy-go v1.22.1
2626
github.com/buengese/sgzip v0.1.1
27-
github.com/cloudinary/cloudinary-go/v2 v2.9.0
27+
github.com/cloudinary/cloudinary-go/v2 v2.9.1
2828
github.com/cloudsoda/go-smb2 v0.0.0-20250124173933-e6bbeea507ed
2929
github.com/colinmarc/hdfs/v2 v2.4.0
3030
github.com/coreos/go-semver v0.3.1

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,8 +179,8 @@ github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtM
179179
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
180180
github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
181181
github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
182-
github.com/cloudinary/cloudinary-go/v2 v2.9.0 h1:8C76QklmuV4qmKAC7cUnu9D68X9kCkFMuLspPikECCo=
183-
github.com/cloudinary/cloudinary-go/v2 v2.9.0/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
182+
github.com/cloudinary/cloudinary-go/v2 v2.9.1 h1:YmR1+ayli8daanfUP8lKjOAFyK/wNJGBcLIUgK9YX8U=
183+
github.com/cloudinary/cloudinary-go/v2 v2.9.1/go.mod h1:ireC4gqVetsjVhYlwjUJwKTbZuWjEIynbR9zQTlqsvo=
184184
github.com/cloudsoda/go-smb2 v0.0.0-20250124173933-e6bbeea507ed h1:KrdJUJWhJ1UWhvaP6SBsvG356KjqfdDjcS/4xTswAU4=
185185
github.com/cloudsoda/go-smb2 v0.0.0-20250124173933-e6bbeea507ed/go.mod h1:0aLYPsmguHbok591y6hI5yAqU0drbUzrPEO10ZpgTTw=
186186
github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=

0 commit comments

Comments
 (0)