Skip to content

Commit 1a395b3

Browse files
wxiaoguangwxiaoguang
committed
fix url detection
1 parent 24642bb commit 1a395b3

File tree

6 files changed

+149
-77
lines changed

6 files changed

+149
-77
lines changed

‎custom/conf/app.example.ini‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1769,10 +1769,7 @@ LEVEL = Info
17691769
;SENDMAIL_CONVERT_CRLF = true
17701770
;;
17711771
;; convert links of attached images to inline images. Only for images hosted in this gitea instance.
1772-
;BASE64_EMBED_IMAGES = false
1773-
;;
1774-
;; The maximum size of sum of all images in a single email. Default is 9.5MB
1775-
;BASE64_EMBED_IMAGES_MAX_SIZE_PER_EMAIL = 9961472
1772+
;EMBED_ATTACHMENT_IMAGES = false
17761773

17771774
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
17781775
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

‎modules/httplib/url.go‎

Lines changed: 65 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -102,25 +102,77 @@ func MakeAbsoluteURL(ctx context.Context, link string) string {
102102
return GuessCurrentHostURL(ctx) + "/" + strings.TrimPrefix(link, "/")
103103
}
104104

105-
func IsCurrentGiteaSiteURL(ctx context.Context, s string) bool {
105+
type urlType int
106+
107+
const (
108+
urlTypeGiteaAbsolute urlType = iota + 1 // "http://gitea/subpath"
109+
urlTypeGiteaPageRelative // "/subpath"
110+
urlTypeGiteaSiteRelative // "?key=val"
111+
urlTypeUnknown // "http://other"
112+
)
113+
114+
func detectURLRoutePath(ctx context.Context, s string) (routePath string, ut urlType) {
106115
u, err := url.Parse(s)
107116
if err != nil {
108-
return false
117+
return "", urlTypeUnknown
109118
}
119+
cleanedPath := ""
110120
if u.Path != "" {
111-
cleanedPath := util.PathJoinRelX(u.Path)
112-
if cleanedPath == "" || cleanedPath == "." {
113-
u.Path = "/"
114-
} else {
115-
u.Path = "/" + cleanedPath + "/"
116-
}
121+
cleanedPath = util.PathJoinRelX(u.Path)
122+
cleanedPath = util.Iif(cleanedPath == ".", "", "/"+cleanedPath)
117123
}
118124
if urlIsRelative(s, u) {
119-
return u.Path == "" || strings.HasPrefix(strings.ToLower(u.Path), strings.ToLower(setting.AppSubURL+"/"))
120-
}
121-
if u.Path == "" {
122-
u.Path = "/"
125+
if u.Path == "" {
126+
return "", urlTypeGiteaPageRelative
127+
}
128+
if strings.HasPrefix(strings.ToLower(cleanedPath+"/"), strings.ToLower(setting.AppSubURL+"/")) {
129+
return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaSiteRelative
130+
}
131+
return "", urlTypeUnknown
123132
}
133+
u.Path = cleanedPath + "/"
124134
urlLower := strings.ToLower(u.String())
125-
return strings.HasPrefix(urlLower, strings.ToLower(setting.AppURL)) || strings.HasPrefix(urlLower, strings.ToLower(GuessCurrentAppURL(ctx)))
135+
if strings.HasPrefix(urlLower, strings.ToLower(setting.AppURL)) {
136+
return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaAbsolute
137+
}
138+
guessedCurURL := GuessCurrentAppURL(ctx)
139+
if strings.HasPrefix(urlLower, strings.ToLower(guessedCurURL)) {
140+
return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaAbsolute
141+
}
142+
return "", urlTypeUnknown
143+
}
144+
145+
func IsCurrentGiteaSiteURL(ctx context.Context, s string) bool {
146+
_, ut := detectURLRoutePath(ctx, s)
147+
return ut != urlTypeUnknown
148+
}
149+
150+
type GiteaSiteURL struct {
151+
RoutePath string
152+
OwnerName string
153+
RepoName string
154+
RepoSubPath string
155+
}
156+
157+
func ParseGiteaSiteURL(ctx context.Context, s string) *GiteaSiteURL {
158+
routePath, ut := detectURLRoutePath(ctx, s)
159+
if ut == urlTypeUnknown || ut == urlTypeGiteaPageRelative {
160+
return nil
161+
}
162+
ret := &GiteaSiteURL{RoutePath: routePath}
163+
fields := strings.SplitN(strings.TrimPrefix(ret.RoutePath, "/"), "/", 3)
164+
165+
// TODO: now it only does a quick check for some known reserved paths, should do more strict checks in the future
166+
if fields[0] == "attachments" {
167+
return ret
168+
}
169+
if len(fields) < 2 {
170+
return ret
171+
}
172+
ret.OwnerName = fields[0]
173+
ret.RepoName = fields[1]
174+
if len(fields) == 3 {
175+
ret.RepoSubPath = "/" + fields[2]
176+
}
177+
return ret
126178
}

‎modules/httplib/url_test.go‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,3 +122,26 @@ func TestIsCurrentGiteaSiteURL(t *testing.T) {
122122
assert.True(t, IsCurrentGiteaSiteURL(ctx, "https://user-host"))
123123
assert.False(t, IsCurrentGiteaSiteURL(ctx, "https://forwarded-host"))
124124
}
125+
126+
func TestParseGiteaSiteURL(t *testing.T) {
127+
defer test.MockVariableValue(&setting.AppURL, "http://localhost:3000/sub/")()
128+
defer test.MockVariableValue(&setting.AppSubURL, "/sub")()
129+
ctx := t.Context()
130+
tests := []struct {
131+
url string
132+
exp *GiteaSiteURL
133+
}{
134+
{"http://localhost:3000/sub?k=v", &GiteaSiteURL{RoutePath: ""}},
135+
{"http://localhost:3000/sub/", &GiteaSiteURL{RoutePath: ""}},
136+
{"http://localhost:3000/sub/foo", &GiteaSiteURL{RoutePath: "/foo"}},
137+
{"http://localhost:3000/sub/foo/bar", &GiteaSiteURL{RoutePath: "/foo/bar", OwnerName: "foo", RepoName: "bar"}},
138+
{"http://localhost:3000/sub/foo/bar/", &GiteaSiteURL{RoutePath: "/foo/bar", OwnerName: "foo", RepoName: "bar"}},
139+
{"http://localhost:3000/sub/attachments/bar", &GiteaSiteURL{RoutePath: "/attachments/bar"}},
140+
{"http://localhost:3000/other", nil},
141+
{"http://other/", nil},
142+
}
143+
for _, test := range tests {
144+
su := ParseGiteaSiteURL(ctx, test.url)
145+
assert.Equal(t, test.exp, su, "URL = %s", test.url)
146+
}
147+
}

‎modules/setting/mailer.go‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,18 @@ import (
1919
// Mailer represents mail service.
2020
type Mailer struct {
2121
// Mailer
22-
Name string `ini:"NAME"`
23-
From string `ini:"FROM"`
24-
EnvelopeFrom string `ini:"ENVELOPE_FROM"`
25-
OverrideEnvelopeFrom bool `ini:"-"`
26-
FromName string `ini:"-"`
27-
FromEmail string `ini:"-"`
28-
SendAsPlainText bool `ini:"SEND_AS_PLAIN_TEXT"`
29-
SubjectPrefix string `ini:"SUBJECT_PREFIX"`
30-
OverrideHeader map[string][]string `ini:"-"`
31-
Base64EmbedImages bool `ini:"BASE64_EMBED_IMAGES"`
32-
Base64EmbedImagesMaxSizePerEmail int64 `ini:"BASE64_EMBED_IMAGES_MAX_SIZE_PER_EMAIL"`
22+
Name string `ini:"NAME"`
23+
From string `ini:"FROM"`
24+
EnvelopeFrom string `ini:"ENVELOPE_FROM"`
25+
OverrideEnvelopeFrom bool `ini:"-"`
26+
FromName string `ini:"-"`
27+
FromEmail string `ini:"-"`
28+
SendAsPlainText bool `ini:"SEND_AS_PLAIN_TEXT"`
29+
SubjectPrefix string `ini:"SUBJECT_PREFIX"`
30+
OverrideHeader map[string][]string `ini:"-"`
31+
32+
// Embed attachment images as inline base64 img src attribute
33+
EmbedAttachmentImages bool
3334

3435
// SMTP sender
3536
Protocol string `ini:"PROTOCOL"`
@@ -152,8 +153,6 @@ func loadMailerFrom(rootCfg ConfigProvider) {
152153
sec.Key("SENDMAIL_TIMEOUT").MustDuration(5 * time.Minute)
153154
sec.Key("SENDMAIL_CONVERT_CRLF").MustBool(true)
154155
sec.Key("FROM").MustString(sec.Key("USER").String())
155-
sec.Key("BASE64_EMBED_IMAGES").MustBool(false)
156-
sec.Key("BASE64_EMBED_IMAGES_MAX_SIZE_PER_EMAIL").MustInt64(9.5 * 1024 * 1024)
157156

158157
// Now map the values on to the MailService
159158
MailService = &Mailer{}

‎services/mailer/mail.go‎

Lines changed: 37 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,14 @@ package mailer
66

77
import (
88
"bytes"
9+
"code.gitea.io/gitea/modules/httplib"
10+
"code.gitea.io/gitea/modules/typesniffer"
911
"context"
1012
"encoding/base64"
1113
"fmt"
1214
"html/template"
1315
"io"
1416
"mime"
15-
"net/http"
1617
"regexp"
1718
"strings"
1819
texttmpl "text/template"
@@ -54,42 +55,43 @@ func sanitizeSubject(subject string) string {
5455
}
5556

5657
type mailAttachmentBase64Embedder struct {
57-
doer *user_model.User
58-
repo *repo_model.Repository
59-
maxSize int64
58+
doer *user_model.User
59+
repo *repo_model.Repository
60+
maxSize int64
61+
estimateSize int64
6062
}
6163

6264
func newMailAttachmentBase64Embedder(doer *user_model.User, repo *repo_model.Repository, maxSize int64) *mailAttachmentBase64Embedder {
6365
return &mailAttachmentBase64Embedder{doer: doer, repo: repo, maxSize: maxSize}
6466
}
6567

66-
func (b64embedder *mailAttachmentBase64Embedder) Base64InlineImages(ctx context.Context, body string) (string, error) {
67-
doc, err := html.Parse(strings.NewReader(body))
68+
func (b64embedder *mailAttachmentBase64Embedder) Base64InlineImages(ctx context.Context, body template.HTML) (template.HTML, error) {
69+
doc, err := html.Parse(strings.NewReader(string(body)))
6870
if err != nil {
69-
return "", fmt.Errorf("%w", err)
71+
return "", fmt.Errorf("html.Parse failed: %w", err)
7072
}
7173

72-
var totalEmbeddedImagesSize int64
74+
b64embedder.estimateSize = int64(len(string(body)))
7375

7476
var processNode func(*html.Node)
7577
processNode = func(n *html.Node) {
7678
if n.Type == html.ElementNode {
7779
if n.Data == "img" {
7880
for i, attr := range n.Attr {
7981
if attr.Key == "src" {
80-
attachmentPath := attr.Val
81-
dataURI, err := b64embedder.AttachmentSrcToBase64DataURI(ctx, attachmentPath, &totalEmbeddedImagesSize)
82+
attachmentSrc := attr.Val
83+
dataURI, err := b64embedder.AttachmentSrcToBase64DataURI(ctx, attachmentSrc)
8284
if err != nil {
83-
log.Trace("attachmentSrcToDataURI not possible: %v", err) // Not an error, just skip. This is probably an image from outside the gitea instance.
84-
continue
85+
// Not an error, just skip. This is probably an image from outside the gitea instance.
86+
log.Trace("Unable to embed attachment %q to mail body: %v", attachmentSrc, err)
87+
} else {
88+
n.Attr[i].Val = dataURI
8589
}
86-
n.Attr[i].Val = dataURI
8790
break
8891
}
8992
}
9093
}
9194
}
92-
9395
for c := n.FirstChild; c != nil; c = c.NextSibling {
9496
processNode(c)
9597
}
@@ -100,22 +102,24 @@ func (b64embedder *mailAttachmentBase64Embedder) Base64InlineImages(ctx context.
100102
var buf bytes.Buffer
101103
err = html.Render(&buf, doc)
102104
if err != nil {
103-
log.Error("Failed to render modified HTML: %v", err)
104-
return "", err
105+
return "", fmt.Errorf("html.Render failed: %w", err)
105106
}
106-
return buf.String(), nil
107+
return template.HTML(buf.String()), nil
107108
}
108109

109-
func (b64embedder *mailAttachmentBase64Embedder) AttachmentSrcToBase64DataURI(ctx context.Context, attachmentPath string, totalEmbeddedImagesSize *int64) (string, error) {
110-
if !strings.HasPrefix(attachmentPath, setting.AppURL) { // external image
111-
return "", fmt.Errorf("external image")
112-
}
113-
parts := strings.Split(attachmentPath, "/attachments/")
114-
if len(parts) <= 1 {
115-
return "", fmt.Errorf("invalid attachment path: %s", attachmentPath)
110+
func (b64embedder *mailAttachmentBase64Embedder) AttachmentSrcToBase64DataURI(ctx context.Context, attachmentSrc string) (string, error) {
111+
parsedSrc := httplib.ParseGiteaSiteURL(ctx, attachmentSrc)
112+
var attachmentUUID string
113+
if parsedSrc != nil {
114+
var ok bool
115+
attachmentUUID, ok = strings.CutPrefix(parsedSrc.RoutePath, "/attachments/")
116+
if !ok {
117+
attachmentUUID, ok = strings.CutPrefix(parsedSrc.RepoSubPath, "/attachments/")
118+
}
119+
if !ok {
120+
return "", fmt.Errorf("not an attachment")
121+
}
116122
}
117-
118-
attachmentUUID := parts[len(parts)-1]
119123
attachment, err := repo_model.GetAttachmentByUUID(ctx, attachmentUUID)
120124
if err != nil {
121125
return "", err
@@ -124,6 +128,10 @@ func (b64embedder *mailAttachmentBase64Embedder) AttachmentSrcToBase64DataURI(ct
124128
if attachment.RepoID != b64embedder.repo.ID {
125129
return "", fmt.Errorf("attachment does not belong to the repository")
126130
}
131+
if attachment.Size+b64embedder.estimateSize > b64embedder.maxSize {
132+
return "", fmt.Errorf("total embedded images exceed max limit")
133+
}
134+
b64embedder.estimateSize += attachment.Size
127135

128136
fr, err := storage.Attachments.Open(attachment.RelativePath())
129137
if err != nil {
@@ -134,26 +142,16 @@ func (b64embedder *mailAttachmentBase64Embedder) AttachmentSrcToBase64DataURI(ct
134142
lr := &io.LimitedReader{R: fr, N: b64embedder.maxSize + 1}
135143
content, err := io.ReadAll(lr)
136144
if err != nil {
137-
return "", err
138-
}
139-
if int64(len(content)) > b64embedder.maxSize {
140-
return "", fmt.Errorf("file size exceeds the embedded image max limit \\(%d bytes\\)", b64embedder.maxSize)
141-
}
142-
143-
if *totalEmbeddedImagesSize+int64(len(content)) > setting.MailService.Base64EmbedImagesMaxSizePerEmail {
144-
return "", fmt.Errorf("total embedded images exceed max limit: %d > %d", *totalEmbeddedImagesSize+int64(len(content)), setting.MailService.Base64EmbedImagesMaxSizePerEmail)
145+
return "", fmt.Errorf("LimitedReader ReadAll: %w", err)
145146
}
146-
*totalEmbeddedImagesSize += int64(len(content))
147-
148-
mimeType := http.DetectContentType(content)
149147

150-
if !strings.HasPrefix(mimeType, "image/") {
148+
mimeType := typesniffer.DetectContentType(content)
149+
if !mimeType.IsImage() {
151150
return "", fmt.Errorf("not an image")
152151
}
153152

154153
encoded := base64.StdEncoding.EncodeToString(content)
155154
dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, encoded)
156-
157155
return dataURI, nil
158156
}
159157

‎services/mailer/mail_issue_common.go‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import (
77
"bytes"
88
"context"
99
"fmt"
10-
"html/template"
1110
"strconv"
1211
"strings"
1312
"time"
@@ -26,6 +25,10 @@ import (
2625
"code.gitea.io/gitea/services/mailer/token"
2726
)
2827

28+
// maxEmailBodySize is the approximate maximum size of an email body in bytes
29+
// Many e-mail service providers have limitations on the size of the email body, it's usually from 10MB to 25MB
30+
const maxEmailBodySize = 9_000_000
31+
2932
func fallbackMailSubject(issue *issues_model.Issue) string {
3033
return fmt.Sprintf("[%s] %s (#%d)", issue.Repo.FullName(), issue.Title, issue.Index)
3134
}
@@ -65,19 +68,19 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipient
6568

6669
// This is the body of the new issue or comment, not the mail body
6770
rctx := renderhelper.NewRenderContextRepoComment(ctx.Context, ctx.Issue.Repo).WithUseAbsoluteLink(true)
68-
body, err := markdown.RenderString(rctx,
69-
ctx.Content)
71+
body, err := markdown.RenderString(rctx, ctx.Content)
7072
if err != nil {
7173
return nil, err
7274
}
7375

74-
if setting.MailService.Base64EmbedImages {
75-
bodyStr := string(body)
76-
bodyStr, err = Base64InlineImages(bodyStr, ctx)
76+
if setting.MailService.EmbedAttachmentImages {
77+
attEmbedder := newMailAttachmentBase64Embedder(ctx.Doer, ctx.Issue.Repo, maxEmailBodySize)
78+
bodyAfterEmbedding, err := attEmbedder.Base64InlineImages(ctx, body)
7779
if err != nil {
78-
return nil, err
80+
log.Error("Failed to embed images in mail body: %v", err)
81+
} else {
82+
body = bodyAfterEmbedding
7983
}
80-
body = template.HTML(bodyStr)
8184
}
8285
actType, actName, tplName := actionToTemplate(ctx.Issue, ctx.ActionType, commentType, reviewType)
8386

0 commit comments

Comments
 (0)