diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 899209874f7cc..178d7a13631fe 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -1767,6 +1767,9 @@ LEVEL = Info ;; ;; convert \r\n to \n for Sendmail ;SENDMAIL_CONVERT_CRLF = true +;; +;; convert links of attached images to inline images. Only for images hosted in this gitea instance. +;EMBED_ATTACHMENT_IMAGES = false ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/modules/httplib/url.go b/modules/httplib/url.go index f543c09190673..5d5b64dc0cfbf 100644 --- a/modules/httplib/url.go +++ b/modules/httplib/url.go @@ -102,25 +102,77 @@ func MakeAbsoluteURL(ctx context.Context, link string) string { return GuessCurrentHostURL(ctx) + "/" + strings.TrimPrefix(link, "/") } -func IsCurrentGiteaSiteURL(ctx context.Context, s string) bool { +type urlType int + +const ( + urlTypeGiteaAbsolute urlType = iota + 1 // "http://gitea/subpath" + urlTypeGiteaPageRelative // "/subpath" + urlTypeGiteaSiteRelative // "?key=val" + urlTypeUnknown // "http://other" +) + +func detectURLRoutePath(ctx context.Context, s string) (routePath string, ut urlType) { u, err := url.Parse(s) if err != nil { - return false + return "", urlTypeUnknown } + cleanedPath := "" if u.Path != "" { - cleanedPath := util.PathJoinRelX(u.Path) - if cleanedPath == "" || cleanedPath == "." { - u.Path = "/" - } else { - u.Path = "/" + cleanedPath + "/" - } + cleanedPath = util.PathJoinRelX(u.Path) + cleanedPath = util.Iif(cleanedPath == ".", "", "/"+cleanedPath) } if urlIsRelative(s, u) { - return u.Path == "" || strings.HasPrefix(strings.ToLower(u.Path), strings.ToLower(setting.AppSubURL+"/")) - } - if u.Path == "" { - u.Path = "/" + if u.Path == "" { + return "", urlTypeGiteaPageRelative + } + if strings.HasPrefix(strings.ToLower(cleanedPath+"/"), strings.ToLower(setting.AppSubURL+"/")) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaSiteRelative + } + return "", urlTypeUnknown } + u.Path = cleanedPath + "/" urlLower := strings.ToLower(u.String()) - return strings.HasPrefix(urlLower, strings.ToLower(setting.AppURL)) || strings.HasPrefix(urlLower, strings.ToLower(GuessCurrentAppURL(ctx))) + if strings.HasPrefix(urlLower, strings.ToLower(setting.AppURL)) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaAbsolute + } + guessedCurURL := GuessCurrentAppURL(ctx) + if strings.HasPrefix(urlLower, strings.ToLower(guessedCurURL)) { + return cleanedPath[len(setting.AppSubURL):], urlTypeGiteaAbsolute + } + return "", urlTypeUnknown +} + +func IsCurrentGiteaSiteURL(ctx context.Context, s string) bool { + _, ut := detectURLRoutePath(ctx, s) + return ut != urlTypeUnknown +} + +type GiteaSiteURL struct { + RoutePath string + OwnerName string + RepoName string + RepoSubPath string +} + +func ParseGiteaSiteURL(ctx context.Context, s string) *GiteaSiteURL { + routePath, ut := detectURLRoutePath(ctx, s) + if ut == urlTypeUnknown || ut == urlTypeGiteaPageRelative { + return nil + } + ret := &GiteaSiteURL{RoutePath: routePath} + fields := strings.SplitN(strings.TrimPrefix(ret.RoutePath, "/"), "/", 3) + + // TODO: now it only does a quick check for some known reserved paths, should do more strict checks in the future + if fields[0] == "attachments" { + return ret + } + if len(fields) < 2 { + return ret + } + ret.OwnerName = fields[0] + ret.RepoName = fields[1] + if len(fields) == 3 { + ret.RepoSubPath = "/" + fields[2] + } + return ret } diff --git a/modules/httplib/url_test.go b/modules/httplib/url_test.go index cb8fac0a2198c..d57653646b306 100644 --- a/modules/httplib/url_test.go +++ b/modules/httplib/url_test.go @@ -122,3 +122,26 @@ func TestIsCurrentGiteaSiteURL(t *testing.T) { assert.True(t, IsCurrentGiteaSiteURL(ctx, "https://user-host")) assert.False(t, IsCurrentGiteaSiteURL(ctx, "https://forwarded-host")) } + +func TestParseGiteaSiteURL(t *testing.T) { + defer test.MockVariableValue(&setting.AppURL, "http://localhost:3000/sub/")() + defer test.MockVariableValue(&setting.AppSubURL, "/sub")() + ctx := t.Context() + tests := []struct { + url string + exp *GiteaSiteURL + }{ + {"http://localhost:3000/sub?k=v", &GiteaSiteURL{RoutePath: ""}}, + {"http://localhost:3000/sub/", &GiteaSiteURL{RoutePath: ""}}, + {"http://localhost:3000/sub/foo", &GiteaSiteURL{RoutePath: "/foo"}}, + {"http://localhost:3000/sub/foo/bar", &GiteaSiteURL{RoutePath: "/foo/bar", OwnerName: "foo", RepoName: "bar"}}, + {"http://localhost:3000/sub/foo/bar/", &GiteaSiteURL{RoutePath: "/foo/bar", OwnerName: "foo", RepoName: "bar"}}, + {"http://localhost:3000/sub/attachments/bar", &GiteaSiteURL{RoutePath: "/attachments/bar"}}, + {"http://localhost:3000/other", nil}, + {"http://other/", nil}, + } + for _, test := range tests { + su := ParseGiteaSiteURL(ctx, test.url) + assert.Equal(t, test.exp, su, "URL = %s", test.url) + } +} diff --git a/modules/setting/mailer.go b/modules/setting/mailer.go index 4c3dff6850947..e79ff304474b8 100644 --- a/modules/setting/mailer.go +++ b/modules/setting/mailer.go @@ -13,7 +13,7 @@ import ( "code.gitea.io/gitea/modules/log" - shellquote "github.com/kballard/go-shellquote" + "github.com/kballard/go-shellquote" ) // Mailer represents mail service. @@ -29,6 +29,9 @@ type Mailer struct { SubjectPrefix string `ini:"SUBJECT_PREFIX"` OverrideHeader map[string][]string `ini:"-"` + // Embed attachment images as inline base64 img src attribute + EmbedAttachmentImages bool + // SMTP sender Protocol string `ini:"PROTOCOL"` SMTPAddr string `ini:"SMTP_ADDR"` diff --git a/services/mailer/mail.go b/services/mailer/mail.go index 7db259ac2c48f..f7e5b0c9f029f 100644 --- a/services/mailer/mail.go +++ b/services/mailer/mail.go @@ -6,16 +6,26 @@ package mailer import ( "bytes" + "context" + "encoding/base64" + "fmt" "html/template" + "io" "mime" "regexp" "strings" texttmpl "text/template" + repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/httplib" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/typesniffer" sender_service "code.gitea.io/gitea/services/mailer/sender" + + "golang.org/x/net/html" ) const mailMaxSubjectRunes = 256 // There's no actual limit for subject in RFC 5322 @@ -44,6 +54,107 @@ func sanitizeSubject(subject string) string { return mime.QEncoding.Encode("utf-8", string(runes)) } +type mailAttachmentBase64Embedder struct { + doer *user_model.User + repo *repo_model.Repository + maxSize int64 + estimateSize int64 +} + +func newMailAttachmentBase64Embedder(doer *user_model.User, repo *repo_model.Repository, maxSize int64) *mailAttachmentBase64Embedder { + return &mailAttachmentBase64Embedder{doer: doer, repo: repo, maxSize: maxSize} +} + +func (b64embedder *mailAttachmentBase64Embedder) Base64InlineImages(ctx context.Context, body template.HTML) (template.HTML, error) { + doc, err := html.Parse(strings.NewReader(string(body))) + if err != nil { + return "", fmt.Errorf("html.Parse failed: %w", err) + } + + b64embedder.estimateSize = int64(len(string(body))) + + var processNode func(*html.Node) + processNode = func(n *html.Node) { + if n.Type == html.ElementNode { + if n.Data == "img" { + for i, attr := range n.Attr { + if attr.Key == "src" { + attachmentSrc := attr.Val + dataURI, err := b64embedder.AttachmentSrcToBase64DataURI(ctx, attachmentSrc) + if err != nil { + // Not an error, just skip. This is probably an image from outside the gitea instance. + log.Trace("Unable to embed attachment %q to mail body: %v", attachmentSrc, err) + } else { + n.Attr[i].Val = dataURI + } + break + } + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + processNode(c) + } + } + + processNode(doc) + + var buf bytes.Buffer + err = html.Render(&buf, doc) + if err != nil { + return "", fmt.Errorf("html.Render failed: %w", err) + } + return template.HTML(buf.String()), nil +} + +func (b64embedder *mailAttachmentBase64Embedder) AttachmentSrcToBase64DataURI(ctx context.Context, attachmentSrc string) (string, error) { + parsedSrc := httplib.ParseGiteaSiteURL(ctx, attachmentSrc) + var attachmentUUID string + if parsedSrc != nil { + var ok bool + attachmentUUID, ok = strings.CutPrefix(parsedSrc.RoutePath, "/attachments/") + if !ok { + attachmentUUID, ok = strings.CutPrefix(parsedSrc.RepoSubPath, "/attachments/") + } + if !ok { + return "", fmt.Errorf("not an attachment") + } + } + attachment, err := repo_model.GetAttachmentByUUID(ctx, attachmentUUID) + if err != nil { + return "", err + } + + if attachment.RepoID != b64embedder.repo.ID { + return "", fmt.Errorf("attachment does not belong to the repository") + } + if attachment.Size+b64embedder.estimateSize > b64embedder.maxSize { + return "", fmt.Errorf("total embedded images exceed max limit") + } + + fr, err := storage.Attachments.Open(attachment.RelativePath()) + if err != nil { + return "", err + } + defer fr.Close() + + lr := &io.LimitedReader{R: fr, N: b64embedder.maxSize + 1} + content, err := io.ReadAll(lr) + if err != nil { + return "", fmt.Errorf("LimitedReader ReadAll: %w", err) + } + + mimeType := typesniffer.DetectContentType(content) + if !mimeType.IsImage() { + return "", fmt.Errorf("not an image") + } + + encoded := base64.StdEncoding.EncodeToString(content) + dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType.GetMimeType(), encoded) + b64embedder.estimateSize += int64(len(dataURI)) + return dataURI, nil +} + func fromDisplayName(u *user_model.User) string { if setting.MailService.FromDisplayNameFormatTemplate != nil { var ctx bytes.Buffer diff --git a/services/mailer/mail_issue_common.go b/services/mailer/mail_issue_common.go index 23ca4c3f1538d..85fe7c1f9adf4 100644 --- a/services/mailer/mail_issue_common.go +++ b/services/mailer/mail_issue_common.go @@ -25,6 +25,10 @@ import ( "code.gitea.io/gitea/services/mailer/token" ) +// maxEmailBodySize is the approximate maximum size of an email body in bytes +// Many e-mail service providers have limitations on the size of the email body, it's usually from 10MB to 25MB +const maxEmailBodySize = 9_000_000 + func fallbackMailSubject(issue *issues_model.Issue) string { return fmt.Sprintf("[%s] %s (#%d)", issue.Repo.FullName(), issue.Title, issue.Index) } @@ -64,12 +68,20 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipient // This is the body of the new issue or comment, not the mail body rctx := renderhelper.NewRenderContextRepoComment(ctx.Context, ctx.Issue.Repo).WithUseAbsoluteLink(true) - body, err := markdown.RenderString(rctx, - ctx.Content) + body, err := markdown.RenderString(rctx, ctx.Content) if err != nil { return nil, err } + if setting.MailService.EmbedAttachmentImages { + attEmbedder := newMailAttachmentBase64Embedder(ctx.Doer, ctx.Issue.Repo, maxEmailBodySize) + bodyAfterEmbedding, err := attEmbedder.Base64InlineImages(ctx, body) + if err != nil { + log.Error("Failed to embed images in mail body: %v", err) + } else { + body = bodyAfterEmbedding + } + } actType, actName, tplName := actionToTemplate(ctx.Issue, ctx.ActionType, commentType, reviewType) if actName != "new" { diff --git a/services/mailer/mail_test.go b/services/mailer/mail_test.go index 1860257e2e74e..85ee3455459d8 100644 --- a/services/mailer/mail_test.go +++ b/services/mailer/mail_test.go @@ -6,6 +6,7 @@ package mailer import ( "bytes" "context" + "encoding/base64" "fmt" "html/template" "io" @@ -23,9 +24,12 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/services/attachment" sender_service "code.gitea.io/gitea/services/mailer/sender" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const subjectTpl = ` @@ -53,22 +57,44 @@ const bodyTpl = ` func prepareMailerTest(t *testing.T) (doer *user_model.User, repo *repo_model.Repository, issue *issues_model.Issue, comment *issues_model.Comment) { assert.NoError(t, unittest.PrepareTestDatabase()) - mailService := setting.Mailer{ - From: "test@gitea.com", - } - - setting.MailService = &mailService + setting.MailService = &setting.Mailer{From: "test@gitea.com"} setting.Domain = "localhost" + setting.AppURL = "https://try.gitea.io/" doer = unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) repo = unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1, Owner: doer}) issue = unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 1, Repo: repo, Poster: doer}) - assert.NoError(t, issue.LoadRepo(db.DefaultContext)) comment = unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 2, Issue: issue}) + require.NoError(t, issue.LoadRepo(db.DefaultContext)) return doer, repo, issue, comment } -func TestComposeIssueCommentMessage(t *testing.T) { +func prepareMailerBase64Test(t *testing.T) (doer *user_model.User, repo *repo_model.Repository, issue *issues_model.Issue, att1, att2 *repo_model.Attachment) { + user, repo, issue, comment := prepareMailerTest(t) + setting.MailService.EmbedAttachmentImages = true + + att1, err := attachment.NewAttachment(t.Context(), &repo_model.Attachment{ + RepoID: repo.ID, + IssueID: issue.ID, + UploaderID: user.ID, + CommentID: comment.ID, + Name: "test.png", + }, bytes.NewReader([]byte("\x89\x50\x4e\x47\x0d\x0a\x1a\x0a")), 8) + require.NoError(t, err) + + att2, err = attachment.NewAttachment(t.Context(), &repo_model.Attachment{ + RepoID: repo.ID, + IssueID: issue.ID, + UploaderID: user.ID, + CommentID: comment.ID, + Name: "test.png", + }, bytes.NewReader([]byte("\x89\x50\x4e\x47\x0d\x0a\x1a\x0a"+strings.Repeat("\x00", 1024))), 8+1024) + require.NoError(t, err) + + return user, repo, issue, att1, att2 +} + +func TestComposeIssueComment(t *testing.T) { doer, _, issue, comment := prepareMailerTest(t) markup.Init(&markup.RenderHelperFuncs{ @@ -109,7 +135,8 @@ func TestComposeIssueCommentMessage(t *testing.T) { assert.Len(t, gomailMsg.GetGenHeader("List-Unsubscribe"), 2) // url + mailto var buf bytes.Buffer - gomailMsg.WriteTo(&buf) + _, err = gomailMsg.WriteTo(&buf) + require.NoError(t, err) b, err := io.ReadAll(quotedprintable.NewReader(&buf)) assert.NoError(t, err) @@ -404,9 +431,9 @@ func TestGenerateMessageIDForRelease(t *testing.T) { } func TestFromDisplayName(t *testing.T) { - template, err := texttmpl.New("mailFrom").Parse("{{ .DisplayName }}") + tmpl, err := texttmpl.New("mailFrom").Parse("{{ .DisplayName }}") assert.NoError(t, err) - setting.MailService = &setting.Mailer{FromDisplayNameFormatTemplate: template} + setting.MailService = &setting.Mailer{FromDisplayNameFormatTemplate: tmpl} defer func() { setting.MailService = nil }() tests := []struct { @@ -435,9 +462,9 @@ func TestFromDisplayName(t *testing.T) { } t.Run("template with all available vars", func(t *testing.T) { - template, err = texttmpl.New("mailFrom").Parse("{{ .DisplayName }} (by {{ .AppName }} on [{{ .Domain }}])") + tmpl, err = texttmpl.New("mailFrom").Parse("{{ .DisplayName }} (by {{ .AppName }} on [{{ .Domain }}])") assert.NoError(t, err) - setting.MailService = &setting.Mailer{FromDisplayNameFormatTemplate: template} + setting.MailService = &setting.Mailer{FromDisplayNameFormatTemplate: tmpl} oldAppName := setting.AppName setting.AppName = "Code IT" oldDomain := setting.Domain @@ -450,3 +477,72 @@ func TestFromDisplayName(t *testing.T) { assert.EqualValues(t, "Mister X (by Code IT on [code.it])", fromDisplayName(&user_model.User{FullName: "Mister X", Name: "tmp"})) }) } + +func TestEmbedBase64Images(t *testing.T) { + user, repo, issue, att1, att2 := prepareMailerBase64Test(t) + ctx := &mailCommentContext{Context: t.Context(), Issue: issue, Doer: user} + + imgExternalURL := "https://via.placeholder.com/10" + imgExternalImg := fmt.Sprintf(``, imgExternalURL) + + att1URL := setting.AppURL + repo.Owner.Name + "/" + repo.Name + "/attachments/" + att1.UUID + att1Img := fmt.Sprintf(``, att1URL) + att1Base64 := "data:image/png;base64,iVBORw0KGgo=" + att1ImgBase64 := fmt.Sprintf(``, att1Base64) + + att2URL := setting.AppURL + repo.Owner.Name + "/" + repo.Name + "/attachments/" + att2.UUID + att2Img := fmt.Sprintf(``, att2URL) + att2File, err := storage.Attachments.Open(att2.RelativePath()) + require.NoError(t, err) + defer att2File.Close() + att2Bytes, err := io.ReadAll(att2File) + require.NoError(t, err) + require.Greater(t, len(att2Bytes), 1024) + att2Base64 := "data:image/png;base64," + base64.StdEncoding.EncodeToString(att2Bytes) + att2ImgBase64 := fmt.Sprintf(``, att2Base64) + + t.Run("ComposeMessage", func(t *testing.T) { + subjectTemplates = texttmpl.Must(texttmpl.New("issue/new").Parse(subjectTpl)) + bodyTemplates = template.Must(template.New("issue/new").Parse(bodyTpl)) + + issue.Content = fmt.Sprintf(`MSG-BEFORE MSG-AFTER`, att1.UUID) + require.NoError(t, issues_model.UpdateIssueCols(t.Context(), issue, "content")) + + recipients := []*user_model.User{{Name: "Test", Email: "test@gitea.com"}} + msgs, err := composeIssueCommentMessages(&mailCommentContext{ + Context: t.Context(), + Issue: issue, + Doer: user, + ActionType: activities_model.ActionCreateIssue, + Content: issue.Content, + }, "en-US", recipients, false, "issue create") + require.NoError(t, err) + + mailBody := msgs[0].Body + assert.Regexp(t, `MSG-BEFORE ]+> MSG-AFTER`, mailBody) + }) + + t.Run("EmbedInstanceImageSkipExternalImage", func(t *testing.T) { + mailBody := "

Test1

" + imgExternalImg + "

Test2

" + att1Img + "

Test3

" + expectedMailBody := "

Test1

" + imgExternalImg + "

Test2

" + att1ImgBase64 + "

Test3

" + b64embedder := newMailAttachmentBase64Embedder(user, repo, 1024) + resultMailBody, err := b64embedder.Base64InlineImages(ctx, template.HTML(mailBody)) + require.NoError(t, err) + assert.Equal(t, expectedMailBody, string(resultMailBody)) + }) + + t.Run("LimitedEmailBodySize", func(t *testing.T) { + mailBody := fmt.Sprintf("%s%s", att1Img, att2Img) + b64embedder := newMailAttachmentBase64Embedder(user, repo, 1024) + resultMailBody, err := b64embedder.Base64InlineImages(ctx, template.HTML(mailBody)) + require.NoError(t, err) + expected := fmt.Sprintf("%s%s", att1ImgBase64, att2Img) + assert.Equal(t, expected, string(resultMailBody)) + + b64embedder = newMailAttachmentBase64Embedder(user, repo, 4096) + resultMailBody, err = b64embedder.Base64InlineImages(ctx, template.HTML(mailBody)) + require.NoError(t, err) + expected = fmt.Sprintf("%s%s", att1ImgBase64, att2ImgBase64) + assert.Equal(t, expected, string(resultMailBody)) + }) +}