Option BASE64_EMBED_IMAGES (default false) in mail settings to inline image attachments + tests

Author: sommerf-lf

custom/conf/app.example.ini

@@ -1704,6 +1704,9 @@ LEVEL = Info
 ;;
 ;; convert \r\n to \n for Sendmail
 ;SENDMAIL_CONVERT_CRLF = true
+;;
+;; convert links of attached images to inline images
+;BASE64_EMBED_IMAGES = true
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

models/fixtures/attachment.yml

@@ -153,3 +153,16 @@
   download_count: 0
   size: 0
   created_unix: 946684800
+
+-
+  id: 13
+  uuid: 1b267670-1793-4cd0-abc1-449269b7cff9
+  repo_id: 1
+  issue_id: 2
+  release_id: 0
+  uploader_id: 2
+  comment_id: 0
+  name: gitea.png
+  download_count: 0
+  size: 1458
+  created_unix: 946684800

modules/setting/mailer.go

@@ -28,6 +28,7 @@ type Mailer struct {
 	SendAsPlainText      bool                `ini:"SEND_AS_PLAIN_TEXT"`
 	SubjectPrefix        string              `ini:"SUBJECT_PREFIX"`
 	OverrideHeader       map[string][]string `ini:"-"`
+	Base64EmbedImages    bool                `ini:"BASE64_EMBED_IMAGES"`
 
 	// SMTP sender
 	Protocol             string `ini:"PROTOCOL"`
@@ -150,6 +151,7 @@ func loadMailerFrom(rootCfg ConfigProvider) {
 	sec.Key("SENDMAIL_TIMEOUT").MustDuration(5 * time.Minute)
 	sec.Key("SENDMAIL_CONVERT_CRLF").MustBool(true)
 	sec.Key("FROM").MustString(sec.Key("USER").String())
+	sec.Key("BASE64_EMBED_IMAGES").MustBool(false)
 
 	// Now map the values on to the MailService
 	MailService = &Mailer{}

services/mailer/mail.go

@@ -7,9 +7,12 @@ package mailer
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
 	"html/template"
+	"io"
 	"mime"
+	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
@@ -26,11 +29,13 @@ import (
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/translation"
 	incoming_payload "code.gitea.io/gitea/services/mailer/incoming/payload"
 	"code.gitea.io/gitea/services/mailer/token"
 
+	"golang.org/x/net/html"
 	"gopkg.in/gomail.v2"
 )
 
@@ -195,7 +200,7 @@ func SendCollaboratorMail(u, doer *user_model.User, repo *repo_model.Repository)
 	SendAsync(msg)
 }
 
-func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipients []*user_model.User, fromMention bool, info string) ([]*Message, error) {
+func composeIssueCommentMessages(ctx *MailCommentContext, lang string, recipients []*user_model.User, fromMention bool, info string) ([]*Message, error) {
 	var (
 		subject string
 		link    string
@@ -232,6 +237,15 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipient
 		return nil, err
 	}
 
+	if setting.MailService.Base64EmbedImages {
+		bodyStr := string(body)
+		bodyStr, err = Base64InlineImages(bodyStr, ctx)
+		if err != nil {
+			return nil, err
+		}
+		body = template.HTML(bodyStr)
+	}
+
 	actType, actName, tplName := actionToTemplate(ctx.Issue, ctx.ActionType, commentType, reviewType)
 
 	if actName != "new" {
@@ -363,6 +377,81 @@ func composeIssueCommentMessages(ctx *mailCommentContext, lang string, recipient
 	return msgs, nil
 }
 
+func Base64InlineImages(body string, ctx *MailCommentContext) (string, error) {
+	doc, err := html.Parse(strings.NewReader(body))
+	if err != nil {
+		log.Error("Failed to parse HTML body: %v", err)
+		return "", err
+	}
+
+	var processNode func(*html.Node)
+	processNode = func(n *html.Node) {
+		if n.Type == html.ElementNode {
+			if n.Data == "img" {
+				for i, attr := range n.Attr {
+					if attr.Key == "src" {
+						attachmentPath := attr.Val
+						dataURI, err := AttachmentSrcToBase64DataURI(attachmentPath, ctx)
+						if err != nil {
+							log.Trace("attachmentSrcToDataURI not possible: %v", err) // Not an error, just skip. This is probably an image from outside the gitea instance.
+							continue
+						}
+						log.Trace("Old value of src attribute: %s, new value (first 100 characters): %s", attr.Val, dataURI[:100])
+						n.Attr[i].Val = dataURI
+					}
+				}
+			}
+		}
+
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			processNode(c)
+		}
+	}
+
+	processNode(doc)
+
+	var buf bytes.Buffer
+	err = html.Render(&buf, doc)
+	if err != nil {
+		log.Error("Failed to render modified HTML: %v", err)
+		return "", err
+	}
+	return buf.String(), nil
+}
+
+func AttachmentSrcToBase64DataURI(attachmentPath string, ctx *MailCommentContext) (string, error) {
+	if !strings.HasPrefix(attachmentPath, setting.AppURL) { // external image
+		return "", fmt.Errorf("external image")
+	}
+	parts := strings.Split(attachmentPath, "/attachments/")
+	if len(parts) <= 1 {
+		return "", fmt.Errorf("invalid attachment path: %s", attachmentPath)
+	}
+
+	attachmentUUID := parts[len(parts)-1]
+	attachment, err := repo_model.GetAttachmentByUUID(ctx, attachmentUUID)
+	if err != nil {
+		return "", err
+	}
+
+	fr, err := storage.Attachments.Open(attachment.RelativePath())
+	if err != nil {
+		return "", err
+	}
+	defer fr.Close()
+
+	content, err := io.ReadAll(fr)
+	if err != nil {
+		return "", err
+	}
+
+	mimeType := http.DetectContentType(content)
+	encoded := base64.StdEncoding.EncodeToString(content)
+	dataURI := fmt.Sprintf("data:%s;base64,%s", mimeType, encoded)
+
+	return dataURI, nil
+}
+
 func generateMessageIDForIssue(issue *issues_model.Issue, comment *issues_model.Comment, actionType activities_model.ActionType) string {
 	var path string
 	if issue.IsPull {
@@ -394,7 +483,7 @@ func generateMessageIDForRelease(release *repo_model.Release) string {
 	return fmt.Sprintf("<%s/releases/%d@%s>", release.Repo.FullName(), release.ID, setting.Domain)
 }
 
-func generateAdditionalHeaders(ctx *mailCommentContext, reason string, recipient *user_model.User) map[string]string {
+func generateAdditionalHeaders(ctx *MailCommentContext, reason string, recipient *user_model.User) map[string]string {
 	repo := ctx.Issue.Repo
 
 	return map[string]string{
@@ -458,7 +547,7 @@ func SendIssueAssignedMail(ctx context.Context, issue *issues_model.Issue, doer
 	}
 
 	for lang, tos := range langMap {
-		msgs, err := composeIssueCommentMessages(&mailCommentContext{
+		msgs, err := composeIssueCommentMessages(&MailCommentContext{
 			Context:    ctx,
 			Issue:      issue,
 			Doer:       doer,

services/mailer/mail_comment.go

@@ -26,7 +26,7 @@ func MailParticipantsComment(ctx context.Context, c *issues_model.Comment, opTyp
 		content = ""
 	}
 	if err := mailIssueCommentToParticipants(
-		&mailCommentContext{
+		&MailCommentContext{
 			Context:    ctx,
 			Issue:      issue,
 			Doer:       c.Poster,
@@ -49,7 +49,7 @@ func MailMentionsComment(ctx context.Context, pr *issues_model.PullRequest, c *i
 	visited := make(container.Set[int64], len(mentions)+1)
 	visited.Add(c.Poster.ID)
 	if err = mailIssueCommentBatch(
-		&mailCommentContext{
+		&MailCommentContext{
 			Context:    ctx,
 			Issue:      pr.Issue,
 			Doer:       c.Poster,

services/mailer/mail_issue.go

@@ -22,7 +22,7 @@ func fallbackMailSubject(issue *issues_model.Issue) string {
 	return fmt.Sprintf("[%s] %s (#%d)", issue.Repo.FullName(), issue.Title, issue.Index)
 }
 
-type mailCommentContext struct {
+type MailCommentContext struct {
 	context.Context
 	Issue                 *issues_model.Issue
 	Doer                  *user_model.User
@@ -41,7 +41,7 @@ const (
 // This function sends two list of emails:
 // 1. Repository watchers (except for WIP pull requests) and users who are participated in comments.
 // 2. Users who are not in 1. but get mentioned in current issue/comment.
-func mailIssueCommentToParticipants(ctx *mailCommentContext, mentions []*user_model.User) error {
+func mailIssueCommentToParticipants(ctx *MailCommentContext, mentions []*user_model.User) error {
 	// Required by the mail composer; make sure to load these before calling the async function
 	if err := ctx.Issue.LoadRepo(ctx); err != nil {
 		return fmt.Errorf("LoadRepo: %w", err)
@@ -120,7 +120,7 @@ func mailIssueCommentToParticipants(ctx *mailCommentContext, mentions []*user_mo
 	return nil
 }
 
-func mailIssueCommentBatch(ctx *mailCommentContext, users []*user_model.User, visited container.Set[int64], fromMention bool) error {
+func mailIssueCommentBatch(ctx *MailCommentContext, users []*user_model.User, visited container.Set[int64], fromMention bool) error {
 	checkUnit := unit.TypeIssues
 	if ctx.Issue.IsPull {
 		checkUnit = unit.TypePullRequests
@@ -186,7 +186,7 @@ func MailParticipants(ctx context.Context, issue *issues_model.Issue, doer *user
 	}
 	forceDoerNotification := opType == activities_model.ActionAutoMergePullRequest
 	if err := mailIssueCommentToParticipants(
-		&mailCommentContext{
+		&MailCommentContext{
 			Context:               ctx,
 			Issue:                 issue,
 			Doer:                  doer,

services/mailer/mail_test.go

@@ -83,7 +83,7 @@ func TestComposeIssueCommentMessage(t *testing.T) {
 	bodyTemplates = template.Must(template.New("issue/comment").Parse(bodyTpl))
 
 	recipients := []*user_model.User{{Name: "Test", Email: "[email protected]"}, {Name: "Test2", Email: "[email protected]"}}
-	msgs, err := composeIssueCommentMessages(&mailCommentContext{
+	msgs, err := composeIssueCommentMessages(&MailCommentContext{
 		Context: context.TODO(), // TODO: use a correct context
 		Issue:   issue, Doer: doer, ActionType: activities_model.ActionCommentIssue,
 		Content: fmt.Sprintf("test @%s %s#%d body", doer.Name, issue.Repo.FullName(), issue.Index),
@@ -129,7 +129,7 @@ func TestComposeIssueMessage(t *testing.T) {
 	bodyTemplates = template.Must(template.New("issue/new").Parse(bodyTpl))
 
 	recipients := []*user_model.User{{Name: "Test", Email: "[email protected]"}, {Name: "Test2", Email: "[email protected]"}}
-	msgs, err := composeIssueCommentMessages(&mailCommentContext{
+	msgs, err := composeIssueCommentMessages(&MailCommentContext{
 		Context: context.TODO(), // TODO: use a correct context
 		Issue:   issue, Doer: doer, ActionType: activities_model.ActionCreateIssue,
 		Content: "test body",
@@ -176,14 +176,14 @@ func TestTemplateSelection(t *testing.T) {
 		assert.Contains(t, wholemsg, expBody)
 	}
 
-	msg := testComposeIssueCommentMessage(t, &mailCommentContext{
+	msg := testComposeIssueCommentMessage(t, &MailCommentContext{
 		Context: context.TODO(), // TODO: use a correct context
 		Issue:   issue, Doer: doer, ActionType: activities_model.ActionCreateIssue,
 		Content: "test body",
 	}, recipients, false, "TestTemplateSelection")
 	expect(t, msg, "issue/new/subject", "issue/new/body")
 
-	msg = testComposeIssueCommentMessage(t, &mailCommentContext{
+	msg = testComposeIssueCommentMessage(t, &MailCommentContext{
 		Context: context.TODO(), // TODO: use a correct context
 		Issue:   issue, Doer: doer, ActionType: activities_model.ActionCommentIssue,
 		Content: "test body", Comment: comment,
@@ -192,14 +192,14 @@ func TestTemplateSelection(t *testing.T) {
 
 	pull := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 2, Repo: repo, Poster: doer})
 	comment = unittest.AssertExistsAndLoadBean(t, &issues_model.Comment{ID: 4, Issue: pull})
-	msg = testComposeIssueCommentMessage(t, &mailCommentContext{
+	msg = testComposeIssueCommentMessage(t, &MailCommentContext{
 		Context: context.TODO(), // TODO: use a correct context
 		Issue:   pull, Doer: doer, ActionType: activities_model.ActionCommentPull,
 		Content: "test body", Comment: comment,
 	}, recipients, false, "TestTemplateSelection")
 	expect(t, msg, "pull/comment/subject", "pull/comment/body")
 
-	msg = testComposeIssueCommentMessage(t, &mailCommentContext{
+	msg = testComposeIssueCommentMessage(t, &MailCommentContext{
 		Context: context.TODO(), // TODO: use a correct context
 		Issue:   issue, Doer: doer, ActionType: activities_model.ActionCloseIssue,
 		Content: "test body", Comment: comment,
@@ -218,7 +218,7 @@ func TestTemplateServices(t *testing.T) {
 		bodyTemplates = template.Must(template.New("issue/default").Parse(tplBody))
 
 		recipients := []*user_model.User{{Name: "Test", Email: "[email protected]"}}
-		msg := testComposeIssueCommentMessage(t, &mailCommentContext{
+		msg := testComposeIssueCommentMessage(t, &MailCommentContext{
 			Context: context.TODO(), // TODO: use a correct context
 			Issue:   issue, Doer: doer, ActionType: actionType,
 			Content: "test body", Comment: comment,
@@ -252,7 +252,7 @@ func TestTemplateServices(t *testing.T) {
 		"//Re: //")
 }
 
-func testComposeIssueCommentMessage(t *testing.T, ctx *mailCommentContext, recipients []*user_model.User, fromMention bool, info string) *Message {
+func testComposeIssueCommentMessage(t *testing.T, ctx *MailCommentContext, recipients []*user_model.User, fromMention bool, info string) *Message {
 	msgs, err := composeIssueCommentMessages(ctx, "en-US", recipients, fromMention, info)
 	assert.NoError(t, err)
 	assert.Len(t, msgs, 1)
@@ -262,7 +262,7 @@ func testComposeIssueCommentMessage(t *testing.T, ctx *mailCommentContext, recip
 func TestGenerateAdditionalHeaders(t *testing.T) {
 	doer, _, issue, _ := prepareMailerTest(t)
 
-	ctx := &mailCommentContext{Context: context.TODO() /* TODO: use a correct context */, Issue: issue, Doer: doer}
+	ctx := &MailCommentContext{Context: context.TODO() /* TODO: use a correct context */, Issue: issue, Doer: doer}
 	recipient := &user_model.User{Name: "test", Email: "[email protected]"}
 
 	headers := generateAdditionalHeaders(ctx, "dummy-reason", recipient)

tests/integration/email_embed_b64_images_test.go

@@ -0,0 +1,71 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package integration
+
+import (
+	"context"
+	"testing"
+
+	issues_model "code.gitea.io/gitea/models/issues"
+	repo_model "code.gitea.io/gitea/models/repo"
+	"code.gitea.io/gitea/models/unittest"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/setting"
+	mail "code.gitea.io/gitea/services/mailer"
+	"code.gitea.io/gitea/tests"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEmailEmbedBase64Images(t *testing.T) {
+	defer tests.PrepareTestEnv(t)()
+	tests.PrepareAttachmentsStorage(t)
+
+	user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
+	repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1, Owner: user})
+	issue := unittest.AssertExistsAndLoadBean(t, &issues_model.Issue{ID: 2, Repo: repo, Poster: user})
+
+	attachment := unittest.AssertExistsAndLoadBean(t, &repo_model.Attachment{ID: 13, IssueID: issue.ID, RepoID: repo.ID})
+	ctx0 := context.Background()
+
+	ctx := &mail.MailCommentContext{Context: ctx0 /* TODO: use a correct context */, Issue: issue, Doer: user}
+
+	img1ExternalURL := "https://via.placeholder.com/10"
+	img1ExternalImg := "<img src=\"" + img1ExternalURL + "\"/>"
+
+	img2InternalURL := setting.AppURL + repo.Owner.Name + "/" + repo.Name + "/attachments/" + attachment.UUID
+	img2InternalImg := "<img src=\"" + img2InternalURL + "\"/>"
+	img2InternalBase64 := "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAFAAAAAxCAYAAABNuS5SAAAAAXNSR0IArs4c6QAABWxJREFUaEPVmm1sU1UYx/9n3RsuxmS+ECdC71YYbCjohmMDNxyJONBNxaIo29qNlZqYbTFhi0FkmZpgogH0w1g7t46ZmEA0KIl+MQsvwQSM8QOJSGDtFO0XFTQE0tG11/Rqa9fel3Pf7+6H7cN9zjnP8zv/53nOvbcElFe7n/mOsKimNAfAfpWTm3NgtCP4Df0Ya1q2DpXdl2OL7yYEbrC4O+FlwBMiif/cH6Grc4RZEotj2lphsddAyB9gSbSlqvvvZx/tqQCQ5x4pBVgUGelrAqIgQJePYY10Rslam1d7w841fSWJsW5/qZIpVI0hBM5sgCyIy8/EVc1s0OCmh3eFt9X0mwYwK4V3+h2LZtnYVYPiV73MUw91hV9c+4Z1AM6HtE2nvmllR/il2jetAZAGHgu8M+4J7U0PgmacaqkJTPDkSvdv22v3PmBWDUylsMtnPwiQHqFAky2b776ZADdWtv26o25gkQUACndcMXgJx80E+MSKV35pW//2YssClIJnNsCG5dt/dj3+7hJTAbp8zAyAfL70tDrA+vJt0+76/XazAfIemGngma3A9eXOYGf9e9wJ2oyDNNdEhGoYPUB7P0D269VpxeatW/r8VNeG98vmNUAzVVhb1nLF03jAYRZAkhurVK1AowDeteBeRGZvYSZ6MyXKmtJnLns3HlpqFkDuZYLaFDYC4FhXMCuTkzUvec+MGjgvAPLBS9IcmuzB+akTZpRfbk3tAPqZU2BRr0ckYgDTVWg5BSYJ00LR44lEDN7E2X2Y/HECvo6LyLMVpNw0EOTtgCdUQDzDJXfcJgX/V+Y0YrRHGb3qIK36+DZZb5BzXukLqYeNo2HcGzpNo0KtFSgELx3MBy+fRXHR/aLu6QWSCqCZacwPkIXbz52bU5eYStPtOkcciLPavWjPBHgSQIPQVtKmslYqFIJy5tIxjJ7un+NmY0UbWtcN0CQJZ6OVIrO+ykkFTwNRag7aKGlqX+ZcK0rq0LflE6ol1EJMZ5H6qNQ+zNwiBAvEPDACohi8q39ewlufN6F13SAaK3ZwroZ+v4DB4y2yUloXgPSdlBwKeIK9QqDVqpBGfXw2gTN7cOqnTzm3pOqibgDpIf6Lj0+RagCKBX4jcg3dE9XYvMoL52N9vPt3I3Id3RNVogC1hJdwIuu7sPMobEV/MbNUxWSu0UzAEyqUuwm0HTXz2VfIv4Rd76ZRrFq8gddEd4CJVZs+dBQsLIxFFEBUPIQm7WrKmuFtPCi4hhRktfD4sk70tzFq0lEuSaW1L30dvQHylSxRgGrSUQ5AmqcOx8Iq7Gk+Jjjtzo+XIRafFax/eqiPtwbq1V3FgNIAlOqueqtPqGlKKpAvcK1TWy3Aw5O9ODf1JQa3fo0Hi8uzXNZLfbIUKKzM0jUAe15OymbaqgVolvo0ASgGTo5ShSDG2RhyiI3qSCL26l/NBos9gSlKYVpn5ABMzil1nOHrumJj9Uxf3RWopovTgMyEkzlGC3gkl1071jF9Tkg0uipQDUAaRb52pAo3Z65zpmakryEK1AKi1BGGTx1aqO/OiK3wo+4rid8OCV66K5ADOGZfjSj5gbZ2itkNuS6gME/6x/haAKR5fWcIQK1UmA5233MnYL+nkqo7K9k4GniGpXAyACVdWSr4rdW78fQjr6bMvr18HP6Tr0sNk7xvSYB6KFGShAIDWniGKzCxYLuPaSbAFwriMmSIHHimAPxPhTEAOYYQkbGIPRyyDQxA1rdPw5pIZhwuH2MpiCzIC+Oe4GcyeHOmpgHklOhnjoKFU67TWtvns8X5vl3fR5XMaypAPbszLQy5NS9zXksA5NQ4vGw5SPQibeBq7dSCS67/D+Q9UQwzW88cAAAAAElFTkSuQmCC"
+	img2InternalBase64Img := "<img src=\"" + img2InternalBase64 + "\"/>"
+
+	// 1st Test: convert internal image to base64
+	t.Run("replaceSpecifiedBase64ImagesInternal", func(t *testing.T) {
+		defer tests.PrintCurrentTest(t)()
+
+		resultImg1Internal, err := mail.AttachmentSrcToBase64DataURI(img2InternalURL, ctx)
+		assert.NoError(t, err)
+		assert.Equal(t, img2InternalBase64, resultImg1Internal) // replace cause internal image
+	})
+
+	// 2nd Test: convert external image to base64 -> abort cause external image
+	t.Run("replaceSpecifiedBase64ImagesExternal", func(t *testing.T) {
+		defer tests.PrintCurrentTest(t)()
+
+		resultImg1External, err := mail.AttachmentSrcToBase64DataURI(img1ExternalURL, ctx)
+		assert.Error(t, err)
+		assert.Equal(t, "", resultImg1External) // don't replace cause external image
+	})
+
+	// 3rd Test: generate email body with 1 internal and 1 external image, expect the result to have the internal image replaced with base64 data and the external not replaced
+	t.Run("generateEmailBody", func(t *testing.T) {
+		defer tests.PrintCurrentTest(t)()
+
+		mailBody := "<html><head></head><body><p>Test1</p>" + img1ExternalImg + "<p>Test2</p>" + img2InternalImg + "<p>Test3</p></body></html>"
+		expectedMailBody := "<html><head></head><body><p>Test1</p>" + img1ExternalImg + "<p>Test2</p>" + img2InternalBase64Img + "<p>Test3</p></body></html>"
+		resultMailBody, err := mail.Base64InlineImages(mailBody, ctx)
+
+		assert.NoError(t, err)
+		assert.Equal(t, expectedMailBody, resultMailBody)
+	})
+}