Merge pull request #2 from simplesurance/CORE-10934-attachments-streaming-new

[CORE-10934]-Redusing-memory-attachments-streaming

Author: YuriiRubakha

cnunkedreader.go

@@ -0,0 +1,31 @@
+package email
+
+import "io"
+
+// ChunkedReader provides reading by specified portion size.
+// Only last chunk can be lesser or zero-size same time with EOF or other error
+type ChunkedReader struct {
+	r        io.Reader
+	chunkLen int
+}
+
+func (cr *ChunkedReader) Read(b []byte) (int, error) {
+	accumulatedBytes := 0
+	if len(b) < cr.chunkLen {
+		return 0, io.ErrShortBuffer
+	}
+	var err error
+	var n int
+	for accumulatedBytes < cr.chunkLen && err == nil {
+		n, err = cr.r.Read(b[accumulatedBytes:])
+		accumulatedBytes += n
+	}
+	return accumulatedBytes, err
+}
+
+func NewChunkedReader(r io.Reader, chunkLen int) *ChunkedReader {
+	return &ChunkedReader{
+		r:        r,
+		chunkLen: chunkLen,
+	}
+}

email.go

@@ -262,15 +262,11 @@ func parseMIMEParts(hs textproto.MIMEHeader, b io.Reader) ([]*part, error) {
 // Required parameters include an io.Reader, the desired filename for the attachment, and the Content-Type
 // The function will return the created Attachment for reference, as well as nil for the error, if successful.
 func (e *Email) Attach(r io.Reader, filename string, c string) (a *Attachment, err error) {
-	var buffer bytes.Buffer
-	if _, err = io.Copy(&buffer, r); err != nil {
-		return
-	}
 	at := &Attachment{
 		Filename:    filename,
 		ContentType: c,
 		Header:      textproto.MIMEHeader{},
-		Content:     buffer.Bytes(),
+		Content:     r,
 	}
 	e.Attachments = append(e.Attachments, at)
 	return at, nil
@@ -472,7 +468,9 @@ func (e *Email) Bytes() ([]byte, error) {
 						return nil, err
 					}
 					// Write the base64Wrapped content to the part
-					base64Wrap(ap, a.Content)
+					if err = streamBase64Wrap(ap, a.Content); err != nil {
+						return nil, err
+					}
 				}
 
 				if isMixed || isAlternative {
@@ -494,7 +492,9 @@ func (e *Email) Bytes() ([]byte, error) {
 			return nil, err
 		}
 		// Write the base64Wrapped content to the part
-		base64Wrap(ap, a.Content)
+		if err = streamBase64Wrap(ap, a.Content); err != nil {
+			return nil, err
+		}
 	}
 	if isMixed || isAlternative || isRelated {
 		if err := w.Close(); err != nil {
@@ -702,7 +702,7 @@ type Attachment struct {
 	Filename    string
 	ContentType string
 	Header      textproto.MIMEHeader
-	Content     []byte
+	Content     io.Reader
 	HTMLRelated bool
 }
 
@@ -751,6 +751,46 @@ func base64Wrap(w io.Writer, b []byte) {
 	}
 }
 
+// streamBase64Wrap encodes the attachment content, provided as stream, and wraps it according to RFC 2045 standards (every 76 chars)
+// The output is then written to the specified io.Writer
+func streamBase64Wrap(w io.Writer, r io.Reader) error {
+	// 57 raw bytes per 76-byte base64 line.
+	const maxRaw = 57
+	// Buffer for each line, including trailing CRLF.
+	wrBuffer := make([]byte, MaxLineLength+len("\r\n"))
+	copy(wrBuffer[MaxLineLength:], "\r\n")
+	rdBuffer := make([]byte, maxRaw)
+	var lastRead int
+	var err error
+	cr := NewChunkedReader(r, maxRaw)
+	for {
+		//Reading next 57-bytes chunk.
+		if lastRead, err = cr.Read(rdBuffer); err != nil && !errors.Is(err, io.EOF) {
+			return err
+		}
+		//In case of last chunk jump to last chunk processing
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		//normal chunk processing. It's len=maxRaw exactly
+		base64.StdEncoding.Encode(wrBuffer, rdBuffer)
+		if _, err := w.Write(wrBuffer); err != nil {
+			return err
+		}
+
+	}
+	//last chunk processing. It can be 0<=size<=maxRaw
+	if lastRead > 0 {
+		out := wrBuffer[:base64.StdEncoding.EncodedLen(lastRead)]
+		base64.StdEncoding.Encode(out, rdBuffer[:lastRead])
+		out = append(out, "\r\n"...)
+		if _, err := w.Write(out); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // headerToBytes renders "header" to "buff". If there are multiple values for a
 // field, multiple "Field: value\r\n" lines will be emitted.
 func headerToBytes(buff io.Writer, header textproto.MIMEHeader) {

email_test.go

@@ -717,13 +717,19 @@ TGV0J3MganVzdCBwcmV0ZW5kIHRoaXMgaXMgcmF3IEpQRUcgZGF0YS4=
 	if e.Attachments[0].Filename != a.Filename {
 		t.Fatalf("Incorrect attachment filename %s != %s", e.Attachments[0].Filename, a.Filename)
 	}
-	if !bytes.Equal(e.Attachments[0].Content, a.Content) {
+	var b1, b2 []byte
+	b1, _ = io.ReadAll(e.Attachments[0].Content)
+	b2, _ = io.ReadAll(a.Content)
+	if !bytes.Equal(b1, b2) {
 		t.Fatalf("Incorrect attachment content %#q != %#q", e.Attachments[0].Content, a.Content)
 	}
 	if e.Attachments[1].Filename != b.Filename {
 		t.Fatalf("Incorrect attachment filename %s != %s", e.Attachments[1].Filename, b.Filename)
 	}
-	if !bytes.Equal(e.Attachments[1].Content, b.Content) {
+	var b3, b4 []byte
+	b3, _ = io.ReadAll(e.Attachments[1].Content)
+	b4, _ = io.ReadAll(b.Content)
+	if !bytes.Equal(b3, b4) {
 		t.Fatalf("Incorrect attachment content %#q != %#q", e.Attachments[1].Content, b.Content)
 	}
 }