[FIX] EmailSubmission/set fails when underlying mail has LF only headers (#2772)

Author: chibenwa

server/apps/postgres-app/src/test/java/org/apache/james/DistributedPostgresJamesServerTest.java

@@ -110,7 +110,7 @@ void guiceServerShouldUpdateQuota(GuiceJamesServer jamesServer) throws Exception
         int imapPort = jamesServer.getProbe(ImapGuiceProbe.class).getImapPort();
         smtpMessageSender.connect(JAMES_SERVER_HOST, jamesServer.getProbe(SmtpGuiceProbe.class).getSmtpPort())
             .authenticate(USER, PASSWORD)
-            .sendMessageWithHeaders(USER, USER, "header: toto\\r\\n\\r\\n" + Strings.repeat("0123456789\n", 1024));
+            .sendMessageWithHeaders(USER, USER, "header: toto\r\n\r\n" + Strings.repeat("0123456789\r\n", 1024));
         AWAIT.until(() -> testIMAPClient.connect(JAMES_SERVER_HOST, imapPort)
             .login(USER, PASSWORD)
             .select(TestIMAPClient.INBOX)

server/blob/blob-common/src/main/java/org/apache/james/blob/api/Store.java

@@ -34,6 +34,7 @@
 
 import com.github.fge.lambdas.Throwing;
 import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
 import com.google.common.hash.HashCode;
 import com.google.common.hash.HashFunction;
 import com.google.common.io.ByteProcessor;
@@ -90,6 +91,7 @@ public Impl(BlobPartsId.Factory<I> idFactory, Encoder<T> encoder, Decoder<T> dec
 
         @Override
         public Mono<I> save(T t) {
+            Preconditions.checkNotNull(t);
             return Flux.fromStream(encoder.encode(t))
                 .flatMapSequential(this::saveEntry)
                 .collectMap(Tuple2::getT1, Tuple2::getT2)

server/blob/blob-memory/src/main/java/org/apache/james/blob/memory/MemoryBlobStoreDAO.java

@@ -99,9 +99,22 @@ public Mono<Void> save(BucketName bucketName, BlobId blobId, ByteSource content)
                     throw new ObjectStoreIOException("IOException occured", e);
                 }
             })
+            .map(bytes -> checkContentSize(content, bytes))
             .flatMap(bytes -> save(bucketName, blobId, bytes));
     }
 
+    private static byte[] checkContentSize(ByteSource content, byte[] bytes) {
+        try {
+            long preComputedSize = content.size();
+            long realSize = bytes.length;
+            Preconditions.checkArgument(content.size() == realSize,
+                "Difference in size between the pre-computed content can cause other blob stores to fail thus we need to test for alignment. Expecting " + realSize + " but pre-computed size was " + preComputedSize);
+            return bytes;
+        } catch (IOException e) {
+            throw new ObjectStoreIOException("IOException occured", e);
+        }
+    }
+
     @Override
     public Mono<Void> delete(BucketName bucketName, BlobId blobId) {
         Preconditions.checkNotNull(bucketName);

server/blob/mail-store/src/main/java/org/apache/james/blob/mail/MimeMessageStore.java

@@ -26,6 +26,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.SequenceInputStream;
 import java.util.Map;
 import java.util.UUID;
@@ -35,6 +36,7 @@
 import jakarta.mail.MessagingException;
 import jakarta.mail.internet.MimeMessage;
 
+import org.apache.commons.io.output.CountingOutputStream;
 import org.apache.commons.lang3.tuple.Pair;
 import org.apache.james.blob.api.BlobStore;
 import org.apache.james.blob.api.BlobType;
@@ -81,6 +83,7 @@ static class MimeMessageEncoder implements Store.Impl.Encoder<MimeMessage> {
         @Override
         public Stream<Pair<BlobType, Store.Impl.ValueToSave>> encode(MimeMessage message) {
             Preconditions.checkNotNull(message);
+
             return Stream.of(
                 Pair.of(HEADER_BLOB_TYPE, (bucketName, blobStore) -> {
                     try {
@@ -107,7 +110,14 @@ public InputStream openStream() throws IOException {
                         @Override
                         public long size() throws IOException {
                             try {
-                                return message.getSize();
+                                int size = message.getSize();
+                                if (size < 0) {
+                                    // Size is unknown: we need to compute it
+                                    CountingOutputStream countingOutputStream = new CountingOutputStream(OutputStream.nullOutputStream());
+                                    openStream().transferTo(countingOutputStream);
+                                    return countingOutputStream.getCount();
+                                }
+                                return size;
                             } catch (MessagingException e) {
                                 throw new IOException("Failed accessing body size", e);
                             }

server/blob/mail-store/src/test/java/org/apache/james/blob/mail/MimeMessageStoreTest.java

@@ -23,6 +23,8 @@
 import static org.assertj.core.api.Assertions.assertThatCode;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 
 import jakarta.mail.internet.MimeMessage;
@@ -34,6 +36,8 @@
 import org.apache.james.blob.api.Store;
 import org.apache.james.blob.memory.MemoryBlobStoreFactory;
 import org.apache.james.core.builder.MimeMessageBuilder;
+import org.apache.james.server.core.MimeMessageSource;
+import org.apache.james.server.core.MimeMessageWrapper;
 import org.apache.james.util.MimeMessageUtil;
 import org.assertj.core.api.SoftAssertions;
 import org.junit.jupiter.api.BeforeEach;
@@ -102,6 +106,134 @@ void readShouldNotReturnDeletedMessage() throws Exception {
             .isInstanceOf(ObjectNotFoundException.class);
     }
 
+    @Test
+    void shouldSupportStoringMimeMessageWrapperWithLFInHeaders() {
+        MimeMessageSource mimeMessageSource = new MimeMessageSource() {
+            private byte[] bytes = "h1: v1\nh2: v2\r\n\r\nkrd2\r\nuhwevre\r\n".getBytes(StandardCharsets.UTF_8);
+
+            @Override
+            public String getSourceId() {
+                return "ABC";
+            }
+
+            @Override
+            public InputStream getInputStream() {
+                return new ByteArrayInputStream(bytes);
+            }
+
+            @Override
+            public long getMessageSize() {
+                return bytes.length;
+            }
+        };
+        MimeMessage message = new MimeMessageWrapper(mimeMessageSource);
+
+        assertThatCode(() -> testee.save(message).block()).doesNotThrowAnyException();
+    }
+
+    @Test
+    void shouldSupportStoringMimeMessageWrapperWithOnlyOneLine() {
+        MimeMessageSource mimeMessageSource = new MimeMessageSource() {
+            private byte[] bytes = "header: toto\\r\\n\\r\\n0123456789".getBytes(StandardCharsets.UTF_8);
+
+            @Override
+            public String getSourceId() {
+                return "ABC";
+            }
+
+            @Override
+            public InputStream getInputStream() {
+                return new ByteArrayInputStream(bytes);
+            }
+
+            @Override
+            public long getMessageSize() {
+                return bytes.length;
+            }
+        };
+        MimeMessage message = new MimeMessageWrapper(mimeMessageSource);
+
+        assertThatCode(() -> testee.save(message).block()).doesNotThrowAnyException();
+    }
+
+    @Test
+    void shouldSupportStoringMimeMessageWrapperAfterHeaderModification() throws Exception {
+        MimeMessageSource mimeMessageSource = new MimeMessageSource() {
+            private byte[] bytes = "h1: v1\r\nh2: v2\r\n\r\nkrd2\r\nuhwevre\r\n".getBytes(StandardCharsets.UTF_8);
+
+            @Override
+            public String getSourceId() {
+                return "ABC";
+            }
+
+            @Override
+            public InputStream getInputStream() {
+                return new ByteArrayInputStream(bytes);
+            }
+
+            @Override
+            public long getMessageSize() {
+                return bytes.length;
+            }
+        };
+        MimeMessage message = new MimeMessageWrapper(mimeMessageSource);
+        message.addHeader("toto", "tata");
+
+        assertThatCode(() -> testee.save(message).block()).doesNotThrowAnyException();
+    }
+
+    @Test
+    void shouldSupportStoringMimeMessageWrapperAfterHeaderModificationAndLF() throws Exception {
+        MimeMessageSource mimeMessageSource = new MimeMessageSource() {
+            private byte[] bytes = "h1: v1\nh2: v2\r\n\r\nkrd2\r\nuhwevre\r\n".getBytes(StandardCharsets.UTF_8);
+
+            @Override
+            public String getSourceId() {
+                return "ABC";
+            }
+
+            @Override
+            public InputStream getInputStream() {
+                return new ByteArrayInputStream(bytes);
+            }
+
+            @Override
+            public long getMessageSize() {
+                return bytes.length;
+            }
+        };
+        MimeMessage message = new MimeMessageWrapper(mimeMessageSource);
+        message.addHeader("toto", "tata");
+
+        assertThatCode(() -> testee.save(message).block()).doesNotThrowAnyException();
+    }
+
+    @Test
+    void shouldSupportStoringMimeMessageWrapperWithOnlyOneLineAbdAdditionalHeader() throws Exception {
+        MimeMessageSource mimeMessageSource = new MimeMessageSource() {
+            private byte[] bytes = "header: toto\\r\\n\\r\\n0123456789".getBytes(StandardCharsets.UTF_8);
+
+            @Override
+            public String getSourceId() {
+                return "ABC";
+            }
+
+            @Override
+            public InputStream getInputStream() {
+                return new ByteArrayInputStream(bytes);
+            }
+
+            @Override
+            public long getMessageSize() {
+                return bytes.length;
+            }
+        };
+        MimeMessage message = new MimeMessageWrapper(mimeMessageSource);
+        message.addHeader("toto", "tata");
+
+        assertThatCode(() -> testee.save(message).block()).doesNotThrowAnyException();
+    }
+
     @Test
     void deleteShouldNotThrowWhenCalledOnNonExistingData() throws Exception {
         MimeMessagePartsId parts = MimeMessagePartsId.builder()

server/container/core/src/main/java/org/apache/james/server/core/MimeMessageWrapper.java

@@ -215,6 +215,20 @@ protected void loadHeaders() throws MessagingException {
         }
     }
 
+    protected long loadHeadersCounting() throws MessagingException {
+        if (source != null) {
+            try (InputStream in = source.getInputStream();
+                 CountingInputStream countingInputStream = new CountingInputStream(in)) {
+                headers = createInternetHeaders(countingInputStream);
+                return countingInputStream.getCount();
+            } catch (IOException ioe) {
+                throw new MessagingException("Unable to parse headers from stream: " + ioe.getMessage(), ioe);
+            }
+        } else {
+            throw new MessagingException("loadHeaders called for a message with no source, contentStream or stream");
+        }
+    }
+
     /**
      * Load the complete MimeMessage from the internal source.
      * 
@@ -361,12 +375,8 @@ public int getSize() throws MessagingException {
         if (source != null && !bodyModified) {
             try {
                 long fullSize = source.getMessageSize();
-                if (headers == null) {
-                    loadHeaders();
-                }
-                // 2 == CRLF
-                return Math.max(0, (int) (fullSize - initialHeaderSize - HEADER_BODY_SEPARATOR_SIZE));
-
+                long l = loadHeadersCounting();
+                return Math.max(0, (int) (fullSize - l));
             } catch (IOException e) {
                 throw new MessagingException("Unable to calculate message size");
             }