Pre-process iCalendars line by line to avoid OOM on large files

lib/src/androidTest/kotlin/at/bitfire/synctools/icalendar/validation/ICalPreprocessorInstrumentedTest.kt

@@ -0,0 +1,94 @@
+/*
+ * This file is part of bitfireAT/synctools which is released under GPLv3.
+ * Copyright © All Contributors. See the LICENSE and AUTHOR files in the root directory for details.
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+package at.bitfire.synctools.icalendar.validation
+
+import org.junit.Test
+import java.io.Reader
+import java.util.UUID
+
+class ICalPreprocessorInstrumentedTest {
+
+    class VCalendarReaderGenerator(val eventCount: Int = Int.MAX_VALUE) : Reader() {
+        private var stage = 0 // 0 = header, 1 = events, 2 = footer, 3 = done
+        private var eventIdx = 0
+        private var current: String? = null
+        private var pos = 0
+
+        override fun reset() {
+            stage = 0
+            eventIdx = 0
+            current = null
+            pos = 0
+        }
+
+        override fun read(cbuf: CharArray, off: Int, len: Int): Int {
+            var charsRead = 0
+            while (charsRead < len) {
+                if (current == null || pos >= current!!.length) {
+                    current = when (stage) {
+                        0 -> {
+                            stage = 1
+                            """
+                        BEGIN:VCALENDAR
+                        PRODID:-//xyz Corp//NONSGML PDA Calendar Version 1.0//EN
+                        VERSION:2.0
+                        """.trimIndent() + "\n"
+                        }
+                        1 -> {
+                            if (eventIdx < eventCount) {
+                                val event = """
+                                BEGIN:VEVENT
+                                DTSTAMP:19960704T120000Z
+                                UID:${UUID.randomUUID()}
+                                ORGANIZER:mailto:jsmith@example.com
+                                DTSTART:19960918T143000Z
+                                DTEND:19960920T220000Z
+                                STATUS:CONFIRMED
+                                CATEGORIES:CONFERENCE
+                                SUMMARY:Event $eventIdx
+                                DESCRIPTION:Event $eventIdx description
+                                END:VEVENT
+                            """.trimIndent() + "\n"
+                                eventIdx++
+                                event
+                            } else {
+                                stage = 2
+                                null
+                            }
+                        }
+                        2 -> {
+                            stage = 3
+                            "END:VCALENDAR\n"
+                        }
+                        else -> return if (charsRead == 0) -1 else charsRead
+                    }
+                    pos = 0
+                    if (current == null) continue // move to next stage
+                }
+                val charsLeft = current!!.length - pos
+                val toRead = minOf(len - charsRead, charsLeft)
+                current!!.toCharArray(pos, pos + toRead).copyInto(cbuf, off + charsRead)
+                pos += toRead
+                charsRead += toRead
+            }
+            return charsRead
+        }
+
+        override fun close() {
+            // No resources to release
+            current = null
+        }
+    }
+
+    @Test
+    fun testParse_SuperLargeFiles() {
+        val preprocessor = ICalPreprocessor()
+        val reader = VCalendarReaderGenerator()
+        preprocessor.preprocessStream(reader)
+        // no exception called
+    }
+}

lib/src/main/kotlin/at/bitfire/synctools/icalendar/validation/FixInvalidDayOffsetPreprocessor.kt

@@ -10,7 +10,7 @@ package at.bitfire.synctools.icalendar.validation
  * Fixes durations with day offsets with the 'T' prefix.
  * See also https://github.com/bitfireAT/ical4android/issues/77
  */
-class FixInvalidDayOffsetPreprocessor : StreamPreprocessor() {
+class FixInvalidDayOffsetPreprocessor : StreamPreprocessor {
 
     override fun regexpForProblem() = Regex(
         // Examples:

lib/src/main/kotlin/at/bitfire/synctools/icalendar/validation/FixInvalidUtcOffsetPreprocessor.kt

@@ -16,7 +16,7 @@ import java.util.logging.Logger
  * Rewrites values of all TZOFFSETFROM and TZOFFSETTO properties which match [TZOFFSET_REGEXP]
  * so that an hour value of 00 is inserted.
  */
-class FixInvalidUtcOffsetPreprocessor: StreamPreprocessor() {
+class FixInvalidUtcOffsetPreprocessor: StreamPreprocessor {
 
     private val logger
         get() = Logger.getLogger(javaClass.name)

lib/src/main/kotlin/at/bitfire/synctools/icalendar/validation/ICalPreprocessor.kt

@@ -6,16 +6,24 @@
 
 package at.bitfire.synctools.icalendar.validation
 
+import android.util.Log
 import androidx.annotation.VisibleForTesting
+import at.bitfire.synctools.utils.SequenceReader
 import net.fortuna.ical4j.model.Calendar
 import net.fortuna.ical4j.model.Property
 import net.fortuna.ical4j.transform.rfc5545.CreatedPropertyRule
 import net.fortuna.ical4j.transform.rfc5545.DateListPropertyRule
 import net.fortuna.ical4j.transform.rfc5545.DatePropertyRule
 import net.fortuna.ical4j.transform.rfc5545.Rfc5545PropertyRule
+import java.io.BufferedReader
+import java.io.IOException
 import java.io.Reader
+import java.io.StringReader
 import java.util.logging.Level
 import java.util.logging.Logger
+import kotlin.collections.joinToString
+import kotlin.sequences.chunked
+import kotlin.sequences.map
 
 /**
  * Applies some rules to increase compatibility of parsed (incoming) iCalendars:
@@ -40,18 +48,50 @@ class ICalPreprocessor {
         FixInvalidDayOffsetPreprocessor()   // fix things like DURATION:PT2D
     )
 
+    /**
+     * Applies [streamPreprocessors] to a given [String] by calling `fixString()` on each of them.
+     */
+    private fun applyPreprocessors(input: String): String {
+        var newString = input
+        for (preprocessor in streamPreprocessors)
+            newString = preprocessor.fixString(newString)
+        return newString
+    }
+
     /**
      * Applies [streamPreprocessors] to a given [Reader] that reads an iCalendar object
      * in order to repair some things that must be fixed before parsing.
      *
-     * @param original    original iCalendar object
-     * @return            the potentially repaired iCalendar object
+     * The original reader content is processed in chunks of [chunkSize] lines to avoid loading
+     * the whole content into memory at once. If the given [Reader] does not support `reset()`,
+     * the whole content will be loaded into memory anyway.
+     *
+     * @param original original iCalendar object
+     * @return The potentially repaired iCalendar object.
+     * If [original] supports `reset()`,  the returned [Reader] will be a [SequenceReader].
+     * Otherwise, it will be a [StringReader].
      */
-    fun preprocessStream(original: Reader): Reader {
-        var reader = original
-        for (preprocessor in streamPreprocessors)
-            reader = preprocessor.preprocess(reader)
-        return reader
+    fun preprocessStream(original: Reader, chunkSize: Int = 1_000): Reader {
+        val resetSupported = try {
+            original.reset()
+            Log.d("StreamPreprocessor", "Reader supports reset()")
+            true
+        } catch(e: IOException) {
+            // reset is not supported. String will be loaded into memory completely
+            Log.w("StreamPreprocessor", "Reader does not support reset()", e)
+            false
+        }
+
+        if (resetSupported) {
+            val chunkedFixedLines = BufferedReader(original)
+                .lineSequence()
+                .chunked(chunkSize)
+                .map { chunk -> applyPreprocessors(chunk.joinToString("\n")) }
+            return SequenceReader(chunkedFixedLines)
+        } else {
+            // The reader doesn't support reset, so we need to load the whole content into memory
+            return StringReader(applyPreprocessors(original.readText()))
+        }
     }
 
 

lib/src/main/kotlin/at/bitfire/synctools/icalendar/validation/StreamPreprocessor.kt

@@ -6,51 +6,18 @@
 
 package at.bitfire.synctools.icalendar.validation
 
-import java.io.IOException
-import java.io.Reader
-import java.io.StringReader
-import java.util.Scanner
+interface StreamPreprocessor {
 
-abstract class StreamPreprocessor {
-
-    abstract fun regexpForProblem(): Regex?
+    fun regexpForProblem(): Regex?
 
     /**
      * Fixes an iCalendar string.
+     * The icalendar may not be complete, but just a chunk.
+     * Lines won't be incomplete.
      *
      * @param original The complete iCalendar string
      * @return The complete iCalendar string, but fixed
      */
-    abstract fun fixString(original: String): String
-
-    fun preprocess(reader: Reader): Reader {
-        var result: String? = null
-
-        val resetSupported = try {
-            reader.reset()
-            true
-        } catch(_: IOException) {
-            false
-        }
-
-        if (resetSupported) {
-            val regex = regexpForProblem()
-            // reset is supported, no need to copy the whole stream to another String (unless we have to fix the TZOFFSET)
-            if (regex == null || Scanner(reader).findWithinHorizon(regex.toPattern(), 0) != null) {
-                reader.reset()
-                result = fixString(reader.readText())
-            }
-        } else
-            // reset not supported, always generate a new String that will be returned
-            result = fixString(reader.readText())
-
-        if (result != null)
-            // modified or reset not supported, return new stream
-            return StringReader(result)
-
-        // not modified, return original iCalendar
-        reader.reset()
-        return reader
-    }
+    fun fixString(original: String): String
 
 }

lib/src/main/kotlin/at/bitfire/synctools/utils/SequenceReader.kt

@@ -0,0 +1,43 @@
+/*
+ * This file is part of bitfireAT/synctools which is released under GPLv3.
+ * Copyright © All Contributors. See the LICENSE and AUTHOR files in the root directory for details.
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+package at.bitfire.synctools.utils
+
+import java.io.Reader
+
+/**
+ * A [Reader] that allows loading data from a [Sequence] of [String]s.
+ */
+class SequenceReader(lines: Sequence<String>) : Reader() {
+    private var iterator = lines.iterator()
+    private var currentLine: String? = null
+    private var currentPos = 0
+    private var closed = false
+
+    override fun read(cbuf: CharArray, off: Int, len: Int): Int {
+        check(!closed) { "Reader closed" }
+        var charsRead = 0
+        while (charsRead < len) {
+            if (currentLine == null || currentPos >= currentLine!!.length) {
+                if (!iterator.hasNext()) {
+                    if (charsRead == 0) return -1
+                    break
+                }
+                currentLine = iterator.next() + "\n"
+                currentPos = 0
+            }
+            val charsToCopy = minOf(len - charsRead, currentLine!!.length - currentPos)
+            currentLine!!.toCharArray(currentPos, currentPos + charsToCopy).copyInto(cbuf, off + charsRead)
+            currentPos += charsToCopy
+            charsRead += charsToCopy
+        }
+        return charsRead
+    }
+
+    override fun close() {
+        closed = true
+    }
+}