Make whitespace handling conform to the XML standard. No longer recognize LFCR as single newline. Support #x85 and #2028

Author: pdvrieze

Changelog.md

@@ -20,6 +20,10 @@ Changes:
   only applies if there are 2 or more characters and this would collapse to an
   empty string). The old behaviour only allowed a single character without
   surrounding whitespace.
+- Standards compliant line ending handling. \n\r is not collapsed as a single
+  line end anymore. Characters #x85 and #2028 are now handled as line end (unless
+  preceded by \r)
+  
 - Update kotlinx.io support to 0.9.0, atomicfu to 0.31.0, kotlinx.serialization
   to 1.10.0, kotlinx.benchmark to 0.4.16, kotlin to 2.3.10, junit to 5.14.3.
 - Always expand entities in attribute values (causing an exception if the entity

benchmark/src/jvmTest/kotlin/nl/adaptivity/xmlutil/benchmark/test/SerializationTest.kt

@@ -54,7 +54,7 @@ open class SerializationTest : Serialization() {
     @Test
     fun testAttributePositionRegression() {
         val schemaName = "/xsts/ibmData/valid/S3_12/s3_12v03.xsd"
-        val xml = XML.v1()
+        val xml = XML.v1 { defaultToGenericParser = true }
         val schemaText = String(javaClass.getResourceAsStream(schemaName)!!.readAllBytes())
         val schema = xml.decodeFromString<XSSchema>(schemaText)
 

core/base/src/commonMain/kotlin/nl/adaptivity/xmlutil/core/InOutBuffer.kt

@@ -206,7 +206,7 @@ public interface InOutBuffer {
         val firstChar = readChar()
         require (isXmlWhitespace(firstChar)) { "Expected whitespace, but found non-whitespace: '$firstChar'" }
 
-        while (peek().let { it == '\t'.code ||  it == '\r'.code || it == ' '.code || it == '\r'.code }) {
+        while (peek().let { it == '\t'.code || it == '\r'.code || it == ' '.code || it == '\n'.code || it == 0x85 || it == 0x2028 }) {
             markPeekedAsRead() //needs line ending handling
         }
     }
@@ -220,7 +220,7 @@ public interface InOutBuffer {
         while (c >= 0) {
             when (c.toChar()) {
                 '\t', ' ' -> cnt += 1
-                '\n', '\r' -> {
+                '\n', '\r', '\u0085', '\u2028' -> {
                     if (cnt > 0) skip(cnt)
                     cnt = 0
                     markPeekedAsRead() // does newlines for us

core/base/src/commonMain/kotlin/nl/adaptivity/xmlutil/core/KtXmlReader.kt

@@ -578,8 +578,6 @@ public class KtXmlReader(
                     inOutBuffer.resumeCopySequence()
                 }
 
-                '\r' -> throw AssertionError("Carriage returns should have been normalized out here")
-
                 delimiter -> return
                 else -> inOutBuffer.markPeekedAsRead()
             }

core/base/src/commonMain/kotlin/nl/adaptivity/xmlutil/core/KtXmlWriter.kt

@@ -545,7 +545,7 @@ public class KtXmlWriter(
 
         for (c in text) {
             when (c) {
-                ' ', '\t', '\r', '\n' -> {}
+                ' ', '\t', '\r', '\n', '\u0085', '\u2028' -> {}
                 else -> throw IllegalArgumentException("\"$text\" is not ignorable whitespace")
             }
         }

core/base/src/commonMain/kotlin/nl/adaptivity/xmlutil/core/internal/StringInOutBuffer.kt

@@ -162,8 +162,10 @@ public class StringInOutBuffer(public val input: CharSequence): InOutBuffer {
     private fun peekCommon(bufPos: Int): Int {
         return if (bufPos >= input.length) -1
         else {
-            val c = input[bufPos]
-            if (c == '\r') '\n'.code else c.code
+            when (val c = input[bufPos]) {
+                '\r', '\u0085', '\u2028' -> '\n'.code
+                else -> c.code
+            }
         }
     }
 
@@ -201,66 +203,59 @@ public class StringInOutBuffer(public val input: CharSequence): InOutBuffer {
     }
 
     override fun markPeekedAsRead() {
-        fun handleLineEnd(complement: Int, oldPos: Int) {
-            if (peek(1) == complement) {
-                pauseCopySequence()
-                addToCopySequence('\n')
-                offset = oldPos + 2
-                resumeCopySequence()
-            } else {
-                if (copySequenceState == State.ACTIVE && complement == '\n'.code) {
-                    pauseCopySequence()
-                    addToCopySequence('\n')
-                    offset = oldPos + 1
-                    resumeCopySequence()
-                } else {
-                    offset = oldPos + 1
-                }
-            }
-
-            line += 1
-            lastColumnStart = offset
-        }
 
         val oldPos = offset
         val peeked = input[oldPos]
         when (peeked) {
-            '\r' -> handleLineEnd('\n'.code, oldPos)
-            '\n' -> handleLineEnd('\r'.code, oldPos)
+            '\r' -> handle2CharLineEnd(oldPos)
+            '\n', '\u0085', '\u2028' -> handleLineEnd(oldPos + 1)
             else -> offset = oldPos + 1
         }
     }
 
-    /** Does never read more than needed  */
-    override fun read(): Int {
-        fun handleLineEnd(complement: Int, oldPos: Int) {
-            val inc = if(peek(2) == complement) 2 else 1
-            val newPos = oldPos + inc
-            if (copySequenceState == State.ACTIVE && (inc == 2 || complement == '\n'.code)) {
-                pauseCopySequence()
-                addToCopySequence('\n')
-                offset = newPos
-                resumeCopySequence()
-            } else {
-                offset = newPos
-            }
-            lastColumnStart = offsetBase + newPos
-            line += 1
+    private fun handleLineEnd(newPos: Int) {
+        // this cannot use peek as peek normalized (so the test will not work)
+        if (copySequenceState == State.ACTIVE && input[offset] != '\n') {
+            pauseCopySequence()
+            addToCopySequence('\n')
+            offset = newPos
+            resumeCopySequence()
+        } else {
+            offset = newPos
         }
+        lastColumnStart = newPos
+        line += 1
+    }
 
+    private fun handle2CharLineEnd(oldPos: Int) {
+        val nextChar = peek(1)
+        val inc = if(nextChar == 0xA || nextChar == 0x85) 2 else 1
+        handleLineEnd(oldPos + inc)
+    }
+
+
+    /** Does never read more than needed  */
+    override fun read(): Int {
         // In this case we *may* need the right buffer, otherwise not
         // optimize this implementation for the "happy" path
         val oldPos = offset
         if (oldPos >= input.length) return -1
 
         when (val char = input[oldPos]) {
             '\r' -> { // as \r is always transformed to \n, this requires a stringBuilder.
-                handleLineEnd('\n'.code, oldPos)
+                handle2CharLineEnd(oldPos)
+                return '\n'.code
+            }
+
+            '\u0085',
+            '\u2028' -> {
+                handleLineEnd(oldPos + 1)
                 return '\n'.code
             }
 
             '\n' -> {
-                handleLineEnd('\r'.code, oldPos)
+                offset = oldPos + 1
+                lastColumnStart = oldPos + 1
                 return '\n'.code
             }
 

core/base/src/commonMain/kotlin/nl/adaptivity/xmlutil/core/internal/SwappedInOutBuffer.kt

@@ -240,17 +240,20 @@ public class SwappedInOutBuffer(public val reader: Reader): InOutBuffer {
         return peekCommon(srcBufPos + offset)
     }
 
+    private fun rawPeek(bufPos: Int):Int = when {
+        bufPos >= srcBufCount -> -1
+        bufPos >= BUF_SIZE -> bufRight[bufPos - BUF_SIZE].code
+        else -> bufLeft[bufPos].code
+    }
+
     private fun peekCommon(bufPos: Int): Int {
         // end of buffer. This implies bufPos < 2 * BUF_SIZE
         if (bufPos >= srcBufCount) return -1
-        val c = when {
-            bufPos >= BUF_SIZE -> bufRight[bufPos - BUF_SIZE]
-            else -> bufLeft[bufPos]
-        }
+        val c = rawPeek(bufPos)
 
         return when (c) {
-            '\r' -> '\n'.code
-            else -> c.code
+            '\r'.code, 0x85, 0x2028 -> '\n'.code
+            else -> c
         }
     }
 
@@ -301,58 +304,51 @@ public class SwappedInOutBuffer(public val reader: Reader): InOutBuffer {
     }
 
     override fun markPeekedAsRead() {
-        fun handleLineEnd(complement: Int, oldPos: Int) {
-            if (peek(1) == complement) {
-                if (copySequenceState == State.ACTIVE) {
-                    pauseCopySequence()
-                    srcBufPos = oldPos + 2
-                    addToCopySequence('\n')
-                    resumeCopySequence()
-                } else {
-                    srcBufPos = oldPos + 2
-                }
-            } else {
-                if (copySequenceState == State.ACTIVE && complement == '\n'.code) {
-                    pauseCopySequence()
-                    addToCopySequence('\n')
-                    srcBufPos = oldPos + 1
-                    resumeCopySequence()
-                } else {
-                    srcBufPos = oldPos + 1
-                }
-            }
-
-            line += 1
-            lastColumnStart = offset
-        }
 
         val oldPos = srcBufPos
-        val peeked = if (oldPos < BUF_SIZE) bufLeft[oldPos] else bufRight[oldPos - BUF_SIZE]
+        val peeked = rawPeek (oldPos).toChar()
         when (peeked) {
-            '\r' -> handleLineEnd('\n'.code, oldPos)
-            '\n' -> handleLineEnd('\r'.code, oldPos)
+            '\r' -> handle2CharLineEnd(oldPos)
+            '\u0085', '\u2028' -> {
+                bufLeft[oldPos] = '\n'
+                handleLineEnd(oldPos + 1)
+            }
+
+            '\n' -> handleLineEnd(oldPos + 1)
+
             else -> srcBufPos = oldPos + 1
         }
         if (srcBufPos >= BUF_SIZE) swapInputBuffer()
     }
 
-    /** Does never read more than needed  */
-    override fun read(): Int {
-        fun handleLineEnd(complement: Int, oldPos: Int) {
-            val inc = if(peek(2) == complement) 2 else 1
-            val newPos = oldPos + inc
-            if (copySequenceState == State.ACTIVE && (inc == 2 || complement == '\n'.code)) {
-                pauseCopySequence()
-                addToCopySequence('\n')
-                srcBufPos = newPos
-                resumeCopySequence()
-            } else {
-                srcBufPos = newPos
+    private fun handleLineEnd(newPos: Int) {
+        if (copySequenceState == State.ACTIVE && rawPeek(srcBufPos) != '\n'.code) {
+            pauseCopySequence()
+            addToCopySequence('\n')
+            srcBufPos = newPos
+            resumeCopySequence()
+        } else {
+            srcBufPos = newPos
+        }
+        lastColumnStart = offsetBase + newPos
+        line += 1
+    }
+
+    private fun handle2CharLineEnd(oldPos: Int) {
+        val nextChar = peek(1)
+        val inc = when (nextChar) {
+            0xA, 0x85 -> 2
+
+            else -> {
+                bufLeft[oldPos] = '\n'
+                1
             }
-            lastColumnStart = offsetBase + newPos
-            line += 1
         }
+        handleLineEnd(oldPos + inc)
+    }
 
+    /** Does never read more than needed  */
+    override fun read(): Int {
         // In this case we *may* need the right buffer, otherwise not
         // optimize this implementation for the "happy" path
         var oldPos = srcBufPos
@@ -367,12 +363,19 @@ public class SwappedInOutBuffer(public val reader: Reader): InOutBuffer {
 
         when (val char = bufLeft[oldPos]) {
             '\r' -> { // as \r is always transformed to \n, this requires a stringBuilder.
-                handleLineEnd('\n'.code, oldPos)
+                handle2CharLineEnd(oldPos)
+                return '\n'.code
+            }
+
+            '\u0085',
+            '\u2028' -> {
+                bufLeft[srcBufPos] = '\n'
+                handleLineEnd(oldPos + 1)
                 return '\n'.code
             }
 
             '\n' -> {
-                handleLineEnd('\r'.code, oldPos)
+                handleLineEnd(oldPos + 1)
                 return '\n'.code
             }
 
@@ -383,7 +386,6 @@ public class SwappedInOutBuffer(public val reader: Reader): InOutBuffer {
         }
     }
 
-
     private fun readUntilFullOrEOF(buffer: CharArray): Int {
         val bufSize = buffer.size
         var totalRead: Int = reader.read(buffer, 0, bufSize)

core/base/src/commonTest/kotlin/nl/adaptivity/xmlutil/test/TestKtXmlReader.kt

@@ -27,6 +27,7 @@ import nl.adaptivity.xmlutil.core.KtXmlWriter
 import nl.adaptivity.xmlutil.core.impl.multiplatform.StringReader
 import nl.adaptivity.xmlutil.core.impl.multiplatform.StringWriter
 import nl.adaptivity.xmlutil.core.impl.multiplatform.use
+import nl.adaptivity.xmlutil.core.internal.StringInOutBuffer
 import nl.adaptivity.xmlutil.test.multiplatform.Target
 import nl.adaptivity.xmlutil.test.multiplatform.testTarget
 import kotlin.test.Test
@@ -81,7 +82,7 @@ class TestKtXmlReader : TestCommonReader() {
     }
 
     @Test
-    fun testXmlDecl() {
+    fun testXmlDeclReader() {
         val reader = KtXmlReader(StringReader("<?xml version=\"1.1\" standalone=\"yes\"?>\r<foo>bar</foo>"))
         assertEquals(EventType.START_DOCUMENT, reader.next())
         assertEquals("1.1", reader.version)
@@ -100,6 +101,54 @@ class TestKtXmlReader : TestCommonReader() {
         assertEquals(2, reader.getLineNumber())
     }
 
+    @Test
+    fun testXmlDeclString() {
+        val reader = KtXmlReader(StringInOutBuffer("<?xml version=\"1.1\" standalone=\"yes\"?>\r<foo>bar</foo>"))
+        assertEquals(EventType.START_DOCUMENT, reader.next())
+        assertEquals("1.1", reader.version)
+        assertEquals(true, reader.standalone)
+        assertEquals(39, reader.getColumnNumber())
+        assertEquals(1, reader.getLineNumber())
+
+        assertEquals(EventType.IGNORABLE_WHITESPACE, reader.next())
+        assertEquals("\n", reader.text)
+        assertEquals(1, reader.getColumnNumber())
+        assertEquals(2, reader.getLineNumber())
+
+        assertEquals(EventType.START_ELEMENT, reader.next())
+        assertEquals("foo", reader.localName)
+        assertEquals(6, reader.getColumnNumber())
+        assertEquals(2, reader.getLineNumber())
+    }
+
+    fun testParseNewline(newLine: String, count: Int = 1) {
+        val xml = "<tag>$newLine</tag>"
+        val reader1 = KtXmlReader(StringReader(xml))
+        assertEquals(EventType.START_ELEMENT, reader1.nextTag())
+        assertEquals(EventType.IGNORABLE_WHITESPACE, reader1.next())
+        assertEquals("\n".repeat(count), reader1.text)
+        assertEquals(EventType.END_ELEMENT, reader1.next())
+        assertEquals(1+count, reader1.getLineNumber())
+        assertEquals(7, reader1.getColumnNumber())
+
+        val reader2 = KtXmlReader(StringInOutBuffer(xml))
+        assertEquals(EventType.START_ELEMENT, reader2.nextTag())
+        assertEquals(EventType.IGNORABLE_WHITESPACE, reader2.next())
+        assertEquals("\n".repeat(count), reader2.text)
+        assertEquals(EventType.END_ELEMENT, reader2.next())
+        assertEquals(1+count, reader2.getLineNumber())
+        assertEquals(7, reader2.getColumnNumber())
+    }
+
+    @Test fun testParseNewlineLF() = testParseNewline("\n")
+    @Test fun testParseNewlineCR() = testParseNewline("\r")
+    @Test fun testParseNewlineCRLF() = testParseNewline("\r\n")
+    @Test fun testParseNewlineCR85() = testParseNewline("\r\u0085")
+    @Test fun testParseNewlineCR2028() = testParseNewline("\r\u2028")
+    @Test fun testParseNewline85() = testParseNewline("\u0085")
+    @Test fun testParseNewline2028() = testParseNewline("\u2028")
+    @Test fun testParseNewlineLFCR() = testParseNewline("\n\r", 2)
+
     @Test
     fun testUnquotedAttributeValues() {
         val xml = "<tag attr='foo' attr2=b&lt;ar attr3=baz/>"